Repository navigation
chore(main): release 0.245.0 #3326
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Verify | |
| on: | |
| pull_request: | |
| # Labelling a pull request `full-verify` (or removing the label) changes how much it runs. | |
| types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] | |
| merge_group: | |
| # Nightly, main also runs the 5k and 10k-model compile jobs that pull requests and merge groups | |
| # leave out, and opens or updates an issue when it fails. | |
| schedule: | |
| - cron: "17 3 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| # Each merge group has its own gh-readonly-queue ref, so queued groups never cancel each other; the | |
| # queue itself discards a group's run when it rebuilds the group. A push to a pull request cancels | |
| # its older run. Adding or removing any label other than `full-verify` starts a run in a group of its | |
| # own that verifies nothing, so it never cancels the pull request's real run. The nightly run has | |
| # its own group, so a manual run on main and the nightly never cancel each other. | |
| concurrency: | |
| group: >- | |
| verify-${{ github.workflow }}-${{ github.event.merge_group.head_ref || github.ref }}${{ | |
| (github.event.action == 'labeled' || github.event.action == 'unlabeled') | |
| && github.event.label.name != 'full-verify' && format('-label-{0}', github.run_id) || '' }}${{ | |
| github.event_name == 'schedule' && '-nightly' || '' }} | |
| cancel-in-progress: ${{ github.event_name != 'merge_group' }} | |
| jobs: | |
| changes: | |
| name: Classify verification scope | |
| # A label other than `full-verify` changes nothing this workflow verifies. | |
| if: ${{ !((github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name != 'full-verify') }} | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| outputs: | |
| full: ${{ steps.depth.outputs.full }} | |
| partial: ${{ steps.depth.outputs.partial }} | |
| checks: ${{ steps.scope.outputs.checks }} | |
| compiler_differential: ${{ steps.scope.outputs.compiler_differential }} | |
| duckdb: ${{ steps.scope.outputs.duckdb }} | |
| postgres: ${{ steps.scope.outputs.postgres }} | |
| clickhouse: ${{ steps.scope.outputs.clickhouse }} | |
| performance: ${{ steps.scope.outputs.performance }} | |
| format_performance: ${{ steps.scope.outputs.format_performance }} | |
| rules_performance: ${{ steps.scope.outputs.rules_performance }} | |
| wheels: ${{ steps.scope.outputs.wheels }} | |
| recovery: ${{ steps.scope.outputs.recovery }} | |
| ui: ${{ steps.scope.outputs.ui }} | |
| podman: ${{ steps.scope.outputs.podman }} | |
| cold_models: ${{ steps.tiers.outputs.cold_models }} | |
| dense_models: ${{ steps.tiers.outputs.dense_models }} | |
| cache_models: ${{ steps.tiers.outputs.cache_models }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Select relevant suites | |
| id: scope | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} | |
| HEAD_REF: ${{ github.event.pull_request.head.ref }} | |
| HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} | |
| run: | | |
| write_scope() { | |
| { | |
| echo "duckdb=$1" | |
| echo "postgres=$2" | |
| echo "performance=$3" | |
| echo "format_performance=$4" | |
| echo "rules_performance=$5" | |
| echo "checks=$6" | |
| echo "wheels=$7" | |
| echo "compiler_differential=$8" | |
| echo "clickhouse=$9" | |
| echo "recovery=${10}" | |
| echo "ui=${11}" | |
| echo "podman=${12}" | |
| } >> "${GITHUB_OUTPUT}" | |
| } | |
| if [[ "${EVENT_NAME}" != "pull_request" && "${EVENT_NAME}" != "merge_group" ]]; then | |
| write_scope true true true true true true true true true true true true | |
| exit 0 | |
| fi | |
| # A merge group's head is its base plus every queued pull request ahead of and including | |
| # this one, so the base..head diff covers the whole group. Anything that prevents an exact | |
| # diff runs everything rather than risk skipping a suite. Three dots compare with the merge | |
| # base, so changes that landed on main after a branch started are not counted as its own. | |
| changed_paths="${RUNNER_TEMP}/changed-paths" | |
| if [[ -z "${BASE_SHA}" || -z "${HEAD_SHA}" ]] || \ | |
| ! git diff --name-only -z "${BASE_SHA}...${HEAD_SHA}" > "${changed_paths}"; then | |
| echo "Could not diff ${BASE_SHA:-<none>}...${HEAD_SHA:-<none>}; running every suite" >&2 | |
| write_scope true true true true true true true true true true true true | |
| exit 0 | |
| fi | |
| # Succeeds when every change is a release metadata change: the manifest, the changelog, | |
| # or version lines of the package and lock files, so no dependency or build setting | |
| # moves. Keep the file list in sync with the release checks in release-please.yml. | |
| release_metadata_only() { | |
| local path diff offending changed_any=false | |
| local metadata_paths="${RUNNER_TEMP}/release-metadata-paths" | |
| local version='[0-9]+\.[0-9]+\.[0-9]+' | |
| local version_line="^[-+](version = \"${version}\"" | |
| version_line+="| \"[a-z0-9_-]+==${version}\", # x-release-please-version)\$" | |
| git diff --name-only --no-renames -z "${BASE_SHA}...${HEAD_SHA}" > "${metadata_paths}" \ | |
| || return 1 | |
| while IFS= read -r -d '' path; do | |
| changed_any=true | |
| case "${path}" in | |
| .release-please-manifest.json|CHANGELOG.md) ;; | |
| crates/sqlbuild-cli/Cargo.toml|crates/sqlbuild-python/Cargo.toml|Cargo.lock|pyproject.toml|uv.lock) | |
| diff="$(git diff -U0 --text --no-color --no-renames "${BASE_SHA}...${HEAD_SHA}" -- "${path}")" \ | |
| || return 1 | |
| offending="$(awk '/^@@/ { hunk = 1; next } hunk && /^[-+]/' <<< "${diff}" \ | |
| | grep -Ev "${version_line}" || true)" | |
| if [[ -n "${offending}" ]]; then | |
| echo "${path} changes more than version lines:" >&2 | |
| echo "${offending}" >&2 | |
| return 1 | |
| fi | |
| ;; | |
| *) return 1 ;; | |
| esac | |
| done < "${metadata_paths}" | |
| [[ "${changed_any}" == "true" ]] | |
| } | |
| # Release Please pull requests change only release metadata on top of product code that | |
| # already passed every suite before it reached main, so the suites are skipped. Their | |
| # required Verify status is posted only by the trusted release checks in | |
| # release-please.yml after the metadata, version and distribution checks pass; the | |
| # verify job below reports under another name on Release Please branches. | |
| # A merge group is judged by its content alone: one whose whole diff is release metadata | |
| # carries only a release pull request whose product code already passed on main. | |
| if { [[ "${EVENT_NAME}" == "merge_group" ]] \ | |
| || [[ "${EVENT_NAME}" == "pull_request" \ | |
| && "${HEAD_REPOSITORY}" == "${GITHUB_REPOSITORY}" \ | |
| && "${HEAD_REF}" == release-please--branches--* ]]; } && release_metadata_only; then | |
| echo "The change is only release metadata; skipping the suites" \ | |
| | tee -a "${GITHUB_STEP_SUMMARY}" | |
| write_scope false false false false false false false false false false false false | |
| exit 0 | |
| fi | |
| duckdb=false | |
| postgres=false | |
| clickhouse=false | |
| performance=false | |
| format_performance=false | |
| rules_performance=false | |
| wheels=false | |
| compiler_differential=false | |
| recovery=false | |
| ui=false | |
| podman=false | |
| broad=false | |
| changed=false | |
| # A change that only touches the website, root-level Markdown, the README demo GIFs or the | |
| # skill pages generated from the website docs skips the Python checks and test suites. | |
| # The website builds in its own workflow. | |
| docs_only=true | |
| # A change that only touches the web UI (and documentation) runs the UI checks and the | |
| # wheels that embed its build, but not the Python and native checks it cannot affect. | |
| ui_only=true | |
| while IFS= read -r -d '' path; do | |
| changed=true | |
| case "${path}" in | |
| website/*|.github/workflows/website.yml|.github/demos/*) ;; | |
| src/sqlbuild/.agents/skills/sqlbuild/references/docs/*.md) ;; | |
| */*) docs_only=false ;; | |
| *.md) ;; | |
| *) docs_only=false ;; | |
| esac | |
| case "${path}" in | |
| ui/*|website/*|.github/workflows/website.yml|.github/demos/*) ;; | |
| src/sqlbuild/.agents/skills/sqlbuild/references/docs/*.md) ;; | |
| */*) ui_only=false ;; | |
| *.md) ;; | |
| *) ui_only=false ;; | |
| esac | |
| case "${path}" in | |
| # Rust test crates and modules that only `cargo test` compiles; they are not in the | |
| # native build identity and never reach a wheel. | |
| crates/sqlbuild-cli/tests/*|crates/sqlbuild-orchestration/tests/*) ;; | |
| crates/*|ui/*|Cargo.toml|Cargo.lock|pyproject.toml|.github/workflows/verify.yml|.github/workflows/build-distributions.yml|deploy/Dockerfile|deploy/project/*|deploy/compose*|compose.yml|.dockerignore|Makefile|tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py|tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/*|benchmarks/duckshop/edge_cases/orchestration/container_deployment.json) | |
| wheels=true | |
| ;; | |
| esac | |
| case "${path}" in | |
| ui/*|clients/typescript/*|.github/workflows/verify.yml|website/src/assets/logo-light.png|website/src/assets/logo-dark.png|website/public/favicon.png) | |
| # The web UI's own checks and unit tests; the wheels embed its build. It compiles | |
| # against the generated control client, so client changes run them too, and it | |
| # imports the website's logos and favicon, so moving them fails its build. | |
| ui=true | |
| ;; | |
| esac | |
| case "${path}" in | |
| crates/sqlbuild-orchestration/*|crates/sqlbuild-process-ownership/*|crates/sqlbuild-cli/*|Cargo.toml|Cargo.lock|.github/workflows/verify.yml) | |
| recovery=true | |
| ;; | |
| esac | |
| case "${path}" in | |
| # The reference stack and the Docker launcher, which Podman serves through its | |
| # Docker-compatible API. | |
| # Every path here also selects the wheels the job installs. | |
| deploy/Dockerfile|deploy/project/*|deploy/compose*|compose.yml|.dockerignore|crates/sqlbuild-cli/src/workers/*|crates/sqlbuild-cli/src/server_config/*|tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py|tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/*|.github/workflows/verify.yml) | |
| podman=true | |
| ;; | |
| esac | |
| case "${path}" in | |
| scripts/compiler_differential/*|scripts/run_compiler_differential.py|tests/*/scripts/compiler_differential/*|tests/goldens/*|tests/e2e/fixtures/*|website/examples/*|.github/workflows/verify.yml|Makefile) | |
| compiler_differential=true | |
| ;; | |
| esac | |
| case "${path}" in | |
| crates/sqlbuild-cli/tests/*|crates/sqlbuild-orchestration/tests/*) | |
| # Native checks and the scheduler recovery suite run these tests; the Python, | |
| # warehouse, wheel and timing suites cannot see them. | |
| ;; | |
| crates/*|Cargo.toml|Cargo.lock|pyproject.toml|uv.lock) | |
| broad=true | |
| performance=true | |
| format_performance=true | |
| rules_performance=true | |
| ;; | |
| src/sqlbuild/compiler/*|src/sqlbuild/adapter/type_system/*) | |
| broad=true | |
| performance=true | |
| rules_performance=true | |
| ;; | |
| benchmarks/duckshop-10k/*|src/sqlbuild/executor/testing/*|scripts/cold_compile_performance/*|scripts/compile_performance_ratio/*|scripts/compare_compile_performance.py|tests/e2e/src/sqlbuild/cli/commands/main/compile/*) | |
| broad=true | |
| performance=true | |
| ;; | |
| src/sqlbuild/executor/build/*|src/sqlbuild/executor/run/*|src/sqlbuild/presentation/*) | |
| # Inspection, plan, and build guards run inside the fresh-process compile jobs. | |
| broad=true | |
| performance=true | |
| ;; | |
| src/sqlbuild/rule_engine/*|src/sqlbuild/lint/*|scripts/rules_benchmark/*|scripts/benchmark_rules_ci.py) | |
| broad=true | |
| performance=true | |
| format_performance=true | |
| rules_performance=true | |
| ;; | |
| tests/e2e/src/sqlbuild/cli/commands/main/format/*) | |
| duckdb=true | |
| format_performance=true | |
| ;; | |
| tests/e2e/src/sqlbuild/cli/commands/main/postgres/*|tests/integration/src/sqlbuild/adapters/postgres/*|tests/integration/src/sqlbuild/runtime/execution_history/*) | |
| postgres=true | |
| ;; | |
| src/sqlbuild/adapters/clickhouse/*|tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/*|tests/integration/src/sqlbuild/adapters/clickhouse/*) | |
| clickhouse=true | |
| ;; | |
| tests/e2e/*) | |
| duckdb=true | |
| ;; | |
| tests/unit/*|tests/integration/*) | |
| ;; | |
| ui/*) | |
| ;; | |
| .github/workflows/verify.yml|Makefile) | |
| performance=true | |
| format_performance=true | |
| rules_performance=true | |
| ;; | |
| .github/*|docs/*|website/*|*.md|.gitignore|LICENSE) | |
| ;; | |
| *) | |
| broad=true | |
| ;; | |
| esac | |
| done < "${changed_paths}" | |
| if [[ "${changed}" != "true" ]]; then | |
| write_scope true true true true true true true true true true true true | |
| exit 0 | |
| fi | |
| checks=true | |
| if [[ "${docs_only}" == "true" || "${ui_only}" == "true" ]]; then | |
| checks=false | |
| fi | |
| if [[ "${broad}" == "true" ]]; then | |
| duckdb=true | |
| postgres=true | |
| clickhouse=true | |
| format_performance=true | |
| compiler_differential=true | |
| fi | |
| write_scope \ | |
| "${duckdb}" \ | |
| "${postgres}" \ | |
| "${performance}" \ | |
| "${format_performance}" \ | |
| "${rules_performance}" \ | |
| "${checks}" \ | |
| "${wheels}" \ | |
| "${compiler_differential}" \ | |
| "${clickhouse}" \ | |
| "${recovery}" \ | |
| "${ui}" \ | |
| "${podman}" | |
| # Pull requests and merge groups run the 3k-model compile jobs and the 1k dense job; the | |
| # nightly run and manual runs add 5k and 10k. Budgets and ratio gates are the same in both. | |
| - name: Choose the compile-performance model counts | |
| id: tiers | |
| env: | |
| ALL: ${{ github.event_name != 'pull_request' && github.event_name != 'merge_group' }} | |
| run: | | |
| if [[ "${ALL}" == "true" ]]; then | |
| echo "cold_models=[3000, 5000, 10000]" | |
| echo "dense_models=[1000, 3000, 5000, 10000]" | |
| echo "cache_models=[3000, 5000, 10000]" | |
| else | |
| echo "cold_models=[3000]" | |
| echo "dense_models=[1000, 3000]" | |
| echo "cache_models=[3000]" | |
| fi >> "${GITHUB_OUTPUT}" | |
| # Pull requests run the fast Linux core: static checks, unit and integration tests, native | |
| # workspace checks and the UI checks. The rest of the matrix (wheels, Compose, Windows, macOS, | |
| # DuckDB E2E, warehouse, compiler-golden and timing jobs) runs only once a pull request is | |
| # labelled `full-verify` and is not a draft, and always on merge groups and manual runs. A | |
| # run that leaves out suites its change needs reports under another name, so the required | |
| # Verify stays missing and blocks the merge until the full matrix has passed. When the | |
| # repository variable SQLBUILD_MERGE_QUEUE is true, the merge queue runs the full matrix on | |
| # every group before it merges, so a passing fast core reports as Verify and each pull | |
| # request runs the heavy suites once, in the queue. Unsetting the variable restores the old | |
| # behaviour. | |
| # GitHub takes the required Verify from the newest Verify check suite on the head. Re-running | |
| # an older suite never repaints it: to recover a head whose newest suite reported another | |
| # name, start a new suite (push, or remove and re-add full-verify), not a rerun. | |
| # The label and draft state come from the pull request as it is now, not the event payload: | |
| # `gh pr create --label` can deliver `labeled` before `opened`, and the later run cancels the | |
| # earlier one, so the surviving run must still see the label. | |
| - name: Choose the fast core or the full matrix | |
| id: depth | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| EVENT_NAME: ${{ github.event_name }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| HEAVY_SELECTED: >- | |
| ${{ steps.scope.outputs.duckdb == 'true' || steps.scope.outputs.postgres == 'true' || steps.scope.outputs.clickhouse == 'true' || steps.scope.outputs.performance == 'true' || steps.scope.outputs.format_performance == 'true' || steps.scope.outputs.rules_performance == 'true' || steps.scope.outputs.wheels == 'true' || steps.scope.outputs.compiler_differential == 'true' || steps.scope.outputs.recovery == 'true' }} | |
| run: | | |
| FULL=true | |
| if [[ "${EVENT_NAME}" == "pull_request" ]]; then | |
| FULL="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" \ | |
| --jq 'any(.labels[]; .name == "full-verify") and (.draft | not)')" | |
| fi | |
| partial=false | |
| if [[ "${FULL}" != "true" && "${HEAVY_SELECTED}" == "true" ]]; then | |
| partial=true | |
| echo "Fast core only: the full-verify label, or the merge queue when it verifies groups, runs the full matrix" \ | |
| | tee -a "${GITHUB_STEP_SUMMARY}" | |
| fi | |
| { | |
| echo "full=${FULL}" | |
| echo "partial=${partial}" | |
| } >> "${GITHUB_OUTPUT}" | |
| lockfile: | |
| name: Lockfile consistency | |
| if: ${{ !((github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name != 'full-verify') }} | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| # Release Please pushes its version bump before release-please.yml refreshes Cargo.lock and | |
| # uv.lock in a commit of its own, so a bump head always disagrees with its lockfiles. That | |
| # head waits for the refresh commit, whose run checks the lockfiles. The refresh step itself | |
| # runs cargo metadata --locked and uv sync --locked before it commits, and the merge queue | |
| # always checks the lockfiles. | |
| - name: Wait for the release lockfile refresh | |
| id: release | |
| if: >- | |
| github.event_name == 'pull_request' | |
| && github.event.pull_request.head.repo.full_name == github.repository | |
| && startsWith(github.event.pull_request.head.ref, 'release-please--branches--') | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| subject="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${HEAD_SHA}" \ | |
| --jq '.commit.message | split("\n")[0]')" | |
| if [[ "${subject}" != "chore: refresh release lockfiles" ]]; then | |
| echo "pending=true" >> "${GITHUB_OUTPUT}" | |
| echo "Version bump ${HEAD_SHA}: the lockfile refresh commit's run checks the lockfiles" \ | |
| | tee -a "${GITHUB_STEP_SUMMARY}" | |
| fi | |
| - uses: actions/checkout@v4 | |
| if: steps.release.outputs.pending != 'true' | |
| - uses: ./.github/actions/setup-uv | |
| if: steps.release.outputs.pending != 'true' | |
| with: | |
| version: "0.12.3" | |
| - uses: dtolnay/rust-toolchain@master | |
| if: steps.release.outputs.pending != 'true' | |
| with: | |
| toolchain: stable | |
| - name: Install Python | |
| if: steps.release.outputs.pending != 'true' | |
| run: uv python install 3.12 | |
| - name: Verify lockfile | |
| if: steps.release.outputs.pending != 'true' | |
| run: uv lock --check --python 3.12 | |
| - name: Verify native workspace lockfile | |
| if: steps.release.outputs.pending != 'true' | |
| run: cargo metadata --locked --format-version 1 > /dev/null | |
| # The extension and CLI build once here; the static job and every test shard install these | |
| # wheels instead of compiling them again. | |
| native_build: | |
| name: Build the extension and CLI wheels | |
| needs: [changes, lockfile] | |
| if: needs.changes.outputs.checks == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| # Restores the native dependency builds that rust-cache.yml saves on main; workspace crates | |
| # always build from source. Verify never saves: a pull request or merge queue cache is | |
| # visible only to its own ref. Jobs that run the same cargo builds share a key. | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-checks | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install the build tools without building the packages | |
| run: uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli | |
| - name: Build the extension and CLI wheels | |
| run: | | |
| uv run --no-sync maturin build --release --compatibility linux --out dist | |
| uv run --no-sync maturin build --manifest-path crates/sqlbuild-cli/Cargo.toml --release --compatibility linux --out dist | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: verify-native-wheels | |
| path: dist/*.whl | |
| if-no-files-found: error | |
| compression-level: 0 | |
| # Long enough for "Re-run failed jobs" on a shard, which reuses this run's wheels. | |
| retention-days: 3 | |
| overwrite: true | |
| checks: | |
| name: Static checks | |
| needs: [changes, lockfile, native_build] | |
| if: needs.changes.outputs.checks == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 30 | |
| env: | |
| # A syncing `uv run` would replace the installed wheels with a fresh editable build. | |
| UV_NO_SYNC: "1" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| # Clippy in check-ci builds its own check artifacts from the same cache as the native checks. | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-checks | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies without building the packages | |
| run: uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: verify-native-wheels | |
| path: dist | |
| - name: Install the prebuilt extension and CLI | |
| run: uv pip install --no-deps dist/*.whl | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| cache: npm | |
| cache-dependency-path: clients/typescript/package-lock.json | |
| - name: Fetch the base for the new-duplicate check | |
| run: git fetch --no-tags --depth=1 origin "$BASE_REF" | |
| env: | |
| BASE_REF: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || 'main:refs/remotes/origin/main' }} | |
| - name: Run static checks | |
| run: make check-ci | |
| env: | |
| DUPES_BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || 'origin/main' }} | |
| tests: | |
| # check-test-shards, part of the static checks, proves that the TEST_SHARD_*_PATHS lists in | |
| # the Makefile run every test once; each of those lists needs an entry here. | |
| name: Unit + integration tests (${{ matrix.name }}) | |
| needs: [changes, lockfile, native_build] | |
| if: needs.changes.outputs.checks == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: core | |
| target: test-shard-core | |
| workers: "4" | |
| - name: publication | |
| target: test-shard-publication | |
| workers: "2" | |
| - name: warehouse history | |
| target: test-shard-history | |
| workers: "4" | |
| env: | |
| UV_NO_SYNC: "1" | |
| PYTEST_XDIST_AUTO_NUM_WORKERS: ${{ matrix.workers }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies without building the packages | |
| run: uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: verify-native-wheels | |
| path: dist | |
| - name: Install the prebuilt extension and CLI | |
| run: uv pip install --no-deps dist/*.whl | |
| - name: Prepare Docker worker tests | |
| run: | | |
| docker info > /dev/null | |
| docker pull alpine:3.23 | |
| - name: Run unit and integration tests | |
| env: | |
| SQLBUILD_REQUIRE_DOCKER_TESTS: "1" | |
| run: make ${{ matrix.target }} | |
| native_checks: | |
| name: Native workspace checks | |
| needs: [changes, lockfile] | |
| if: needs.changes.outputs.checks == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-checks | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install native test dependencies without rebuilding the packages | |
| run: uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli | |
| - name: Run native checks | |
| env: | |
| UV_NO_SYNC: "1" | |
| PYO3_PYTHON: ${{ github.workspace }}/.venv/bin/python | |
| run: make rust-check | |
| e2e_shards: | |
| name: DuckDB E2E (${{ matrix.name }}) | |
| needs: [changes, lockfile] | |
| if: needs.changes.outputs.duckdb == 'true' && needs.changes.outputs.full == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: build core | |
| target: test-e2e-duckdb-build-core | |
| - name: build incremental | |
| target: test-e2e-duckdb-build-incremental | |
| - name: CLI data | |
| target: test-e2e-duckdb-cli-data | |
| - name: CLI commands | |
| target: test-e2e-duckdb-cli | |
| - name: CLI inspection | |
| target: test-e2e-duckdb-inspection | |
| - name: integrations | |
| target: test-e2e-duckdb-integrations | |
| env: | |
| PYTEST_XDIST_AUTO_NUM_WORKERS: "4" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-linux | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync --locked --all-groups --all-extras | |
| - name: Report runner CPU topology | |
| run: | | |
| echo "nproc=$(nproc)" | |
| echo "online_processors=$(getconf _NPROCESSORS_ONLN)" | |
| lscpu | |
| uv run python - <<'PY' | |
| import os | |
| print(f"os.cpu_count={os.cpu_count()}") | |
| print(f"sched_getaffinity={sorted(os.sched_getaffinity(0))}") | |
| print( | |
| "PYTEST_XDIST_AUTO_NUM_WORKERS=" | |
| f"{os.environ.get('PYTEST_XDIST_AUTO_NUM_WORKERS', '<unset>')}" | |
| ) | |
| PY | |
| - name: Validate diagnostic encryption recipient | |
| if: matrix.target == 'test-e2e-duckdb-cli' | |
| run: openssl x509 -in .github/diagnostics-recipient.pem -noout | |
| - name: Run DuckDB E2E shard | |
| env: | |
| SQLBUILD_COMPARISON_ENCRYPTED_DIR: ${{ matrix.target == 'test-e2e-duckdb-cli' && format('{0}/comparison-encrypted', runner.temp) || '' }} | |
| SQLBUILD_COMPARISON_RECIPIENT: ${{ github.workspace }}/.github/diagnostics-recipient.pem | |
| run: make ${{ matrix.target }} | |
| - name: Retain encrypted comparison failures | |
| if: failure() && matrix.target == 'test-e2e-duckdb-cli' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: encrypted-comparison-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: ${{ runner.temp }}/comparison-encrypted/failure-*/failure.cms | |
| if-no-files-found: warn | |
| compression-level: 0 | |
| retention-days: 7 | |
| compiler_differential: | |
| name: Compiler golden outputs | |
| needs: [changes, lockfile] | |
| if: needs.changes.outputs.compiler_differential == 'true' && needs.changes.outputs.full == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-linux | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync --locked --all-groups --all-extras | |
| - name: Check fixtures, examples, generated seeds and the failure corpus against the goldens | |
| run: >- | |
| make compiler-differential | |
| COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage | |
| --require-render-coverage --require-analysis-coverage | |
| --evidence-dir ${{ runner.temp }}/compiler-differential-evidence" | |
| - name: Upload failing projects' targets, stderr and compiled-test stats | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: compiler-differential-evidence-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: ${{ runner.temp }}/compiler-differential-evidence | |
| if-no-files-found: warn | |
| retention-days: 30 | |
| compiler_differential_python_versions: | |
| name: Native compiler on Python ${{ matrix.python }} | |
| needs: [changes, lockfile] | |
| if: needs.changes.outputs.compiler_differential == 'true' && needs.changes.outputs.full == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: ["3.13", "3.14"] | |
| env: | |
| UV_PYTHON: ${{ matrix.python }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-linux | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install ${{ matrix.python }} | |
| - name: Install dependencies | |
| run: uv sync --locked --all-groups --all-extras --python ${{ matrix.python }} | |
| - name: Check fixtures, examples, the failure corpus and a few seeds against the goldens | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| run: >- | |
| uv run --no-sync python -m scripts.run_compiler_differential | |
| --goldens check | |
| --corpus fixtures examples failures seeds --seeds 4 --jobs 4 | |
| native_discovery_windows: | |
| name: Native compiler on Windows | |
| needs: [changes, lockfile] | |
| if: needs.changes.outputs.compiler_differential == 'true' && needs.changes.outputs.full == 'true' | |
| runs-on: windows-2025 | |
| timeout-minutes: 45 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-windows | |
| save-if: false | |
| - name: Test the native discovery crate (walk, junctions, ordering, read errors) | |
| run: cargo test --locked -p sqlbuild-discovery | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install SQLBuild and pytest | |
| run: | | |
| uv sync --locked --no-dev --python 3.12 | |
| uv pip install "pytest>=8.4.2" "pytest-xdist>=3.8.0" | |
| - name: Check the native YAML and TOML readers and the lineage fingerprint with this Python | |
| run: >- | |
| uv run --no-sync pytest | |
| tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py | |
| tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py | |
| tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py | |
| -n auto --dist loadfile | |
| - name: Compile and build playground projects | |
| run: | | |
| project="${RUNNER_TEMP}/waffle-shop" | |
| uv run --no-sync sqb playground "${project}" | |
| uv run --no-sync sqb --project-dir "${project}" --no-color compile | |
| if ! test -f "${project}/target/cache/compiler-native-v1/macro-calls.bin"; then | |
| echo "The compile did not write the macro-call store; macro bridge debug log:" | |
| rm -rf "${project}/target" | |
| uv run --no-sync sqb --project-dir "${project}" --no-color --debug compile 2>&1 \ | |
| | grep "macro_bridge" || true | |
| exit 1 | |
| fi | |
| uv run --no-sync sqb --project-dir "${project}" --no-color build | |
| nodes="${RUNNER_TEMP}/python-nodes" | |
| uv run --no-sync sqb playground --template python_nodes "${nodes}" | |
| uv run --no-sync sqb --project-dir "${nodes}" --no-color compile | |
| scheduler_recovery: | |
| name: Scheduler recovery (${{ matrix.name }}) | |
| needs: [changes, lockfile] | |
| if: needs.changes.outputs.recovery == 'true' && needs.changes.outputs.full == 'true' | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 75 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: macOS | |
| runner: macos-15 | |
| test_threads: 3 | |
| - name: Windows | |
| runner: windows-2025 | |
| test_threads: 2 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-recovery-${{ matrix.name }} | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Start a native PostgreSQL server | |
| run: | | |
| if [[ "${RUNNER_OS}" == "macOS" ]]; then | |
| brew install --quiet postgresql@16 | |
| bin="$(brew --prefix postgresql@16)/bin" | |
| else | |
| bin="$(cygpath -u "${PGBIN}")" | |
| fi | |
| data="${RUNNER_TEMP}/postgres-data" | |
| "${bin}/initdb" -D "${data}" -U postgres --auth=trust -E UTF8 > /dev/null | |
| "${bin}/pg_ctl" -D "${data}" -o "-p 5433 -c listen_addresses=127.0.0.1" \ | |
| -l "${RUNNER_TEMP}/postgres.log" -w start | |
| - name: Worker claim ownership, recovery and stop | |
| run: cargo test --locked -p sqlbuild-process-ownership | |
| - name: sqlbuild forwarder arguments, exit codes, streams, signals and missing sqb | |
| run: cargo test --locked -p sqlbuild-cli --test launcher | |
| - name: Control store, scheduler and recovery contracts on SQLite and PostgreSQL | |
| env: | |
| SQLBUILD_TEST_CONTROL_POSTGRES_URL: postgresql://postgres@127.0.0.1:5433/postgres | |
| run: cargo test --locked -p sqlbuild-orchestration --lib -- --test-threads=${{ matrix.test_threads }} | |
| - name: PostgreSQL server log | |
| if: failure() | |
| run: cat "${RUNNER_TEMP}/postgres.log" || true | |
| format_performance: | |
| name: Formatter performance guards | |
| needs: [changes, lockfile] | |
| if: needs.changes.outputs.format_performance == 'true' && needs.changes.outputs.full == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-linux | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync --locked --all-groups --all-extras --python 3.12 | |
| - name: Run formatter performance guards | |
| timeout-minutes: 10 | |
| run: >- | |
| uv run pytest | |
| tests/e2e/src/sqlbuild/cli/commands/main/format/test_format_performance.py | |
| -n auto --dist loadfile -m performance | |
| performance: | |
| name: Performance guards | |
| needs: [changes, lockfile] | |
| # Temporarily disabled while the Blacksmith runner allowance is unavailable. | |
| # These budgets are calibrated for the 8-vCPU Blacksmith runner. | |
| if: false | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-linux | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync --locked --all-groups --all-extras | |
| - name: Run performance guards | |
| run: make test-e2e-performance | |
| cold_compile_performance: | |
| name: Fresh-process compile (${{ matrix.models }} models) | |
| needs: [changes, lockfile] | |
| if: needs.changes.outputs.performance == 'true' && needs.changes.outputs.full == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| models: ${{ fromJSON(needs.changes.outputs.cold_models) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-linux | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync --locked --all-groups --all-extras | |
| - name: Run fresh-process compile guard under 4 GiB | |
| env: | |
| BENCHMARK_MODELS: ${{ matrix.models }} | |
| MEMORY_LIMIT_BYTES: "4294967296" | |
| run: | | |
| sudo systemd-run \ | |
| --unit="sqlbuild-cold-${BENCHMARK_MODELS}-${GITHUB_RUN_ID}" \ | |
| --uid="$(id -u)" \ | |
| --gid="$(id -g)" \ | |
| --same-dir \ | |
| --setenv="HOME=${HOME}" \ | |
| --setenv="PATH=${PATH}" \ | |
| --setenv="SQLBUILD_REQUIRE_MEMORY_LIMIT_BYTES=${MEMORY_LIMIT_BYTES}" \ | |
| --property="MemoryMax=${MEMORY_LIMIT_BYTES}" \ | |
| --property="MemorySwapMax=0" \ | |
| --wait \ | |
| --collect \ | |
| --pipe \ | |
| make test-e2e-cold-compile-performance \ | |
| SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}" | |
| dense_compile_performance: | |
| name: Dense all-rules compile (${{ matrix.models }} models) | |
| needs: [changes, lockfile] | |
| if: needs.changes.outputs.performance == 'true' && needs.changes.outputs.full == 'true' | |
| runs-on: ubuntu-24.04 | |
| # The 3k and 10k cells also run the delta edit-loop guard, about five minutes at 10k. | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| models: ${{ fromJSON(needs.changes.outputs.dense_models) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-linux | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync --locked --all-groups --all-extras | |
| # Each timed compile runs in its own 4 GiB transient scope; pytest stays outside it. | |
| - name: Run dense all-rules and delta edit-loop guards with each compile under 4 GiB | |
| env: | |
| BENCHMARK_MODELS: ${{ matrix.models }} | |
| SQLBUILD_COMPILE_MEMORY_LIMIT_BYTES: "4294967296" | |
| run: | | |
| make test-e2e-dense-compile-performance \ | |
| SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}" | |
| - name: Run diagnostic-heavy compile guards | |
| if: matrix.models == 1000 | |
| run: make test-e2e-diagnostic-compile-performance | |
| compile_performance_ratio: | |
| name: Same-runner compile ratio (${{ matrix.label || matrix.kind }} 3000 models) | |
| needs: [changes, lockfile] | |
| if: needs.changes.outputs.performance == 'true' && needs.changes.outputs.full == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - kind: dense | |
| max_ratio: "1.10" | |
| edit_max_ratio: "1.20" | |
| edit_max_cpu_ratio: "1.75" | |
| - kind: fresh | |
| max_ratio: "1.10" | |
| - kind: dense | |
| label: contracts CPU vs base | |
| gate_phase: contracts_cpu_ms | |
| max_ratio: "1.10" | |
| - kind: dense | |
| label: model analysis CPU vs base | |
| gate_phase: model_analysis_cpu_ms | |
| compile_args: --compile-arg=--select=* | |
| max_ratio: "1.10" | |
| - kind: dense | |
| label: custom rules CPU vs base | |
| gate_phase: custom_rules_cpu_ms | |
| # Temporary migration allowance; restore 1.10 after custom-host projection optimization. | |
| # Whole-command CPU, wall-time, memory and correctness gates retain their limits. | |
| max_ratio: "1.35" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check out the comparison base | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || 'main' }} | |
| path: base | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| base/uv.lock | |
| base/Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-linux | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Seed the base build directory with the cached dependency builds | |
| run: | | |
| if [[ -d target ]]; then | |
| cp -a target base/target | |
| fi | |
| - name: Install head dependencies | |
| run: uv sync --locked --all-groups --all-extras | |
| - name: Install base dependencies | |
| working-directory: base | |
| run: uv sync --locked --all-groups --all-extras | |
| - name: Compare base and head phase CPU | |
| if: matrix.gate_phase | |
| env: | |
| KIND: ${{ matrix.kind }} | |
| MAX_RATIO: ${{ matrix.max_ratio }} | |
| GATE_PHASE: ${{ matrix.gate_phase }} | |
| COMPILE_ARGS: ${{ matrix.compile_args }} | |
| run: | | |
| uv run python -m scripts.compare_compile_performance \ | |
| --kind "${KIND}" \ | |
| --modes cold \ | |
| --max-ratio "${MAX_RATIO}" \ | |
| --models 3000 \ | |
| --base-root base \ | |
| --base-python base/.venv/bin/python \ | |
| --head-python .venv/bin/python \ | |
| --gate-phase "${GATE_PHASE}" \ | |
| ${COMPILE_ARGS:+"${COMPILE_ARGS}"} | |
| - name: Compare base and head cold, warm and one-model edit compiles | |
| if: ${{ !matrix.gate_phase }} | |
| env: | |
| KIND: ${{ matrix.kind }} | |
| MAX_RATIO: ${{ matrix.max_ratio }} | |
| COLD_MAX_RATIO: ${{ matrix.cold_max_ratio }} | |
| COLD_MAX_CPU_RATIO: ${{ matrix.cold_max_cpu_ratio }} | |
| EDIT_MAX_RATIO: ${{ matrix.edit_max_ratio }} | |
| EDIT_MAX_CPU_RATIO: ${{ matrix.edit_max_cpu_ratio }} | |
| run: | | |
| uv run python -m scripts.compare_compile_performance \ | |
| --kind "${KIND}" \ | |
| --modes cold warm edit \ | |
| ${COLD_MAX_RATIO:+--cold-max-ratio "$COLD_MAX_RATIO"} \ | |
| ${COLD_MAX_CPU_RATIO:+--cold-max-cpu-ratio "$COLD_MAX_CPU_RATIO"} \ | |
| ${EDIT_MAX_RATIO:+--edit-max-ratio "$EDIT_MAX_RATIO"} \ | |
| ${EDIT_MAX_CPU_RATIO:+--edit-max-cpu-ratio "$EDIT_MAX_CPU_RATIO"} \ | |
| --max-ratio "${MAX_RATIO}" \ | |
| --models 3000 \ | |
| --base-root base \ | |
| --base-python base/.venv/bin/python \ | |
| --head-python .venv/bin/python | |
| cache_compile_performance: | |
| name: Fresh-process compile cache (${{ matrix.models }} models) | |
| needs: [changes, lockfile] | |
| if: needs.changes.outputs.performance == 'true' && needs.changes.outputs.full == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| models: ${{ fromJSON(needs.changes.outputs.cache_models) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-linux | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync --locked --all-groups --all-extras | |
| - name: Run fresh-process compile cache guard under 4 GiB | |
| timeout-minutes: 15 | |
| env: | |
| BENCHMARK_MODELS: ${{ matrix.models }} | |
| MEMORY_LIMIT_BYTES: "4294967296" | |
| run: | | |
| sudo systemd-run \ | |
| --unit="sqlbuild-cache-${BENCHMARK_MODELS}-${GITHUB_RUN_ID}" \ | |
| --uid="$(id -u)" \ | |
| --gid="$(id -g)" \ | |
| --same-dir \ | |
| --setenv="HOME=${HOME}" \ | |
| --setenv="PATH=${PATH}" \ | |
| --setenv="SQLBUILD_REQUIRE_MEMORY_LIMIT_BYTES=${MEMORY_LIMIT_BYTES}" \ | |
| --property="MemoryMax=${MEMORY_LIMIT_BYTES}" \ | |
| --property="MemorySwapMax=0" \ | |
| --wait \ | |
| --collect \ | |
| --pipe \ | |
| make test-e2e-cache-compile-performance \ | |
| SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}" | |
| - name: Run varied 1k warm-cache guard under 4 GiB | |
| if: matrix.models == 3000 | |
| timeout-minutes: 15 | |
| env: | |
| MEMORY_LIMIT_BYTES: "4294967296" | |
| run: | | |
| sudo systemd-run \ | |
| --unit="sqlbuild-varied-cache-${GITHUB_RUN_ID}" \ | |
| --uid="$(id -u)" \ | |
| --gid="$(id -g)" \ | |
| --same-dir \ | |
| --setenv="HOME=${HOME}" \ | |
| --setenv="PATH=${PATH}" \ | |
| --setenv="SQLBUILD_REQUIRE_MEMORY_LIMIT_BYTES=${MEMORY_LIMIT_BYTES}" \ | |
| --property="MemoryMax=${MEMORY_LIMIT_BYTES}" \ | |
| --property="MemorySwapMax=0" \ | |
| --wait \ | |
| --collect \ | |
| --pipe \ | |
| make test-e2e-varied-cache-performance | |
| rules_performance: | |
| name: Rules performance guards | |
| needs: [changes, lockfile] | |
| # Temporarily disabled while the Blacksmith runner allowance is unavailable. | |
| # These budgets are calibrated for the 8-vCPU Blacksmith runner. | |
| if: false | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-linux | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync --locked --all-groups --all-extras | |
| - name: Run required Rules performance profile | |
| timeout-minutes: 10 | |
| run: >- | |
| uv run python -m scripts.benchmark_rules_ci | |
| --output "${RUNNER_TEMP}/rules-performance-ci.json" | |
| --summary-output "${RUNNER_TEMP}/rules-performance-ci.md" | |
| --max-seconds 420 | |
| - name: Publish benchmark summary | |
| if: always() | |
| run: | | |
| if [[ -f "${RUNNER_TEMP}/rules-performance-ci.md" ]]; then | |
| cat "${RUNNER_TEMP}/rules-performance-ci.md" >> "${GITHUB_STEP_SUMMARY}" | |
| fi | |
| - name: Upload benchmark evidence | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: rules-performance-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: | | |
| ${{ runner.temp }}/rules-performance-ci.json | |
| ${{ runner.temp }}/rules-performance-ci.md | |
| if-no-files-found: error | |
| retention-days: 90 | |
| postgres: | |
| name: PostgreSQL integration and E2E tests | |
| needs: [changes, lockfile] | |
| if: needs.changes.outputs.postgres == 'true' && needs.changes.outputs.full == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-linux | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync --locked --all-groups --all-extras | |
| - name: Verify Docker | |
| run: docker info | |
| - name: Run PostgreSQL tests | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| TESTCONTAINERS_RYUK_DISABLED: "true" | |
| run: >- | |
| uv run pytest tests | |
| -m "real_warehouse and postgres" | |
| -vv --color=yes -n auto --dist loadfile | |
| clickhouse: | |
| name: ClickHouse integration and E2E tests | |
| needs: [changes, lockfile] | |
| if: needs.changes.outputs.clickhouse == 'true' && needs.changes.outputs.full == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: verify-linux | |
| save-if: false | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync --locked --all-groups --all-extras | |
| - name: Verify Docker | |
| run: docker info | |
| - name: Run ClickHouse tests | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| TESTCONTAINERS_RYUK_DISABLED: "true" | |
| run: >- | |
| uv run pytest tests | |
| -m "real_warehouse and clickhouse" | |
| -vv --color=yes -n auto --dist loadfile | |
| wheels: | |
| name: Wheels | |
| needs: [changes] | |
| if: needs.changes.outputs.wheels == 'true' && needs.changes.outputs.full == 'true' | |
| uses: ./.github/workflows/build-distributions.yml | |
| with: | |
| sdist: false | |
| upload: true | |
| retention-days: 3 | |
| # Release Distributions builds and smoke tests the slower dist profile for every release. | |
| cargo-profile: release | |
| # Pull requests and merge groups build the wheel Compose installs; manual runs build every | |
| # platform, and Release Distributions builds every platform for each release. | |
| platforms: >- | |
| ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') | |
| && 'linux-x86_64' || '' }} | |
| compose: | |
| name: Packaged Compose lifecycle | |
| needs: [changes, lockfile, wheels] | |
| if: needs.changes.outputs.wheels == 'true' && needs.changes.outputs.full == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| - name: Install Python test dependencies without rebuilding the packages | |
| run: | | |
| uv python install 3.12 | |
| uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: dist-wheel-linux-x86_64 | |
| path: dist | |
| - name: Install the same wheels used by the images | |
| run: uv pip install --no-deps dist/*.whl | |
| - name: Verify packaged Compose deployment and operational recovery | |
| run: make test-compose-packaged | |
| podman: | |
| name: Packaged Compose lifecycle on Podman | |
| needs: [changes, lockfile, wheels] | |
| if: needs.changes.outputs.podman == 'true' && needs.changes.outputs.wheels == 'true' && needs.changes.outputs.full == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 25 | |
| env: | |
| # The Docker CLI and its Compose plugin drive rootful Podman through its API socket. | |
| DOCKER_HOST: unix:///run/podman/podman.sock | |
| DOCKER_SOCKET: /run/podman/podman.sock | |
| # Podman has no BuildKit API: buildx would build in its own container and never load the image. | |
| DOCKER_BUILDKIT: "0" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| - name: Install Python test dependencies without rebuilding the packages | |
| run: | | |
| uv python install 3.12 | |
| uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: dist-wheel-linux-x86_64 | |
| path: dist | |
| - name: Install the same wheels used by the images | |
| run: uv pip install --no-deps dist/*.whl | |
| - name: Start the Podman API socket with group access for the runner | |
| run: | | |
| sudo systemctl start podman.socket | |
| sudo chmod 0711 /run/podman | |
| sudo chgrp "$(id -g)" /run/podman/podman.sock | |
| sudo chmod 0660 /run/podman/podman.sock | |
| podman --version | |
| docker version --format 'Podman API {{.Server.Version}}' | |
| docker compose version | |
| - name: Verify packaged Compose deployment and operational recovery | |
| run: make test-compose-packaged | |
| ui: | |
| name: UI checks and unit tests | |
| needs: changes | |
| if: needs.changes.outputs.ui == 'true' | |
| runs-on: ubuntu-24.04 | |
| defaults: | |
| run: | |
| working-directory: ui | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| cache: npm | |
| cache-dependency-path: ui/package-lock.json | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Type-check | |
| run: npm run check | |
| - name: Unit tests | |
| run: npm test | |
| - name: Build the static bundle sqb server embeds | |
| run: npm run build | |
| - name: Install the layout test's browser | |
| run: npx playwright install --with-deps chromium | |
| - name: Layout test (every page with awkward mock data at 1440 and 1920 px) | |
| run: npx playwright test --project=layout | |
| ui_timings: | |
| # The timing guards in ui/e2e/perf.spec.ts, with the CPU slowed 6x. They go red over budget and | |
| # report their numbers, but stay out of Verify's needs: a slow hosted runner must not block a merge. | |
| name: UI work timings (not required) | |
| needs: changes | |
| if: needs.changes.outputs.ui == 'true' | |
| runs-on: ubuntu-24.04 | |
| defaults: | |
| run: | |
| working-directory: ui | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| cache: npm | |
| cache-dependency-path: ui/package-lock.json | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Build the static bundle sqb server embeds | |
| run: npm run build | |
| - name: Install the timing test's browser | |
| run: npx playwright install --with-deps chromium | |
| - name: Time the slow paths (graph gestures, Checks reopen, plan sections) | |
| shell: bash | |
| run: | | |
| set -o pipefail | |
| npx playwright test --project=perf --no-deps --reporter=list 2>&1 | tee perf.log | |
| - name: Report the timings | |
| if: always() | |
| shell: bash | |
| run: | | |
| { | |
| echo "### UI work timings" | |
| echo '```' | |
| grep -E 'CPU slowed|second visit' perf.log || echo "No timings were recorded." | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| verify: | |
| # The required Verify of a Release Please pull request is the commit status that the trusted | |
| # release checks in release-please.yml post on the branch's current head, after the lockfile | |
| # refresh and every dispatched check. This job reports under another name there, so its quick | |
| # metadata-only pass can never satisfy Verify first; a skipped job would count as passing, so | |
| # the job always runs and only its name changes. Until those checks post Verify on the head, | |
| # the pull request waits for an expected check, and every new push to the branch waits again. | |
| # The same holds for a run started by an unrelated label, and for a fast-core run that leaves | |
| # out suites its change needs, unless the merge queue (SQLBUILD_MERGE_QUEUE) runs them. | |
| name: >- | |
| ${{ (github.event_name == 'pull_request' | |
| && github.event.pull_request.head.repo.full_name == github.repository | |
| && startsWith(github.event.pull_request.head.ref, 'release-please--branches--')) | |
| && 'Release pull request suites' | |
| || ((github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name != 'full-verify') && 'Label change (nothing verified)' | |
| || needs.changes.outputs.partial == 'true' && vars.SQLBUILD_MERGE_QUEUE != 'true' | |
| && 'Fast core (label full-verify for Verify)' | |
| || 'Verify' }} | |
| if: always() | |
| needs: | |
| [ | |
| changes, | |
| lockfile, | |
| native_build, | |
| checks, | |
| tests, | |
| native_checks, | |
| e2e_shards, | |
| compiler_differential, | |
| compiler_differential_python_versions, | |
| native_discovery_windows, | |
| scheduler_recovery, | |
| format_performance, | |
| performance, | |
| cold_compile_performance, | |
| dense_compile_performance, | |
| compile_performance_ratio, | |
| cache_compile_performance, | |
| rules_performance, | |
| postgres, | |
| clickhouse, | |
| wheels, | |
| compose, | |
| podman, | |
| ui, | |
| ] | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Confirm all required verification jobs passed | |
| env: | |
| CHANGES_RESULT: ${{ needs.changes.result }} | |
| CHECKS_SELECTED: ${{ needs.changes.outputs.checks }} | |
| LOCKFILE_RESULT: ${{ needs.lockfile.result }} | |
| NATIVE_BUILD_RESULT: ${{ needs.native_build.result }} | |
| CHECKS_RESULT: ${{ needs.checks.result }} | |
| TESTS_RESULT: ${{ needs.tests.result }} | |
| NATIVE_CHECKS_RESULT: ${{ needs.native_checks.result }} | |
| E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }} | |
| COMPILER_DIFFERENTIAL_RESULT: ${{ needs.compiler_differential.result }} | |
| COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT: ${{ needs.compiler_differential_python_versions.result }} | |
| NATIVE_DISCOVERY_WINDOWS_RESULT: ${{ needs.native_discovery_windows.result }} | |
| SCHEDULER_RECOVERY_RESULT: ${{ needs.scheduler_recovery.result }} | |
| FORMAT_PERFORMANCE_RESULT: ${{ needs.format_performance.result }} | |
| PERFORMANCE_RESULT: ${{ needs.performance.result }} | |
| COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }} | |
| DENSE_COMPILE_PERFORMANCE_RESULT: ${{ needs.dense_compile_performance.result }} | |
| COMPILE_PERFORMANCE_RATIO_RESULT: ${{ needs.compile_performance_ratio.result }} | |
| CACHE_COMPILE_PERFORMANCE_RESULT: ${{ needs.cache_compile_performance.result }} | |
| RULES_PERFORMANCE_RESULT: ${{ needs.rules_performance.result }} | |
| POSTGRES_RESULT: ${{ needs.postgres.result }} | |
| CLICKHOUSE_RESULT: ${{ needs.clickhouse.result }} | |
| WHEELS_RESULT: ${{ needs.wheels.result }} | |
| COMPOSE_RESULT: ${{ needs.compose.result }} | |
| PODMAN_RESULT: ${{ needs.podman.result }} | |
| UI_SELECTED: ${{ needs.changes.outputs.ui }} | |
| UI_RESULT: ${{ needs.ui.result }} | |
| LABEL_ONLY: ${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name != 'full-verify' }} | |
| PARTIAL: ${{ needs.changes.outputs.partial }} | |
| DUCKDB_SELECTED: ${{ needs.changes.outputs.duckdb }} | |
| COMPILER_DIFFERENTIAL_SELECTED: ${{ needs.changes.outputs.compiler_differential }} | |
| RECOVERY_SELECTED: ${{ needs.changes.outputs.recovery }} | |
| FORMAT_PERFORMANCE_SELECTED: ${{ needs.changes.outputs.format_performance }} | |
| PERFORMANCE_SELECTED: ${{ needs.changes.outputs.performance }} | |
| POSTGRES_SELECTED: ${{ needs.changes.outputs.postgres }} | |
| CLICKHOUSE_SELECTED: ${{ needs.changes.outputs.clickhouse }} | |
| WHEELS_SELECTED: ${{ needs.changes.outputs.wheels }} | |
| PODMAN_SELECTED: ${{ needs.changes.outputs.podman }} | |
| run: | | |
| if [[ "${LABEL_ONLY}" == "true" ]]; then | |
| echo "A label other than full-verify changed; this run verifies nothing" | |
| exit 0 | |
| fi | |
| if [[ "${CHANGES_RESULT}" != "success" || "${LOCKFILE_RESULT}" != "success" ]]; then | |
| echo "Required classification or lockfile checks did not pass" >&2 | |
| exit 1 | |
| fi | |
| for result in "${NATIVE_BUILD_RESULT}" "${CHECKS_RESULT}" "${TESTS_RESULT}" "${NATIVE_CHECKS_RESULT}"; do | |
| if [[ "${result}" != "success" ]] && \ | |
| ! [[ "${CHECKS_SELECTED}" == "false" && "${result}" == "skipped" ]]; then | |
| echo "Required static or native checks did not pass: ${result}" >&2 | |
| exit 1 | |
| fi | |
| done | |
| if [[ "${WHEELS_RESULT}" == "success" && "${COMPOSE_RESULT}" != "success" ]]; then | |
| echo "Required packaged Compose verification did not pass: ${COMPOSE_RESULT}" >&2 | |
| exit 1 | |
| fi | |
| for result in \ | |
| "${E2E_SHARDS_RESULT}" \ | |
| "${COMPILER_DIFFERENTIAL_RESULT}" \ | |
| "${COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT}" \ | |
| "${NATIVE_DISCOVERY_WINDOWS_RESULT}" \ | |
| "${SCHEDULER_RECOVERY_RESULT}" \ | |
| "${FORMAT_PERFORMANCE_RESULT}" \ | |
| "${PERFORMANCE_RESULT}" \ | |
| "${COLD_COMPILE_PERFORMANCE_RESULT}" \ | |
| "${DENSE_COMPILE_PERFORMANCE_RESULT}" \ | |
| "${COMPILE_PERFORMANCE_RATIO_RESULT}" \ | |
| "${CACHE_COMPILE_PERFORMANCE_RESULT}" \ | |
| "${RULES_PERFORMANCE_RESULT}" \ | |
| "${POSTGRES_RESULT}" \ | |
| "${CLICKHOUSE_RESULT}" \ | |
| "${WHEELS_RESULT}" \ | |
| "${COMPOSE_RESULT}" \ | |
| "${PODMAN_RESULT}"; do | |
| if [[ "${result}" != "success" && "${result}" != "skipped" ]]; then | |
| echo "A selected verification suite did not pass: ${result}" >&2 | |
| exit 1 | |
| fi | |
| done | |
| if [[ "${UI_RESULT}" != "success" ]] && \ | |
| ! [[ "${UI_SELECTED}" != "true" && "${UI_RESULT}" == "skipped" ]]; then | |
| echo "Required UI checks did not pass: ${UI_RESULT}" >&2 | |
| exit 1 | |
| fi | |
| # A full run must have run every suite its change selected; a skip here is never a pass. | |
| if [[ "${PARTIAL}" != "true" ]]; then | |
| for pair in \ | |
| "${DUCKDB_SELECTED}:${E2E_SHARDS_RESULT}" \ | |
| "${COMPILER_DIFFERENTIAL_SELECTED}:${COMPILER_DIFFERENTIAL_RESULT}" \ | |
| "${COMPILER_DIFFERENTIAL_SELECTED}:${COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT}" \ | |
| "${COMPILER_DIFFERENTIAL_SELECTED}:${NATIVE_DISCOVERY_WINDOWS_RESULT}" \ | |
| "${RECOVERY_SELECTED}:${SCHEDULER_RECOVERY_RESULT}" \ | |
| "${FORMAT_PERFORMANCE_SELECTED}:${FORMAT_PERFORMANCE_RESULT}" \ | |
| "${PERFORMANCE_SELECTED}:${COLD_COMPILE_PERFORMANCE_RESULT}" \ | |
| "${PERFORMANCE_SELECTED}:${DENSE_COMPILE_PERFORMANCE_RESULT}" \ | |
| "${PERFORMANCE_SELECTED}:${COMPILE_PERFORMANCE_RATIO_RESULT}" \ | |
| "${PERFORMANCE_SELECTED}:${CACHE_COMPILE_PERFORMANCE_RESULT}" \ | |
| "${POSTGRES_SELECTED}:${POSTGRES_RESULT}" \ | |
| "${CLICKHOUSE_SELECTED}:${CLICKHOUSE_RESULT}" \ | |
| "${WHEELS_SELECTED}:${WHEELS_RESULT}" \ | |
| "${WHEELS_SELECTED}:${COMPOSE_RESULT}" \ | |
| "${PODMAN_SELECTED}:${PODMAN_RESULT}"; do | |
| if [[ "${pair%%:*}" == "true" && "${pair#*:}" != "success" ]]; then | |
| echo "A suite this change needs did not run to success: ${pair#*:}" >&2 | |
| exit 1 | |
| fi | |
| done | |
| else | |
| echo "Fast core passed; the full matrix runs on full-verify, or in the merge queue when it verifies groups" | |
| exit 0 | |
| fi | |
| echo "All path-relevant verification jobs passed" | |
| report_nightly: | |
| name: Report nightly failure | |
| needs: verify | |
| if: ${{ !cancelled() && github.event_name == 'schedule' && needs.verify.result == 'failure' }} | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| issues: write | |
| pull-requests: write | |
| steps: | |
| - name: Open or update the nightly Verify issue | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: ${{ github.repository }} | |
| LABEL: nightly-verify | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| { | |
| echo "Nightly Verify failed at ${GITHUB_SHA} on main." | |
| echo | |
| echo "Run: ${RUN_URL}" | |
| echo | |
| echo "The nightly run adds the 5k and 10k-model compile jobs and every wheel platform." | |
| } > report.md | |
| gh label create "${LABEL}" --color B60205 \ | |
| --description "The nightly Verify run on main failed" 2>/dev/null || true | |
| issue="$(gh issue list --label "${LABEL}" --state open --limit 1 \ | |
| --json number --jq '.[0].number // empty')" | |
| if [[ -n "${issue}" ]]; then | |
| gh issue comment "${issue}" --body-file report.md | |
| echo "Updated issue #${issue}" >> "${GITHUB_STEP_SUMMARY}" | |
| else | |
| gh issue create --title "Nightly Verify failed on main" \ | |
| --label "${LABEL}" --body-file report.md | |
| fi | |
| # The open issue holds releases (release-please.yml); stop a release pull request that | |
| # already passed its checks from merging before Release Please runs again. | |
| release_pr="$(gh pr list --state open --label "autorelease: pending" \ | |
| --json number --jq '.[0].number // empty')" | |
| if [[ -n "${release_pr}" ]]; then | |
| gh pr merge "${release_pr}" --disable-auto 2>/dev/null || true | |
| fi |