Repository navigation
ci: report release performance runs that do not finish #528
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release Performance | |
| run-name: >- | |
| ${{ github.event.pull_request.title || github.event.head_commit.message || github.workflow }} | |
| # Every push to main compares main with the latest release on the same runner. A regression fails | |
| # the run and opens or updates one issue labelled performance-regression; it does not gate the | |
| # release pull request. A main run that does not finish, such as one that errors, loses its runner | |
| # or times out, opens or updates one issue labelled release-performance-unfinished. Pull requests | |
| # that change the comparison itself run it too. | |
| on: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| baseline: | |
| description: "Published sqlbuild version to compare against (default: the release before the candidate)" | |
| required: false | |
| type: string | |
| pull_request: | |
| paths: | |
| - .github/workflows/release-performance.yml | |
| - .github/actions/dist-build-memory/** | |
| - scripts/compare_release_performance.py | |
| - scripts/release_performance/** | |
| - scripts/compile_performance_ratio/_helpers/edit.py | |
| - scripts/compile_performance_ratio/_helpers/measure.py | |
| - scripts/compile_performance_ratio/_helpers/report.py | |
| - scripts/compile_performance_ratio/constants.py | |
| - scripts/compile_performance_ratio/exceptions.py | |
| - scripts/compile_performance_ratio/models.py | |
| - scripts/cold_compile_performance/_helpers/dense_project.py | |
| - scripts/cold_compile_performance/constants.py | |
| permissions: | |
| contents: read | |
| # Every main commit gets its own comparison, so a later push never cancels the run that would | |
| # report an earlier regression. Pull request runs supersede older runs of the same pull request. | |
| concurrency: | |
| group: >- | |
| release-performance-${{ github.event_name }}-${{ | |
| github.event_name == 'push' && github.sha || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| compare: | |
| name: Compare with the previous release | |
| runs-on: ubuntu-24.04 | |
| # A backstop above the slowest normal install and build (about 51 minutes together) plus the | |
| # comparison's own limit, so a hung comparison stops at its step and still uploads evidence. | |
| timeout-minutes: 130 | |
| outputs: | |
| regressed: ${{ steps.compare.outputs.regressed }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # Release tags identify the previous release before its wheels reach PyPI. | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| version: "0.12.3" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| uv.lock | |
| Cargo.lock | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - name: Install Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync --locked --all-groups --all-extras | |
| # The candidate uses the released wheels' cargo profile, so both sides of the comparison match. | |
| # That build can need more memory than the hosted runner has; see the action. | |
| - name: Add swap and limit cargo jobs for the candidate build | |
| uses: ./.github/actions/dist-build-memory | |
| with: | |
| step: start | |
| - name: Build candidate wheel | |
| env: | |
| MATURIN_PEP517_ARGS: --profile dist | |
| run: | | |
| uv build --wheel --out-dir "${RUNNER_TEMP}/candidate" | |
| uv build --wheel crates/sqlbuild-cli --out-dir "${RUNNER_TEMP}/candidate" | |
| - name: Report memory use of the candidate build | |
| if: always() | |
| uses: ./.github/actions/dist-build-memory | |
| with: | |
| step: report | |
| # A release tag created while the candidate built names the baseline's benchmark generator. | |
| - name: Fetch release tags | |
| run: git fetch --tags --force --quiet origin | |
| # Temporary: one-model edit compiles use MAX_EDIT_COMPILE_TIME_RATIO (1.35x) because an edited | |
| # model recompiles uncached; tighten it back to 1.15x once native rendering is the default. | |
| # Comparisons take 10 to 21 minutes, plus up to 30 while a new baseline reaches PyPI. | |
| - name: Compare candidate and previous release on this runner | |
| id: compare | |
| timeout-minutes: 65 | |
| env: | |
| BASELINE: ${{ inputs.baseline }} | |
| EVENT_NAME: ${{ github.event_name }} | |
| run: | | |
| wheels=("${RUNNER_TEMP}"/candidate/sqlbuild-*.whl) | |
| evidence="${RUNNER_TEMP}/release-performance" | |
| mkdir -p "${evidence}" | |
| args=( | |
| --candidate "${wheels[0]}" | |
| --work-dir "${RUNNER_TEMP}/release-performance-work" | |
| --output "${evidence}/release-performance.json" | |
| ) | |
| if [[ -n "${BASELINE}" ]]; then | |
| args+=(--baseline "${BASELINE}") | |
| fi | |
| # A main commit that is not itself a release still carries the latest release's version, | |
| # so it is compared with that release once PyPI serves it, and with the newest published | |
| # release before it until then. A release commit bumps the version and is compared with | |
| # the newest published release before it. | |
| if [[ "${EVENT_NAME}" == "push" ]]; then | |
| project_version() { | |
| python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["project"]["version"])' | |
| } | |
| version="$(project_version < pyproject.toml)" | |
| previous="$(git show HEAD^:pyproject.toml | project_version)" || previous="" | |
| if [[ "${version}" == "${previous}" ]]; then | |
| args+=(--same-version-baseline) | |
| fi | |
| fi | |
| # The comparison's own exit status, not tee's, decides the outcome. | |
| set +e | |
| uv run python -m scripts.compare_release_performance "${args[@]}" \ | |
| | tee "${evidence}/release-performance.md" | |
| status="${PIPESTATUS[0]}" | |
| set -e | |
| regressed=false | |
| if [[ "${status}" != "0" && -f "${evidence}/release-performance.json" ]] && \ | |
| jq -e 'any(.verdicts[]; .regressed)' "${evidence}/release-performance.json" > /dev/null; then | |
| regressed=true | |
| fi | |
| echo "status=${status}" >> "${GITHUB_OUTPUT}" | |
| echo "regressed=${regressed}" >> "${GITHUB_OUTPUT}" | |
| - name: Upload comparison evidence | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: release-performance-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: ${{ runner.temp }}/release-performance | |
| if-no-files-found: warn | |
| retention-days: 90 | |
| - name: Fail on a regression or comparison error | |
| if: steps.compare.outputs.status != '0' | |
| env: | |
| STATUS: ${{ steps.compare.outputs.status }} | |
| run: | | |
| echo "The release performance comparison failed with exit status ${STATUS:-unknown}" >&2 | |
| exit 1 | |
| report: | |
| name: Report regression or unfinished comparison | |
| needs: compare | |
| if: ${{ always() && github.event_name == 'push' && needs.compare.result != 'success' }} | |
| runs-on: ubuntu-24.04 | |
| # Overlapping main runs report one at a time so they update a single issue. | |
| concurrency: | |
| group: release-performance-report | |
| cancel-in-progress: false | |
| permissions: | |
| actions: read | |
| checks: read | |
| contents: read | |
| issues: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: ${{ github.repository }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| if: needs.compare.outputs.regressed == 'true' | |
| with: | |
| name: release-performance-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: evidence | |
| - name: Open or update the performance regression issue | |
| if: needs.compare.outputs.regressed == 'true' | |
| env: | |
| LABEL: performance-regression | |
| run: | | |
| { | |
| echo "Release performance regressed at ${GITHUB_SHA} on main." | |
| echo | |
| echo "Comparison run: ${RUN_URL}" | |
| echo | |
| cat evidence/release-performance.md | |
| } > report.md | |
| gh label create "${LABEL}" --color B60205 \ | |
| --description "Main is slower or larger than the latest release" 2>/dev/null || true | |
| issue="$(gh issue list --label "${LABEL}" --state open --limit 1 \ | |
| --json number --jq '.[0].number // empty')" | |
| if [[ -n "${issue}" ]]; then | |
| gh issue comment "${issue}" --body-file report.md | |
| echo "Updated issue #${issue}" >> "${GITHUB_STEP_SUMMARY}" | |
| else | |
| gh issue create --title "Release performance regression on main" \ | |
| --label "${LABEL}" --body-file report.md | |
| fi | |
| - uses: actions/checkout@v4 | |
| if: needs.compare.outputs.regressed != 'true' | |
| with: | |
| sparse-checkout: scripts | |
| # A failed compare job, or one stopped by its own timeout, did not finish. A run cancelled by | |
| # a newer run or by hand is also `cancelled`, but without the timeout annotation, so the | |
| # script prints nothing for it. | |
| - name: Open or update the unfinished comparison issue | |
| if: needs.compare.outputs.regressed != 'true' | |
| env: | |
| LABEL: release-performance-unfinished | |
| RESULT: ${{ needs.compare.result }} | |
| REGRESSED: ${{ needs.compare.outputs.regressed }} | |
| run: | | |
| job="$(gh api "repos/${GH_REPO}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \ | |
| --jq '.jobs[] | select(.name == "Compare with the previous release") | .id')" | |
| if [[ -z "${job}" ]]; then | |
| echo "::error::The compare job is not listed in run attempt ${GITHUB_RUN_ATTEMPT}" | |
| exit 1 | |
| fi | |
| gh api "repos/${GH_REPO}/check-runs/${job}/annotations" > annotations.json | |
| reason="$(python3 -m scripts.report_unfinished_release_performance \ | |
| --result "${RESULT}" --regressed "${REGRESSED}" --annotations annotations.json)" | |
| if [[ -z "${reason}" ]]; then | |
| echo "The comparison was cancelled by a newer run or by hand; nothing to report." \ | |
| >> "${GITHUB_STEP_SUMMARY}" | |
| exit 0 | |
| fi | |
| { | |
| echo "The release performance comparison did not finish at ${GITHUB_SHA} on main." | |
| echo | |
| echo "Comparison run: ${RUN_URL} (its job summary lists the phases that started)" | |
| echo | |
| echo "${reason}" | |
| } > report.md | |
| gh label create "${LABEL}" --color FBCA04 \ | |
| --description "A main release performance comparison did not finish" 2>/dev/null || true | |
| issue="$(gh issue list --label "${LABEL}" --state open --limit 1 \ | |
| --json number --jq '.[0].number // empty')" | |
| if [[ -n "${issue}" ]]; then | |
| gh issue comment "${issue}" --body-file report.md | |
| echo "Updated issue #${issue}" >> "${GITHUB_STEP_SUMMARY}" | |
| else | |
| gh issue create --title "Release performance comparison did not finish on main" \ | |
| --label "${LABEL}" --body-file report.md | |
| fi |