bench.yml

# Speed comparison of a pull request against its base, required on master: it
# fails only on a large, clear slowdown (a workload more than 2x slower than the
# base, --fail-above 2), which the operation-count tests cannot see (for
# example slow pure-Python code). Smaller changes are flagged SLOWER in the job
# summary and the "bench-ab" artifact for a maintainer. It does not post PR
# comments, because pull requests from forks get a read-only token. Script and
# statistics: .github/scripts/bench_ab.py.
name: Speed comparison

on:
  pull_request:
    branches: [ master ]

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  POETRY_VERSION: "2.4.3"

jobs:
  bench:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
    - uses: actions/checkout@v7
      with:
        fetch-depth: 0  # the base commit must be present for git worktree
    - name: Install Poetry
      run: pipx install "poetry==${POETRY_VERSION}"
    - uses: actions/setup-python@v7
      with:
        python-version: "3.13"
        cache: poetry
    - name: Install dependencies
      run: |
        poetry install
        .github/scripts/use_cpu_torch.sh "poetry run python"
    # Head is github.sha, the test merge of the pull request into master, so the
    # difference from the base commit is exactly the pull request's change.
    - name: Compare base and head (alternating rounds)
      env:
        HEAD_SHA: ${{ github.sha }}
      # HEAD^1 of the test merge is the master tip it merges into.
      run: >
        poetry run python .github/scripts/bench_ab.py
        --base "$(git rev-parse HEAD^1)" --head "$HEAD_SHA" --fail-above 2
        --out bench-out --summary "$GITHUB_STEP_SUMMARY"
    - uses: actions/upload-artifact@v7
      if: always()
      with:
        name: bench-ab
        path: |
          bench-out/bench_ab.md
          bench-out/bench_ab.json