Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 66 additions & 8 deletions .github/workflows/release-performance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ run-name: >-

# Every push to main compares main with the latest release on the same runner. A regression fails
# the run and opens or updates one issue labelled performance-regression; it does not gate the
# release pull request. Pull requests that change the comparison itself run it too.
# release pull request. A main run that does not finish, such as one that errors, loses its runner
# or times out, opens or updates one issue labelled release-performance-unfinished. Pull requests
# that change the comparison itself run it too.
on:
push:
branches:
Expand Down Expand Up @@ -45,7 +47,9 @@ jobs:
compare:
name: Compare with the previous release
runs-on: ubuntu-24.04
timeout-minutes: 90
# A backstop above the slowest normal install and build (about 51 minutes together) plus the
# comparison's own limit, so a hung comparison stops at its step and still uploads evidence.
timeout-minutes: 130
outputs:
regressed: ${{ steps.compare.outputs.regressed }}
steps:
Expand Down Expand Up @@ -89,8 +93,10 @@ jobs:
run: git fetch --tags --force --quiet origin
# Temporary: one-model edit compiles use MAX_EDIT_COMPILE_TIME_RATIO (1.35x) because an edited
# model recompiles uncached; tighten it back to 1.15x once native rendering is the default.
# Comparisons take 10 to 21 minutes, plus up to 30 while a new baseline reaches PyPI.
- name: Compare candidate and previous release on this runner
id: compare
timeout-minutes: 65
env:
BASELINE: ${{ inputs.baseline }}
EVENT_NAME: ${{ github.event_name }}
Expand Down Expand Up @@ -150,27 +156,33 @@ jobs:
exit 1

report:
name: Report regression
name: Report regression or unfinished comparison
needs: compare
if: ${{ always() && github.event_name == 'push' && needs.compare.outputs.regressed == 'true' }}
if: ${{ always() && github.event_name == 'push' && needs.compare.result != 'success' }}
runs-on: ubuntu-24.04
# Overlapping main runs report one at a time so they update a single regression issue.
# Overlapping main runs report one at a time so they update a single issue.
concurrency:
group: release-performance-report
cancel-in-progress: false
permissions:
actions: read
checks: read
contents: read
issues: write
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
steps:
- uses: actions/download-artifact@v4
if: needs.compare.outputs.regressed == 'true'
with:
name: release-performance-${{ github.run_id }}-${{ github.run_attempt }}
path: evidence
- name: Open or update the performance regression issue
if: needs.compare.outputs.regressed == 'true'
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
LABEL: performance-regression
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
{
echo "Release performance regressed at ${GITHUB_SHA} on main."
Expand All @@ -190,3 +202,49 @@ jobs:
gh issue create --title "Release performance regression on main" \
--label "${LABEL}" --body-file report.md
fi
- uses: actions/checkout@v4
if: needs.compare.outputs.regressed != 'true'
with:
sparse-checkout: scripts
# A failed compare job, or one stopped by its own timeout, did not finish. A run cancelled by
# a newer run or by hand is also `cancelled`, but without the timeout annotation, so the
# script prints nothing for it.
- name: Open or update the unfinished comparison issue
if: needs.compare.outputs.regressed != 'true'
env:
LABEL: release-performance-unfinished
RESULT: ${{ needs.compare.result }}
REGRESSED: ${{ needs.compare.outputs.regressed }}
run: |
job="$(gh api "repos/${GH_REPO}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
--jq '.jobs[] | select(.name == "Compare with the previous release") | .id')"
if [[ -z "${job}" ]]; then
echo "::error::The compare job is not listed in run attempt ${GITHUB_RUN_ATTEMPT}"
exit 1
fi
gh api "repos/${GH_REPO}/check-runs/${job}/annotations" > annotations.json
reason="$(python3 -m scripts.report_unfinished_release_performance \
--result "${RESULT}" --regressed "${REGRESSED}" --annotations annotations.json)"
if [[ -z "${reason}" ]]; then
echo "The comparison was cancelled by a newer run or by hand; nothing to report." \
>> "${GITHUB_STEP_SUMMARY}"
exit 0
fi
{
echo "The release performance comparison did not finish at ${GITHUB_SHA} on main."
echo
echo "Comparison run: ${RUN_URL} (its job summary lists the phases that started)"
echo
echo "${reason}"
} > report.md
gh label create "${LABEL}" --color FBCA04 \
--description "A main release performance comparison did not finish" 2>/dev/null || true
issue="$(gh issue list --label "${LABEL}" --state open --limit 1 \
--json number --jq '.[0].number // empty')"
if [[ -n "${issue}" ]]; then
gh issue comment "${issue}" --body-file report.md
echo "Updated issue #${issue}" >> "${GITHUB_STEP_SUMMARY}"
else
gh issue create --title "Release performance comparison did not finish on main" \
--label "${LABEL}" --body-file report.md
fi
2 changes: 2 additions & 0 deletions scripts/compile_performance_ratio/_helpers/measure.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ def run_checkout_generator(
arguments: tuple[str, ...],
environment: dict[str, str],
library_paths: tuple[Path, ...],
timeout_seconds: float | None = None,
) -> subprocess.CompletedProcess[str]:
"""Run another checkout's generator with `python`, importing `library_paths` before its own."""

Expand All @@ -91,6 +92,7 @@ def run_checkout_generator(
PYTHONPATH_KEY: os.pathsep.join(str(path) for path in (checkout, *library_paths)),
},
check=False,
timeout=timeout_seconds,
)


Expand Down
24 changes: 15 additions & 9 deletions scripts/release_performance/_helpers/benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project
from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit
from scripts.compile_performance_ratio._helpers.measure import run_checkout_generator
from scripts.release_performance._helpers.versions import library_paths
from scripts.release_performance._helpers.versions import library_paths, setup_timeout_error
from scripts.release_performance.constants import (
BASELINE_GENERATOR_ENTRY,
BUILD_PROJECT,
Expand All @@ -29,6 +29,7 @@
INSPECTION_PROJECT,
INSPECTION_WARMUPS,
PRISTINE_DIRECTORY,
SETUP_COMMAND_TIMEOUT_SECONDS,
STDERR_TAIL_CHARACTERS,
TARGET_DIRECTORY,
TIME_FORMAT,
Expand Down Expand Up @@ -174,14 +175,19 @@ def compare_command(
def _run_baseline_generator(
*, source: Path, python: Path, entry: str, arguments: tuple[str, ...], description: str
) -> None:
completed: subprocess.CompletedProcess[str] = run_checkout_generator(
checkout=source,
python=Path(sys.executable).absolute(),
entry=entry,
arguments=arguments,
environment=_environment(),
library_paths=library_paths(python=python),
)
paths: tuple[Path, ...] = library_paths(python=python)
try:
completed: subprocess.CompletedProcess[str] = run_checkout_generator(
checkout=source,
python=Path(sys.executable).absolute(),
entry=entry,
arguments=arguments,
environment=_environment(),
library_paths=paths,
timeout_seconds=SETUP_COMMAND_TIMEOUT_SECONDS,
)
except subprocess.TimeoutExpired:
raise setup_timeout_error(phase=f"Baseline {description} generation") from None
if completed.returncode != 0:
raise ReleasePerformanceError(
f"Baseline {description} generation with the generator in {source} failed with exit "
Expand Down
96 changes: 66 additions & 30 deletions scripts/release_performance/_helpers/versions.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
from __future__ import annotations

import hashlib
import io
import json
import platform
import re
Expand All @@ -24,6 +23,7 @@
PYPI_SIMPLE_JSON,
PYPI_SIMPLE_URL,
RELEASE_TAG_PREFIX,
SETUP_COMMAND_TIMEOUT_SECONDS,
STDERR_TAIL_CHARACTERS,
UNIVERSAL_WHEEL_MACHINE,
WHEEL_MACHINE_ALIASES,
Expand Down Expand Up @@ -107,21 +107,33 @@ def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path:
"""Extract the source tree of release tag v<version> from repo_dir into destination."""

tag: str = RELEASE_TAG_PREFIX + version
completed: subprocess.CompletedProcess[bytes] = subprocess.run(
["git", "-C", str(repo_dir), "archive", "--format=tar", tag],
capture_output=True,
check=False,
)
if completed.returncode != 0:
raise ReleasePerformanceError(
f"Release tag {tag} is not available in {repo_dir}, so the baseline's own benchmark "
"generator cannot be used; fetch the release tags or pass --baseline-source: "
+ _tail(completed.stderr.decode("utf-8", errors="replace"))
)
shutil.rmtree(destination, ignore_errors=True)
destination.mkdir(parents=True)
with tarfile.open(fileobj=io.BytesIO(completed.stdout)) as archive:
archive.extractall(destination, filter="data")
tar: Path = destination.with_name(destination.name + ".tar")
try:
completed: subprocess.CompletedProcess[str] = run_setup_command(
command=[
"git",
"-C",
str(repo_dir),
"archive",
"--format=tar",
"--output",
str(tar),
tag,
],
phase=f"Extracting release tag {tag}",
)
if completed.returncode != 0:
raise ReleasePerformanceError(
f"Release tag {tag} is not available in {repo_dir}, so the baseline's own "
"benchmark generator cannot be used; fetch the release tags or pass "
"--baseline-source: " + _tail(completed.stderr)
)
with tarfile.open(tar) as archive:
archive.extractall(destination, filter="data")
finally:
tar.unlink(missing_ok=True)
return destination


Expand All @@ -143,7 +155,9 @@ def install_published(*, version: str, venv_dir: Path, python: str, wait_seconds
deadline: float = time.monotonic() + wait_seconds
while True:
attempt: subprocess.CompletedProcess[str] = _uv_pip_install(
venv_dir=venv_dir, requirements=(f"{PACKAGE_NAME}=={version}",)
venv_dir=venv_dir,
requirements=(f"{PACKAGE_NAME}=={version}",),
phase=f"Installing {PACKAGE_NAME} {version} from the index",
)
if attempt.returncode == 0:
return _sqb(venv_dir=venv_dir)
Expand Down Expand Up @@ -174,6 +188,7 @@ def install_wheel(*, wheel: Path, venv_dir: Path, python: str) -> Path:
completed: subprocess.CompletedProcess[str] = _uv_pip_install(
venv_dir=venv_dir,
requirements=("--find-links", str(wheel.resolve().parent), str(wheel.resolve())),
phase=f"Installing {wheel.name}",
)
if completed.returncode != 0:
raise ReleasePerformanceError(f"Installing {wheel} failed: {_tail(completed.stderr)}")
Expand All @@ -183,8 +198,8 @@ def install_wheel(*, wheel: Path, venv_dir: Path, python: str) -> Path:
def installed_version(*, sqb: Path) -> str:
"""Return the version an installed sqb reports."""

completed: subprocess.CompletedProcess[str] = subprocess.run(
[str(sqb), "--version"], capture_output=True, text=True, check=False
completed: subprocess.CompletedProcess[str] = run_setup_command(
command=[str(sqb), "--version"], phase=f"Reading the version of {sqb}"
)
match: re.Match[str] | None = _RELEASE_VERSION.search(completed.stdout)
if completed.returncode != 0 or match is None:
Expand All @@ -195,8 +210,9 @@ def installed_version(*, sqb: Path) -> str:
def library_paths(*, python: Path) -> tuple[Path, ...]:
"""Return the package directories of `python`'s environment, which must match this Python."""

completed: subprocess.CompletedProcess[str] = subprocess.run(
[str(python), "-c", _LIBRARY_PATHS_PROBE], capture_output=True, text=True, check=False
completed: subprocess.CompletedProcess[str] = run_setup_command(
command=[str(python), "-c", _LIBRARY_PATHS_PROBE],
phase=f"Reading the package directories of {python}",
)
if completed.returncode != 0:
raise ReleasePerformanceError(
Expand All @@ -213,6 +229,29 @@ def library_paths(*, python: Path) -> tuple[Path, ...]:
return tuple(dict.fromkeys(Path(str(path)) for path in probe["paths"]))


def run_setup_command(*, command: list[str], phase: str) -> subprocess.CompletedProcess[str]:
"""Run a setup command; one that hangs is stopped and fails with the phase it blocked."""

try:
return subprocess.run(
command,
capture_output=True,
text=True,
check=False,
timeout=SETUP_COMMAND_TIMEOUT_SECONDS,
)
except subprocess.TimeoutExpired:
raise setup_timeout_error(phase=phase) from None


def setup_timeout_error(*, phase: str) -> ReleasePerformanceError:
"""Return the error for a setup phase that exceeded SETUP_COMMAND_TIMEOUT_SECONDS."""

return ReleasePerformanceError(
f"{phase} did not finish within {SETUP_COMMAND_TIMEOUT_SECONDS:g}s and was stopped"
)


def existing_sqb(*, venv_dir: Path) -> Path:
"""Return the sqb entry point of an existing virtual environment."""

Expand Down Expand Up @@ -247,6 +286,7 @@ def _install_from_wheel_urls(*, version: str, venv_dir: Path) -> bool:
completed: subprocess.CompletedProcess[str] = _uv_pip_install(
venv_dir=venv_dir,
requirements=("--find-links", str(download_dir), f"{PACKAGE_NAME}=={version}"),
phase=f"Installing {PACKAGE_NAME} {version} from its wheel URLs",
)
if completed.returncode != 0:
print(f"Wheel URL install failed: {_tail(completed.stderr)}", file=sys.stderr)
Expand Down Expand Up @@ -280,11 +320,9 @@ def _platform_matches(*, platform_tag: str, machine: str, system: str) -> bool:

def _create_venv(*, venv_dir: Path, python: str) -> None:
shutil.rmtree(venv_dir, ignore_errors=True)
completed: subprocess.CompletedProcess[str] = subprocess.run(
[_uv(), "venv", "--quiet", "--python", python, str(venv_dir)],
capture_output=True,
text=True,
check=False,
completed: subprocess.CompletedProcess[str] = run_setup_command(
command=[_uv(), "venv", "--quiet", "--python", python, str(venv_dir)],
phase=f"Creating the Python {python} environment {venv_dir}",
)
if completed.returncode != 0:
raise ReleasePerformanceError(
Expand All @@ -293,10 +331,10 @@ def _create_venv(*, venv_dir: Path, python: str) -> None:


def _uv_pip_install(
*, venv_dir: Path, requirements: tuple[str, ...]
*, venv_dir: Path, requirements: tuple[str, ...], phase: str
) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[
return run_setup_command(
command=[
_uv(),
"pip",
"install",
Expand All @@ -305,9 +343,7 @@ def _uv_pip_install(
str(venv_dir / "bin" / "python"),
*requirements,
],
capture_output=True,
text=True,
check=False,
phase=phase,
)


Expand Down
Loading
Loading