Skip to content

Commit d3dda67

Browse files
authored
ci: report release performance runs that do not finish (#1536)
A main comparison that hung ran into the job timeout, lost its log and evidence upload, and reported nothing. The comparison step now has its own limit below the job's, setup commands stop with an error naming their phase, each phase is listed in the job summary as it starts, and a main run that errors, loses its runner or times out opens or updates one release-performance-unfinished issue. Runs cancelled by concurrency or by hand are not reported.
1 parent 3b9422e commit d3dda67

21 files changed

Lines changed: 499 additions & 58 deletions

File tree

‎.github/workflows/release-performance.yml‎

Lines changed: 66 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@ run-name: >-
44
55
# Every push to main compares main with the latest release on the same runner. A regression fails
66
# the run and opens or updates one issue labelled performance-regression; it does not gate the
7-
# release pull request. Pull requests that change the comparison itself run it too.
7+
# release pull request. A main run that does not finish, such as one that errors, loses its runner
8+
# or times out, opens or updates one issue labelled release-performance-unfinished. Pull requests
9+
# that change the comparison itself run it too.
810
on:
911
push:
1012
branches:
@@ -45,7 +47,9 @@ jobs:
4547
compare:
4648
name: Compare with the previous release
4749
runs-on: ubuntu-24.04
48-
timeout-minutes: 90
50+
# A backstop above the slowest normal install and build (about 51 minutes together) plus the
51+
# comparison's own limit, so a hung comparison stops at its step and still uploads evidence.
52+
timeout-minutes: 130
4953
outputs:
5054
regressed: ${{ steps.compare.outputs.regressed }}
5155
steps:
@@ -89,8 +93,10 @@ jobs:
8993
run: git fetch --tags --force --quiet origin
9094
# Temporary: one-model edit compiles use MAX_EDIT_COMPILE_TIME_RATIO (1.35x) because an edited
9195
# model recompiles uncached; tighten it back to 1.15x once native rendering is the default.
96+
# Comparisons take 10 to 21 minutes, plus up to 30 while a new baseline reaches PyPI.
9297
- name: Compare candidate and previous release on this runner
9398
id: compare
99+
timeout-minutes: 65
94100
env:
95101
BASELINE: ${{ inputs.baseline }}
96102
EVENT_NAME: ${{ github.event_name }}
@@ -150,27 +156,33 @@ jobs:
150156
exit 1
151157
152158
report:
153-
name: Report regression
159+
name: Report regression or unfinished comparison
154160
needs: compare
155-
if: ${{ always() && github.event_name == 'push' && needs.compare.outputs.regressed == 'true' }}
161+
if: ${{ always() && github.event_name == 'push' && needs.compare.result != 'success' }}
156162
runs-on: ubuntu-24.04
157-
# Overlapping main runs report one at a time so they update a single regression issue.
163+
# Overlapping main runs report one at a time so they update a single issue.
158164
concurrency:
159165
group: release-performance-report
160166
cancel-in-progress: false
161167
permissions:
168+
actions: read
169+
checks: read
170+
contents: read
162171
issues: write
172+
env:
173+
GH_TOKEN: ${{ github.token }}
174+
GH_REPO: ${{ github.repository }}
175+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
163176
steps:
164177
- uses: actions/download-artifact@v4
178+
if: needs.compare.outputs.regressed == 'true'
165179
with:
166180
name: release-performance-${{ github.run_id }}-${{ github.run_attempt }}
167181
path: evidence
168182
- name: Open or update the performance regression issue
183+
if: needs.compare.outputs.regressed == 'true'
169184
env:
170-
GH_TOKEN: ${{ github.token }}
171-
GH_REPO: ${{ github.repository }}
172185
LABEL: performance-regression
173-
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
174186
run: |
175187
{
176188
echo "Release performance regressed at ${GITHUB_SHA} on main."
@@ -190,3 +202,49 @@ jobs:
190202
gh issue create --title "Release performance regression on main" \
191203
--label "${LABEL}" --body-file report.md
192204
fi
205+
- uses: actions/checkout@v4
206+
if: needs.compare.outputs.regressed != 'true'
207+
with:
208+
sparse-checkout: scripts
209+
# A failed compare job, or one stopped by its own timeout, did not finish. A run cancelled by
210+
# a newer run or by hand is also `cancelled`, but without the timeout annotation, so the
211+
# script prints nothing for it.
212+
- name: Open or update the unfinished comparison issue
213+
if: needs.compare.outputs.regressed != 'true'
214+
env:
215+
LABEL: release-performance-unfinished
216+
RESULT: ${{ needs.compare.result }}
217+
REGRESSED: ${{ needs.compare.outputs.regressed }}
218+
run: |
219+
job="$(gh api "repos/${GH_REPO}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
220+
--jq '.jobs[] | select(.name == "Compare with the previous release") | .id')"
221+
if [[ -z "${job}" ]]; then
222+
echo "::error::The compare job is not listed in run attempt ${GITHUB_RUN_ATTEMPT}"
223+
exit 1
224+
fi
225+
gh api "repos/${GH_REPO}/check-runs/${job}/annotations" > annotations.json
226+
reason="$(python3 -m scripts.report_unfinished_release_performance \
227+
--result "${RESULT}" --regressed "${REGRESSED}" --annotations annotations.json)"
228+
if [[ -z "${reason}" ]]; then
229+
echo "The comparison was cancelled by a newer run or by hand; nothing to report." \
230+
>> "${GITHUB_STEP_SUMMARY}"
231+
exit 0
232+
fi
233+
{
234+
echo "The release performance comparison did not finish at ${GITHUB_SHA} on main."
235+
echo
236+
echo "Comparison run: ${RUN_URL} (its job summary lists the phases that started)"
237+
echo
238+
echo "${reason}"
239+
} > report.md
240+
gh label create "${LABEL}" --color FBCA04 \
241+
--description "A main release performance comparison did not finish" 2>/dev/null || true
242+
issue="$(gh issue list --label "${LABEL}" --state open --limit 1 \
243+
--json number --jq '.[0].number // empty')"
244+
if [[ -n "${issue}" ]]; then
245+
gh issue comment "${issue}" --body-file report.md
246+
echo "Updated issue #${issue}" >> "${GITHUB_STEP_SUMMARY}"
247+
else
248+
gh issue create --title "Release performance comparison did not finish on main" \
249+
--label "${LABEL}" --body-file report.md
250+
fi

‎scripts/compile_performance_ratio/_helpers/measure.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ def run_checkout_generator(
7878
arguments: tuple[str, ...],
7979
environment: dict[str, str],
8080
library_paths: tuple[Path, ...],
81+
timeout_seconds: float | None = None,
8182
) -> subprocess.CompletedProcess[str]:
8283
"""Run another checkout's generator with `python`, importing `library_paths` before its own."""
8384

@@ -91,6 +92,7 @@ def run_checkout_generator(
9192
PYTHONPATH_KEY: os.pathsep.join(str(path) for path in (checkout, *library_paths)),
9293
},
9394
check=False,
95+
timeout=timeout_seconds,
9496
)
9597

9698

‎scripts/release_performance/_helpers/benchmark.py‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project
1515
from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit
1616
from scripts.compile_performance_ratio._helpers.measure import run_checkout_generator
17-
from scripts.release_performance._helpers.versions import library_paths
17+
from scripts.release_performance._helpers.versions import library_paths, setup_timeout_error
1818
from scripts.release_performance.constants import (
1919
BASELINE_GENERATOR_ENTRY,
2020
BUILD_PROJECT,
@@ -29,6 +29,7 @@
2929
INSPECTION_PROJECT,
3030
INSPECTION_WARMUPS,
3131
PRISTINE_DIRECTORY,
32+
SETUP_COMMAND_TIMEOUT_SECONDS,
3233
STDERR_TAIL_CHARACTERS,
3334
TARGET_DIRECTORY,
3435
TIME_FORMAT,
@@ -174,14 +175,19 @@ def compare_command(
174175
def _run_baseline_generator(
175176
*, source: Path, python: Path, entry: str, arguments: tuple[str, ...], description: str
176177
) -> None:
177-
completed: subprocess.CompletedProcess[str] = run_checkout_generator(
178-
checkout=source,
179-
python=Path(sys.executable).absolute(),
180-
entry=entry,
181-
arguments=arguments,
182-
environment=_environment(),
183-
library_paths=library_paths(python=python),
184-
)
178+
paths: tuple[Path, ...] = library_paths(python=python)
179+
try:
180+
completed: subprocess.CompletedProcess[str] = run_checkout_generator(
181+
checkout=source,
182+
python=Path(sys.executable).absolute(),
183+
entry=entry,
184+
arguments=arguments,
185+
environment=_environment(),
186+
library_paths=paths,
187+
timeout_seconds=SETUP_COMMAND_TIMEOUT_SECONDS,
188+
)
189+
except subprocess.TimeoutExpired:
190+
raise setup_timeout_error(phase=f"Baseline {description} generation") from None
185191
if completed.returncode != 0:
186192
raise ReleasePerformanceError(
187193
f"Baseline {description} generation with the generator in {source} failed with exit "

‎scripts/release_performance/_helpers/versions.py‎

Lines changed: 66 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
from __future__ import annotations
44

55
import hashlib
6-
import io
76
import json
87
import platform
98
import re
@@ -24,6 +23,7 @@
2423
PYPI_SIMPLE_JSON,
2524
PYPI_SIMPLE_URL,
2625
RELEASE_TAG_PREFIX,
26+
SETUP_COMMAND_TIMEOUT_SECONDS,
2727
STDERR_TAIL_CHARACTERS,
2828
UNIVERSAL_WHEEL_MACHINE,
2929
WHEEL_MACHINE_ALIASES,
@@ -107,21 +107,33 @@ def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path:
107107
"""Extract the source tree of release tag v<version> from repo_dir into destination."""
108108

109109
tag: str = RELEASE_TAG_PREFIX + version
110-
completed: subprocess.CompletedProcess[bytes] = subprocess.run(
111-
["git", "-C", str(repo_dir), "archive", "--format=tar", tag],
112-
capture_output=True,
113-
check=False,
114-
)
115-
if completed.returncode != 0:
116-
raise ReleasePerformanceError(
117-
f"Release tag {tag} is not available in {repo_dir}, so the baseline's own benchmark "
118-
"generator cannot be used; fetch the release tags or pass --baseline-source: "
119-
+ _tail(completed.stderr.decode("utf-8", errors="replace"))
120-
)
121110
shutil.rmtree(destination, ignore_errors=True)
122111
destination.mkdir(parents=True)
123-
with tarfile.open(fileobj=io.BytesIO(completed.stdout)) as archive:
124-
archive.extractall(destination, filter="data")
112+
tar: Path = destination.with_name(destination.name + ".tar")
113+
try:
114+
completed: subprocess.CompletedProcess[str] = run_setup_command(
115+
command=[
116+
"git",
117+
"-C",
118+
str(repo_dir),
119+
"archive",
120+
"--format=tar",
121+
"--output",
122+
str(tar),
123+
tag,
124+
],
125+
phase=f"Extracting release tag {tag}",
126+
)
127+
if completed.returncode != 0:
128+
raise ReleasePerformanceError(
129+
f"Release tag {tag} is not available in {repo_dir}, so the baseline's own "
130+
"benchmark generator cannot be used; fetch the release tags or pass "
131+
"--baseline-source: " + _tail(completed.stderr)
132+
)
133+
with tarfile.open(tar) as archive:
134+
archive.extractall(destination, filter="data")
135+
finally:
136+
tar.unlink(missing_ok=True)
125137
return destination
126138

127139

@@ -143,7 +155,9 @@ def install_published(*, version: str, venv_dir: Path, python: str, wait_seconds
143155
deadline: float = time.monotonic() + wait_seconds
144156
while True:
145157
attempt: subprocess.CompletedProcess[str] = _uv_pip_install(
146-
venv_dir=venv_dir, requirements=(f"{PACKAGE_NAME}=={version}",)
158+
venv_dir=venv_dir,
159+
requirements=(f"{PACKAGE_NAME}=={version}",),
160+
phase=f"Installing {PACKAGE_NAME} {version} from the index",
147161
)
148162
if attempt.returncode == 0:
149163
return _sqb(venv_dir=venv_dir)
@@ -174,6 +188,7 @@ def install_wheel(*, wheel: Path, venv_dir: Path, python: str) -> Path:
174188
completed: subprocess.CompletedProcess[str] = _uv_pip_install(
175189
venv_dir=venv_dir,
176190
requirements=("--find-links", str(wheel.resolve().parent), str(wheel.resolve())),
191+
phase=f"Installing {wheel.name}",
177192
)
178193
if completed.returncode != 0:
179194
raise ReleasePerformanceError(f"Installing {wheel} failed: {_tail(completed.stderr)}")
@@ -183,8 +198,8 @@ def install_wheel(*, wheel: Path, venv_dir: Path, python: str) -> Path:
183198
def installed_version(*, sqb: Path) -> str:
184199
"""Return the version an installed sqb reports."""
185200

186-
completed: subprocess.CompletedProcess[str] = subprocess.run(
187-
[str(sqb), "--version"], capture_output=True, text=True, check=False
201+
completed: subprocess.CompletedProcess[str] = run_setup_command(
202+
command=[str(sqb), "--version"], phase=f"Reading the version of {sqb}"
188203
)
189204
match: re.Match[str] | None = _RELEASE_VERSION.search(completed.stdout)
190205
if completed.returncode != 0 or match is None:
@@ -195,8 +210,9 @@ def installed_version(*, sqb: Path) -> str:
195210
def library_paths(*, python: Path) -> tuple[Path, ...]:
196211
"""Return the package directories of `python`'s environment, which must match this Python."""
197212

198-
completed: subprocess.CompletedProcess[str] = subprocess.run(
199-
[str(python), "-c", _LIBRARY_PATHS_PROBE], capture_output=True, text=True, check=False
213+
completed: subprocess.CompletedProcess[str] = run_setup_command(
214+
command=[str(python), "-c", _LIBRARY_PATHS_PROBE],
215+
phase=f"Reading the package directories of {python}",
200216
)
201217
if completed.returncode != 0:
202218
raise ReleasePerformanceError(
@@ -213,6 +229,29 @@ def library_paths(*, python: Path) -> tuple[Path, ...]:
213229
return tuple(dict.fromkeys(Path(str(path)) for path in probe["paths"]))
214230

215231

232+
def run_setup_command(*, command: list[str], phase: str) -> subprocess.CompletedProcess[str]:
233+
"""Run a setup command; one that hangs is stopped and fails with the phase it blocked."""
234+
235+
try:
236+
return subprocess.run(
237+
command,
238+
capture_output=True,
239+
text=True,
240+
check=False,
241+
timeout=SETUP_COMMAND_TIMEOUT_SECONDS,
242+
)
243+
except subprocess.TimeoutExpired:
244+
raise setup_timeout_error(phase=phase) from None
245+
246+
247+
def setup_timeout_error(*, phase: str) -> ReleasePerformanceError:
248+
"""Return the error for a setup phase that exceeded SETUP_COMMAND_TIMEOUT_SECONDS."""
249+
250+
return ReleasePerformanceError(
251+
f"{phase} did not finish within {SETUP_COMMAND_TIMEOUT_SECONDS:g}s and was stopped"
252+
)
253+
254+
216255
def existing_sqb(*, venv_dir: Path) -> Path:
217256
"""Return the sqb entry point of an existing virtual environment."""
218257

@@ -247,6 +286,7 @@ def _install_from_wheel_urls(*, version: str, venv_dir: Path) -> bool:
247286
completed: subprocess.CompletedProcess[str] = _uv_pip_install(
248287
venv_dir=venv_dir,
249288
requirements=("--find-links", str(download_dir), f"{PACKAGE_NAME}=={version}"),
289+
phase=f"Installing {PACKAGE_NAME} {version} from its wheel URLs",
250290
)
251291
if completed.returncode != 0:
252292
print(f"Wheel URL install failed: {_tail(completed.stderr)}", file=sys.stderr)
@@ -280,11 +320,9 @@ def _platform_matches(*, platform_tag: str, machine: str, system: str) -> bool:
280320

281321
def _create_venv(*, venv_dir: Path, python: str) -> None:
282322
shutil.rmtree(venv_dir, ignore_errors=True)
283-
completed: subprocess.CompletedProcess[str] = subprocess.run(
284-
[_uv(), "venv", "--quiet", "--python", python, str(venv_dir)],
285-
capture_output=True,
286-
text=True,
287-
check=False,
323+
completed: subprocess.CompletedProcess[str] = run_setup_command(
324+
command=[_uv(), "venv", "--quiet", "--python", python, str(venv_dir)],
325+
phase=f"Creating the Python {python} environment {venv_dir}",
288326
)
289327
if completed.returncode != 0:
290328
raise ReleasePerformanceError(
@@ -293,10 +331,10 @@ def _create_venv(*, venv_dir: Path, python: str) -> None:
293331

294332

295333
def _uv_pip_install(
296-
*, venv_dir: Path, requirements: tuple[str, ...]
334+
*, venv_dir: Path, requirements: tuple[str, ...], phase: str
297335
) -> subprocess.CompletedProcess[str]:
298-
return subprocess.run(
299-
[
336+
return run_setup_command(
337+
command=[
300338
_uv(),
301339
"pip",
302340
"install",
@@ -305,9 +343,7 @@ def _uv_pip_install(
305343
str(venv_dir / "bin" / "python"),
306344
*requirements,
307345
],
308-
capture_output=True,
309-
text=True,
310-
check=False,
346+
phase=phase,
311347
)
312348

313349

0 commit comments

Comments
 (0)