From cbaa0e68f0a1f413d8e851bb386f1af7378d3288 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 03:53:30 +0000 Subject: [PATCH] ci: report release performance runs that do not finish A main comparison that hung ran into the job timeout, lost its log and evidence upload, and reported nothing. The comparison step now has its own limit below the job's, setup commands stop with an error naming their phase, each phase is listed in the job summary as it starts, and a main run that errors, loses its runner or times out opens or updates one release-performance-unfinished issue. Runs cancelled by concurrency or by hand are not reported. --- .github/workflows/release-performance.yml | 74 ++++++++- .../_helpers/measure.py | 2 + .../release_performance/_helpers/benchmark.py | 24 +-- .../release_performance/_helpers/versions.py | 96 ++++++++---- .../release_performance/_helpers/workflow.py | 37 +++-- scripts/release_performance/constants.py | 2 + .../release_performance_outcome/__init__.py | 1 + .../_helpers/__init__.py | 0 .../_helpers/outcome.py | 22 +++ .../release_performance_outcome/constants.py | 8 + .../main/__init__.py | 0 .../main/report.py | 41 +++++ .../report_unfinished_release_performance.py | 15 ++ .../test_compare_release_performance.py | 4 + .../_helpers/_test_types.py | 9 ++ .../release_performance/_helpers/helpers.py | 29 ++++ .../_helpers/test_benchmark.py | 36 +++++ .../release_performance_outcome/__init__.py | 0 .../_helpers/__init__.py | 0 .../_helpers/_test_types.py | 12 ++ .../_helpers/test_outcome.py | 145 ++++++++++++++++++ 21 files changed, 499 insertions(+), 58 deletions(-) create mode 100644 scripts/release_performance_outcome/__init__.py create mode 100644 scripts/release_performance_outcome/_helpers/__init__.py create mode 100644 scripts/release_performance_outcome/_helpers/outcome.py create mode 100644 scripts/release_performance_outcome/constants.py create mode 100644 scripts/release_performance_outcome/main/__init__.py create mode 100644 scripts/release_performance_outcome/main/report.py create mode 100644 scripts/report_unfinished_release_performance.py create mode 100644 tests/unit/scripts/release_performance_outcome/__init__.py create mode 100644 tests/unit/scripts/release_performance_outcome/_helpers/__init__.py create mode 100644 tests/unit/scripts/release_performance_outcome/_helpers/_test_types.py create mode 100644 tests/unit/scripts/release_performance_outcome/_helpers/test_outcome.py diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml index 855c7caa5d..0c2d533ac3 100644 --- a/.github/workflows/release-performance.yml +++ b/.github/workflows/release-performance.yml @@ -4,7 +4,9 @@ run-name: >- # Every push to main compares main with the latest release on the same runner. A regression fails # the run and opens or updates one issue labelled performance-regression; it does not gate the -# release pull request. Pull requests that change the comparison itself run it too. +# release pull request. A main run that does not finish, such as one that errors, loses its runner +# or times out, opens or updates one issue labelled release-performance-unfinished. Pull requests +# that change the comparison itself run it too. on: push: branches: @@ -45,7 +47,9 @@ jobs: compare: name: Compare with the previous release runs-on: ubuntu-24.04 - timeout-minutes: 90 + # A backstop above the slowest normal install and build (about 51 minutes together) plus the + # comparison's own limit, so a hung comparison stops at its step and still uploads evidence. + timeout-minutes: 130 outputs: regressed: ${{ steps.compare.outputs.regressed }} steps: @@ -89,8 +93,10 @@ jobs: run: git fetch --tags --force --quiet origin # Temporary: one-model edit compiles use MAX_EDIT_COMPILE_TIME_RATIO (1.35x) because an edited # model recompiles uncached; tighten it back to 1.15x once native rendering is the default. + # Comparisons take 10 to 21 minutes, plus up to 30 while a new baseline reaches PyPI. - name: Compare candidate and previous release on this runner id: compare + timeout-minutes: 65 env: BASELINE: ${{ inputs.baseline }} EVENT_NAME: ${{ github.event_name }} @@ -150,27 +156,33 @@ jobs: exit 1 report: - name: Report regression + name: Report regression or unfinished comparison needs: compare - if: ${{ always() && github.event_name == 'push' && needs.compare.outputs.regressed == 'true' }} + if: ${{ always() && github.event_name == 'push' && needs.compare.result != 'success' }} runs-on: ubuntu-24.04 - # Overlapping main runs report one at a time so they update a single regression issue. + # Overlapping main runs report one at a time so they update a single issue. concurrency: group: release-performance-report cancel-in-progress: false permissions: + actions: read + checks: read + contents: read issues: write + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} steps: - uses: actions/download-artifact@v4 + if: needs.compare.outputs.regressed == 'true' with: name: release-performance-${{ github.run_id }}-${{ github.run_attempt }} path: evidence - name: Open or update the performance regression issue + if: needs.compare.outputs.regressed == 'true' env: - GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} LABEL: performance-regression - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | { echo "Release performance regressed at ${GITHUB_SHA} on main." @@ -190,3 +202,49 @@ jobs: gh issue create --title "Release performance regression on main" \ --label "${LABEL}" --body-file report.md fi + - uses: actions/checkout@v4 + if: needs.compare.outputs.regressed != 'true' + with: + sparse-checkout: scripts + # A failed compare job, or one stopped by its own timeout, did not finish. A run cancelled by + # a newer run or by hand is also `cancelled`, but without the timeout annotation, so the + # script prints nothing for it. + - name: Open or update the unfinished comparison issue + if: needs.compare.outputs.regressed != 'true' + env: + LABEL: release-performance-unfinished + RESULT: ${{ needs.compare.result }} + REGRESSED: ${{ needs.compare.outputs.regressed }} + run: | + job="$(gh api "repos/${GH_REPO}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \ + --jq '.jobs[] | select(.name == "Compare with the previous release") | .id')" + if [[ -z "${job}" ]]; then + echo "::error::The compare job is not listed in run attempt ${GITHUB_RUN_ATTEMPT}" + exit 1 + fi + gh api "repos/${GH_REPO}/check-runs/${job}/annotations" > annotations.json + reason="$(python3 -m scripts.report_unfinished_release_performance \ + --result "${RESULT}" --regressed "${REGRESSED}" --annotations annotations.json)" + if [[ -z "${reason}" ]]; then + echo "The comparison was cancelled by a newer run or by hand; nothing to report." \ + >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + { + echo "The release performance comparison did not finish at ${GITHUB_SHA} on main." + echo + echo "Comparison run: ${RUN_URL} (its job summary lists the phases that started)" + echo + echo "${reason}" + } > report.md + gh label create "${LABEL}" --color FBCA04 \ + --description "A main release performance comparison did not finish" 2>/dev/null || true + issue="$(gh issue list --label "${LABEL}" --state open --limit 1 \ + --json number --jq '.[0].number // empty')" + if [[ -n "${issue}" ]]; then + gh issue comment "${issue}" --body-file report.md + echo "Updated issue #${issue}" >> "${GITHUB_STEP_SUMMARY}" + else + gh issue create --title "Release performance comparison did not finish on main" \ + --label "${LABEL}" --body-file report.md + fi diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index 05add8ed2a..ed67532c46 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -78,6 +78,7 @@ def run_checkout_generator( arguments: tuple[str, ...], environment: dict[str, str], library_paths: tuple[Path, ...], + timeout_seconds: float | None = None, ) -> subprocess.CompletedProcess[str]: """Run another checkout's generator with `python`, importing `library_paths` before its own.""" @@ -91,6 +92,7 @@ def run_checkout_generator( PYTHONPATH_KEY: os.pathsep.join(str(path) for path in (checkout, *library_paths)), }, check=False, + timeout=timeout_seconds, ) diff --git a/scripts/release_performance/_helpers/benchmark.py b/scripts/release_performance/_helpers/benchmark.py index f6c4f81769..5b8627f6c7 100644 --- a/scripts/release_performance/_helpers/benchmark.py +++ b/scripts/release_performance/_helpers/benchmark.py @@ -14,7 +14,7 @@ from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit from scripts.compile_performance_ratio._helpers.measure import run_checkout_generator -from scripts.release_performance._helpers.versions import library_paths +from scripts.release_performance._helpers.versions import library_paths, setup_timeout_error from scripts.release_performance.constants import ( BASELINE_GENERATOR_ENTRY, BUILD_PROJECT, @@ -29,6 +29,7 @@ INSPECTION_PROJECT, INSPECTION_WARMUPS, PRISTINE_DIRECTORY, + SETUP_COMMAND_TIMEOUT_SECONDS, STDERR_TAIL_CHARACTERS, TARGET_DIRECTORY, TIME_FORMAT, @@ -174,14 +175,19 @@ def compare_command( def _run_baseline_generator( *, source: Path, python: Path, entry: str, arguments: tuple[str, ...], description: str ) -> None: - completed: subprocess.CompletedProcess[str] = run_checkout_generator( - checkout=source, - python=Path(sys.executable).absolute(), - entry=entry, - arguments=arguments, - environment=_environment(), - library_paths=library_paths(python=python), - ) + paths: tuple[Path, ...] = library_paths(python=python) + try: + completed: subprocess.CompletedProcess[str] = run_checkout_generator( + checkout=source, + python=Path(sys.executable).absolute(), + entry=entry, + arguments=arguments, + environment=_environment(), + library_paths=paths, + timeout_seconds=SETUP_COMMAND_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + raise setup_timeout_error(phase=f"Baseline {description} generation") from None if completed.returncode != 0: raise ReleasePerformanceError( f"Baseline {description} generation with the generator in {source} failed with exit " diff --git a/scripts/release_performance/_helpers/versions.py b/scripts/release_performance/_helpers/versions.py index 946f42440b..9b0cb2a316 100644 --- a/scripts/release_performance/_helpers/versions.py +++ b/scripts/release_performance/_helpers/versions.py @@ -3,7 +3,6 @@ from __future__ import annotations import hashlib -import io import json import platform import re @@ -24,6 +23,7 @@ PYPI_SIMPLE_JSON, PYPI_SIMPLE_URL, RELEASE_TAG_PREFIX, + SETUP_COMMAND_TIMEOUT_SECONDS, STDERR_TAIL_CHARACTERS, UNIVERSAL_WHEEL_MACHINE, WHEEL_MACHINE_ALIASES, @@ -107,21 +107,33 @@ def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path: """Extract the source tree of release tag v from repo_dir into destination.""" tag: str = RELEASE_TAG_PREFIX + version - completed: subprocess.CompletedProcess[bytes] = subprocess.run( - ["git", "-C", str(repo_dir), "archive", "--format=tar", tag], - capture_output=True, - check=False, - ) - if completed.returncode != 0: - raise ReleasePerformanceError( - f"Release tag {tag} is not available in {repo_dir}, so the baseline's own benchmark " - "generator cannot be used; fetch the release tags or pass --baseline-source: " - + _tail(completed.stderr.decode("utf-8", errors="replace")) - ) shutil.rmtree(destination, ignore_errors=True) destination.mkdir(parents=True) - with tarfile.open(fileobj=io.BytesIO(completed.stdout)) as archive: - archive.extractall(destination, filter="data") + tar: Path = destination.with_name(destination.name + ".tar") + try: + completed: subprocess.CompletedProcess[str] = run_setup_command( + command=[ + "git", + "-C", + str(repo_dir), + "archive", + "--format=tar", + "--output", + str(tar), + tag, + ], + phase=f"Extracting release tag {tag}", + ) + if completed.returncode != 0: + raise ReleasePerformanceError( + f"Release tag {tag} is not available in {repo_dir}, so the baseline's own " + "benchmark generator cannot be used; fetch the release tags or pass " + "--baseline-source: " + _tail(completed.stderr) + ) + with tarfile.open(tar) as archive: + archive.extractall(destination, filter="data") + finally: + tar.unlink(missing_ok=True) return destination @@ -143,7 +155,9 @@ def install_published(*, version: str, venv_dir: Path, python: str, wait_seconds deadline: float = time.monotonic() + wait_seconds while True: attempt: subprocess.CompletedProcess[str] = _uv_pip_install( - venv_dir=venv_dir, requirements=(f"{PACKAGE_NAME}=={version}",) + venv_dir=venv_dir, + requirements=(f"{PACKAGE_NAME}=={version}",), + phase=f"Installing {PACKAGE_NAME} {version} from the index", ) if attempt.returncode == 0: return _sqb(venv_dir=venv_dir) @@ -174,6 +188,7 @@ def install_wheel(*, wheel: Path, venv_dir: Path, python: str) -> Path: completed: subprocess.CompletedProcess[str] = _uv_pip_install( venv_dir=venv_dir, requirements=("--find-links", str(wheel.resolve().parent), str(wheel.resolve())), + phase=f"Installing {wheel.name}", ) if completed.returncode != 0: raise ReleasePerformanceError(f"Installing {wheel} failed: {_tail(completed.stderr)}") @@ -183,8 +198,8 @@ def install_wheel(*, wheel: Path, venv_dir: Path, python: str) -> Path: def installed_version(*, sqb: Path) -> str: """Return the version an installed sqb reports.""" - completed: subprocess.CompletedProcess[str] = subprocess.run( - [str(sqb), "--version"], capture_output=True, text=True, check=False + completed: subprocess.CompletedProcess[str] = run_setup_command( + command=[str(sqb), "--version"], phase=f"Reading the version of {sqb}" ) match: re.Match[str] | None = _RELEASE_VERSION.search(completed.stdout) if completed.returncode != 0 or match is None: @@ -195,8 +210,9 @@ def installed_version(*, sqb: Path) -> str: def library_paths(*, python: Path) -> tuple[Path, ...]: """Return the package directories of `python`'s environment, which must match this Python.""" - completed: subprocess.CompletedProcess[str] = subprocess.run( - [str(python), "-c", _LIBRARY_PATHS_PROBE], capture_output=True, text=True, check=False + completed: subprocess.CompletedProcess[str] = run_setup_command( + command=[str(python), "-c", _LIBRARY_PATHS_PROBE], + phase=f"Reading the package directories of {python}", ) if completed.returncode != 0: raise ReleasePerformanceError( @@ -213,6 +229,29 @@ def library_paths(*, python: Path) -> tuple[Path, ...]: return tuple(dict.fromkeys(Path(str(path)) for path in probe["paths"])) +def run_setup_command(*, command: list[str], phase: str) -> subprocess.CompletedProcess[str]: + """Run a setup command; one that hangs is stopped and fails with the phase it blocked.""" + + try: + return subprocess.run( + command, + capture_output=True, + text=True, + check=False, + timeout=SETUP_COMMAND_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + raise setup_timeout_error(phase=phase) from None + + +def setup_timeout_error(*, phase: str) -> ReleasePerformanceError: + """Return the error for a setup phase that exceeded SETUP_COMMAND_TIMEOUT_SECONDS.""" + + return ReleasePerformanceError( + f"{phase} did not finish within {SETUP_COMMAND_TIMEOUT_SECONDS:g}s and was stopped" + ) + + def existing_sqb(*, venv_dir: Path) -> Path: """Return the sqb entry point of an existing virtual environment.""" @@ -247,6 +286,7 @@ def _install_from_wheel_urls(*, version: str, venv_dir: Path) -> bool: completed: subprocess.CompletedProcess[str] = _uv_pip_install( venv_dir=venv_dir, requirements=("--find-links", str(download_dir), f"{PACKAGE_NAME}=={version}"), + phase=f"Installing {PACKAGE_NAME} {version} from its wheel URLs", ) if completed.returncode != 0: print(f"Wheel URL install failed: {_tail(completed.stderr)}", file=sys.stderr) @@ -280,11 +320,9 @@ def _platform_matches(*, platform_tag: str, machine: str, system: str) -> bool: def _create_venv(*, venv_dir: Path, python: str) -> None: shutil.rmtree(venv_dir, ignore_errors=True) - completed: subprocess.CompletedProcess[str] = subprocess.run( - [_uv(), "venv", "--quiet", "--python", python, str(venv_dir)], - capture_output=True, - text=True, - check=False, + completed: subprocess.CompletedProcess[str] = run_setup_command( + command=[_uv(), "venv", "--quiet", "--python", python, str(venv_dir)], + phase=f"Creating the Python {python} environment {venv_dir}", ) if completed.returncode != 0: raise ReleasePerformanceError( @@ -293,10 +331,10 @@ def _create_venv(*, venv_dir: Path, python: str) -> None: def _uv_pip_install( - *, venv_dir: Path, requirements: tuple[str, ...] + *, venv_dir: Path, requirements: tuple[str, ...], phase: str ) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [ + return run_setup_command( + command=[ _uv(), "pip", "install", @@ -305,9 +343,7 @@ def _uv_pip_install( str(venv_dir / "bin" / "python"), *requirements, ], - capture_output=True, - text=True, - check=False, + phase=phase, ) diff --git a/scripts/release_performance/_helpers/workflow.py b/scripts/release_performance/_helpers/workflow.py index b05be29949..5aa7d0bff6 100644 --- a/scripts/release_performance/_helpers/workflow.py +++ b/scripts/release_performance/_helpers/workflow.py @@ -7,6 +7,7 @@ import os import sys import tempfile +import time from pathlib import Path from scripts.compile_performance_ratio._helpers.report import cpu_model @@ -44,6 +45,7 @@ BASELINE_SOURCE_DIRECTORY, BENCHMARK_COMMANDS, CANDIDATE_LABEL, + PROGRESS_HEADING, RELEASE_TAG_PREFIX, REPO_ROOT, ) @@ -66,10 +68,12 @@ def run_release_comparison(*, options: ComparisonOptions) -> int: try: if options.work_dir is not None: options.work_dir.mkdir(parents=True, exist_ok=True) - comparison: ReleaseComparison = _compare(options=options, root=options.work_dir) + comparison: ReleaseComparison = _compare( + options=options, root=options.work_dir, summary=summary + ) else: with tempfile.TemporaryDirectory(prefix="sqlbuild-release-performance-") as root: - comparison = _compare(options=options, root=Path(root)) + comparison = _compare(options=options, root=Path(root), summary=summary) except (ReleasePerformanceError, BenchmarkEditError, OSError) as error: print(f"Release performance comparison failed: {error}", file=sys.stderr) _ = append_summary(path=summary, markdown=error_markdown(message=str(error))) @@ -87,8 +91,9 @@ def run_release_comparison(*, options: ComparisonOptions) -> int: return 0 -def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: - print(f"Installing candidate {options.candidate}", file=sys.stderr) +def _compare(*, options: ComparisonOptions, root: Path, summary: Path | None) -> ReleaseComparison: + _ = append_summary(path=summary, markdown=PROGRESS_HEADING) + _progress(summary=summary, message=f"Installing candidate {options.candidate}") candidate: InstalledVersion = _install( label=CANDIDATE_LABEL, spec=options.candidate, @@ -99,7 +104,7 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: baseline_spec: str = options.baseline or resolve_baseline( candidate=candidate.version, same_version=options.same_version_baseline ) - print(f"Installing baseline {baseline_spec}", file=sys.stderr) + _progress(summary=summary, message=f"Installing baseline {baseline_spec}") baseline: InstalledVersion = _install( label=BASELINE_LABEL, spec=baseline_spec, @@ -107,12 +112,12 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: python=options.python, wait_seconds=BASELINE_PUBLICATION_WAIT_SECONDS, ) - print( - f"Comparing candidate {candidate.version} with baseline {baseline.version}", - file=sys.stderr, + _progress( + summary=summary, + message=f"Comparing candidate {candidate.version} with baseline {baseline.version}", ) load_before: tuple[float, float, float] = os.getloadavg() - print("Generating candidate benchmark projects", file=sys.stderr) + _progress(summary=summary, message="Generating candidate benchmark projects") pristine: dict[str, Path] = { CANDIDATE_LABEL: write_pristine_projects( root=root, @@ -126,7 +131,9 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: else RELEASE_TAG_PREFIX + baseline.version ) write_dense_project(pristine=pristine[CANDIDATE_LABEL], models=options.dense_models) - print(f"Generating baseline benchmark projects with {baseline_generator}", file=sys.stderr) + _progress( + summary=summary, message=f"Generating baseline benchmark projects with {baseline_generator}" + ) baseline_source: Path = ( options.baseline_source.absolute() if options.baseline_source is not None @@ -152,7 +159,7 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: versions: tuple[InstalledVersion, InstalledVersion] = (baseline, candidate) projects: dict[str, dict[str, Path]] = {} for version in versions: - print(f"Warming caches for {version.label} {version.version}", file=sys.stderr) + _progress(summary=summary, message=f"Warming caches for {version.label} {version.version}") projects[version.label] = prepare_version_projects( version=version, pristine=pristine[version.label], root=root ) @@ -166,6 +173,7 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: print(f"Skipping {command.name}: {reason}", file=sys.stderr) skipped.append(SkippedCommand(name=command.name, reason=reason)) continue + _progress(summary=summary, message=f"Timing `{command.name}`") measured.append( compare_command( command=command, versions=versions, projects=projects, root=root, runs=options.runs @@ -187,6 +195,13 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: ) +def _progress(*, summary: Path | None, message: str) -> None: + print(message, file=sys.stderr, flush=True) + _ = append_summary( + path=summary, markdown=f"- {time.strftime('%H:%M:%S', time.gmtime())} UTC: {message}" + ) + + def _install( *, label: str, spec: str, root: Path, python: str, wait_seconds: float ) -> InstalledVersion: diff --git a/scripts/release_performance/constants.py b/scripts/release_performance/constants.py index 6656111a0a..d513816c06 100644 --- a/scripts/release_performance/constants.py +++ b/scripts/release_performance/constants.py @@ -23,6 +23,8 @@ DEFAULT_BUILD_MODELS: int = 1_000 DEFAULT_DENSE_MODELS: int = 3_000 COMMAND_TIMEOUT_SECONDS: float = 900.0 +SETUP_COMMAND_TIMEOUT_SECONDS: float = 2_400.0 +PROGRESS_HEADING: str = "### Release performance progress\n" PACKAGE_NAME: str = "sqlbuild" PYPI_SIMPLE_URL: str = "https://pypi.org/simple/sqlbuild/" diff --git a/scripts/release_performance_outcome/__init__.py b/scripts/release_performance_outcome/__init__.py new file mode 100644 index 0000000000..01d3e466ce --- /dev/null +++ b/scripts/release_performance_outcome/__init__.py @@ -0,0 +1 @@ +"""Decide whether a main release performance comparison run needs an unfinished report.""" diff --git a/scripts/release_performance_outcome/_helpers/__init__.py b/scripts/release_performance_outcome/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/release_performance_outcome/_helpers/outcome.py b/scripts/release_performance_outcome/_helpers/outcome.py new file mode 100644 index 0000000000..cc3df34ff2 --- /dev/null +++ b/scripts/release_performance_outcome/_helpers/outcome.py @@ -0,0 +1,22 @@ +"""Classify a compare job result as finished, cancelled from outside, or unfinished.""" + +from __future__ import annotations + +from scripts.release_performance_outcome.constants import ( + CANCELLED_RESULT, + FAILURE_RESULT, + JOB_TIMEOUT_ANNOTATION, + SUCCESS_RESULT, +) + + +def unfinished_reason(*, result: str, regressed: bool, annotations: tuple[str, ...]) -> str | None: + """Return why a non-regressed compare job failed or hit its own timeout, else None.""" + + if result == SUCCESS_RESULT or regressed: + return None + timed_out: bool = any(JOB_TIMEOUT_ANNOTATION in message for message in annotations) + if result != FAILURE_RESULT and not (result == CANCELLED_RESULT and timed_out): + return None + details: str = "".join(f"\n- {message}" for message in annotations) + return f"The comparison job ended with result `{result}`.{details}" diff --git a/scripts/release_performance_outcome/constants.py b/scripts/release_performance_outcome/constants.py new file mode 100644 index 0000000000..af2ddbe0ee --- /dev/null +++ b/scripts/release_performance_outcome/constants.py @@ -0,0 +1,8 @@ +"""Job results and annotations that tell an unfinished comparison from a cancelled one.""" + +SUCCESS_RESULT: str = "success" +FAILURE_RESULT: str = "failure" +CANCELLED_RESULT: str = "cancelled" +REGRESSED_OUTPUT: str = "true" +FAILURE_ANNOTATION_LEVEL: str = "failure" +JOB_TIMEOUT_ANNOTATION: str = "has exceeded the maximum execution time" diff --git a/scripts/release_performance_outcome/main/__init__.py b/scripts/release_performance_outcome/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/release_performance_outcome/main/report.py b/scripts/release_performance_outcome/main/report.py new file mode 100644 index 0000000000..376e5815ef --- /dev/null +++ b/scripts/release_performance_outcome/main/report.py @@ -0,0 +1,41 @@ +"""Print why a main comparison run did not finish, or nothing when it needs no report.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import cast + +from scripts.release_performance_outcome._helpers.outcome import unfinished_reason +from scripts.release_performance_outcome.constants import ( + FAILURE_ANNOTATION_LEVEL, + REGRESSED_OUTPUT, +) + + +def report_unfinished_comparison(argv: list[str] | None = None) -> int: + """Print the reason a compare job result needs an issue; print nothing otherwise.""" + + parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--result", required=True, help="The compare job's result.") + parser.add_argument("--regressed", default="", help="The compare job's regressed output.") + parser.add_argument( + "--annotations", type=Path, required=True, help="The compare job's check-run annotations." + ) + arguments: argparse.Namespace = parser.parse_args(argv) + annotations: list[dict[str, object]] = cast( + list[dict[str, object]], json.loads(Path(arguments.annotations).read_text("utf-8")) + ) + reason: str | None = unfinished_reason( + result=str(arguments.result), + regressed=arguments.regressed == REGRESSED_OUTPUT, + annotations=tuple( + str(annotation.get("message", "")) + for annotation in annotations + if annotation.get("annotation_level") == FAILURE_ANNOTATION_LEVEL + ), + ) + if reason is not None: + print(reason) + return 0 diff --git a/scripts/report_unfinished_release_performance.py b/scripts/report_unfinished_release_performance.py new file mode 100644 index 0000000000..f99f7a62f1 --- /dev/null +++ b/scripts/report_unfinished_release_performance.py @@ -0,0 +1,15 @@ +"""Direct wrapper for deciding whether a main comparison run needs an unfinished report.""" + +from __future__ import annotations + +from scripts.release_performance_outcome.main.report import report_unfinished_comparison + + +def main(argv: list[str] | None = None) -> int: + """Print why the comparison did not finish, or nothing.""" + + return report_unfinished_comparison(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/e2e/scripts/release_performance/test_compare_release_performance.py b/tests/e2e/scripts/release_performance/test_compare_release_performance.py index 4938bac819..4ea5745642 100644 --- a/tests/e2e/scripts/release_performance/test_compare_release_performance.py +++ b/tests/e2e/scripts/release_performance/test_compare_release_performance.py @@ -40,6 +40,10 @@ "| `build (empty warehouse)` |", "| `lineage column trace` |", "Benchmark projects are generated per side", + "### Release performance progress", + " UTC: Generating baseline benchmark projects with ", + " UTC: Warming caches for candidate ", + " UTC: Timing `dense compile (no cache)`", "**Result: ", ), expected_marked_projects=( diff --git a/tests/unit/scripts/release_performance/_helpers/_test_types.py b/tests/unit/scripts/release_performance/_helpers/_test_types.py index 6a4994d5ed..dee19eeed1 100644 --- a/tests/unit/scripts/release_performance/_helpers/_test_types.py +++ b/tests/unit/scripts/release_performance/_helpers/_test_types.py @@ -1,6 +1,8 @@ """Test-case models for the release performance comparison.""" +from collections.abc import Callable from dataclasses import dataclass +from pathlib import Path from scripts.release_performance.models import BenchmarkCommand, CommandComparison, SkippedCommand @@ -100,3 +102,10 @@ class ReleaseSourceTestCase: requested_version: str expected_files: tuple[str, ...] expected_error_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class HangingSetupPhaseTestCase: + description: str + run_phase: Callable[[Path], object] + expected_error: Callable[[Path], str] diff --git a/tests/unit/scripts/release_performance/_helpers/helpers.py b/tests/unit/scripts/release_performance/_helpers/helpers.py index 8128989b16..699b014a7a 100644 --- a/tests/unit/scripts/release_performance/_helpers/helpers.py +++ b/tests/unit/scripts/release_performance/_helpers/helpers.py @@ -4,6 +4,8 @@ import sys from pathlib import Path +from scripts.release_performance._helpers.benchmark import write_baseline_pristine_projects +from scripts.release_performance._helpers.versions import installed_version from scripts.release_performance.models import CommandComparison, CommandSample MIB: int = 1024 * 1024 @@ -31,6 +33,12 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models: FAILING_GENERATOR: str = """def write_pristine_projects(*, root, inspection_models, build_models): raise RuntimeError("baseline generator exploded") """ +HANGING_GENERATOR: str = """import time + + +def write_pristine_projects(*, root, inspection_models, build_models): + time.sleep(60) +""" MARKER_DENSE_GENERATOR: str = """from pathlib import Path @@ -138,3 +146,24 @@ def tagged_repository(*, root: Path, tag: str) -> Path: ): _ = subprocess.run(["git", "-C", str(root), *command], check=True, capture_output=True) return root + + +def generate_with_hanging_generator(root: Path) -> object: + """Generate baseline projects with a generator that never returns.""" + + return write_baseline_pristine_projects( + source=write_baseline_source(root=root / "source", generator=HANGING_GENERATOR), + python=write_baseline_environment(root=root / "venv"), + root=root / "generated", + inspection_models=30, + build_models=10, + ) + + +def read_hanging_sqb_version(root: Path) -> object: + """Read the version of an sqb that never exits.""" + + sqb: Path = root / "sqb" + _ = sqb.write_text("#!/bin/sh\nexec sleep 60\n", encoding="utf-8") + sqb.chmod(0o755) + return installed_version(sqb=sqb) diff --git a/tests/unit/scripts/release_performance/_helpers/test_benchmark.py b/tests/unit/scripts/release_performance/_helpers/test_benchmark.py index 8bb9bbf53b..eb79ed173f 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_benchmark.py +++ b/tests/unit/scripts/release_performance/_helpers/test_benchmark.py @@ -6,6 +6,7 @@ import pytest +from scripts.release_performance._helpers import benchmark, versions from scripts.release_performance._helpers.benchmark import ( write_baseline_dense_project, write_baseline_pristine_projects, @@ -15,6 +16,7 @@ from tests.unit.scripts.release_performance._helpers._test_types import ( BaselineDenseGeneratorTestCase, BaselineGeneratorTestCase, + HangingSetupPhaseTestCase, ReleaseSourceTestCase, ) from tests.unit.scripts.release_performance._helpers.helpers import ( @@ -25,6 +27,8 @@ FAILING_GENERATOR, MARKER_DENSE_GENERATOR, MARKER_GENERATOR, + generate_with_hanging_generator, + read_hanging_sqb_version, tagged_repository, write_baseline_dense_source, write_baseline_environment, @@ -175,5 +179,37 @@ def test_given_release_tag_when_extracting_source_then_returns_its_tree( assert all(fragment in error for fragment in test_case.expected_error_fragments), error +@pytest.mark.parametrize( + "test_case", + ( + HangingSetupPhaseTestCase( + description="a hanging baseline generator is stopped and named", + run_phase=generate_with_hanging_generator, + expected_error=lambda _root: ( + "Baseline benchmark generation did not finish within 2s and was stopped" + ), + ), + HangingSetupPhaseTestCase( + description="a hanging sqb --version is stopped and named", + run_phase=read_hanging_sqb_version, + expected_error=lambda root: ( + f"Reading the version of {root / 'sqb'} did not finish within 2s and was stopped" + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_a_hanging_setup_command_when_comparing_then_fails_naming_its_phase( + test_case: HangingSetupPhaseTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(benchmark, "SETUP_COMMAND_TIMEOUT_SECONDS", 2.0) + monkeypatch.setattr(versions, "SETUP_COMMAND_TIMEOUT_SECONDS", 2.0) + + with pytest.raises(ReleasePerformanceError) as raised: + _ = test_case.run_phase(tmp_path) + + assert str(raised.value) == test_case.expected_error(tmp_path) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/release_performance_outcome/__init__.py b/tests/unit/scripts/release_performance_outcome/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/release_performance_outcome/_helpers/__init__.py b/tests/unit/scripts/release_performance_outcome/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/release_performance_outcome/_helpers/_test_types.py b/tests/unit/scripts/release_performance_outcome/_helpers/_test_types.py new file mode 100644 index 0000000000..cd7582d2f1 --- /dev/null +++ b/tests/unit/scripts/release_performance_outcome/_helpers/_test_types.py @@ -0,0 +1,12 @@ +"""Test-case models for classifying main comparison runs.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class UnfinishedComparisonTestCase: + description: str + result: str + regressed: str + annotations: tuple[tuple[str, str], ...] + expected_output: str diff --git a/tests/unit/scripts/release_performance_outcome/_helpers/test_outcome.py b/tests/unit/scripts/release_performance_outcome/_helpers/test_outcome.py new file mode 100644 index 0000000000..85605098f0 --- /dev/null +++ b/tests/unit/scripts/release_performance_outcome/_helpers/test_outcome.py @@ -0,0 +1,145 @@ +"""Tests for deciding whether a main comparison run did not finish and needs an issue.""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests.unit.scripts.release_performance_outcome._helpers._test_types import ( + UnfinishedComparisonTestCase, +) + +_REPO_ROOT: Path = Path(__file__).resolve().parents[5] +# Annotations in the form GitHub writes them on failed, cancelled and timed-out jobs. +_JOB_TIMEOUT: tuple[str, str] = ( + "failure", + "The job has exceeded the maximum execution time of 1h30m0s", +) +_SUPERSEDED: tuple[tuple[str, str], ...] = ( + ( + "failure", + "Canceling since a higher priority waiting request for " + "release-performance-pull_request-refs/pull/1515/merge exists", + ), + ("warning", "No files were found with the provided path: /home/runner/work/_temp/x."), + ("failure", "Process completed with exit code 2."), + ("failure", "The operation was canceled."), +) +_STEP_TIMEOUT: tuple[str, str] = ( + "failure", + "The action 'Compare candidate and previous release on this runner' has timed out after " + "65 minutes.", +) +_CANCELLED_BY_HAND: tuple[str, str] = ("failure", "The run was canceled forcefully by @someone.") + + +@pytest.mark.parametrize( + "test_case", + ( + UnfinishedComparisonTestCase( + description="a job stopped by its own timeout is reported", + result="cancelled", + regressed="", + annotations=(("warning", "Node.js 20 is deprecated."), _JOB_TIMEOUT), + expected_output=( + "The comparison job ended with result `cancelled`.\n" + "- The job has exceeded the maximum execution time of 1h30m0s\n" + ), + ), + UnfinishedComparisonTestCase( + description="a step timeout fails the job and is reported", + result="failure", + regressed="", + annotations=(_STEP_TIMEOUT,), + expected_output=( + "The comparison job ended with result `failure`.\n" + "- The action 'Compare candidate and previous release on this runner' has timed " + "out after 65 minutes.\n" + ), + ), + UnfinishedComparisonTestCase( + description="a lost runner fails the job and is reported", + result="failure", + regressed="", + annotations=( + ( + "failure", + "The hosted runner: GitHub Actions 7 lost communication with the server.", + ), + ), + expected_output=( + "The comparison job ended with result `failure`.\n" + "- The hosted runner: GitHub Actions 7 lost communication with the server.\n" + ), + ), + UnfinishedComparisonTestCase( + description="a run superseded through concurrency is not reported", + result="cancelled", + regressed="", + annotations=_SUPERSEDED, + expected_output="", + ), + UnfinishedComparisonTestCase( + description="a run cancelled by hand is not reported", + result="cancelled", + regressed="", + annotations=(_CANCELLED_BY_HAND,), + expected_output="", + ), + UnfinishedComparisonTestCase( + description="a regression fails the job but has its own issue", + result="failure", + regressed="true", + annotations=(("failure", "Process completed with exit code 1."),), + expected_output="", + ), + UnfinishedComparisonTestCase( + description="a finished comparison is not reported", + result="success", + regressed="false", + annotations=(), + expected_output="", + ), + ), + ids=lambda case: case.description, +) +def test_given_a_compare_job_result_when_classifying_then_reports_only_unfinished_runs( + test_case: UnfinishedComparisonTestCase, tmp_path: Path +) -> None: + annotations: Path = tmp_path / "annotations.json" + _ = annotations.write_text( + json.dumps( + [ + {"annotation_level": level, "message": message} + for level, message in test_case.annotations + ] + ), + encoding="utf-8", + ) + + # -S drops site-packages: the report job runs this with the runner's Python and no project + # dependencies installed. + completed: subprocess.CompletedProcess[str] = subprocess.run( + [ + sys.executable, + "-S", + "-m", + "scripts.report_unfinished_release_performance", + "--result", + test_case.result, + "--regressed", + test_case.regressed, + "--annotations", + str(annotations), + ], + cwd=_REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert (completed.returncode, completed.stdout) == (0, test_case.expected_output), ( + completed.stderr + )