diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml index 855c7caa5d..0c2d533ac3 100644 --- a/.github/workflows/release-performance.yml +++ b/.github/workflows/release-performance.yml @@ -4,7 +4,9 @@ run-name: >- # Every push to main compares main with the latest release on the same runner. A regression fails # the run and opens or updates one issue labelled performance-regression; it does not gate the -# release pull request. Pull requests that change the comparison itself run it too. +# release pull request. A main run that does not finish, such as one that errors, loses its runner +# or times out, opens or updates one issue labelled release-performance-unfinished. Pull requests +# that change the comparison itself run it too. on: push: branches: @@ -45,7 +47,9 @@ jobs: compare: name: Compare with the previous release runs-on: ubuntu-24.04 - timeout-minutes: 90 + # A backstop above the slowest normal install and build (about 51 minutes together) plus the + # comparison's own limit, so a hung comparison stops at its step and still uploads evidence. + timeout-minutes: 130 outputs: regressed: ${{ steps.compare.outputs.regressed }} steps: @@ -89,8 +93,10 @@ jobs: run: git fetch --tags --force --quiet origin # Temporary: one-model edit compiles use MAX_EDIT_COMPILE_TIME_RATIO (1.35x) because an edited # model recompiles uncached; tighten it back to 1.15x once native rendering is the default. + # Comparisons take 10 to 21 minutes, plus up to 30 while a new baseline reaches PyPI. - name: Compare candidate and previous release on this runner id: compare + timeout-minutes: 65 env: BASELINE: ${{ inputs.baseline }} EVENT_NAME: ${{ github.event_name }} @@ -150,27 +156,33 @@ jobs: exit 1 report: - name: Report regression + name: Report regression or unfinished comparison needs: compare - if: ${{ always() && github.event_name == 'push' && needs.compare.outputs.regressed == 'true' }} + if: ${{ always() && github.event_name == 'push' && needs.compare.result != 'success' }} runs-on: ubuntu-24.04 - # Overlapping main runs report one at a time so they update a single regression issue. + # Overlapping main runs report one at a time so they update a single issue. concurrency: group: release-performance-report cancel-in-progress: false permissions: + actions: read + checks: read + contents: read issues: write + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} steps: - uses: actions/download-artifact@v4 + if: needs.compare.outputs.regressed == 'true' with: name: release-performance-${{ github.run_id }}-${{ github.run_attempt }} path: evidence - name: Open or update the performance regression issue + if: needs.compare.outputs.regressed == 'true' env: - GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} LABEL: performance-regression - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | { echo "Release performance regressed at ${GITHUB_SHA} on main." @@ -190,3 +202,49 @@ jobs: gh issue create --title "Release performance regression on main" \ --label "${LABEL}" --body-file report.md fi + - uses: actions/checkout@v4 + if: needs.compare.outputs.regressed != 'true' + with: + sparse-checkout: scripts + # A failed compare job, or one stopped by its own timeout, did not finish. A run cancelled by + # a newer run or by hand is also `cancelled`, but without the timeout annotation, so the + # script prints nothing for it. + - name: Open or update the unfinished comparison issue + if: needs.compare.outputs.regressed != 'true' + env: + LABEL: release-performance-unfinished + RESULT: ${{ needs.compare.result }} + REGRESSED: ${{ needs.compare.outputs.regressed }} + run: | + job="$(gh api "repos/${GH_REPO}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \ + --jq '.jobs[] | select(.name == "Compare with the previous release") | .id')" + if [[ -z "${job}" ]]; then + echo "::error::The compare job is not listed in run attempt ${GITHUB_RUN_ATTEMPT}" + exit 1 + fi + gh api "repos/${GH_REPO}/check-runs/${job}/annotations" > annotations.json + reason="$(python3 -m scripts.report_unfinished_release_performance \ + --result "${RESULT}" --regressed "${REGRESSED}" --annotations annotations.json)" + if [[ -z "${reason}" ]]; then + echo "The comparison was cancelled by a newer run or by hand; nothing to report." \ + >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + { + echo "The release performance comparison did not finish at ${GITHUB_SHA} on main." + echo + echo "Comparison run: ${RUN_URL} (its job summary lists the phases that started)" + echo + echo "${reason}" + } > report.md + gh label create "${LABEL}" --color FBCA04 \ + --description "A main release performance comparison did not finish" 2>/dev/null || true + issue="$(gh issue list --label "${LABEL}" --state open --limit 1 \ + --json number --jq '.[0].number // empty')" + if [[ -n "${issue}" ]]; then + gh issue comment "${issue}" --body-file report.md + echo "Updated issue #${issue}" >> "${GITHUB_STEP_SUMMARY}" + else + gh issue create --title "Release performance comparison did not finish on main" \ + --label "${LABEL}" --body-file report.md + fi diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index 05add8ed2a..ed67532c46 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -78,6 +78,7 @@ def run_checkout_generator( arguments: tuple[str, ...], environment: dict[str, str], library_paths: tuple[Path, ...], + timeout_seconds: float | None = None, ) -> subprocess.CompletedProcess[str]: """Run another checkout's generator with `python`, importing `library_paths` before its own.""" @@ -91,6 +92,7 @@ def run_checkout_generator( PYTHONPATH_KEY: os.pathsep.join(str(path) for path in (checkout, *library_paths)), }, check=False, + timeout=timeout_seconds, ) diff --git a/scripts/release_performance/_helpers/benchmark.py b/scripts/release_performance/_helpers/benchmark.py index f6c4f81769..5b8627f6c7 100644 --- a/scripts/release_performance/_helpers/benchmark.py +++ b/scripts/release_performance/_helpers/benchmark.py @@ -14,7 +14,7 @@ from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit from scripts.compile_performance_ratio._helpers.measure import run_checkout_generator -from scripts.release_performance._helpers.versions import library_paths +from scripts.release_performance._helpers.versions import library_paths, setup_timeout_error from scripts.release_performance.constants import ( BASELINE_GENERATOR_ENTRY, BUILD_PROJECT, @@ -29,6 +29,7 @@ INSPECTION_PROJECT, INSPECTION_WARMUPS, PRISTINE_DIRECTORY, + SETUP_COMMAND_TIMEOUT_SECONDS, STDERR_TAIL_CHARACTERS, TARGET_DIRECTORY, TIME_FORMAT, @@ -174,14 +175,19 @@ def compare_command( def _run_baseline_generator( *, source: Path, python: Path, entry: str, arguments: tuple[str, ...], description: str ) -> None: - completed: subprocess.CompletedProcess[str] = run_checkout_generator( - checkout=source, - python=Path(sys.executable).absolute(), - entry=entry, - arguments=arguments, - environment=_environment(), - library_paths=library_paths(python=python), - ) + paths: tuple[Path, ...] = library_paths(python=python) + try: + completed: subprocess.CompletedProcess[str] = run_checkout_generator( + checkout=source, + python=Path(sys.executable).absolute(), + entry=entry, + arguments=arguments, + environment=_environment(), + library_paths=paths, + timeout_seconds=SETUP_COMMAND_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + raise setup_timeout_error(phase=f"Baseline {description} generation") from None if completed.returncode != 0: raise ReleasePerformanceError( f"Baseline {description} generation with the generator in {source} failed with exit " diff --git a/scripts/release_performance/_helpers/versions.py b/scripts/release_performance/_helpers/versions.py index 946f42440b..9b0cb2a316 100644 --- a/scripts/release_performance/_helpers/versions.py +++ b/scripts/release_performance/_helpers/versions.py @@ -3,7 +3,6 @@ from __future__ import annotations import hashlib -import io import json import platform import re @@ -24,6 +23,7 @@ PYPI_SIMPLE_JSON, PYPI_SIMPLE_URL, RELEASE_TAG_PREFIX, + SETUP_COMMAND_TIMEOUT_SECONDS, STDERR_TAIL_CHARACTERS, UNIVERSAL_WHEEL_MACHINE, WHEEL_MACHINE_ALIASES, @@ -107,21 +107,33 @@ def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path: """Extract the source tree of release tag v from repo_dir into destination.""" tag: str = RELEASE_TAG_PREFIX + version - completed: subprocess.CompletedProcess[bytes] = subprocess.run( - ["git", "-C", str(repo_dir), "archive", "--format=tar", tag], - capture_output=True, - check=False, - ) - if completed.returncode != 0: - raise ReleasePerformanceError( - f"Release tag {tag} is not available in {repo_dir}, so the baseline's own benchmark " - "generator cannot be used; fetch the release tags or pass --baseline-source: " - + _tail(completed.stderr.decode("utf-8", errors="replace")) - ) shutil.rmtree(destination, ignore_errors=True) destination.mkdir(parents=True) - with tarfile.open(fileobj=io.BytesIO(completed.stdout)) as archive: - archive.extractall(destination, filter="data") + tar: Path = destination.with_name(destination.name + ".tar") + try: + completed: subprocess.CompletedProcess[str] = run_setup_command( + command=[ + "git", + "-C", + str(repo_dir), + "archive", + "--format=tar", + "--output", + str(tar), + tag, + ], + phase=f"Extracting release tag {tag}", + ) + if completed.returncode != 0: + raise ReleasePerformanceError( + f"Release tag {tag} is not available in {repo_dir}, so the baseline's own " + "benchmark generator cannot be used; fetch the release tags or pass " + "--baseline-source: " + _tail(completed.stderr) + ) + with tarfile.open(tar) as archive: + archive.extractall(destination, filter="data") + finally: + tar.unlink(missing_ok=True) return destination @@ -143,7 +155,9 @@ def install_published(*, version: str, venv_dir: Path, python: str, wait_seconds deadline: float = time.monotonic() + wait_seconds while True: attempt: subprocess.CompletedProcess[str] = _uv_pip_install( - venv_dir=venv_dir, requirements=(f"{PACKAGE_NAME}=={version}",) + venv_dir=venv_dir, + requirements=(f"{PACKAGE_NAME}=={version}",), + phase=f"Installing {PACKAGE_NAME} {version} from the index", ) if attempt.returncode == 0: return _sqb(venv_dir=venv_dir) @@ -174,6 +188,7 @@ def install_wheel(*, wheel: Path, venv_dir: Path, python: str) -> Path: completed: subprocess.CompletedProcess[str] = _uv_pip_install( venv_dir=venv_dir, requirements=("--find-links", str(wheel.resolve().parent), str(wheel.resolve())), + phase=f"Installing {wheel.name}", ) if completed.returncode != 0: raise ReleasePerformanceError(f"Installing {wheel} failed: {_tail(completed.stderr)}") @@ -183,8 +198,8 @@ def install_wheel(*, wheel: Path, venv_dir: Path, python: str) -> Path: def installed_version(*, sqb: Path) -> str: """Return the version an installed sqb reports.""" - completed: subprocess.CompletedProcess[str] = subprocess.run( - [str(sqb), "--version"], capture_output=True, text=True, check=False + completed: subprocess.CompletedProcess[str] = run_setup_command( + command=[str(sqb), "--version"], phase=f"Reading the version of {sqb}" ) match: re.Match[str] | None = _RELEASE_VERSION.search(completed.stdout) if completed.returncode != 0 or match is None: @@ -195,8 +210,9 @@ def installed_version(*, sqb: Path) -> str: def library_paths(*, python: Path) -> tuple[Path, ...]: """Return the package directories of `python`'s environment, which must match this Python.""" - completed: subprocess.CompletedProcess[str] = subprocess.run( - [str(python), "-c", _LIBRARY_PATHS_PROBE], capture_output=True, text=True, check=False + completed: subprocess.CompletedProcess[str] = run_setup_command( + command=[str(python), "-c", _LIBRARY_PATHS_PROBE], + phase=f"Reading the package directories of {python}", ) if completed.returncode != 0: raise ReleasePerformanceError( @@ -213,6 +229,29 @@ def library_paths(*, python: Path) -> tuple[Path, ...]: return tuple(dict.fromkeys(Path(str(path)) for path in probe["paths"])) +def run_setup_command(*, command: list[str], phase: str) -> subprocess.CompletedProcess[str]: + """Run a setup command; one that hangs is stopped and fails with the phase it blocked.""" + + try: + return subprocess.run( + command, + capture_output=True, + text=True, + check=False, + timeout=SETUP_COMMAND_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + raise setup_timeout_error(phase=phase) from None + + +def setup_timeout_error(*, phase: str) -> ReleasePerformanceError: + """Return the error for a setup phase that exceeded SETUP_COMMAND_TIMEOUT_SECONDS.""" + + return ReleasePerformanceError( + f"{phase} did not finish within {SETUP_COMMAND_TIMEOUT_SECONDS:g}s and was stopped" + ) + + def existing_sqb(*, venv_dir: Path) -> Path: """Return the sqb entry point of an existing virtual environment.""" @@ -247,6 +286,7 @@ def _install_from_wheel_urls(*, version: str, venv_dir: Path) -> bool: completed: subprocess.CompletedProcess[str] = _uv_pip_install( venv_dir=venv_dir, requirements=("--find-links", str(download_dir), f"{PACKAGE_NAME}=={version}"), + phase=f"Installing {PACKAGE_NAME} {version} from its wheel URLs", ) if completed.returncode != 0: print(f"Wheel URL install failed: {_tail(completed.stderr)}", file=sys.stderr) @@ -280,11 +320,9 @@ def _platform_matches(*, platform_tag: str, machine: str, system: str) -> bool: def _create_venv(*, venv_dir: Path, python: str) -> None: shutil.rmtree(venv_dir, ignore_errors=True) - completed: subprocess.CompletedProcess[str] = subprocess.run( - [_uv(), "venv", "--quiet", "--python", python, str(venv_dir)], - capture_output=True, - text=True, - check=False, + completed: subprocess.CompletedProcess[str] = run_setup_command( + command=[_uv(), "venv", "--quiet", "--python", python, str(venv_dir)], + phase=f"Creating the Python {python} environment {venv_dir}", ) if completed.returncode != 0: raise ReleasePerformanceError( @@ -293,10 +331,10 @@ def _create_venv(*, venv_dir: Path, python: str) -> None: def _uv_pip_install( - *, venv_dir: Path, requirements: tuple[str, ...] + *, venv_dir: Path, requirements: tuple[str, ...], phase: str ) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [ + return run_setup_command( + command=[ _uv(), "pip", "install", @@ -305,9 +343,7 @@ def _uv_pip_install( str(venv_dir / "bin" / "python"), *requirements, ], - capture_output=True, - text=True, - check=False, + phase=phase, ) diff --git a/scripts/release_performance/_helpers/workflow.py b/scripts/release_performance/_helpers/workflow.py index b05be29949..5aa7d0bff6 100644 --- a/scripts/release_performance/_helpers/workflow.py +++ b/scripts/release_performance/_helpers/workflow.py @@ -7,6 +7,7 @@ import os import sys import tempfile +import time from pathlib import Path from scripts.compile_performance_ratio._helpers.report import cpu_model @@ -44,6 +45,7 @@ BASELINE_SOURCE_DIRECTORY, BENCHMARK_COMMANDS, CANDIDATE_LABEL, + PROGRESS_HEADING, RELEASE_TAG_PREFIX, REPO_ROOT, ) @@ -66,10 +68,12 @@ def run_release_comparison(*, options: ComparisonOptions) -> int: try: if options.work_dir is not None: options.work_dir.mkdir(parents=True, exist_ok=True) - comparison: ReleaseComparison = _compare(options=options, root=options.work_dir) + comparison: ReleaseComparison = _compare( + options=options, root=options.work_dir, summary=summary + ) else: with tempfile.TemporaryDirectory(prefix="sqlbuild-release-performance-") as root: - comparison = _compare(options=options, root=Path(root)) + comparison = _compare(options=options, root=Path(root), summary=summary) except (ReleasePerformanceError, BenchmarkEditError, OSError) as error: print(f"Release performance comparison failed: {error}", file=sys.stderr) _ = append_summary(path=summary, markdown=error_markdown(message=str(error))) @@ -87,8 +91,9 @@ def run_release_comparison(*, options: ComparisonOptions) -> int: return 0 -def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: - print(f"Installing candidate {options.candidate}", file=sys.stderr) +def _compare(*, options: ComparisonOptions, root: Path, summary: Path | None) -> ReleaseComparison: + _ = append_summary(path=summary, markdown=PROGRESS_HEADING) + _progress(summary=summary, message=f"Installing candidate {options.candidate}") candidate: InstalledVersion = _install( label=CANDIDATE_LABEL, spec=options.candidate, @@ -99,7 +104,7 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: baseline_spec: str = options.baseline or resolve_baseline( candidate=candidate.version, same_version=options.same_version_baseline ) - print(f"Installing baseline {baseline_spec}", file=sys.stderr) + _progress(summary=summary, message=f"Installing baseline {baseline_spec}") baseline: InstalledVersion = _install( label=BASELINE_LABEL, spec=baseline_spec, @@ -107,12 +112,12 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: python=options.python, wait_seconds=BASELINE_PUBLICATION_WAIT_SECONDS, ) - print( - f"Comparing candidate {candidate.version} with baseline {baseline.version}", - file=sys.stderr, + _progress( + summary=summary, + message=f"Comparing candidate {candidate.version} with baseline {baseline.version}", ) load_before: tuple[float, float, float] = os.getloadavg() - print("Generating candidate benchmark projects", file=sys.stderr) + _progress(summary=summary, message="Generating candidate benchmark projects") pristine: dict[str, Path] = { CANDIDATE_LABEL: write_pristine_projects( root=root, @@ -126,7 +131,9 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: else RELEASE_TAG_PREFIX + baseline.version ) write_dense_project(pristine=pristine[CANDIDATE_LABEL], models=options.dense_models) - print(f"Generating baseline benchmark projects with {baseline_generator}", file=sys.stderr) + _progress( + summary=summary, message=f"Generating baseline benchmark projects with {baseline_generator}" + ) baseline_source: Path = ( options.baseline_source.absolute() if options.baseline_source is not None @@ -152,7 +159,7 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: versions: tuple[InstalledVersion, InstalledVersion] = (baseline, candidate) projects: dict[str, dict[str, Path]] = {} for version in versions: - print(f"Warming caches for {version.label} {version.version}", file=sys.stderr) + _progress(summary=summary, message=f"Warming caches for {version.label} {version.version}") projects[version.label] = prepare_version_projects( version=version, pristine=pristine[version.label], root=root ) @@ -166,6 +173,7 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: print(f"Skipping {command.name}: {reason}", file=sys.stderr) skipped.append(SkippedCommand(name=command.name, reason=reason)) continue + _progress(summary=summary, message=f"Timing `{command.name}`") measured.append( compare_command( command=command, versions=versions, projects=projects, root=root, runs=options.runs @@ -187,6 +195,13 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: ) +def _progress(*, summary: Path | None, message: str) -> None: + print(message, file=sys.stderr, flush=True) + _ = append_summary( + path=summary, markdown=f"- {time.strftime('%H:%M:%S', time.gmtime())} UTC: {message}" + ) + + def _install( *, label: str, spec: str, root: Path, python: str, wait_seconds: float ) -> InstalledVersion: diff --git a/scripts/release_performance/constants.py b/scripts/release_performance/constants.py index 6656111a0a..d513816c06 100644 --- a/scripts/release_performance/constants.py +++ b/scripts/release_performance/constants.py @@ -23,6 +23,8 @@ DEFAULT_BUILD_MODELS: int = 1_000 DEFAULT_DENSE_MODELS: int = 3_000 COMMAND_TIMEOUT_SECONDS: float = 900.0 +SETUP_COMMAND_TIMEOUT_SECONDS: float = 2_400.0 +PROGRESS_HEADING: str = "### Release performance progress\n" PACKAGE_NAME: str = "sqlbuild" PYPI_SIMPLE_URL: str = "https://pypi.org/simple/sqlbuild/" diff --git a/scripts/release_performance_outcome/__init__.py b/scripts/release_performance_outcome/__init__.py new file mode 100644 index 0000000000..01d3e466ce --- /dev/null +++ b/scripts/release_performance_outcome/__init__.py @@ -0,0 +1 @@ +"""Decide whether a main release performance comparison run needs an unfinished report.""" diff --git a/scripts/release_performance_outcome/_helpers/__init__.py b/scripts/release_performance_outcome/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/release_performance_outcome/_helpers/outcome.py b/scripts/release_performance_outcome/_helpers/outcome.py new file mode 100644 index 0000000000..cc3df34ff2 --- /dev/null +++ b/scripts/release_performance_outcome/_helpers/outcome.py @@ -0,0 +1,22 @@ +"""Classify a compare job result as finished, cancelled from outside, or unfinished.""" + +from __future__ import annotations + +from scripts.release_performance_outcome.constants import ( + CANCELLED_RESULT, + FAILURE_RESULT, + JOB_TIMEOUT_ANNOTATION, + SUCCESS_RESULT, +) + + +def unfinished_reason(*, result: str, regressed: bool, annotations: tuple[str, ...]) -> str | None: + """Return why a non-regressed compare job failed or hit its own timeout, else None.""" + + if result == SUCCESS_RESULT or regressed: + return None + timed_out: bool = any(JOB_TIMEOUT_ANNOTATION in message for message in annotations) + if result != FAILURE_RESULT and not (result == CANCELLED_RESULT and timed_out): + return None + details: str = "".join(f"\n- {message}" for message in annotations) + return f"The comparison job ended with result `{result}`.{details}" diff --git a/scripts/release_performance_outcome/constants.py b/scripts/release_performance_outcome/constants.py new file mode 100644 index 0000000000..af2ddbe0ee --- /dev/null +++ b/scripts/release_performance_outcome/constants.py @@ -0,0 +1,8 @@ +"""Job results and annotations that tell an unfinished comparison from a cancelled one.""" + +SUCCESS_RESULT: str = "success" +FAILURE_RESULT: str = "failure" +CANCELLED_RESULT: str = "cancelled" +REGRESSED_OUTPUT: str = "true" +FAILURE_ANNOTATION_LEVEL: str = "failure" +JOB_TIMEOUT_ANNOTATION: str = "has exceeded the maximum execution time" diff --git a/scripts/release_performance_outcome/main/__init__.py b/scripts/release_performance_outcome/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/release_performance_outcome/main/report.py b/scripts/release_performance_outcome/main/report.py new file mode 100644 index 0000000000..376e5815ef --- /dev/null +++ b/scripts/release_performance_outcome/main/report.py @@ -0,0 +1,41 @@ +"""Print why a main comparison run did not finish, or nothing when it needs no report.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import cast + +from scripts.release_performance_outcome._helpers.outcome import unfinished_reason +from scripts.release_performance_outcome.constants import ( + FAILURE_ANNOTATION_LEVEL, + REGRESSED_OUTPUT, +) + + +def report_unfinished_comparison(argv: list[str] | None = None) -> int: + """Print the reason a compare job result needs an issue; print nothing otherwise.""" + + parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--result", required=True, help="The compare job's result.") + parser.add_argument("--regressed", default="", help="The compare job's regressed output.") + parser.add_argument( + "--annotations", type=Path, required=True, help="The compare job's check-run annotations." + ) + arguments: argparse.Namespace = parser.parse_args(argv) + annotations: list[dict[str, object]] = cast( + list[dict[str, object]], json.loads(Path(arguments.annotations).read_text("utf-8")) + ) + reason: str | None = unfinished_reason( + result=str(arguments.result), + regressed=arguments.regressed == REGRESSED_OUTPUT, + annotations=tuple( + str(annotation.get("message", "")) + for annotation in annotations + if annotation.get("annotation_level") == FAILURE_ANNOTATION_LEVEL + ), + ) + if reason is not None: + print(reason) + return 0 diff --git a/scripts/report_unfinished_release_performance.py b/scripts/report_unfinished_release_performance.py new file mode 100644 index 0000000000..f99f7a62f1 --- /dev/null +++ b/scripts/report_unfinished_release_performance.py @@ -0,0 +1,15 @@ +"""Direct wrapper for deciding whether a main comparison run needs an unfinished report.""" + +from __future__ import annotations + +from scripts.release_performance_outcome.main.report import report_unfinished_comparison + + +def main(argv: list[str] | None = None) -> int: + """Print why the comparison did not finish, or nothing.""" + + return report_unfinished_comparison(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/e2e/scripts/release_performance/test_compare_release_performance.py b/tests/e2e/scripts/release_performance/test_compare_release_performance.py index 4938bac819..4ea5745642 100644 --- a/tests/e2e/scripts/release_performance/test_compare_release_performance.py +++ b/tests/e2e/scripts/release_performance/test_compare_release_performance.py @@ -40,6 +40,10 @@ "| `build (empty warehouse)` |", "| `lineage column trace` |", "Benchmark projects are generated per side", + "### Release performance progress", + " UTC: Generating baseline benchmark projects with ", + " UTC: Warming caches for candidate ", + " UTC: Timing `dense compile (no cache)`", "**Result: ", ), expected_marked_projects=( diff --git a/tests/unit/scripts/release_performance/_helpers/_test_types.py b/tests/unit/scripts/release_performance/_helpers/_test_types.py index 6a4994d5ed..dee19eeed1 100644 --- a/tests/unit/scripts/release_performance/_helpers/_test_types.py +++ b/tests/unit/scripts/release_performance/_helpers/_test_types.py @@ -1,6 +1,8 @@ """Test-case models for the release performance comparison.""" +from collections.abc import Callable from dataclasses import dataclass +from pathlib import Path from scripts.release_performance.models import BenchmarkCommand, CommandComparison, SkippedCommand @@ -100,3 +102,10 @@ class ReleaseSourceTestCase: requested_version: str expected_files: tuple[str, ...] expected_error_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class HangingSetupPhaseTestCase: + description: str + run_phase: Callable[[Path], object] + expected_error: Callable[[Path], str] diff --git a/tests/unit/scripts/release_performance/_helpers/helpers.py b/tests/unit/scripts/release_performance/_helpers/helpers.py index 8128989b16..699b014a7a 100644 --- a/tests/unit/scripts/release_performance/_helpers/helpers.py +++ b/tests/unit/scripts/release_performance/_helpers/helpers.py @@ -4,6 +4,8 @@ import sys from pathlib import Path +from scripts.release_performance._helpers.benchmark import write_baseline_pristine_projects +from scripts.release_performance._helpers.versions import installed_version from scripts.release_performance.models import CommandComparison, CommandSample MIB: int = 1024 * 1024 @@ -31,6 +33,12 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models: FAILING_GENERATOR: str = """def write_pristine_projects(*, root, inspection_models, build_models): raise RuntimeError("baseline generator exploded") """ +HANGING_GENERATOR: str = """import time + + +def write_pristine_projects(*, root, inspection_models, build_models): + time.sleep(60) +""" MARKER_DENSE_GENERATOR: str = """from pathlib import Path @@ -138,3 +146,24 @@ def tagged_repository(*, root: Path, tag: str) -> Path: ): _ = subprocess.run(["git", "-C", str(root), *command], check=True, capture_output=True) return root + + +def generate_with_hanging_generator(root: Path) -> object: + """Generate baseline projects with a generator that never returns.""" + + return write_baseline_pristine_projects( + source=write_baseline_source(root=root / "source", generator=HANGING_GENERATOR), + python=write_baseline_environment(root=root / "venv"), + root=root / "generated", + inspection_models=30, + build_models=10, + ) + + +def read_hanging_sqb_version(root: Path) -> object: + """Read the version of an sqb that never exits.""" + + sqb: Path = root / "sqb" + _ = sqb.write_text("#!/bin/sh\nexec sleep 60\n", encoding="utf-8") + sqb.chmod(0o755) + return installed_version(sqb=sqb) diff --git a/tests/unit/scripts/release_performance/_helpers/test_benchmark.py b/tests/unit/scripts/release_performance/_helpers/test_benchmark.py index 8bb9bbf53b..eb79ed173f 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_benchmark.py +++ b/tests/unit/scripts/release_performance/_helpers/test_benchmark.py @@ -6,6 +6,7 @@ import pytest +from scripts.release_performance._helpers import benchmark, versions from scripts.release_performance._helpers.benchmark import ( write_baseline_dense_project, write_baseline_pristine_projects, @@ -15,6 +16,7 @@ from tests.unit.scripts.release_performance._helpers._test_types import ( BaselineDenseGeneratorTestCase, BaselineGeneratorTestCase, + HangingSetupPhaseTestCase, ReleaseSourceTestCase, ) from tests.unit.scripts.release_performance._helpers.helpers import ( @@ -25,6 +27,8 @@ FAILING_GENERATOR, MARKER_DENSE_GENERATOR, MARKER_GENERATOR, + generate_with_hanging_generator, + read_hanging_sqb_version, tagged_repository, write_baseline_dense_source, write_baseline_environment, @@ -175,5 +179,37 @@ def test_given_release_tag_when_extracting_source_then_returns_its_tree( assert all(fragment in error for fragment in test_case.expected_error_fragments), error +@pytest.mark.parametrize( + "test_case", + ( + HangingSetupPhaseTestCase( + description="a hanging baseline generator is stopped and named", + run_phase=generate_with_hanging_generator, + expected_error=lambda _root: ( + "Baseline benchmark generation did not finish within 2s and was stopped" + ), + ), + HangingSetupPhaseTestCase( + description="a hanging sqb --version is stopped and named", + run_phase=read_hanging_sqb_version, + expected_error=lambda root: ( + f"Reading the version of {root / 'sqb'} did not finish within 2s and was stopped" + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_a_hanging_setup_command_when_comparing_then_fails_naming_its_phase( + test_case: HangingSetupPhaseTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(benchmark, "SETUP_COMMAND_TIMEOUT_SECONDS", 2.0) + monkeypatch.setattr(versions, "SETUP_COMMAND_TIMEOUT_SECONDS", 2.0) + + with pytest.raises(ReleasePerformanceError) as raised: + _ = test_case.run_phase(tmp_path) + + assert str(raised.value) == test_case.expected_error(tmp_path) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/release_performance_outcome/__init__.py b/tests/unit/scripts/release_performance_outcome/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/release_performance_outcome/_helpers/__init__.py b/tests/unit/scripts/release_performance_outcome/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/release_performance_outcome/_helpers/_test_types.py b/tests/unit/scripts/release_performance_outcome/_helpers/_test_types.py new file mode 100644 index 0000000000..cd7582d2f1 --- /dev/null +++ b/tests/unit/scripts/release_performance_outcome/_helpers/_test_types.py @@ -0,0 +1,12 @@ +"""Test-case models for classifying main comparison runs.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class UnfinishedComparisonTestCase: + description: str + result: str + regressed: str + annotations: tuple[tuple[str, str], ...] + expected_output: str diff --git a/tests/unit/scripts/release_performance_outcome/_helpers/test_outcome.py b/tests/unit/scripts/release_performance_outcome/_helpers/test_outcome.py new file mode 100644 index 0000000000..85605098f0 --- /dev/null +++ b/tests/unit/scripts/release_performance_outcome/_helpers/test_outcome.py @@ -0,0 +1,145 @@ +"""Tests for deciding whether a main comparison run did not finish and needs an issue.""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests.unit.scripts.release_performance_outcome._helpers._test_types import ( + UnfinishedComparisonTestCase, +) + +_REPO_ROOT: Path = Path(__file__).resolve().parents[5] +# Annotations in the form GitHub writes them on failed, cancelled and timed-out jobs. +_JOB_TIMEOUT: tuple[str, str] = ( + "failure", + "The job has exceeded the maximum execution time of 1h30m0s", +) +_SUPERSEDED: tuple[tuple[str, str], ...] = ( + ( + "failure", + "Canceling since a higher priority waiting request for " + "release-performance-pull_request-refs/pull/1515/merge exists", + ), + ("warning", "No files were found with the provided path: /home/runner/work/_temp/x."), + ("failure", "Process completed with exit code 2."), + ("failure", "The operation was canceled."), +) +_STEP_TIMEOUT: tuple[str, str] = ( + "failure", + "The action 'Compare candidate and previous release on this runner' has timed out after " + "65 minutes.", +) +_CANCELLED_BY_HAND: tuple[str, str] = ("failure", "The run was canceled forcefully by @someone.") + + +@pytest.mark.parametrize( + "test_case", + ( + UnfinishedComparisonTestCase( + description="a job stopped by its own timeout is reported", + result="cancelled", + regressed="", + annotations=(("warning", "Node.js 20 is deprecated."), _JOB_TIMEOUT), + expected_output=( + "The comparison job ended with result `cancelled`.\n" + "- The job has exceeded the maximum execution time of 1h30m0s\n" + ), + ), + UnfinishedComparisonTestCase( + description="a step timeout fails the job and is reported", + result="failure", + regressed="", + annotations=(_STEP_TIMEOUT,), + expected_output=( + "The comparison job ended with result `failure`.\n" + "- The action 'Compare candidate and previous release on this runner' has timed " + "out after 65 minutes.\n" + ), + ), + UnfinishedComparisonTestCase( + description="a lost runner fails the job and is reported", + result="failure", + regressed="", + annotations=( + ( + "failure", + "The hosted runner: GitHub Actions 7 lost communication with the server.", + ), + ), + expected_output=( + "The comparison job ended with result `failure`.\n" + "- The hosted runner: GitHub Actions 7 lost communication with the server.\n" + ), + ), + UnfinishedComparisonTestCase( + description="a run superseded through concurrency is not reported", + result="cancelled", + regressed="", + annotations=_SUPERSEDED, + expected_output="", + ), + UnfinishedComparisonTestCase( + description="a run cancelled by hand is not reported", + result="cancelled", + regressed="", + annotations=(_CANCELLED_BY_HAND,), + expected_output="", + ), + UnfinishedComparisonTestCase( + description="a regression fails the job but has its own issue", + result="failure", + regressed="true", + annotations=(("failure", "Process completed with exit code 1."),), + expected_output="", + ), + UnfinishedComparisonTestCase( + description="a finished comparison is not reported", + result="success", + regressed="false", + annotations=(), + expected_output="", + ), + ), + ids=lambda case: case.description, +) +def test_given_a_compare_job_result_when_classifying_then_reports_only_unfinished_runs( + test_case: UnfinishedComparisonTestCase, tmp_path: Path +) -> None: + annotations: Path = tmp_path / "annotations.json" + _ = annotations.write_text( + json.dumps( + [ + {"annotation_level": level, "message": message} + for level, message in test_case.annotations + ] + ), + encoding="utf-8", + ) + + # -S drops site-packages: the report job runs this with the runner's Python and no project + # dependencies installed. + completed: subprocess.CompletedProcess[str] = subprocess.run( + [ + sys.executable, + "-S", + "-m", + "scripts.report_unfinished_release_performance", + "--result", + test_case.result, + "--regressed", + test_case.regressed, + "--annotations", + str(annotations), + ], + cwd=_REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert (completed.returncode, completed.stdout) == (0, test_case.expected_output), ( + completed.stderr + )