From c2ac7c3b6b95745508ba681be76fb7fdc0492bad Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Mon, 24 Aug 2026 17:13:20 +0000 Subject: [PATCH] fix: omit baseline-only formats from the file-size report A benchmark run that skips a format (for example vortex-compact) produced no file-size rows for it, so the PR comment rendered every baseline file of that format as shrinking to 0 B (-100.0%). Scope the baseline rows to the (benchmark, scale factor, format) combinations the PR run actually produced, so skipped formats are omitted instead of reported as deleted. Signed-off-by: Joe Isaacs --- scripts/compare-benchmark-jsons.py | 7 +++++-- scripts/tests/test_benchmark_reporting.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/scripts/compare-benchmark-jsons.py b/scripts/compare-benchmark-jsons.py index ffd899b55b5..f86528d2a19 100644 --- a/scripts/compare-benchmark-jsons.py +++ b/scripts/compare-benchmark-jsons.py @@ -861,8 +861,11 @@ def format_file_size_report(base_rows: pd.DataFrame, pr_rows: pd.DataFrame) -> s return "" base_data = {key: value for key, value in base_data.items() if key not in ignored} - pr_scopes = {(benchmark, scale_factor) for benchmark, scale_factor, _file_format, _file_name in pr_data} - base_data = {key: value for key, value in base_data.items() if key[:2] in pr_scopes} + # Scope the baseline to (benchmark, scale factor, format) combinations the PR run + # produced. A format the PR run skipped entirely (for example vortex-compact) would + # otherwise render every one of its baseline files as shrinking to 0 B. + pr_scopes = {(benchmark, scale_factor, file_format) for benchmark, scale_factor, file_format, _file_name in pr_data} + base_data = {key: value for key, value in base_data.items() if key[:3] in pr_scopes} if not base_data: return "_No baseline file sizes found for base commit._" diff --git a/scripts/tests/test_benchmark_reporting.py b/scripts/tests/test_benchmark_reporting.py index 024d5b75e67..e2b829f1d86 100644 --- a/scripts/tests/test_benchmark_reporting.py +++ b/scripts/tests/test_benchmark_reporting.py @@ -662,6 +662,29 @@ def test_file_size_report_ignores_baseline_rows_outside_pr_scope() -> None: assert "| part-0.vortex | 1.0 |" not in report +def test_file_size_report_omits_formats_the_pr_run_skipped() -> None: + compare = load_compare_module() + + report = compare.format_file_size_report( + pd.DataFrame( + [ + file_size_record_for("base-sha", 100, "tpch", "10", "vortex-file-compressed", "part-0.vortex"), + file_size_record_for("base-sha", 80, "tpch", "10", "vortex-compact", "part-0.vortex"), + file_size_record_for("base-sha", 5, "tpch", "10", "vortex-compact", "duckdb.db"), + ] + ), + pd.DataFrame( + [ + file_size_record_for("pr-sha", 125, "tpch", "10", "vortex-file-compressed", "part-0.vortex"), + ] + ), + ) + + assert "File Size Changes (1 files changed, +25.0% overall, 1↑ 0↓)" in report + assert "vortex-compact" not in report + assert "-100.0%" not in report + + def test_capture_file_sizes_emits_shared_benchmark_rows(tmp_path: Path) -> None: data_dir = tmp_path / "data" format_dir = data_dir / "tpch" / "10" / "vortex-file-compressed"