diff --git a/scripts/compare-benchmark-jsons.py b/scripts/compare-benchmark-jsons.py index ffd899b55b5..f86528d2a19 100644 --- a/scripts/compare-benchmark-jsons.py +++ b/scripts/compare-benchmark-jsons.py @@ -861,8 +861,11 @@ def format_file_size_report(base_rows: pd.DataFrame, pr_rows: pd.DataFrame) -> s return "" base_data = {key: value for key, value in base_data.items() if key not in ignored} - pr_scopes = {(benchmark, scale_factor) for benchmark, scale_factor, _file_format, _file_name in pr_data} - base_data = {key: value for key, value in base_data.items() if key[:2] in pr_scopes} + # Scope the baseline to (benchmark, scale factor, format) combinations the PR run + # produced. A format the PR run skipped entirely (for example vortex-compact) would + # otherwise render every one of its baseline files as shrinking to 0 B. + pr_scopes = {(benchmark, scale_factor, file_format) for benchmark, scale_factor, file_format, _file_name in pr_data} + base_data = {key: value for key, value in base_data.items() if key[:3] in pr_scopes} if not base_data: return "_No baseline file sizes found for base commit._" diff --git a/scripts/tests/test_benchmark_reporting.py b/scripts/tests/test_benchmark_reporting.py index 024d5b75e67..e2b829f1d86 100644 --- a/scripts/tests/test_benchmark_reporting.py +++ b/scripts/tests/test_benchmark_reporting.py @@ -662,6 +662,29 @@ def test_file_size_report_ignores_baseline_rows_outside_pr_scope() -> None: assert "| part-0.vortex | 1.0 |" not in report +def test_file_size_report_omits_formats_the_pr_run_skipped() -> None: + compare = load_compare_module() + + report = compare.format_file_size_report( + pd.DataFrame( + [ + file_size_record_for("base-sha", 100, "tpch", "10", "vortex-file-compressed", "part-0.vortex"), + file_size_record_for("base-sha", 80, "tpch", "10", "vortex-compact", "part-0.vortex"), + file_size_record_for("base-sha", 5, "tpch", "10", "vortex-compact", "duckdb.db"), + ] + ), + pd.DataFrame( + [ + file_size_record_for("pr-sha", 125, "tpch", "10", "vortex-file-compressed", "part-0.vortex"), + ] + ), + ) + + assert "File Size Changes (1 files changed, +25.0% overall, 1↑ 0↓)" in report + assert "vortex-compact" not in report + assert "-100.0%" not in report + + def test_capture_file_sizes_emits_shared_benchmark_rows(tmp_path: Path) -> None: data_dir = tmp_path / "data" format_dir = data_dir / "tpch" / "10" / "vortex-file-compressed"