Skip to content

Commit aeeb41b

Browse files
committed
ci: generate the base benchmark project with the base checkout's generator
The same-runner compile ratio job generated one project with head's generator and compiled it with both builds, so any change that adds syntax to the generators (such as FUNCTION descriptions) made the base build reject the project. compare_compile_performance now accepts --base-root: the base project is written by the base checkout's own write_benchmark_project (run with the head interpreter and the base root on PYTHONPATH), head's project by head's generator, and each build compiles its own project. Base generation failures stop the comparison with the generator's stderr tail, and the report notes per-side generation. Without --base-root the single-project behaviour is kept.
1 parent 23f1620 commit aeeb41b

8 files changed

Lines changed: 239 additions & 16 deletions

File tree

‎.github/workflows/verify.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -460,6 +460,7 @@ jobs:
460460
--kind "${KIND}" \
461461
--max-ratio "${MAX_RATIO}" \
462462
--models 3000 \
463+
--base-root base \
463464
--base-python base/.venv/bin/python \
464465
--head-python .venv/bin/python
465466

‎scripts/compile_performance_ratio/_helpers/measure.py‎

Lines changed: 43 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
"""Generate a benchmark project once and time alternating base and head compiles."""
1+
"""Generate benchmark projects and time alternating base and head compiles."""
22

33
from __future__ import annotations
44

@@ -13,9 +13,11 @@
1313

1414
from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project
1515
from scripts.compile_performance_ratio.constants import (
16+
BASE_GENERATOR_ENTRY,
1617
BASE_LABEL,
1718
COMPILE_ENTRY,
1819
DENSE_KIND,
20+
ERROR_TAIL_CHARACTERS,
1921
EXCLUDED_ENVIRONMENT_KEYS,
2022
EXCLUDED_ENVIRONMENT_PREFIX,
2123
FRESH_AUDIT_SHARE,
@@ -25,6 +27,7 @@
2527
FRESH_SOURCE_SHARE,
2628
FRESH_TEST_SHARE,
2729
HEAD_LABEL,
30+
PYTHONPATH_KEY,
2831
REPORTED_PHASES,
2932
)
3033
from scripts.compile_performance_ratio.exceptions import CompileComparisonError
@@ -52,23 +55,47 @@ def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> Non
5255
)
5356

5457

58+
def write_base_benchmark_project(
59+
*, base_root: Path, python: Path, kind: str, project_dir: Path, models: int
60+
) -> None:
61+
"""Write the base project with the base checkout's own generator, run by `python`."""
62+
63+
completed: subprocess.CompletedProcess[str] = subprocess.run(
64+
[str(python), "-c", BASE_GENERATOR_ENTRY, kind, str(project_dir), str(models)],
65+
cwd=base_root,
66+
capture_output=True,
67+
text=True,
68+
env={**_compile_environment(), PYTHONPATH_KEY: str(base_root)},
69+
check=False,
70+
)
71+
if completed.returncode != 0:
72+
raise CompileComparisonError(
73+
f"base project generation in {base_root} failed with exit {completed.returncode}: "
74+
f"{completed.stderr[-ERROR_TAIL_CHARACTERS:]}"
75+
)
76+
77+
5578
def compare_builds(
5679
*,
5780
kind: str,
5881
models: int,
59-
project_dir: Path,
82+
base_project_dir: Path,
83+
head_project_dir: Path,
6084
base_python: Path,
6185
head_python: Path,
6286
runs: int,
6387
) -> CompileComparison:
6488
"""Alternate base and head cold compiles after one untimed warm-up compile per build."""
6589

66-
builds: tuple[tuple[str, Path], ...] = ((BASE_LABEL, base_python), (HEAD_LABEL, head_python))
67-
for label, python in builds:
90+
builds: tuple[tuple[str, Path, Path], ...] = (
91+
(BASE_LABEL, base_python, base_project_dir),
92+
(HEAD_LABEL, head_python, head_project_dir),
93+
)
94+
for label, python, project_dir in builds:
6895
_compile_once(label=label, python=python, project_dir=project_dir)
6996
results: dict[str, list[CompileRun]] = {BASE_LABEL: [], HEAD_LABEL: []}
7097
for _ in range(runs):
71-
for label, python in builds:
98+
for label, python, project_dir in builds:
7299
results[label].append(
73100
_compile_once(label=label, python=python, project_dir=project_dir)
74101
)
@@ -88,11 +115,7 @@ def compare_builds(
88115

89116
def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun:
90117
shutil.rmtree(project_dir / "target", ignore_errors=True)
91-
environment: dict[str, str] = {
92-
key: value
93-
for key, value in os.environ.items()
94-
if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX)
95-
}
118+
environment: dict[str, str] = _compile_environment()
96119
before: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
97120
started: float = time.perf_counter()
98121
completed: subprocess.CompletedProcess[str] = subprocess.run(
@@ -116,7 +139,8 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun:
116139
after: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
117140
if completed.returncode != 0:
118141
raise CompileComparisonError(
119-
f"{label} compile failed with exit {completed.returncode}: {completed.stderr[-2000:]}"
142+
f"{label} compile failed with exit {completed.returncode}: "
143+
f"{completed.stderr[-ERROR_TAIL_CHARACTERS:]}"
120144
)
121145
payload: dict[str, object] = json.loads(completed.stdout)
122146
timings: object = payload.get("compile_timings", {})
@@ -132,6 +156,14 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun:
132156
)
133157

134158

159+
def _compile_environment() -> dict[str, str]:
160+
return {
161+
key: value
162+
for key, value in os.environ.items()
163+
if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX)
164+
}
165+
166+
135167
def _median_phases(*, runs: list[CompileRun]) -> dict[str, float]:
136168
medians: dict[str, float] = {}
137169
for phase in REPORTED_PHASES:

‎scripts/compile_performance_ratio/_helpers/report.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,23 @@
99
from scripts.compile_performance_ratio.models import CompileComparison
1010

1111

12-
def comparison_markdown(*, comparison: CompileComparison, runs: int, max_ratio: float) -> str:
12+
def comparison_markdown(
13+
*, comparison: CompileComparison, runs: int, max_ratio: float, per_side_projects: bool
14+
) -> str:
1315
"""Summarize medians, ratios and per-phase timings as a Markdown table."""
1416

17+
generation: str = (
18+
"Projects generated per side: base with the base checkout's generator, head with head's."
19+
if per_side_projects
20+
else "One project generated with head's generator and compiled by both builds."
21+
)
1522
lines: list[str] = [
1623
f"### {comparison.kind} {comparison.models} models: head vs base (same runner)",
1724
"",
1825
f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.",
1926
"",
27+
generation,
28+
"",
2029
"| Metric | Base | Head | Ratio |",
2130
"|---|---:|---:|---:|",
2231
f"| Wall (s) | {comparison.base_wall_seconds:.2f} | "

‎scripts/compile_performance_ratio/constants.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,14 @@
1010
EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset({"VIRTUAL_ENV"})
1111
EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_"
1212
COMPILE_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())"
13+
BASE_GENERATOR_ENTRY: str = (
14+
"import sys; from pathlib import Path; "
15+
"from scripts.compile_performance_ratio._helpers.measure import write_benchmark_project; "
16+
"write_benchmark_project("
17+
"kind=sys.argv[1], project_dir=Path(sys.argv[2]), models=int(sys.argv[3]))"
18+
)
19+
PYTHONPATH_KEY: str = "PYTHONPATH"
20+
ERROR_TAIL_CHARACTERS: int = 2000
1321
FRESH_SOURCE_SHARE: float = 713 / 3000
1422
FRESH_SEED_SHARE: float = 141 / 3000
1523
FRESH_FUNCTION_SHARE: float = 71 / 3000

‎scripts/compile_performance_ratio/main/compare.py‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
from scripts.compile_performance_ratio._helpers.measure import (
1212
compare_builds,
13+
write_base_benchmark_project,
1314
write_benchmark_project,
1415
)
1516
from scripts.compile_performance_ratio._helpers.report import append_summary, comparison_markdown
@@ -26,21 +27,39 @@ def compare_compile_performance(argv: list[str] | None = None) -> int:
2627

2728
args: argparse.Namespace = _parse_args(argv)
2829
summary_value: str | None = os.environ.get("GITHUB_STEP_SUMMARY")
30+
per_side_projects: bool = args.base_root is not None
2931
with tempfile.TemporaryDirectory(prefix="sqlbuild-ratio-") as root:
30-
project_dir: Path = Path(root) / f"{args.kind}_{args.models}"
32+
project_name: str = f"{args.kind}_{args.models}"
33+
head_project_dir: Path = Path(root) / "head" / project_name
34+
base_project_dir: Path = (
35+
Path(root) / "base" / project_name if per_side_projects else head_project_dir
36+
)
3137
print(f"Generating {args.kind} project with {args.models} models", file=sys.stderr)
32-
write_benchmark_project(kind=args.kind, project_dir=project_dir, models=args.models)
38+
write_benchmark_project(kind=args.kind, project_dir=head_project_dir, models=args.models)
39+
if args.base_root is not None:
40+
print(f"Generating the base project with {args.base_root}'s generator", file=sys.stderr)
41+
write_base_benchmark_project(
42+
base_root=args.base_root.absolute(),
43+
python=args.head_python.absolute(),
44+
kind=args.kind,
45+
project_dir=base_project_dir,
46+
models=args.models,
47+
)
3348
print(f"Comparing base and head over {args.runs} alternating runs", file=sys.stderr)
3449
comparison: CompileComparison = compare_builds(
3550
kind=args.kind,
3651
models=args.models,
37-
project_dir=project_dir,
52+
base_project_dir=base_project_dir,
53+
head_project_dir=head_project_dir,
3854
base_python=args.base_python,
3955
head_python=args.head_python,
4056
runs=args.runs,
4157
)
4258
markdown: str = comparison_markdown(
43-
comparison=comparison, runs=args.runs, max_ratio=args.max_ratio
59+
comparison=comparison,
60+
runs=args.runs,
61+
max_ratio=args.max_ratio,
62+
per_side_projects=per_side_projects,
4463
)
4564
print(markdown)
4665
_ = append_summary(path=Path(summary_value) if summary_value else None, markdown=markdown)
@@ -64,6 +83,12 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace:
6483
parser.add_argument("--models", type=int, required=True)
6584
parser.add_argument("--base-python", type=Path, required=True)
6685
parser.add_argument("--head-python", type=Path, required=True)
86+
parser.add_argument(
87+
"--base-root",
88+
type=Path,
89+
default=None,
90+
help="Base checkout whose own generator writes the project the base build compiles.",
91+
)
6792
parser.add_argument("--runs", type=int, default=DEFAULT_RUNS)
6893
parser.add_argument("--max-ratio", type=float, default=DEFAULT_MAX_RATIO)
6994
return parser.parse_args(argv)

‎tests/e2e/scripts/compile_performance_ratio/_test_types.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,13 @@ class CompilePerformanceRatioTestCase:
1313
max_ratio: str
1414
expected_return_code: int
1515
expected_fragments: tuple[str, ...]
16+
17+
18+
@dataclass(frozen=True)
19+
class PerSideCompilePerformanceRatioTestCase:
20+
description: str
21+
base_generator: str
22+
expected_return_code: int
23+
expected_fragments: tuple[str, ...]
24+
expected_base_projects: tuple[str, ...]
25+
expected_head_projects: tuple[str, ...]
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
"""Fixtures for per-side compile performance ratio comparisons."""
2+
3+
from __future__ import annotations
4+
5+
import stat
6+
import sys
7+
from pathlib import Path
8+
9+
MARKER_GENERATOR: str = """from pathlib import Path
10+
11+
12+
def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> None:
13+
del models
14+
(project_dir / "models").mkdir(parents=True)
15+
(project_dir / "sqlbuild_project.toml").write_text('name = "orders"\\nadapter = "duckdb"\\n')
16+
(project_dir / "models" / "base_marker.sql").write_text(
17+
'MODEL (description "Base marker");\\n\\nSELECT 1 AS id\\n'
18+
)
19+
(project_dir / "base_marker.txt").write_text(kind)
20+
"""
21+
FAILING_GENERATOR: str = """def write_benchmark_project(*, kind, project_dir, models):
22+
raise RuntimeError("base generator exploded")
23+
"""
24+
_LOGGING_PYTHON: str = """#!/usr/bin/env bash
25+
project=""
26+
previous=""
27+
for argument in "$@"; do
28+
if [ "$previous" = "--project-dir" ]; then project="$argument"; fi
29+
previous="$argument"
30+
done
31+
if [ -n "$project" ]; then
32+
if [ -f "$project/base_marker.txt" ]; then echo base >> "{log}"; else echo head >> "{log}"; fi
33+
fi
34+
exec "{python}" "$@"
35+
"""
36+
37+
38+
def write_fake_base_root(*, root: Path, generator: str) -> Path:
39+
"""Write a base checkout whose only content is a benchmark generator module."""
40+
41+
helpers: Path = root / "scripts" / "compile_performance_ratio" / "_helpers"
42+
helpers.mkdir(parents=True)
43+
for package in (root / "scripts", helpers.parent, helpers):
44+
(package / "__init__.py").write_text("", encoding="utf-8")
45+
(helpers / "measure.py").write_text(generator, encoding="utf-8")
46+
return root
47+
48+
49+
def write_logging_python(*, path: Path, log: Path) -> Path:
50+
"""Write an interpreter wrapper logging which project each compile received."""
51+
52+
path.write_text(_LOGGING_PYTHON.format(log=log, python=sys.executable), encoding="utf-8")
53+
path.chmod(path.stat().st_mode | stat.S_IXUSR)
54+
log.touch()
55+
return path
56+
57+
58+
def logged_projects(log: Path) -> tuple[str, ...]:
59+
"""Return the project kind each logged compile received, in order."""
60+
61+
return tuple(log.read_text(encoding="utf-8").split())

‎tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,14 @@
1111

1212
from tests.e2e.scripts.compile_performance_ratio._test_types import (
1313
CompilePerformanceRatioTestCase,
14+
PerSideCompilePerformanceRatioTestCase,
15+
)
16+
from tests.e2e.scripts.compile_performance_ratio.helpers import (
17+
FAILING_GENERATOR,
18+
MARKER_GENERATOR,
19+
logged_projects,
20+
write_fake_base_root,
21+
write_logging_python,
1422
)
1523

1624
_REPO_ROOT: Path = Path(__file__).resolve().parents[4]
@@ -74,5 +82,74 @@ def test_given_two_builds_when_comparing_compile_performance_then_enforces_ratio
7482
assert fragment in output
7583

7684

85+
@pytest.mark.parametrize(
86+
"test_case",
87+
(
88+
PerSideCompilePerformanceRatioTestCase(
89+
description="each build compiles the project its own generator wrote",
90+
base_generator=MARKER_GENERATOR,
91+
expected_return_code=0,
92+
expected_fragments=("Projects generated per side",),
93+
expected_base_projects=("base", "base"),
94+
expected_head_projects=("head", "head"),
95+
),
96+
PerSideCompilePerformanceRatioTestCase(
97+
description="a failing base generator stops before any compile",
98+
base_generator=FAILING_GENERATOR,
99+
expected_return_code=1,
100+
expected_fragments=("base project generation in", "base generator exploded"),
101+
expected_base_projects=(),
102+
expected_head_projects=(),
103+
),
104+
),
105+
ids=lambda case: case.description,
106+
)
107+
def test_given_base_root_when_comparing_compile_performance_then_each_side_uses_its_project(
108+
test_case: PerSideCompilePerformanceRatioTestCase, tmp_path: Path
109+
) -> None:
110+
base_root: Path = write_fake_base_root(
111+
root=tmp_path / "base", generator=test_case.base_generator
112+
)
113+
base_log: Path = tmp_path / "base.log"
114+
head_log: Path = tmp_path / "head.log"
115+
base_python: Path = write_logging_python(path=tmp_path / "base-python", log=base_log)
116+
head_python: Path = write_logging_python(path=tmp_path / "head-python", log=head_log)
117+
environment: dict[str, str] = dict(os.environ)
118+
_ = environment.pop("GITHUB_STEP_SUMMARY", None)
119+
120+
result: subprocess.CompletedProcess[str] = subprocess.run(
121+
[
122+
sys.executable,
123+
"-m",
124+
"scripts.compare_compile_performance",
125+
"--kind",
126+
"dense",
127+
"--models",
128+
"20",
129+
"--runs",
130+
"1",
131+
"--base-root",
132+
str(base_root),
133+
"--base-python",
134+
str(base_python),
135+
"--head-python",
136+
str(head_python),
137+
"--max-ratio",
138+
"1000",
139+
],
140+
cwd=_REPO_ROOT,
141+
capture_output=True,
142+
text=True,
143+
env=environment,
144+
check=False,
145+
)
146+
147+
output: str = result.stdout + result.stderr
148+
assert result.returncode == test_case.expected_return_code, output
149+
assert all(fragment in output for fragment in test_case.expected_fragments), output
150+
assert logged_projects(base_log) == test_case.expected_base_projects
151+
assert logged_projects(head_log) == test_case.expected_head_projects
152+
153+
77154
if __name__ == "__main__":
78155
pytest.main([__file__, "-vv"])

0 commit comments

Comments
 (0)