perf(ir): make measured startup the same in every run and re-record every budget (LAB-12297)

CONTRIBUTING.md

@@ -340,7 +340,7 @@ make build  # Standard build
 ```bash
 make perf               # Full battery: env fingerprint + timer calibration, serializer benchmarks, GIL scaling
 make perf-compare       # Regression gate: fail on >10% median serializer regression vs baseline
-make perf-ir            # Instruction-count gate: fail on a >=1% per-call regression (orjson 2%; needs valgrind)
+make perf-ir            # Instruction-count gate: fail on a >=1% per-call regression (needs valgrind)
 make perf-ir-update     # Ratchet the committed instruction budgets down to the measured figures
 
 make benchmark          # Serializer benchmarks only + save a local baseline

Makefile

@@ -425,7 +425,7 @@ perf-compare: benchmark-compare ## Perf regression gate: fail on >10% median ser
 # Deterministic hot-path gate: main-thread callgrind instructions per op, fail at >=+1% vs the
 # budgets in tests/performance/ir_baselines.json (keyed per interpreter). Needs valgrind and the
 # release extension `uv sync` builds. See docs/performance.md, "Instruction budgets".
-perf-ir: ## Instruction-count gate: fail on >=1% per-op regression (orjson 2%; needs valgrind)
+perf-ir: ## Instruction-count gate: fail on >=1% per-op regression (needs valgrind)
 	@uv run python tests/performance/ir_budget.py
 
 perf-ir-update: ## Ratchet instruction budgets down to the measured figures (never up)

tests/performance/README.md

@@ -26,9 +26,9 @@ arms (manual; see [docs/free-threading.md](../../docs/free-threading.md#decorate
 Run the whole battery with `make perf`; gate regressions with `make perf-compare`.
 
 **Instruction budgets.** `ir_budget.py` (`make perf-ir`) is the deterministic gate: main-thread
-callgrind instructions per call for each hot path, failing at +1% (the orjson round trip at +2%)
-against `ir_baselines.json`. Budgets belong to the build that recorded them; on another
-interpreter, extension or C library, record a baseline on `main` first. CI runs it on every PR
+callgrind instructions per call for each hot path, failing at +1% against `ir_baselines.json`.
+Budgets belong to the reference machine that recorded them; on any other machine, record a
+baseline on `main` first. CI runs it on every PR
 with `--base`: the PR and its base are measured on the same runner and compared with each other,
 not with the committed budgets, and `ir_report.py` renders the shards' `--report` files as one PR
 comment, which CI posts ([In CI](../../docs/performance.md#in-ci)).

tests/performance/ir_baselines.json

@@ -2,54 +2,54 @@
   "interpreters": {
     "cpython-3.12-x86_64": {
       "budgets": {
-        "byte_storage_store_40b": 2690,
-        "byte_storage_store_64k": 720631,
-        "file_set": 76035,
-        "l1_hit": 2595,
-        "l1_hit_backed": 83015,
-        "l1_hit_backed_auto": 91986,
-        "l2_hit": 172539,
-        "l2_hit_async_metrics": 171976,
-        "minimal_l1_hit": 2595,
-        "miss": 213364,
-        "object_cache_put_full": 165014,
-        "secure_l1_hit": 134323,
-        "serializer_arrow": 1969626,
-        "serializer_arrow_usgs": 61781406,
-        "serializer_auto": 64722,
-        "serializer_auto_usgs": 61793577,
-        "serializer_default": 47895,
-        "serializer_default_records": 1420388,
-        "serializer_encrypted": 89289,
-        "serializer_orjson": 22135
+        "byte_storage_store_40b": 2685,
+        "byte_storage_store_64k": 720570,
+        "file_set": 75737,
+        "l1_hit": 2597,
+        "l1_hit_backed": 80291,
+        "l1_hit_backed_auto": 91959,
+        "l2_hit": 162311,
+        "l2_hit_async_metrics": 162254,
+        "minimal_l1_hit": 2598,
+        "miss": 212074,
+        "object_cache_put_full": 164636,
+        "secure_l1_hit": 124488,
+        "serializer_arrow": 1970058,
+        "serializer_arrow_usgs": 61805038,
+        "serializer_auto": 64453,
+        "serializer_auto_usgs": 61813852,
+        "serializer_default": 47796,
+        "serializer_default_records": 1011782,
+        "serializer_encrypted": 89109,
+        "serializer_orjson": 22227
       },
       "python": "3.12.12"
     },
     "cpython-3.14-x86_64": {
       "budgets": {
-        "byte_storage_store_40b": 2565,
-        "byte_storage_store_64k": 720751,
-        "file_set": 74112,
+        "byte_storage_store_40b": 2559,
+        "byte_storage_store_64k": 720715,
+        "file_set": 73700,
         "l1_hit": 2487,
-        "l1_hit_backed": 83882,
-        "l1_hit_backed_auto": 92950,
-        "l2_hit": 167686,
-        "l2_hit_async_metrics": 167728,
+        "l1_hit_backed": 80911,
+        "l1_hit_backed_auto": 92913,
+        "l2_hit": 157437,
+        "l2_hit_async_metrics": 157718,
         "minimal_l1_hit": 2487,
-        "miss": 204897,
-        "object_cache_put_full": 160608,
-        "secure_l1_hit": 134972,
-        "serializer_arrow": 1980777,
-        "serializer_arrow_usgs": 61324948,
-        "serializer_auto": 65653,
-        "serializer_auto_usgs": 61344459,
-        "serializer_default": 48623,
-        "serializer_default_records": 1424292,
-        "serializer_encrypted": 90918,
-        "serializer_orjson": 22825
+        "miss": 203600,
+        "object_cache_put_full": 160541,
+        "secure_l1_hit": 125093,
+        "serializer_arrow": 1979439,
+        "serializer_arrow_usgs": 61327875,
+        "serializer_auto": 65623,
+        "serializer_auto_usgs": 61331592,
+        "serializer_default": 48501,
+        "serializer_default_records": 1006308,
+        "serializer_encrypted": 90790,
+        "serializer_orjson": 22519
       },
       "python": "3.14.3"
     }
   },
-  "method": "main-thread callgrind Ir per op, (Ir[3000] - Ir[1000]) / 2000, fixed env (PYTHONHASHSEED=0, one BLAS/OMP thread), pinned clocks; written by tests/performance/ir_budget.py --update"
+  "method": "main-thread callgrind Ir per op, (Ir[3000] - Ir[1000]) / 2000 ((Ir[30] - Ir[10]) / 20 on the two *_usgs paths), median of five heap layouts, fixed env (PYTHONHASHSEED=0, one BLAS/OMP thread, pyarrow's jemalloc without its background thread, neither of pyarrow's allocators purging), pinned clocks and OS entropy; budgets written by tests/performance/ir_budget.py --update"
 }

tests/performance/ir_budget.py

@@ -9,7 +9,9 @@
   is not counted either: about half of the real-data Arrow round trip runs on pyarrow's threads.
 - Interpreter startup is ~2 billion instructions. Each path runs at two loop sizes and the
   per-op cost is the difference, ``(Ir[N_HI] - Ir[N_LO]) / (N_HI - N_LO)``, so startup and
-  warmup cancel. The real-data paths run at smaller sizes, and the cheapest paths at larger ones (``LOOPS``).
+  warmup cancel, as long as startup costs the same in every run: the environment keeps pyarrow's
+  allocators off real time (``measure``) and the OS entropy numpy draws at import is seeded
+  (``ir_workload.pin_os_entropy``). The real-data paths run at smaller sizes (``LOOPS``).
 - Code that observes its own wall-clock duration executes more instructions when it runs
   slower, so the measured process pins its main-thread clocks (``ir_workload.pin_main_thread_clocks``).
   Pinned clocks never let the metrics collector's 5 s mode check fire, so the collector stays
@@ -20,8 +22,8 @@
   cache is warmed first, and the heap layout is sampled (``ir_workload.run_workload``, ``measure``
   and ``LAYOUTS``).
 
-Two full runs agree within 0.08% per op (0.01% on 3.12; byte_storage_store_40b within 0.22%), against a
-1% fail threshold.
+Repeated runs agree within 0.04% per op on most paths, against a 1% fail threshold; docs/performance.md
+names the paths that keep more noise, and how much.
 
 The measured process is ``ir_workload.py``. Its text is compiled into every measured run, so
 editing it moves the heap layout the budgets were recorded at; this gate is kept out of that
@@ -69,16 +71,9 @@
 N_LO, N_HI = 1000, 3000
 WARMUP = 50
 # The real-data paths cost about 60M main-thread Ir per op, 300 times a small path, so a run of
-# 3000 would take hours under callgrind. Fewer ops still cancel startup: it is a few hundred
-# thousand Ir of run-to-run spread against a difference of over a billion.
+# 3000 would take hours under callgrind. Fewer ops still resolve them: pyarrow's thread pool moves a
+# run by up to about 2.8M Ir, against a difference of over a billion.
 LOOPS = dict.fromkeys(("serializer_arrow_usgs", "serializer_auto_usgs"), (5, 10, 30))  # path: (warmup, n_lo, n_hi)
-# The cheapest paths run longer. Startup does not cancel exactly: importing pandas loads pyarrow's
-# jemalloc, which starts a background thread, and depending on how that thread is scheduled against
-# the main thread a run starts in one of two states, whose totals differ by up to about 1.2M Ir (with
-# the thread turned off, by under 5,000). Over 2,000 calls of a 2,600 Ir path that is up to 13% in one
-# layout, and such layouts moved byte_storage_store_40b's median by up to 8.4% in 8 of 80 gate runs.
-# Over 299,000 calls its figure held within 0.22% in 40 runs.
-LOOPS |= dict.fromkeys(("l1_hit", "minimal_l1_hit", "byte_storage_store_40b"), (WARMUP, N_LO, 300_000))
 # Heap layout moves per-op counts: the allocators take shorter or longer paths in some heap
 # states. Any code change shifts the layout, even an edit to ir_workload.py's comments, so a single
 # run can fail an untouched path or hide a real regression. Each path runs at these layout shifts
@@ -88,11 +83,6 @@
 LAYOUTS = (0, 48, 80, 336, 880)
 FAIL_PCT = 1.0  # regression at or above this fails the gate
 WARN_PCT = 0.2  # above the A/A floor: report but pass
-# The exception: the orjson round trip's 1 KB output buffer goes to glibc malloc, whose path length
-# depends on heap state that no layout sample pins (longer warmups did not settle it). Its figure
-# moves by more than the 1% threshold between unrelated changes, so it is gated at what this
-# harness resolves for it.
-TOLERANCE_PCT = {"serializer_orjson": (2.0, 1.0)}  # path: (fail, warn)
 # Callgrind runs at a time by default. Each path peaks at 0.5 to 1.0 GiB per run (the L2, miss and
 # secure paths at the top), so the default stays small enough to share the machine with other work
 # rather than growing with its core count; --jobs overrides it. A workload that grows under callgrind
@@ -270,6 +260,15 @@ def measure(
         "PYTHONNOUSERSITE": "1",
         "OPENBLAS_NUM_THREADS": "1",
         "OMP_NUM_THREADS": "1",  # pyarrow converts on a thread pool otherwise: +-0.5% per op
+        # pyarrow's two allocators run on real time. Its bundled jemalloc starts a background thread at
+        # import, and how that thread is scheduled against the main thread puts startup in one of two
+        # states, up to 1.2M Ir apart: background_thread:false stops it. With the thread off, jemalloc
+        # would check its real-time purge deadlines on the allocating thread instead; the two decay
+        # settings stop it purging at all. pyarrow's default pool, mimalloc, purges freed memory after a
+        # real-time delay that the freeing thread checks (10 clock reads per real-data round trip); -1
+        # stops it purging.
+        "JE_ARROW_MALLOC_CONF": "background_thread:false,dirty_decay_ms:-1,muzzy_decay_ms:-1",
+        "MIMALLOC_PURGE_DELAY": "-1",
         # cachekit's L1 sweeper wakes every 30 s of real time. A woken thread takes the GIL at a
         # load-dependent moment, and the sweeper compares expiry times stamped by the pinned
         # main-thread clock against the real one, so it evicts live entries mid-run. A day-long
@@ -340,12 +339,11 @@ def judge(budgets: dict[str, int], measured: dict[str, int]) -> list[dict[str, o
             rows.append({"path": path, "verdict": "FAIL", "ir": ir, "budget": None, "pct": None})
             continue
         pct = (ir - budget) / budget * 100
-        fail_pct, warn_pct = TOLERANCE_PCT.get(path, (FAIL_PCT, WARN_PCT))
-        if pct >= fail_pct:
+        if pct >= FAIL_PCT:
             verdict = "FAIL"
-        elif pct >= warn_pct:
+        elif pct >= WARN_PCT:
             verdict = "WARN"
-        elif pct <= -fail_pct:
+        elif pct <= -FAIL_PCT:
             verdict = "LOWER"  # cheaper: ratchet it down with --update if the change touched this path
         else:
             verdict = "ok"

tests/performance/ir_report.py

@@ -89,7 +89,7 @@ def current(k: int) -> dict | None:
         out.append(f"| {verdict} | `{row['path']}` | {_figure(row.get('ir'))} | {_figure(row.get('budget'))} | {pct} |")
     out += [
         "",
-        "A path fails at +1% or more against the base (+2% for `serializer_orjson`). This check is not required: "
+        "A path fails at +1% or more against the base. This check is not required: "
         "a deliberate cost increase is accepted at merge. Method: `docs/performance.md`, *Instruction budgets*.",
     ]
     return "\n".join(out) + "\n"

tests/performance/ir_workload.py

@@ -217,6 +217,21 @@ def read() -> Any:
         setattr(time, f"{name}_ns", pinned(getattr(time, f"{name}_ns"), int(origin * 1e9), 1000))
 
 
+def pin_os_entropy() -> None:
+    """Give Python code the same OS entropy in every run.
+
+    Importing pandas imports ``numpy.random``, which seeds its global generator from 128 bits of
+    ``secrets.randbits``. The cost of that seeding and the sizes of what it allocates depend on the
+    bits drawn, and the heap layout everything after it runs in moves with them: on CPython 3.14,
+    16 runs of one path spread over 57,000 instructions, and over 557 with this. ``secrets`` reads
+    ``random._urandom``, which is ``os.urandom`` bound at import, so both are replaced. Must run
+    before ``import pandas``. C code that reads OS entropy itself is not covered.
+    """
+    import random
+
+    random._urandom = os.urandom = random.Random(0).randbytes  # pyright: ignore[reportAttributeAccessIssue]
+
+
 def run_workload(path: str, n: int, warmup: int) -> None:
     import gc
 
@@ -230,6 +245,7 @@ def run_workload(path: str, n: int, warmup: int) -> None:
     # main thread gives up the GIL only where the code releases it itself.
     sys.setswitchinterval(1e6)
     pin_main_thread_clocks()
+    pin_os_entropy()
     _shift = [object() for _ in range(int(os.environ.get("IR_BUDGET_LAYOUT", "0")))]  # noqa: F841 (held: see LAYOUTS in ir_budget.py)
     op = build_workload(path)
     for _ in range(warmup):  # identical warmup at both N: first-call costs (L1 fill, lazy imports) cancel

tests/unit/test_ir_budget.py

@@ -57,16 +57,6 @@ def test_compare_thresholds(measured: int, verdict: str, ok: bool) -> None:
     assert passed is ok
 
 
-@pytest.mark.parametrize(
-    ("measured", "verdict", "ok"),
-    [(100_999, "ok", True), (101_000, "WARN", True), (101_999, "WARN", True), (102_000, "FAIL", False), (98_000, "LOWER", True)],
-)
-def test_compare_gates_orjson_at_its_own_tolerance(measured: int, verdict: str, ok: bool) -> None:
-    lines, passed = compare({"serializer_orjson": 100_000}, {"serializer_orjson": measured})
-    assert lines[0].split()[0] == verdict
-    assert passed is ok
-
-
 def test_compare_fails_a_path_without_a_budget() -> None:
     lines, passed = compare({}, {"l1_hit": 1})
     assert not passed
@@ -512,6 +502,24 @@ def test_pinned_clocks_tick_per_read_on_the_main_thread_only(monkeypatch) -> Non
     assert seen[0] > 1.7e9 + 1e6  # the real clock: 2023-11 + 11 days at the very least
 
 
+def test_pinned_os_entropy_starts_numpy_in_the_same_state_in_every_run() -> None:
+    """numpy.random seeds its global generator from OS entropy at import, which moved every path's count."""
+    pytest.importorskip("numpy")  # the free-threaded CI lanes install no numpy
+    probe = (
+        "import sys; sys.path.insert(0, sys.argv[1]); import ir_workload\n"
+        "if sys.argv[2] == 'pin': ir_workload.pin_os_entropy()\n"
+        "import numpy.random; print(numpy.random.get_state()[1][:8].tolist())"
+    )
+    perf = str(Path(ir_workload.__file__).parent)
+
+    def first_words(mode: str) -> str:
+        run = [sys.executable, "-c", probe, perf, mode]
+        return subprocess.run(run, capture_output=True, text=True, check=True).stdout  # noqa: S603 (trusted: this test's probe)
+
+    assert first_words("pin") == first_words("pin")
+    assert first_words("os") != first_words("os")  # unpinned, numpy's start does come from OS entropy
+
+
 @pytest.mark.parametrize("n", [1, 4, len(PATHS) + 1])
 def test_shards_split_the_paths_exactly_once(n: int) -> None:
     """N machines running --shard 1/N .. N/N measure every path that is not skipped, and none twice."""

tests/unit/test_ir_report.py

@@ -144,9 +144,7 @@ def test_a_report_ir_budget_did_not_write_is_no_report(tmp_path, text: str) -> N
 
 def test_the_footnote_states_the_gate_thresholds() -> None:
     body = render(_all_ok(), 4, JOBS, "aaa", "bbb", RUN)
-    assert f"+{ir_budget.FAIL_PCT:g}% or more" in body
-    assert set(ir_budget.TOLERANCE_PCT) == {"serializer_orjson"}  # the footnote names no other exception
-    assert f"(+{ir_budget.TOLERANCE_PCT['serializer_orjson'][0]:g}% for `serializer_orjson`)" in body
+    assert f"+{ir_budget.FAIL_PCT:g}% or more against the base." in body  # one threshold, no per-path exception
 
 
 def test_ci_names_the_marker_and_the_shard_jobs() -> None: