From 1c7665d9a5b417aac4ef9e3f6cd541a5595a35a9 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 21 Aug 2026 12:07:04 -0500 Subject: [PATCH 01/52] Cached benchmark IO sketch --- benchmarks/asv.conf.json | 9 + benchmarks/bench_connectivity.py | 82 +++----- benchmarks/face_bounds.py | 55 +++-- benchmarks/helpers/_fixtures.py | 334 +++++++++++++++++++++++++++++++ benchmarks/mpas_dyamond.py | 34 ++-- benchmarks/mpas_ocean.py | 179 ++++++++++------- benchmarks/quad_hexagon.py | 12 +- 7 files changed, 533 insertions(+), 172 deletions(-) create mode 100644 benchmarks/helpers/_fixtures.py diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 31a43921d..41ee30072 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -61,6 +61,15 @@ // defaults to 10 min "install_timeout": 600, + // Fork each benchmark from one interpreter that has already imported the + // suite, instead of starting a fresh one per benchmark. Saves the ~0.9s + // uxarray import and the numba kernel load on every one of ~160 benchmark + // processes, and lets a fixture loaded at import be inherited rather than + // re-read. asv leaves this at "spawn" by default because fork and threads + // mix badly; nothing in this suite runs a parallel kernel at import time, + // which is what makes it safe here -- keep it that way. + "launch_method": "forkserver", + "benchmark_timeout": 360, // the base URL to show a commit for the project. diff --git a/benchmarks/bench_connectivity.py b/benchmarks/bench_connectivity.py index 5e3f53d02..3a5859ee9 100644 --- a/benchmarks/bench_connectivity.py +++ b/benchmarks/bench_connectivity.py @@ -1,53 +1,27 @@ -import os -import urllib.request -from pathlib import Path - import uxarray as ux -current_path = Path(os.path.dirname(os.path.realpath(__file__))) - -grid_filename_480 = "oQU480.grid.nc" -grid_filename_120 = "oQU120.grid.nc" -filenames = [grid_filename_480, grid_filename_120] - -for filename in filenames: - if not os.path.isfile(current_path / filename): - # downloads the files from Cookbook repo, if they haven't been downloaded locally yet - url = f"https://github.com/ProjectPythia/unstructured-grid-viz-cookbook/raw/main/meshfiles/{filename}" - _, headers = urllib.request.urlretrieve(url, filename=current_path / filename) - -oQU_path_dict = {"480km": current_path / grid_filename_480, - "120km": current_path / grid_filename_120} - -# Paths to grid files on Glade -dyamond_path_dict = {"30km": "/glade/campaign/cisl/vast/uxarray/data/dyamond/30km/grid.nc", - "15km": "/glade/campaign/cisl/vast/uxarray/data/dyamond/15km/grid.nc", - "7.5km": "/glade/campaign/cisl/vast/uxarray/data/dyamond/7.5km/grid.nc", - "3.75km": "/glade/campaign/cisl/vast/uxarray/data/dyamond/3.75km/grid.nc"} +from .helpers._fixtures import ( + ALL_RESOLUTIONS, + GRIDS_BY_RESOLUTION, + CachedFixtures, + cached_topology, +) -# Determines if all file paths exist and are accesible -all_paths_exist = True -for file_path in dyamond_path_dict.values(): - all_paths_exist = all_paths_exist and os.path.exists(file_path) -file_path_dict = oQU_path_dict -if all_paths_exist: - file_path_dict = file_path_dict | dyamond_path_dict - - -class GridBenchmark: +class GridBenchmark(CachedFixtures): """Class used as a template for benchmarks requiring a ``Grid`` in this module across both resolutions.""" param_names = ['resolution', ] # Conditionally available; could get annoying if there are downstream tools relying on it. - if all_paths_exist: - params = [['480km', '120km', '30km', '15km', '7.5km', '3.75km'], ] - else: - params = [['480km', '120km'], ] + params = [ALL_RESOLUTIONS, ] + + # A single connectivity build at 3.75km does not fit in the 360s default + # from ``asv.conf.json``. + timeout = 1200 def setup(self, resolution, *args, **kwargs): - self.uxgrid = ux.open_grid(file_path_dict[resolution]) + self.uxgrid = self.cached_grid(GRIDS_BY_RESOLUTION[resolution]) def teardown(self, resolution, *args, **kwargs): del self.uxgrid @@ -65,16 +39,23 @@ def teardown(self, resolution, *args, **kwargs): _numba_warmed_up = False -def _warmup(uxgrid): +def _warmup(): """Compiles the Numba kernels backing each connectivity variable. - ``_build_node_edge_connectivity`` is not disk-cached, so a fresh benchmark - process would otherwise charge ~240ms of JIT compilation to whichever sample - happened to touch it first. + ``_build_node_edge_connectivity`` is ``@njit`` without ``cache=True``, so a + fresh benchmark process would otherwise charge ~240ms of JIT compilation to + whichever sample happened to touch it first. + + Warmed on the coarsest grid in ``params``, because resolution decides how + long the kernels run but not which signatures compile. Warming at the + benchmark's own resolution instead means eight full connectivity builds + before the sample -- 0.493s against a 0.106s sample at 120km, and the gap + only widens from there. """ global _numba_warmed_up if _numba_warmed_up: return + uxgrid = ux.Grid.from_topology(*cached_topology(GRIDS_BY_RESOLUTION[ALL_RESOLUTIONS[0]])) for name in CONNECTIVITY_NAMES: getattr(uxgrid, name) _numba_warmed_up = True @@ -89,16 +70,11 @@ class Connectivity(GridBenchmark): def setup(self, resolution, *args, **kwargs): # The benchmark grids are MPAS meshes, which carry every connectivity # variable on disk. Reading one would time the MPAS parser rather than - # the construction routines, so reduce the grid down to the minimal - # UGRID topology and let each variable be built on demand. - source_grid = ux.open_grid(file_path_dict[resolution]) - self.topology = ( - source_grid.node_lon.data, - source_grid.node_lat.data, - source_grid.face_node_connectivity.data, - ) - - _warmup(self.minimal_grid()) + # the construction routines, so this takes the minimal UGRID topology + # fixture and lets each variable be built on demand. + self.topology = self.cached_topology(GRIDS_BY_RESOLUTION[resolution]) + + _warmup() self.uxgrid = self.minimal_grid() def minimal_grid(self): diff --git a/benchmarks/face_bounds.py b/benchmarks/face_bounds.py index 8f1e41416..d406723d3 100644 --- a/benchmarks/face_bounds.py +++ b/benchmarks/face_bounds.py @@ -1,18 +1,17 @@ -import os -from pathlib import Path - import uxarray as ux -from .helpers._memsize import grid_nbytes -from .helpers._peakmem import numba_threads, peak_allocated -current_path = Path(os.path.dirname(os.path.realpath(__file__))).parents[0] +from .helpers._fixtures import GRIDS_BY_FORMAT, CachedFixtures +from .helpers._memsize import grid_nbytes +from .helpers._peakmem import numba_threads, peak_allocated, subprocess_peak_rss -grid_quad_hex = current_path / "test" / "meshfiles" / "ugrid" / "quad-hexagon" / "grid.nc" -grid_geoflow = current_path / "test" / "meshfiles" / "ugrid" / "geoflow-small" / "grid.nc" -grid_scrip = current_path / "test" / "meshfiles" / "scrip" / "outCSne8" / "outCSne8.nc" -grid_mpas= current_path / "test" / "meshfiles" / "mpas" / "QU" / "oQU480.231010.nc" +# One grid per reader, from the shared registry. ``mpas`` here is the same mesh +# the oQU ``480km`` benchmarks use, through the copy in the repo. +grid_quad_hex = GRIDS_BY_FORMAT["ugrid-quad-hexagon"] +grid_geoflow = GRIDS_BY_FORMAT["ugrid-geoflow"] +grid_scrip = GRIDS_BY_FORMAT["scrip-outCSne8"] +grid_mpas = GRIDS_BY_FORMAT["mpas-oQU480"] -class FaceBounds: +class FaceBounds(CachedFixtures): params = [grid_quad_hex, grid_geoflow, grid_scrip, grid_mpas] @@ -24,8 +23,8 @@ def setup(self, grid_path): # compiled before anything is measured. ``track_peakmem_*`` would # otherwise charge the first sample for loading them off numba's disk # cache, which inflates the reported peak by ~3%. - ux.open_grid(grid_quad_hex).bounds - self.uxgrid = ux.open_grid(grid_path) + self.cached_grid(grid_quad_hex).bounds + self.uxgrid = self.cached_grid(grid_path) def teardown(self, n): del self.uxgrid @@ -63,21 +62,37 @@ def track_peakmem_face_bounds(self, grid_path): class FaceBoundsColdStartRss: """Peak memory of a cold start: import uxarray, open a grid, get its bounds. - Whole-process ``ru_maxrss``, not tracemalloc -- the ~250MB uxarray import is - part of the number by design, because the cold start is the subject. For the - cost of ``bounds`` alone see ``FaceBounds.track_peakmem_face_bounds``, which - runs one to three orders of magnitude lower. + Whole-process peak resident memory, not tracemalloc -- the ~250MB uxarray + import is part of the number by design, because the cold start is the + subject. For the cost of ``bounds`` alone see + ``FaceBounds.track_peakmem_face_bounds``, which runs one to three orders of + magnitude lower. + + Measured in a subprocess of its own rather than through asv's ``peakmem_*``, + which reports ``ru_maxrss`` for the benchmark process. Under + ``launch_method: forkserver`` that process is forked from an interpreter that + has already imported the suite, so a ``peakmem_*`` here would be reporting a + warm start plus whatever the parent was holding. A fresh interpreter is the + only way to keep measuring what this benchmark is named for. """ params = FaceBounds.params param_names = ["grid_path"] def setup_cache(self): - """Compile the njit kernels before anything is measured.""" + """Compile the njit kernels before anything is measured. + + The subprocess inherits numba's on-disk cache, not this process's + memory, so this keeps compilation out of the measured cold start. + """ for grid_path in self.params: ux.open_grid(grid_path).bounds setup_cache.timeout = 1800 - def peakmem_open_and_bounds(self, grid_path): - ux.open_grid(grid_path).bounds + def track_peakmem_open_and_bounds(self, grid_path): + return subprocess_peak_rss( + f"import uxarray as ux; ux.open_grid({str(grid_path)!r}).bounds" + ) + + track_peakmem_open_and_bounds.unit = "bytes" diff --git a/benchmarks/helpers/_fixtures.py b/benchmarks/helpers/_fixtures.py new file mode 100644 index 000000000..a39975c14 --- /dev/null +++ b/benchmarks/helpers/_fixtures.py @@ -0,0 +1,334 @@ +"""Cached inputs for benchmarks whose subject is not reading a file. + +asv gives every benchmark its own process, so a grid opened in ``setup`` is +opened once per benchmark rather than once per run -- around 160 times across +the suite once the dyamond resolutions are visible. For a mesh on local disk +that is a rounding error. For one on campaign storage it is most of the run, and +it is spent inside the benchmark's own timeout. + +Benchmarks that do measure opening a file keep reading the real thing: +``quad_hexagon``, ``OpenGrid`` in ``mpas_dyamond``, ``import``, and the +cold-start peak-memory benchmarks. They take their paths from the registries +here too, so the suite declares its inputs in one place either way. + +Two flavours, picked per benchmark: + +``topology`` + the three arrays ``Grid.from_topology`` needs and nothing else, for + benchmarks that mean to build the rest themselves -- 2.3MB of the 102MB + 120km MPAS file. +``grid`` / ``dataset`` + everything the reader produced, so a benchmark still gets the + ``face_areas`` and connectivity variables an MPAS file carries on disk + rather than silently measuring their construction. + +One source read produces both, so choosing between them costs nothing. + +Artifacts are keyed on the uxarray build as well as on the file, because an +artifact is one version's reader output and asv walks commits. That means a +fresh read per commit; ``prime`` therefore leaves the dyamond grids out unless +asked, and there is a CLI (``python -m benchmarks.helpers._fixtures``) to fill +the cache from a batch script instead of from inside a benchmark. +""" + +import hashlib +import os +import tempfile +import urllib.request +from pathlib import Path + +import numpy as np +import xarray as xr + +import uxarray as ux + +__all__ = [ + "ALL_RESOLUTIONS", + "DYAMOND_AVAILABLE", + "DYAMOND_GRIDS", + "GRIDS_BY_FORMAT", + "GRIDS_BY_RESOLUTION", + "OQU_DATASETS", + "OQU_GRIDS", + "OQU_RESOLUTIONS", + "QUAD_HEXAGON_DATASET", + "CachedFixtures", + "cache_dir", + "cached_dataset", + "cached_grid", + "cached_topology", + "prime", +] + +BENCHMARK_DIR = Path(__file__).resolve().parents[1] +REPO_DIR = BENCHMARK_DIR.parent + +_COOKBOOK_URL = ( + "https://github.com/ProjectPythia/unstructured-grid-viz-cookbook/raw/main/meshfiles" +) + + +def _cookbook(filename): + """Path to a Cookbook mesh, fetched once if this checkout lacks it.""" + path = BENCHMARK_DIR / filename + if not path.is_file(): + urllib.request.urlretrieve(f"{_COOKBOOK_URL}/{filename}", filename=path) + return path + + +# Grids, and grid/data pairs, by mesh resolution. +OQU_GRIDS = { + "480km": _cookbook("oQU480.grid.nc"), + "120km": _cookbook("oQU120.grid.nc"), +} +OQU_DATASETS = { + "480km": (OQU_GRIDS["480km"], _cookbook("oQU480.data.nc")), + "120km": (OQU_GRIDS["120km"], _cookbook("oQU120.data.nc")), +} + +DYAMOND_GRIDS = { + "30km": Path("/glade/campaign/cisl/vast/uxarray/data/dyamond/30km/grid.nc"), + "15km": Path("/glade/campaign/cisl/vast/uxarray/data/dyamond/15km/grid.nc"), + "7.5km": Path("/glade/campaign/cisl/vast/uxarray/data/dyamond/7.5km/grid.nc"), + "3.75km": Path("/glade/campaign/cisl/vast/uxarray/data/dyamond/3.75km/grid.nc"), +} + +# Asked once here, rather than separately in each module that cares. +DYAMOND_AVAILABLE = all(path.exists() for path in DYAMOND_GRIDS.values()) + +GRIDS_BY_RESOLUTION = dict(OQU_GRIDS) +if DYAMOND_AVAILABLE: + GRIDS_BY_RESOLUTION |= DYAMOND_GRIDS + +# Two ladders rather than one, because which of them a benchmark belongs on is a +# per-benchmark decision: an algorithm that does not care which model wrote the +# mesh can take the wide one, while anything tied to the oQU pair -- or too slow +# to run four dyamond resolutions of -- stays on the narrow one. +OQU_RESOLUTIONS = list(OQU_GRIDS) +ALL_RESOLUTIONS = list(GRIDS_BY_RESOLUTION) + +# Grids by source format, for benchmarks whose axis is the reader rather than the +# mesh size. ``mpas-oQU480`` is the same 1,791-face mesh as ``480km`` above, +# reached through the copy in the repo instead of the Cookbook download. +GRIDS_BY_FORMAT = { + "ugrid-quad-hexagon": REPO_DIR / "test" / "meshfiles" / "ugrid" / "quad-hexagon" / "grid.nc", + "ugrid-geoflow": REPO_DIR / "test" / "meshfiles" / "ugrid" / "geoflow-small" / "grid.nc", + "scrip-outCSne8": REPO_DIR / "test" / "meshfiles" / "scrip" / "outCSne8" / "outCSne8.nc", + "mpas-oQU480": REPO_DIR / "test" / "meshfiles" / "mpas" / "QU" / "oQU480.231010.nc", +} + +QUAD_HEXAGON_DATASET = ( + GRIDS_BY_FORMAT["ugrid-quad-hexagon"], + REPO_DIR / "test" / "meshfiles" / "ugrid" / "quad-hexagon" / "data.nc", +) + +CACHE_DIR_VAR = "UXARRAY_BENCH_CACHE_DIR" +PRIME_VAR = "UXARRAY_BENCH_PRIME" + +# The arguments ``ux.Grid.from_topology`` takes, in order. +_TOPOLOGY_ARRAYS = ("node_lon", "node_lat", "face_node_connectivity") + +# Artifact path -> what it holds, for this process. +_loaded = {} + + +def cache_dir(): + """Directory the cached artifacts live in. + + ``UXARRAY_BENCH_CACHE_DIR`` overrides the default, and on a cluster it + should: the cache only pays off on a filesystem faster than the one holding + the source grids, and putting it somewhere that outlives the job means each + grid is read once per machine rather than once per job. + """ + root = Path(os.environ.get(CACHE_DIR_VAR) or tempfile.gettempdir()) + cached = root / "uxarray-bench-fixtures" + cached.mkdir(parents=True, exist_ok=True) + return cached + + +def _artifact(source, flavour, suffix): + """Where ``flavour`` of ``source`` is cached. + + ``source`` is a grid path, or a (grid, data) pair. Keyed on each file's size + and mtime as well as its path, so a replaced source misses rather than being + served something stale, and on the uxarray version, because the artifact is + that version's reader output. + """ + parts = [ux.__version__] + for path in source: + stat = os.stat(path) + parts.append(f"{os.path.realpath(path)}:{stat.st_size}:{stat.st_mtime_ns}") + digest = hashlib.sha256("|".join(parts).encode()).hexdigest()[:16] + # The dyamond grids are all named ``grid.nc``; the directory above them is + # what tells one resolution from the next. + grid_path = Path(source[0]) + stem = f"{grid_path.parent.name}-{grid_path.stem}" + return cache_dir() / f"{stem}-{flavour}-{digest}{suffix}" + + +def _write(dataset, artifact_path, writer): + """Writes ``dataset`` to ``artifact_path``, atomically. + + Via a scratch name in the same directory, so a process racing this one sees + either no artifact or a complete one, never a half-written file. + """ + scratch = artifact_path.with_name( + f"{artifact_path.stem}.{os.getpid()}.tmp{artifact_path.suffix}" + ) + writer(dataset, scratch) + os.replace(scratch, artifact_path) + + +def _read_dataset(artifact_path): + """Reads back a cached ``xr.Dataset``. + + ``mask_and_scale=False`` is load-bearing: the connectivity variables carry + ``_FillValue``, which xarray would otherwise consume, handing back float64 + where uxarray's njit kernels require int64 -- they fail to type rather than + returning something wrong, but they do fail. + """ + return xr.open_dataset(artifact_path, mask_and_scale=False).load() + + +def _build(source): + """Reads ``source`` and writes every flavour of it, from the one read.""" + grid_path = source[0] + if len(source) == 1: + uxgrid = ux.open_grid(grid_path) + uxgrid._ds.load() + data_ds = None + else: + uxds = ux.open_dataset(*source) + uxds.load() + uxgrid = uxds.uxgrid + uxgrid._ds.load() + # A ``UxDataset`` is an ``xr.Dataset`` subclass, so it writes itself; the + # grid half is cached separately just below. + data_ds = uxds + + _write( + {name: getattr(uxgrid, name).data for name in _TOPOLOGY_ARRAYS}, + _artifact(source[:1], "topology", ".npz"), + # Uncompressed: written once, read by every benchmark process after. + lambda arrays, path: np.savez(path, **arrays), + ) + _write(uxgrid._ds, _artifact(source[:1], "grid", ".nc"), lambda ds, path: ds.to_netcdf(path)) + if data_ds is not None: + _write(data_ds, _artifact(source, "data", ".nc"), lambda ds, path: ds.to_netcdf(path)) + + +def _ensure(source, flavour, suffix): + """Path to a cached artifact, building the source's artifacts if need be.""" + artifact_path = _artifact(source, flavour, suffix) + if not artifact_path.exists(): + _build(source if flavour == "data" else source[:1]) + return artifact_path + + +def cached_topology(grid_path): + """``(node_lon, node_lat, face_node_connectivity)`` for ``grid_path``. + + The arrays are shared rather than copied: ``Grid.from_topology`` wraps + ``node_lon`` and ``node_lat`` without copying them, and the construction + routines assign to ``Grid._ds`` rather than writing through their inputs, + which is what makes one copy per process safe. Treat them as read-only. + """ + artifact_path = _ensure((Path(grid_path),), "topology", ".npz") + if artifact_path not in _loaded: + with np.load(artifact_path) as cached: + _loaded[artifact_path] = tuple(cached[name] for name in _TOPOLOGY_ARRAYS) + return _loaded[artifact_path] + + +def _cached_grid_ds(grid_path): + """The cached internal dataset of ``grid_path``, held for this process.""" + artifact_path = _ensure((Path(grid_path),), "grid", ".nc") + if artifact_path not in _loaded: + _loaded[artifact_path] = _read_dataset(artifact_path) + return _loaded[artifact_path] + + +def cached_grid(grid_path): + """A ``Grid`` carrying everything the reader found in ``grid_path``. + + A fresh ``Grid`` over a shallow copy each call, so a benchmark that + populates or normalizes something does not hand its leftovers to the next + repeat: uxarray assigns new variables into ``_ds`` rather than writing + through the arrays, so the copy isolates that while the data stays shared. + """ + return ux.Grid(_cached_grid_ds(grid_path).copy()) + + +def cached_dataset(grid_path, data_path): + """A ``UxDataset`` over ``data_path``, on the cached grid.""" + source = (Path(grid_path), Path(data_path)) + artifact_path = _ensure(source, "data", ".nc") + if artifact_path not in _loaded: + _loaded[artifact_path] = _read_dataset(artifact_path) + return ux.UxDataset(_loaded[artifact_path].copy(), uxgrid=cached_grid(grid_path)) + + +def prime(include_dyamond=None): + """Fills the cache for every source the fixtures can serve. + + Returns the sources it had to read. Idempotent, and once warm costs a + ``stat`` per file, so it is cheap to call ahead of every run. + + The dyamond grids are left out unless ``UXARRAY_BENCH_PRIME=all`` (or + ``include_dyamond``) asks for them, because a filtered run should not pay + for reading four grids off campaign storage that it will never touch. On a + machine that does have them, prime from the CLI before ``asv run``. + """ + if include_dyamond is None: + include_dyamond = os.environ.get(PRIME_VAR, "").lower() == "all" + + sources = [(path,) for path in OQU_GRIDS.values()] + sources += [(path,) for path in GRIDS_BY_FORMAT.values()] + sources += list(OQU_DATASETS.values()) + if include_dyamond and DYAMOND_AVAILABLE: + sources += [(path,) for path in DYAMOND_GRIDS.values()] + + read = [] + for source in sources: + flavour, suffix = ("data", ".nc") if len(source) == 2 else ("grid", ".nc") + if not _artifact(source, flavour, suffix).exists(): + _build(source) + read.append(source) + return read + + +class CachedFixtures: + """Mixin for benchmarks whose subject is not reading a file. + + Holds the one ``setup_cache`` the suite shares. asv keys ``setup_cache`` on + where it is defined and groups benchmarks by that key, so this single + definition -- inherited by every such class in every module -- runs once per + ``asv run`` rather than once per class. It returns ``None``, which asv reads + as "no cache argument", so the benchmark signatures stay as they are. + + The accessors are re-exported as methods so a ``setup`` reads as + ``self.cached_grid(...)`` instead of importing the module's functions + alongside its registries. + """ + + def setup_cache(self): + prime() + + # Reading grids off campaign storage is the point of the cache, and does not + # fit in a benchmark-sized timeout. + setup_cache.timeout = 7200 + + cached_topology = staticmethod(cached_topology) + cached_grid = staticmethod(cached_grid) + cached_dataset = staticmethod(cached_dataset) + + +if __name__ == "__main__": + # Fills the cache ahead of ``asv run``, so no benchmark -- and not even + # ``setup_cache`` -- pays for reading a source grid. Worth a line in a batch + # script whenever the dyamond grids are in play. + print(f"fixture cache: {cache_dir()}", flush=True) + for source in prime(include_dyamond=True) or [None]: + # Unbuffered and one line per source, so a batch log shows how far the + # reading has got. + print(f" read {' + '.join(Path(p).name for p in source)}" if source else " nothing to do", flush=True) diff --git a/benchmarks/mpas_dyamond.py b/benchmarks/mpas_dyamond.py index 3e4ecc8c0..a91db9f8a 100644 --- a/benchmarks/mpas_dyamond.py +++ b/benchmarks/mpas_dyamond.py @@ -1,31 +1,25 @@ -import os - from asv_runner.benchmarks.mark import skip_benchmark_if, timeout_class_at import uxarray as ux -# Paths to grid files on Glade -grid_path_dict = {"30km": "/glade/campaign/cisl/vast/uxarray/data/dyamond/30km/grid.nc", - "15km": "/glade/campaign/cisl/vast/uxarray/data/dyamond/15km/grid.nc", - "7.5km": "/glade/campaign/cisl/vast/uxarray/data/dyamond/7.5km/grid.nc", - "3.75km": "/glade/campaign/cisl/vast/uxarray/data/dyamond/3.75km/grid.nc"} - - +from .helpers._fixtures import DYAMOND_AVAILABLE, DYAMOND_GRIDS, CachedFixtures -# Determines if all file paths exist and are accesible -all_paths_exist = True -for file_path in grid_path_dict.values(): - all_paths_exist = all_paths_exist and os.path.exists(file_path) +# Paths, and the question of whether this machine can see them, both come from +# ``helpers._fixtures`` -- ``bench_connectivity`` asks for the same four grids. +grid_path_dict = DYAMOND_GRIDS -class BaseGridBenchmark: +class BaseGridBenchmark(CachedFixtures): """Base class for Grid Benchmarks across the four supported resolutions (30km, 15km, 7.5km, 3.75km)""" param_names = ['resolution'] - params = [['30km', '15km', '7.5km', '3.75km'], ] + params = [list(DYAMOND_GRIDS), ] def setup(self, resolution, **kwargs): - self.uxgrid = ux.open_grid(grid_path_dict[resolution]) + # The cached grid, not a fresh read: what these benchmarks measure is + # ``bounds`` and ``to_geodataframe``, not the MPAS reader. ``OpenGrid`` + # below is the one that measures reading, and it opens the real file. + self.uxgrid = self.cached_grid(grid_path_dict[resolution]) def teardown(self, resolution, **kwargs): del self.uxgrid @@ -33,21 +27,21 @@ def teardown(self, resolution, **kwargs): @timeout_class_at(1200) class OpenGrid: param_names = ['resolution'] - params = [['30km', '15km', '7.5km', '3.75km'], ] + params = [list(DYAMOND_GRIDS), ] - @skip_benchmark_if(not all_paths_exist) + @skip_benchmark_if(not DYAMOND_AVAILABLE) def time_open_grid(self, resolution): _ = ux.open_grid(grid_path_dict[resolution]) @timeout_class_at(1200) class Bounds(BaseGridBenchmark): - @skip_benchmark_if(not all_paths_exist) + @skip_benchmark_if(not DYAMOND_AVAILABLE) def time_bounds(self, resolution): _ = self.uxgrid.bounds @timeout_class_at(1200) class GeoDataFrame(BaseGridBenchmark): - @skip_benchmark_if(not all_paths_exist) + @skip_benchmark_if(not DYAMOND_AVAILABLE) def time_to_geodataframe(self, resolution): self.uxgrid.to_geodataframe(exclude_antimeridian=True) diff --git a/benchmarks/mpas_ocean.py b/benchmarks/mpas_ocean.py index cd6177f47..ba352db5f 100644 --- a/benchmarks/mpas_ocean.py +++ b/benchmarks/mpas_ocean.py @@ -1,59 +1,51 @@ -import os -import urllib.request -from pathlib import Path - import numpy as np import uxarray as ux +from .helpers._fixtures import ( + OQU_DATASETS, + OQU_GRIDS, + OQU_RESOLUTIONS, + CachedFixtures, +) from .helpers._memsize import grid_nbytes -from .helpers._peakmem import numba_threads, peak_allocated - -current_path = Path(os.path.dirname(os.path.realpath(__file__))) +from .helpers._peakmem import numba_threads, peak_allocated, subprocess_peak_rss data_var = 'bottomDepth' -grid_filename_480 = "oQU480.grid.nc" -data_filename_480 = "oQU480.data.nc" - -grid_filename_120 = "oQU120.grid.nc" -data_filename_120 = "oQU120.data.nc" - -filenames = [grid_filename_480, data_filename_480, grid_filename_120, data_filename_120] - -for filename in filenames: - if not os.path.isfile(current_path / filename): - # downloads the files from Cookbook repo, if they haven't been downloaded locally yet - url = f"https://github.com/ProjectPythia/unstructured-grid-viz-cookbook/raw/main/meshfiles/{filename}" - _, headers = urllib.request.urlretrieve(url, filename=current_path / filename) - - -file_path_dict = {"480km": [current_path / grid_filename_480, current_path / data_filename_480], - "120km": [current_path / grid_filename_120, current_path / data_filename_120]} +# Paths, and fetching the files in the first place, both live in +# ``helpers._fixtures`` now -- ``bench_connectivity`` draws the same grids from it. +file_path_dict = OQU_DATASETS - -class DatasetBenchmark: +class DatasetBenchmark(CachedFixtures): """Class used as a template for benchmarks requiring a ``UxDataset`` in - this module across both resolutions.""" + this module across both resolutions. + + The dataset comes from the fixture cache rather than a fresh + ``open_dataset``: every benchmark below measures an algorithm over the mesh, + not the reader that produced it. The fixture is what the reader produced, + connectivity and ``face_areas`` included, so nothing here silently starts + measuring construction that used to come off disk. + """ param_names = ['resolution', ] - params = [['480km', '120km'], ] + params = [OQU_RESOLUTIONS, ] def setup(self, resolution, *args, **kwargs): - self.uxds = ux.open_dataset(file_path_dict[resolution][0], file_path_dict[resolution][1]) + self.uxds = self.cached_dataset(*file_path_dict[resolution]) def teardown(self, resolution, *args, **kwargs): del self.uxds -class GridBenchmark: +class GridBenchmark(CachedFixtures): """Class used as a template for benchmarks requiring a ``Grid`` in this module across both resolutions.""" param_names = ['resolution', ] - params = [['480km', '120km'], ] + params = [OQU_RESOLUTIONS, ] def setup(self, resolution, *args, **kwargs): - self.uxgrid = ux.open_grid(file_path_dict[resolution][0]) + self.uxgrid = self.cached_grid(file_path_dict[resolution][0]) def teardown(self, resolution, *args, **kwargs): del self.uxgrid @@ -65,9 +57,11 @@ class FaceAreas(GridBenchmark): def setup(self, resolution, *args, **kwargs): # The coarsest grid, purely to compile the njit kernel - warmup_grid = ux.open_grid(file_path_dict[self.params[0][0]][0]) - _ = warmup_grid.face_areas + _ = self.cached_grid(OQU_GRIDS[OQU_RESOLUTIONS[0]]).face_areas super().setup(resolution, *args, **kwargs) + # MPAS meshes carry ``face_areas`` on disk and the fixture keeps it, so + # it is dropped here to leave the computation to be measured. Safe to do + # to a fixture: each handout is a fresh ``Grid`` over a shallow copy. self.uxgrid._ds = self.uxgrid._ds.drop_vars("face_areas", errors="ignore") def time_face_areas(self, resolution): @@ -91,8 +85,7 @@ class Gradient(DatasetBenchmark): def setup(self, resolution, *args, **kwargs): super().setup(resolution, *args, **kwargs) # Compiles the gradient kernels on the coarsest grid - grid, data = file_path_dict[self.params[0][0]] - _ = ux.open_dataset(grid, data)[data_var].gradient() + _ = self.cached_dataset(*file_path_dict[OQU_RESOLUTIONS[0]])[data_var].gradient() def time_gradient(self, resolution): self.uxds[data_var].gradient() @@ -126,26 +119,44 @@ def track_nbytes_integrate(self, resolution): class GradientColdStartRss: """Peak memory of a cold start: import uxarray, open a dataset, take a gradient. - Whole-process ``ru_maxrss``, not tracemalloc -- the ~250MB uxarray import is - part of the number by design, because the cold start is the subject. For the - gradient's own transient cost see ``Gradient.track_peakmem_gradient``, which - runs one to three orders of magnitude lower. + Whole-process peak resident memory, not tracemalloc -- the ~226MB uxarray + import is part of the number by design, because the cold start is the + subject. For the gradient's own transient cost see + ``Gradient.track_peakmem_gradient``, which runs one to three orders of + magnitude lower. + + Measured in a subprocess of its own rather than through asv's ``peakmem_*``, + which reports ``ru_maxrss`` for the benchmark process. Under + ``launch_method: forkserver`` that process is forked from an interpreter + that has already imported the suite, so ``peakmem_*`` would report a warm + start plus whatever the parent held. A fresh interpreter is the only way to + keep measuring the thing this benchmark is named for. """ param_names = ["resolution"] - params = [["480km", "120km"]] + params = [OQU_RESOLUTIONS] def setup_cache(self): - """Compile the njit kernels before anything is measured.""" + """Compile the njit kernels before anything is measured. + + The subprocess inherits numba's on-disk cache rather than this process's + memory, so this keeps compilation out of the measured cold start. + """ for resolution in self.params[0]: grid, data = file_path_dict[resolution] ux.open_dataset(grid, data)[data_var].gradient() setup_cache.timeout = 1800 - def peakmem_gradient(self, resolution): + def track_peakmem_gradient(self, resolution): grid, data = file_path_dict[resolution] - ux.open_dataset(grid, data)[data_var].gradient() + return subprocess_peak_rss( + "import uxarray as ux\n" + f"uxds = ux.open_dataset({str(grid)!r}, {str(data)!r})\n" + f"uxds[{data_var!r}].gradient()\n" + ) + + track_peakmem_gradient.unit = "bytes" class GeoDataFrame(DatasetBenchmark): @@ -181,11 +192,11 @@ def time_ball_tree(self, resolution): self.uxds.uxgrid.get_ball_tree() -class RemapDownsample: +class RemapDownsample(CachedFixtures): def setup(self): - self.uxds_120 = ux.open_dataset(file_path_dict['120km'][0], file_path_dict['120km'][1]) - self.uxds_480 = ux.open_dataset(file_path_dict['480km'][0], file_path_dict['480km'][1]) + self.uxds_120 = self.cached_dataset(*file_path_dict['120km']) + self.uxds_480 = self.cached_dataset(*file_path_dict['480km']) def teardown(self): del self.uxds_120, self.uxds_480 @@ -199,11 +210,11 @@ def time_inverse_distance_weighted_remapping(self): def time_bilinear_remapping(self): self.uxds_120["bottomDepth"].remap.bilinear(self.uxds_480.uxgrid) -class RemapUpsample: +class RemapUpsample(CachedFixtures): def setup(self): - self.uxds_120 = ux.open_dataset(file_path_dict['120km'][0], file_path_dict['120km'][1]) - self.uxds_480 = ux.open_dataset(file_path_dict['480km'][0], file_path_dict['480km'][1]) + self.uxds_120 = self.cached_dataset(*file_path_dict['120km']) + self.uxds_480 = self.cached_dataset(*file_path_dict['480km']) def teardown(self): del self.uxds_120, self.uxds_480 @@ -236,12 +247,12 @@ def time_cartesian_averaging(self, resolution): self.uxgrid.construct_face_centers(method='cartesian average') -class CheckNorm: +class CheckNorm(CachedFixtures): param_names = ['resolution'] - params = ['480km', '120km'] + params = OQU_RESOLUTIONS def setup(self, resolution): - self.uxgrid = ux.open_grid(file_path_dict[resolution][0]) + self.uxgrid = self.cached_grid(file_path_dict[resolution][0]) def teardown(self, resolution): del self.uxgrid @@ -255,7 +266,7 @@ class CrossSections(DatasetBenchmark): params = DatasetBenchmark.params + [[1, 2, 4]] def setup(self, resolution, lat_step): - self.uxgrid = ux.open_grid(file_path_dict[resolution][0]) + self.uxgrid = self.cached_grid(file_path_dict[resolution][0]) self.uxgrid.normalize_cartesian_coordinates() self.lats = np.arange(-45, 45, lat_step) _ = self.uxgrid.bounds @@ -268,12 +279,12 @@ def time_const_lat(self, resolution, lat_step): self.uxgrid.cross_section.constant_latitude(lat) -class PointInPolygon: +class PointInPolygon(CachedFixtures): param_names = ['resolution'] - params = ['480km', '120km'] + params = OQU_RESOLUTIONS def setup(self, resolution): - self.uxgrid = ux.open_grid(file_path_dict[resolution][0]) + self.uxgrid = self.cached_grid(file_path_dict[resolution][0]) self.uxgrid.normalize_cartesian_coordinates() # Construct variables needed to ensure that the benchmark doesn't measure construction time @@ -299,7 +310,7 @@ def time_face_search_lonlat(self, resolution): class ZonalAverage(DatasetBenchmark): def setup(self, resolution, *args, **kwargs): - self.uxds = ux.open_dataset(file_path_dict[resolution][0], file_path_dict[resolution][1]) + super().setup(resolution, *args, **kwargs) bounds = self.uxds.uxgrid.bounds def time_zonal_average(self, resolution): @@ -308,10 +319,15 @@ def time_zonal_average(self, resolution): class ZonalAveragePeakMem: - """Peak memory of a cold-start non-conservative zonal-mean sweep.""" + """Peak memory of a cold-start non-conservative zonal-mean sweep. + + A fresh interpreter per sample, for the reason spelled out in + :class:`GradientColdStartRss`: the cold start is the subject, and a forked + benchmark process no longer has one. + """ param_names = ["resolution"] - params = [["480km", "120km"]] + params = [OQU_RESOLUTIONS] def setup_cache(self): """Compile the njit kernels before anything is measured.""" @@ -321,18 +337,29 @@ def setup_cache(self): uxds.uxgrid.bounds uxds[data_var].zonal_mean(lat=(-45, 45, 10)) - def peakmem_zonal_average(self, resolution): + setup_cache.timeout = 1800 + + def track_peakmem_zonal_average(self, resolution): grid, data = file_path_dict[resolution] - uxds = ux.open_dataset(grid, data) - uxds.uxgrid.bounds - uxds[data_var].zonal_mean(lat=(-45, 45, 10)) + return subprocess_peak_rss( + "import uxarray as ux\n" + f"uxds = ux.open_dataset({str(grid)!r}, {str(data)!r})\n" + "uxds.uxgrid.bounds\n" + f"uxds[{data_var!r}].zonal_mean(lat=(-45, 45, 10))\n" + ) + + track_peakmem_zonal_average.unit = "bytes" class CrossSectionsPeakMem: - """Peak memory of a cold-start constant-latitude cross-section sweep.""" + """Peak memory of a cold-start constant-latitude cross-section sweep. + + A fresh interpreter per sample, for the reason spelled out in + :class:`GradientColdStartRss`. + """ param_names = ["resolution", "lat_step"] - params = [["480km", "120km"], [1, 2, 4]] + params = [OQU_RESOLUTIONS, [1, 2, 4]] def setup_cache(self): """Compile the njit kernels before anything is measured.""" @@ -342,9 +369,17 @@ def setup_cache(self): uxgrid.bounds uxgrid.cross_section.constant_latitude(0.0) - def peakmem_const_lat(self, resolution, lat_step): - uxgrid = ux.open_grid(file_path_dict[resolution][0]) - uxgrid.normalize_cartesian_coordinates() - uxgrid.bounds - for lat in np.arange(-45, 45, lat_step): - uxgrid.cross_section.constant_latitude(lat) + setup_cache.timeout = 1800 + + def track_peakmem_const_lat(self, resolution, lat_step): + grid = file_path_dict[resolution][0] + return subprocess_peak_rss( + "import numpy as np, uxarray as ux\n" + f"uxgrid = ux.open_grid({str(grid)!r})\n" + "uxgrid.normalize_cartesian_coordinates()\n" + "uxgrid.bounds\n" + f"for lat in np.arange(-45, 45, {lat_step}):\n" + " uxgrid.cross_section.constant_latitude(lat)\n" + ) + + track_peakmem_const_lat.unit = "bytes" diff --git a/benchmarks/quad_hexagon.py b/benchmarks/quad_hexagon.py index 5b03eec6f..a3c5b95ef 100644 --- a/benchmarks/quad_hexagon.py +++ b/benchmarks/quad_hexagon.py @@ -1,14 +1,12 @@ -import os -from pathlib import Path - import uxarray as ux + +from .helpers._fixtures import QUAD_HEXAGON_DATASET from .helpers._memsize import dataset_nbytes, grid_nbytes from .helpers._peakmem import peak_allocated -current_path = Path(os.path.dirname(os.path.realpath(__file__))).parents[0] - -grid_path = current_path / "test" / "meshfiles" / "ugrid" / "quad-hexagon" / "grid.nc" -data_path = current_path / "test" / "meshfiles" / "ugrid" / "quad-hexagon" / "data.nc" +# Opening these files is what this module measures, so it reads them for real +# every time; only the paths come from the shared registry. +grid_path, data_path = QUAD_HEXAGON_DATASET class QuadHexagon: From 787e2d48e2dc6ce45fef54c15e73aa9a8119109b Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 21 Aug 2026 12:49:42 -0500 Subject: [PATCH 02/52] benchmark debugging --- benchmarks/asv.conf.json | 4 +++- benchmarks/helpers/_peakmem.py | 33 +++++++++++++++++++++------------ benchmarks/mpas_ocean.py | 8 ++++++-- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 41ee30072..535ec6b2a 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -70,7 +70,9 @@ // which is what makes it safe here -- keep it that way. "launch_method": "forkserver", - "benchmark_timeout": 360, + // ``benchmark_timeout`` is not a key asv reads -- the one that sets the + // default is ``default_benchmark_timeout`` (asv/config.py) + "default_benchmark_timeout": 360, // the base URL to show a commit for the project. "show_commit_url": "https://github.com/UXARRAY/uxarray/commit/", diff --git a/benchmarks/helpers/_peakmem.py b/benchmarks/helpers/_peakmem.py index 0265d80f8..011e03034 100644 --- a/benchmarks/helpers/_peakmem.py +++ b/benchmarks/helpers/_peakmem.py @@ -1,7 +1,9 @@ import contextlib +import os import subprocess import sys +import tempfile import numba @@ -66,15 +68,22 @@ def subprocess_peak_rss(statement): ``RUSAGE_CHILDREN``, which is a maximum over every child that has exited and so would not isolate this one. """ - reporter = ( - "import resource, sys\n" - f"exec({statement!r})\n" - "sys.stdout.write(str(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss))\n" - ) - completed = subprocess.run( - [sys.executable, "-c", reporter], - capture_output=True, - text=True, - check=True, - ) - return int(completed.stdout) * _MAXRSS_TO_BYTES + with tempfile.TemporaryDirectory() as scratch: + report_path = os.path.join(scratch, "peak_rss") + reporter = ( + "import resource\n" + f"exec({statement!r})\n" + "peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss\n" + # Reported through a file rather than stdout, which is not ours + # alone: uxarray prints coordinate warnings there for some grids, + # and the number would arrive with prose in front of it. + f"open({report_path!r}, 'w').write(str(peak))\n" + ) + subprocess.run( + [sys.executable, "-c", reporter], + capture_output=True, + text=True, + check=True, + ) + with open(report_path) as report: + return int(report.read()) * _MAXRSS_TO_BYTES diff --git a/benchmarks/mpas_ocean.py b/benchmarks/mpas_ocean.py index ba352db5f..b62620d83 100644 --- a/benchmarks/mpas_ocean.py +++ b/benchmarks/mpas_ocean.py @@ -97,8 +97,12 @@ def track_nbytes_gradient(self, resolution): track_nbytes_gradient.unit = "bytes" def track_peakmem_gradient(self, resolution): - """Transient high-water allocation of taking a gradient.""" - return peak_allocated(lambda: self.uxds[data_var].gradient()) + """Transient high-water allocation of taking a gradient. + + The kernel behind ``gradient`` is ``parallel=True``, hence the pinning + """ + with numba_threads(1): + return peak_allocated(lambda: self.uxds[data_var].gradient()) track_peakmem_gradient.unit = "bytes" From 96ce618dfd32171614ffc9a784bb4f3900344b25 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 21 Aug 2026 15:29:08 -0500 Subject: [PATCH 03/52] Sample large benchmarks fewer times --- benchmarks/mpas_ocean.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/benchmarks/mpas_ocean.py b/benchmarks/mpas_ocean.py index b62620d83..cb3a178bf 100644 --- a/benchmarks/mpas_ocean.py +++ b/benchmarks/mpas_ocean.py @@ -13,6 +13,12 @@ data_var = 'bottomDepth' +# Sample budget for the benchmarks with long single call runtimes. +# +# Only the classes whose slowest parameter clears ~0.25s carry this; on the rest +# the cap would never bind and would cost samples for nothing. +SLOW_CALL_REPEAT = (2, 3, 8.0) + # Paths, and fetching the files in the first place, both live in # ``helpers._fixtures`` now -- ``bench_connectivity`` draws the same grids from it. file_path_dict = OQU_DATASETS @@ -165,6 +171,7 @@ def track_peakmem_gradient(self, resolution): class GeoDataFrame(DatasetBenchmark): param_names = DatasetBenchmark.param_names + ['exclude_antimeridian'] + repeat = SLOW_CALL_REPEAT params = DatasetBenchmark.params + [[True, False]] def time_to_geodataframe(self, resolution, exclude_antimeridian): @@ -215,6 +222,7 @@ def time_bilinear_remapping(self): self.uxds_120["bottomDepth"].remap.bilinear(self.uxds_480.uxgrid) class RemapUpsample(CachedFixtures): + repeat = SLOW_CALL_REPEAT def setup(self): self.uxds_120 = self.cached_dataset(*file_path_dict['120km']) @@ -244,6 +252,8 @@ def time_dual_mesh_construction(self, resolution): class ConstructFaceLatLon(GridBenchmark): + repeat = SLOW_CALL_REPEAT + def time_welzl(self, resolution): self.uxgrid.construct_face_centers(method='welzl') @@ -267,6 +277,7 @@ def time_check_norm(self, resolution): class CrossSections(DatasetBenchmark): param_names = DatasetBenchmark.param_names + ['n_lat'] + repeat = SLOW_CALL_REPEAT params = DatasetBenchmark.params + [[1, 2, 4]] def setup(self, resolution, lat_step): From 4060897d7ec66fd9fc00589b7f4e8e0d4b0d2df1 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 21 Aug 2026 16:00:47 -0500 Subject: [PATCH 04/52] Temp fork/thread speedup for connectivity. REEVALUATE OR REMOVE WHEN CONNECTIVITY BECOMES MULTITHREADED --- benchmarks/bench_connectivity.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/benchmarks/bench_connectivity.py b/benchmarks/bench_connectivity.py index 3a5859ee9..6732c395c 100644 --- a/benchmarks/bench_connectivity.py +++ b/benchmarks/bench_connectivity.py @@ -1,3 +1,5 @@ +import numba + import uxarray as ux from .helpers._fixtures import ( @@ -74,6 +76,8 @@ def setup(self, resolution, *args, **kwargs): # fixture and lets each variable be built on demand. self.topology = self.cached_topology(GRIDS_BY_RESOLUTION[resolution]) + # A no-op once the module-level warm below has run; kept so the class is + # still correct if that ever goes away. _warmup() self.uxgrid = self.minimal_grid() @@ -115,3 +119,29 @@ def time_edge_face(self, resolution): def time_node_face(self, resolution): _ = self.uxgrid.node_face_connectivity.compute() + + +# Compiled at import rather than only in ``setup``: under +# ``launch_method: forkserver`` asv imports the suite once and forks every +# benchmark from that interpreter, so kernels compiled here are inherited by all +# of them. A forked child otherwise spends 0.496s of its own on JIT and cache +# loading before it can build anything. +# +# Safe only while the connectivity kernels are serial. Warming a +# ``parallel=True`` kernel -- or even calling ``.compile()`` on one -- launches +# numba's thread pool in the parent, and its OpenMP layer is not fork-safe, with +# no at-fork handler to rebuild it in the child. So check, rather than trust: +# this turns the day chunked connectivity lands into a loud import error instead +# of a hung benchmark. +_warmup() + +try: + numba.threading_layer() +except ValueError: + pass # nothing launched a pool, which is what we want to inherit +else: + raise RuntimeError( + "warming the connectivity kernels started numba's thread pool, which a " + "forked benchmark cannot safely inherit -- move _warmup() back into " + "setup() now that these kernels run in parallel" + ) From 7ab239d6192b0e70c78fb36a11e14ee37e816905 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 21 Aug 2026 16:13:31 -0500 Subject: [PATCH 05/52] Cache benchmark sampling across forks too --- benchmarks/bench_connectivity.py | 27 ++++-------------- benchmarks/geometry_kernels.py | 29 +++++++++++++++++++ benchmarks/geometry_samebody.py | 36 ++++++++++++++++++++---- benchmarks/geometry_samebody_gcagca.py | 39 ++++++++++++++++++++++---- benchmarks/helpers/_warmup.py | 37 ++++++++++++++++++++++++ 5 files changed, 134 insertions(+), 34 deletions(-) create mode 100644 benchmarks/helpers/_warmup.py diff --git a/benchmarks/bench_connectivity.py b/benchmarks/bench_connectivity.py index 6732c395c..ca1b4e20e 100644 --- a/benchmarks/bench_connectivity.py +++ b/benchmarks/bench_connectivity.py @@ -1,5 +1,3 @@ -import numba - import uxarray as ux from .helpers._fixtures import ( @@ -8,6 +6,7 @@ CachedFixtures, cached_topology, ) +from .helpers._warmup import warm_in_parent class GridBenchmark(CachedFixtures): @@ -125,23 +124,7 @@ def time_node_face(self, resolution): # ``launch_method: forkserver`` asv imports the suite once and forks every # benchmark from that interpreter, so kernels compiled here are inherited by all # of them. A forked child otherwise spends 0.496s of its own on JIT and cache -# loading before it can build anything. -# -# Safe only while the connectivity kernels are serial. Warming a -# ``parallel=True`` kernel -- or even calling ``.compile()`` on one -- launches -# numba's thread pool in the parent, and its OpenMP layer is not fork-safe, with -# no at-fork handler to rebuild it in the child. So check, rather than trust: -# this turns the day chunked connectivity lands into a loud import error instead -# of a hung benchmark. -_warmup() - -try: - numba.threading_layer() -except ValueError: - pass # nothing launched a pool, which is what we want to inherit -else: - raise RuntimeError( - "warming the connectivity kernels started numba's thread pool, which a " - "forked benchmark cannot safely inherit -- move _warmup() back into " - "setup() now that these kernels run in parallel" - ) +# loading before it can build anything. Guarded, because this only stays safe +# while the connectivity kernels are serial -- see +# :mod:`benchmarks.helpers._warmup`. +warm_in_parent(_warmup, "the connectivity kernels") diff --git a/benchmarks/geometry_kernels.py b/benchmarks/geometry_kernels.py index 08aea236e..058d555ea 100644 --- a/benchmarks/geometry_kernels.py +++ b/benchmarks/geometry_kernels.py @@ -15,6 +15,8 @@ import numpy as np +from .helpers._warmup import warm_in_parent + def _unit(v): return v / np.linalg.norm(v) @@ -205,3 +207,30 @@ def time_try_gca_const_lat_intersection(self): def time_gca_const_lat_intersection(self): """Layer 3: dispatcher (full public API).""" self.gca_const_lat_intersection(self.gca_cart, _CONST_Z) + + +def _warm_classes(): + """Compiles what each class's ``setup`` compiles, once per process. + + Runs the setups themselves rather than a copy of their warm calls, so this + cannot drift out of step with them. Failures are swallowed on purpose: this + module imports its kernels inside ``setup`` so that a commit missing a symbol + fails one benchmark rather than the whole module's collection, and warming + here must not take that away. + """ + for cls in ( + EFTPrimitives, + AccucrossKernels, + OrientPredicates, + GCAGCAIntersection, + GCAConstLatIntersection, + ): + try: + cls().setup() + except Exception: + pass + + +# Warmed at import so every forked benchmark inherits the compiled kernels; see +# :mod:`benchmarks.helpers._warmup`. +warm_in_parent(_warm_classes, "the geometry kernels") diff --git a/benchmarks/geometry_samebody.py b/benchmarks/geometry_samebody.py index 7712d1c7a..bee8cdf4b 100644 --- a/benchmarks/geometry_samebody.py +++ b/benchmarks/geometry_samebody.py @@ -60,6 +60,8 @@ gca_const_lat_intersection, ) +from .helpers._warmup import warm_in_parent + # --------------------------------------------------------------------------- # L1 (FP64 body) — direct double-precision kernel, verbatim from # fp64_GCAconstLat.hh. Scalar in / scalar out so Numba keeps it in registers, @@ -392,16 +394,33 @@ def main(): # --------------------------------------------------------------------------- +_prepared = None + + +def _prepare(): + """The packed cases, with the batched drivers warmed, once per process. + + Hoisted out of ``setup`` so a forked benchmark inherits them: the seed is + fixed, so the arrays are the same ones ``setup`` used to build, and the + drivers are ``@njit(cache=True)`` either way. + """ + global _prepared + if _prepared is None: + packed = _pack(_make_cases(20_000, seed=20251104)) + A, B, Z, gcas = packed + _batch_fp64_kernel(A, B, Z) + _batch_accux_kernel(A, B, Z) + _batch_fp64_dispatch(gcas, Z) + _batch_accux_dispatch(gcas, Z) + _prepared = packed + return _prepared + + class SameBodyConstLat: """asv: same-body FP64 vs real AccuX at kernel (L1) and dispatch (L3) levels.""" def setup(self): - cases = _make_cases(20_000, seed=20251104) - self.A, self.B, self.Z, self.gcas = _pack(cases) - _batch_fp64_kernel(self.A, self.B, self.Z) - _batch_accux_kernel(self.A, self.B, self.Z) - _batch_fp64_dispatch(self.gcas, self.Z) - _batch_accux_dispatch(self.gcas, self.Z) + self.A, self.B, self.Z, self.gcas = _prepare() def time_fp64_kernel(self): _batch_fp64_kernel(self.A, self.B, self.Z) @@ -416,5 +435,10 @@ def time_accux_dispatch(self): _batch_accux_dispatch(self.gcas, self.Z) +# Prepared at import so every forked benchmark inherits it; see +# :mod:`benchmarks.helpers._warmup`. +warm_in_parent(_prepare, "the const-lat drivers") + + if __name__ == "__main__": main() diff --git a/benchmarks/geometry_samebody_gcagca.py b/benchmarks/geometry_samebody_gcagca.py index 348f55859..e93108025 100644 --- a/benchmarks/geometry_samebody_gcagca.py +++ b/benchmarks/geometry_samebody_gcagca.py @@ -23,6 +23,8 @@ from uxarray.grid.arcs import on_minor_arc from uxarray.grid.intersections import _accux_gca, gca_gca_intersection +from .helpers._warmup import warm_in_parent + @njit(cache=True, inline="always") def _fp64_gca(w0, w1, v0, v1): @@ -307,6 +309,30 @@ def ns(t): print("=" * 70) +_prepared = None + + +def _prepare(): + """The packed cases, with the batched drivers warmed, once per process. + + Building the cases is 4.06s of the 4.27s this used to spend in every + ``setup``; compiling the drivers is 0.08s, since they are all + ``@njit(cache=True)``. The seed is fixed, so hoisting the arrays out of + ``setup`` changes what is measured not at all -- and lets a forked benchmark + inherit them rather than generate them again. + """ + global _prepared + if _prepared is None: + packed = _pack_gca(_make_gca_cases(100_000, seed=20251104)) + wa, wb, va, vb, ga, gb = packed + _batch_fp64_gca_kernel(wa, wb, va, vb) + _batch_accux_gca_kernel(wa, wb, va, vb) + _batch_fp64_gca_dispatch(ga, gb) + _batch_accux_gca_dispatch(ga, gb) + _prepared = packed + return _prepared + + class SameBodyGcaGca: """ asv timing class (Numba warmed in setup, distinct cases) @@ -314,12 +340,7 @@ class SameBodyGcaGca: """ def setup(self): - cases = _make_gca_cases(100_000, seed=20251104) - self.wa, self.wb, self.va, self.vb, self.ga, self.gb = _pack_gca(cases) - _batch_fp64_gca_kernel(self.wa, self.wb, self.va, self.vb) - _batch_accux_gca_kernel(self.wa, self.wb, self.va, self.vb) - _batch_fp64_gca_dispatch(self.ga, self.gb) - _batch_accux_gca_dispatch(self.ga, self.gb) + self.wa, self.wb, self.va, self.vb, self.ga, self.gb = _prepare() def time_fp64_kernel(self): _batch_fp64_gca_kernel(self.wa, self.wb, self.va, self.vb) @@ -334,5 +355,11 @@ def time_accux_dispatch(self): _batch_accux_gca_dispatch(self.ga, self.gb) +# Prepared at import so every forked benchmark inherits it; see +# :mod:`benchmarks.helpers._warmup` for why that is safe only while these +# kernels stay serial. +warm_in_parent(_prepare, "the gca-gca drivers") + + if __name__ == "__main__": main() diff --git a/benchmarks/helpers/_warmup.py b/benchmarks/helpers/_warmup.py new file mode 100644 index 000000000..dbab5692b --- /dev/null +++ b/benchmarks/helpers/_warmup.py @@ -0,0 +1,37 @@ +"""Warming benchmark state in the interpreter every benchmark is forked from. + +Under ``launch_method: forkserver`` asv imports the suite once and forks each +benchmark from that interpreter, so whatever a module prepares at import is +inherited copy-on-write instead of being paid for again by every benchmark +process. On this suite that is 0.5s of JIT and cache loading for the +connectivity kernels and 4.1s of case generation for the gca-gca drivers, each +of which was being repeated per benchmark. + +Only work that leaves no numba thread pool behind may be warmed this way. +Running a ``parallel=True`` kernel -- or merely calling ``.compile()`` on one -- +launches the pool, and numba's OpenMP layer is not fork-safe, with no at-fork +handler to rebuild it in the child. So this checks rather than trusts: the day a +warmed kernel goes parallel becomes a loud import error rather than a benchmark +that hangs on a cluster. +""" + +import numba + +__all__ = ["warm_in_parent"] + + +def warm_in_parent(warm, what): + """Runs ``warm``, then fails if it left a numba thread pool behind. + + ``what`` names the thing being warmed, for the error message. + """ + warm() + try: + numba.threading_layer() + except ValueError: + return # nothing launched a pool, which is what makes this inheritable + raise RuntimeError( + f"warming {what} started numba's thread pool, which a forked benchmark " + "cannot safely inherit -- warm it from setup() instead, now that these " + "kernels run in parallel" + ) From c63676c29a3a9427ac7d2ec832348fc00de09ba1 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 21 Aug 2026 17:37:52 -0500 Subject: [PATCH 06/52] Per-resolution async benchmarks --- benchmarks/helpers/_fixtures.py | 56 +++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/benchmarks/helpers/_fixtures.py b/benchmarks/helpers/_fixtures.py index a39975c14..50722d303 100644 --- a/benchmarks/helpers/_fixtures.py +++ b/benchmarks/helpers/_fixtures.py @@ -32,9 +32,12 @@ """ import hashlib +import multiprocessing import os import tempfile import urllib.request +import uuid +from concurrent.futures import ProcessPoolExecutor from pathlib import Path import numpy as np @@ -172,8 +175,13 @@ def _write(dataset, artifact_path, writer): Via a scratch name in the same directory, so a process racing this one sees either no artifact or a complete one, never a half-written file. """ + if artifact_path.exists(): + # A grid reached through both its own source and a (grid, data) pair + # would otherwise be written twice, and two writers racing on one + # scratch name is worse than wasteful. + return scratch = artifact_path.with_name( - f"{artifact_path.stem}.{os.getpid()}.tmp{artifact_path.suffix}" + f"{artifact_path.stem}.{uuid.uuid4().hex}.tmp{artifact_path.suffix}" ) writer(dataset, scratch) os.replace(scratch, artifact_path) @@ -268,7 +276,7 @@ def cached_dataset(grid_path, data_path): return ux.UxDataset(_loaded[artifact_path].copy(), uxgrid=cached_grid(grid_path)) -def prime(include_dyamond=None): +def prime(include_dyamond=None, workers=1): """Fills the cache for every source the fixtures can serve. Returns the sources it had to read. Idempotent, and once warm costs a @@ -278,6 +286,14 @@ def prime(include_dyamond=None): ``include_dyamond``) asks for them, because a filtered run should not pay for reading four grids off campaign storage that it will never touch. On a machine that does have them, prime from the CLI before ``asv run``. + + ``workers`` reads that many sources at once, which is worth having when the + sources differ wildly in size and live somewhere slow: the small ones finish + while a dyamond grid is still being read, instead of queueing behind it. It + costs an interpreter per worker, so it only pays when a read is slower than + a process start -- the default stays sequential for that reason. Processes + rather than threads because the netCDF/HDF5 stack here is not thread-safe + for concurrent opens: threads produce HDF5 errors, measured, not assumed. """ if include_dyamond is None: include_dyamond = os.environ.get(PRIME_VAR, "").lower() == "all" @@ -288,13 +304,37 @@ def prime(include_dyamond=None): if include_dyamond and DYAMOND_AVAILABLE: sources += [(path,) for path in DYAMOND_GRIDS.values()] - read = [] + missing = [] for source in sources: flavour, suffix = ("data", ".nc") if len(source) == 2 else ("grid", ".nc") if not _artifact(source, flavour, suffix).exists(): + missing.append(source) + + # Reading a (grid, data) pair produces that grid's artifacts too, so a + # grid-only source covered by a pair here would just read the grid a second + # time to write nothing. + paired_grids = {source[0] for source in missing if len(source) == 2} + missing = [ + source for source in missing if len(source) == 2 or source[0] not in paired_grids + ] + + if workers > 1 and len(missing) > 1: + # Largest first: with a pool, the longest read should start earliest, or + # it lands last and everything waits on it. + missing.sort(key=lambda source: -sum(os.path.getsize(path) for path in source)) + # Spawned, not forked: this process has the netCDF/HDF5 library loaded by + # the time it primes, and a fresh interpreter per worker keeps that state + # out of the children. The extra second of startup is nothing against a + # read this is worth parallelizing. + with ProcessPoolExecutor( + max_workers=min(workers, len(missing)), + mp_context=multiprocessing.get_context("spawn"), + ) as pool: + list(pool.map(_build, missing)) + else: + for source in missing: _build(source) - read.append(source) - return read + return missing class CachedFixtures: @@ -328,7 +368,11 @@ def setup_cache(self): # ``setup_cache`` -- pays for reading a source grid. Worth a line in a batch # script whenever the dyamond grids are in play. print(f"fixture cache: {cache_dir()}", flush=True) - for source in prime(include_dyamond=True) or [None]: + # Four at a time only where the dyamond grids are readable, since those are + # the reads worth overlapping: on the oQU pair alone, priming in parallel is + # slower than doing it sequentially (1.69s against 0.52s), because starting + # an interpreter costs more than reading a small local file. + for source in prime(include_dyamond=True, workers=4 if DYAMOND_AVAILABLE else 1) or [None]: # Unbuffered and one line per source, so a batch log shows how far the # reading has got. print(f" read {' + '.join(Path(p).name for p in source)}" if source else " nothing to do", flush=True) From 85f7c0c5a10d3520507f08182f72bc1d0e42a808 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 21 Aug 2026 18:04:09 -0500 Subject: [PATCH 07/52] Benchmark IO caching fixes --- .gitignore | 1 + benchmarks/bench_connectivity.py | 10 +++++- benchmarks/helpers/_fixtures.py | 57 ++++++++++++++++++++++++-------- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index c8309635c..5c52cd586 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,4 @@ docs/user-guide/psi_healpix.nc benchmarks/env benchmarks/results benchmarks/html +benchmarks/_io_cache diff --git a/benchmarks/bench_connectivity.py b/benchmarks/bench_connectivity.py index ca1b4e20e..16dd3b590 100644 --- a/benchmarks/bench_connectivity.py +++ b/benchmarks/bench_connectivity.py @@ -5,6 +5,7 @@ GRIDS_BY_RESOLUTION, CachedFixtures, cached_topology, + preload_topologies, ) from .helpers._warmup import warm_in_parent @@ -127,4 +128,11 @@ def time_node_face(self, resolution): # loading before it can build anything. Guarded, because this only stays safe # while the connectivity kernels are serial -- see # :mod:`benchmarks.helpers._warmup`. -warm_in_parent(_warmup, "the connectivity kernels") +def _warm_parent(): + _warmup() + # And, if asked, the topologies themselves, so a forked benchmark inherits + # them instead of reading its resolution's artifact again. + preload_topologies(GRIDS_BY_RESOLUTION[res] for res in ALL_RESOLUTIONS) + + +warm_in_parent(_warm_parent, "the connectivity kernels") diff --git a/benchmarks/helpers/_fixtures.py b/benchmarks/helpers/_fixtures.py index 50722d303..cd357c439 100644 --- a/benchmarks/helpers/_fixtures.py +++ b/benchmarks/helpers/_fixtures.py @@ -34,7 +34,6 @@ import hashlib import multiprocessing import os -import tempfile import urllib.request import uuid from concurrent.futures import ProcessPoolExecutor @@ -60,6 +59,7 @@ "cached_dataset", "cached_grid", "cached_topology", + "preload_topologies", "prime", ] @@ -127,6 +127,7 @@ def _cookbook(filename): CACHE_DIR_VAR = "UXARRAY_BENCH_CACHE_DIR" PRIME_VAR = "UXARRAY_BENCH_PRIME" +PRELOAD_VAR = "UXARRAY_BENCH_PRELOAD" # The arguments ``ux.Grid.from_topology`` takes, in order. _TOPOLOGY_ARRAYS = ("node_lon", "node_lat", "face_node_connectivity") @@ -136,15 +137,9 @@ def _cookbook(filename): def cache_dir(): - """Directory the cached artifacts live in. - - ``UXARRAY_BENCH_CACHE_DIR`` overrides the default, and on a cluster it - should: the cache only pays off on a filesystem faster than the one holding - the source grids, and putting it somewhere that outlives the job means each - grid is read once per machine rather than once per job. - """ - root = Path(os.environ.get(CACHE_DIR_VAR) or tempfile.gettempdir()) - cached = root / "uxarray-bench-fixtures" + """Directory the cached artifacts live in: ``benchmarks/_io_cache``.""" + root = Path(os.environ.get(CACHE_DIR_VAR) or BENCHMARK_DIR) + cached = root / "_io_cache" cached.mkdir(parents=True, exist_ok=True) return cached @@ -154,10 +149,18 @@ def _artifact(source, flavour, suffix): ``source`` is a grid path, or a (grid, data) pair. Keyed on each file's size and mtime as well as its path, so a replaced source misses rather than being - served something stale, and on the uxarray version, because the artifact is - that version's reader output. + served something stale. + + Deliberately not keyed on the uxarray version. These meshes are stable and + the artifacts are meant to persist -- across jobs, and across the commits asv + walks. Including the version cost a fresh read per commit and produced two + full sets of artifacts here, because a benchmark run imports the uxarray + installed in asv's environment while a direct run from the repo root imports + the working tree, and those report different versions. The tradeoff is that a + change to how a reader parses these files does not invalidate the cache on + its own: delete ``_io_cache`` when that happens. """ - parts = [ux.__version__] + parts = [] for path in source: stat = os.stat(path) parts.append(f"{os.path.realpath(path)}:{stat.st_size}:{stat.st_mtime_ns}") @@ -337,6 +340,34 @@ def prime(include_dyamond=None, workers=1): return missing +def preload_topologies(grid_paths): + """Loads topologies here so forked benchmarks inherit them. + + Reading an artifact caches it on disk, not in the next process: under + ``launch_method: forkserver`` each benchmark is forked from the interpreter + that imported the suite, so it starts with whatever *that* process holds and + reads its own copy of everything else. Which is why a benchmark's memory + appears and then vanishes when it exits -- expected, but at 3.75km it means + re-reading gigabytes for every benchmark in the module. + + Loading them in the parent instead means one read per resolution per run, + shared copy-on-write by every child. Off unless ``UXARRAY_BENCH_PRELOAD`` + asks for it, because the parent then holds every resolution at once and asv's + discovery import pays for it too -- a poor trade unless the reads are slow, + which on campaign storage they are. + + Safe to call at import: reading arrays starts no numba thread pool, which is + what a forked child cannot inherit (see :mod:`benchmarks.helpers._warmup`). + """ + if not os.environ.get(PRELOAD_VAR): + return 0 + loaded = 0 + for grid_path in grid_paths: + cached_topology(grid_path) # held by the process-level memo from here on + loaded += 1 + return loaded + + class CachedFixtures: """Mixin for benchmarks whose subject is not reading a file. From 9cd8fa8c38e0f73f55d4edff30785a1da44fde2f Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 24 Aug 2026 11:20:46 -0500 Subject: [PATCH 08/52] cached IO benchmarks deslopping --- benchmarks/bench_connectivity.py | 29 +---- benchmarks/face_bounds.py | 11 +- benchmarks/geometry_kernels.py | 7 +- benchmarks/geometry_samebody_gcagca.py | 8 +- benchmarks/helpers/_fixtures.py | 150 ++++++++----------------- benchmarks/helpers/_warmup.py | 16 +-- benchmarks/mpas_dyamond.py | 4 +- benchmarks/mpas_ocean.py | 24 +--- 8 files changed, 68 insertions(+), 181 deletions(-) diff --git a/benchmarks/bench_connectivity.py b/benchmarks/bench_connectivity.py index 16dd3b590..4796c1224 100644 --- a/benchmarks/bench_connectivity.py +++ b/benchmarks/bench_connectivity.py @@ -13,13 +13,9 @@ class GridBenchmark(CachedFixtures): """Class used as a template for benchmarks requiring a ``Grid`` in this module across both resolutions.""" - param_names = ['resolution', ] - # Conditionally available; could get annoying if there are downstream tools relying on it. + param_names = ['resolution', ] params = [ALL_RESOLUTIONS, ] - - # A single connectivity build at 3.75km does not fit in the 360s default - # from ``asv.conf.json``. timeout = 1200 def setup(self, resolution, *args, **kwargs): @@ -47,12 +43,6 @@ def _warmup(): ``_build_node_edge_connectivity`` is ``@njit`` without ``cache=True``, so a fresh benchmark process would otherwise charge ~240ms of JIT compilation to whichever sample happened to touch it first. - - Warmed on the coarsest grid in ``params``, because resolution decides how - long the kernels run but not which signatures compile. Warming at the - benchmark's own resolution instead means eight full connectivity builds - before the sample -- 0.493s against a 0.106s sample at 120km, and the gap - only widens from there. """ global _numba_warmed_up if _numba_warmed_up: @@ -64,9 +54,7 @@ def _warmup(): class Connectivity(GridBenchmark): - # Each connectivity variable is cached in ``Grid._ds`` once constructed, so a - # sample may only contain a single call; otherwise every call but the first - # would time a dictionary lookup. + # connectivity is cached in ``Grid._ds`` on construction, so only run them once number = 1 def setup(self, resolution, *args, **kwargs): @@ -121,17 +109,12 @@ def time_node_face(self, resolution): _ = self.uxgrid.node_face_connectivity.compute() -# Compiled at import rather than only in ``setup``: under -# ``launch_method: forkserver`` asv imports the suite once and forks every -# benchmark from that interpreter, so kernels compiled here are inherited by all -# of them. A forked child otherwise spends 0.496s of its own on JIT and cache -# loading before it can build anything. Guarded, because this only stays safe -# while the connectivity kernels are serial -- see -# :mod:`benchmarks.helpers._warmup`. +# Compiled at import rather than in ``setup``. ASV imports the suite once and forks +# every benchmark from that parent, so kernels compiled here are inherited by all +# of them. Only safe while the connectivity kernels are serial def _warm_parent(): _warmup() - # And, if asked, the topologies themselves, so a forked benchmark inherits - # them instead of reading its resolution's artifact again. + # And, if asked, the topologies themselves... preload_topologies(GRIDS_BY_RESOLUTION[res] for res in ALL_RESOLUTIONS) diff --git a/benchmarks/face_bounds.py b/benchmarks/face_bounds.py index d406723d3..f5d58ea47 100644 --- a/benchmarks/face_bounds.py +++ b/benchmarks/face_bounds.py @@ -4,8 +4,6 @@ from .helpers._memsize import grid_nbytes from .helpers._peakmem import numba_threads, peak_allocated, subprocess_peak_rss -# One grid per reader, from the shared registry. ``mpas`` here is the same mesh -# the oQU ``480km`` benchmarks use, through the copy in the repo. grid_quad_hex = GRIDS_BY_FORMAT["ugrid-quad-hexagon"] grid_geoflow = GRIDS_BY_FORMAT["ugrid-geoflow"] grid_scrip = GRIDS_BY_FORMAT["scrip-outCSne8"] @@ -69,11 +67,7 @@ class FaceBoundsColdStartRss: magnitude lower. Measured in a subprocess of its own rather than through asv's ``peakmem_*``, - which reports ``ru_maxrss`` for the benchmark process. Under - ``launch_method: forkserver`` that process is forked from an interpreter that - has already imported the suite, so a ``peakmem_*`` here would be reporting a - warm start plus whatever the parent was holding. A fresh interpreter is the - only way to keep measuring what this benchmark is named for. + which reports ``ru_maxrss`` for the benchmark process. """ params = FaceBounds.params @@ -81,9 +75,6 @@ class FaceBoundsColdStartRss: def setup_cache(self): """Compile the njit kernels before anything is measured. - - The subprocess inherits numba's on-disk cache, not this process's - memory, so this keeps compilation out of the measured cold start. """ for grid_path in self.params: ux.open_grid(grid_path).bounds diff --git a/benchmarks/geometry_kernels.py b/benchmarks/geometry_kernels.py index 058d555ea..ceb9f995c 100644 --- a/benchmarks/geometry_kernels.py +++ b/benchmarks/geometry_kernels.py @@ -213,10 +213,7 @@ def _warm_classes(): """Compiles what each class's ``setup`` compiles, once per process. Runs the setups themselves rather than a copy of their warm calls, so this - cannot drift out of step with them. Failures are swallowed on purpose: this - module imports its kernels inside ``setup`` so that a commit missing a symbol - fails one benchmark rather than the whole module's collection, and warming - here must not take that away. + cannot drift out of step with them. """ for cls in ( EFTPrimitives, @@ -231,6 +228,4 @@ def _warm_classes(): pass -# Warmed at import so every forked benchmark inherits the compiled kernels; see -# :mod:`benchmarks.helpers._warmup`. warm_in_parent(_warm_classes, "the geometry kernels") diff --git a/benchmarks/geometry_samebody_gcagca.py b/benchmarks/geometry_samebody_gcagca.py index e93108025..fd3494ed3 100644 --- a/benchmarks/geometry_samebody_gcagca.py +++ b/benchmarks/geometry_samebody_gcagca.py @@ -317,9 +317,8 @@ def _prepare(): Building the cases is 4.06s of the 4.27s this used to spend in every ``setup``; compiling the drivers is 0.08s, since they are all - ``@njit(cache=True)``. The seed is fixed, so hoisting the arrays out of - ``setup`` changes what is measured not at all -- and lets a forked benchmark - inherit them rather than generate them again. + ``@njit(cache=True)``. This method allows for reuse of cases in forked + benchmarks to reduce time spent on case generation. """ global _prepared if _prepared is None: @@ -355,9 +354,6 @@ def time_accux_dispatch(self): _batch_accux_gca_dispatch(self.ga, self.gb) -# Prepared at import so every forked benchmark inherits it; see -# :mod:`benchmarks.helpers._warmup` for why that is safe only while these -# kernels stay serial. warm_in_parent(_prepare, "the gca-gca drivers") diff --git a/benchmarks/helpers/_fixtures.py b/benchmarks/helpers/_fixtures.py index cd357c439..b2c32df0b 100644 --- a/benchmarks/helpers/_fixtures.py +++ b/benchmarks/helpers/_fixtures.py @@ -1,32 +1,22 @@ """Cached inputs for benchmarks whose subject is not reading a file. -asv gives every benchmark its own process, so a grid opened in ``setup`` is -opened once per benchmark rather than once per run -- around 160 times across -the suite once the dyamond resolutions are visible. For a mesh on local disk -that is a rounding error. For one on campaign storage it is most of the run, and -it is spent inside the benchmark's own timeout. +ASV gives every benchmark its own process, so a grid opened in ``setup`` is +opened once per benchmark rather than once per run. This module provides flexible +access to files across benchmark runs by caching the needed files. -Benchmarks that do measure opening a file keep reading the real thing: -``quad_hexagon``, ``OpenGrid`` in ``mpas_dyamond``, ``import``, and the -cold-start peak-memory benchmarks. They take their paths from the registries -here too, so the suite declares its inputs in one place either way. +Benchmarks that do measure opening a file behave as before. -Two flavours, picked per benchmark: +Two flavors: ``topology`` - the three arrays ``Grid.from_topology`` needs and nothing else, for - benchmarks that mean to build the rest themselves -- 2.3MB of the 102MB - 120km MPAS file. + the three arrays ``Grid.from_topology`` needs and nothing else ``grid`` / ``dataset`` - everything the reader produced, so a benchmark still gets the - ``face_areas`` and connectivity variables an MPAS file carries on disk - rather than silently measuring their construction. + everything the reader produced from ``Grid.open_grid`` and + ``Grid.open_dataset`` -One source read produces both, so choosing between them costs nothing. - -Artifacts are keyed on the uxarray build as well as on the file, because an -artifact is one version's reader output and asv walks commits. That means a -fresh read per commit; ``prime`` therefore leaves the dyamond grids out unless +Artifacts are keyed on both the uxarray build and the files, because an +artifact is one version's reader output and ASV diffs commits. Likewise, there's a +fresh read per commit. ``prime`` therefore leaves the dyamond grids out unless asked, and there is a CLI (``python -m benchmarks.helpers._fixtures``) to fill the cache from a batch script instead of from inside a benchmark. """ @@ -79,7 +69,7 @@ def _cookbook(filename): return path -# Grids, and grid/data pairs, by mesh resolution. +# Grids and grid/data pairs, by mesh resolution. OQU_GRIDS = { "480km": _cookbook("oQU480.grid.nc"), "120km": _cookbook("oQU120.grid.nc"), @@ -96,17 +86,13 @@ def _cookbook(filename): "3.75km": Path("/glade/campaign/cisl/vast/uxarray/data/dyamond/3.75km/grid.nc"), } -# Asked once here, rather than separately in each module that cares. +# Find out which files are actually available DYAMOND_AVAILABLE = all(path.exists() for path in DYAMOND_GRIDS.values()) GRIDS_BY_RESOLUTION = dict(OQU_GRIDS) if DYAMOND_AVAILABLE: GRIDS_BY_RESOLUTION |= DYAMOND_GRIDS -# Two ladders rather than one, because which of them a benchmark belongs on is a -# per-benchmark decision: an algorithm that does not care which model wrote the -# mesh can take the wide one, while anything tied to the oQU pair -- or too slow -# to run four dyamond resolutions of -- stays on the narrow one. OQU_RESOLUTIONS = list(OQU_GRIDS) ALL_RESOLUTIONS = list(GRIDS_BY_RESOLUTION) @@ -129,10 +115,7 @@ def _cookbook(filename): PRIME_VAR = "UXARRAY_BENCH_PRIME" PRELOAD_VAR = "UXARRAY_BENCH_PRELOAD" -# The arguments ``ux.Grid.from_topology`` takes, in order. -_TOPOLOGY_ARRAYS = ("node_lon", "node_lat", "face_node_connectivity") - -# Artifact path -> what it holds, for this process. +# Per path, which artifacts are actually loaded _loaded = {} @@ -144,21 +127,12 @@ def cache_dir(): return cached -def _artifact(source, flavour, suffix): - """Where ``flavour`` of ``source`` is cached. +def _artifact(source, flavor, suffix): + """Where ``flavor`` of ``source`` is cached. ``source`` is a grid path, or a (grid, data) pair. Keyed on each file's size and mtime as well as its path, so a replaced source misses rather than being served something stale. - - Deliberately not keyed on the uxarray version. These meshes are stable and - the artifacts are meant to persist -- across jobs, and across the commits asv - walks. Including the version cost a fresh read per commit and produced two - full sets of artifacts here, because a benchmark run imports the uxarray - installed in asv's environment while a direct run from the repo root imports - the working tree, and those report different versions. The tradeoff is that a - change to how a reader parses these files does not invalidate the cache on - its own: delete ``_io_cache`` when that happens. """ parts = [] for path in source: @@ -169,19 +143,17 @@ def _artifact(source, flavour, suffix): # what tells one resolution from the next. grid_path = Path(source[0]) stem = f"{grid_path.parent.name}-{grid_path.stem}" - return cache_dir() / f"{stem}-{flavour}-{digest}{suffix}" + return cache_dir() / f"{stem}-{flavor}-{digest}{suffix}" def _write(dataset, artifact_path, writer): - """Writes ``dataset`` to ``artifact_path``, atomically. + """Writes ``dataset`` to ``artifact_path`` atomically. Via a scratch name in the same directory, so a process racing this one sees - either no artifact or a complete one, never a half-written file. + either no artifact or a complete one. """ if artifact_path.exists(): - # A grid reached through both its own source and a (grid, data) pair - # would otherwise be written twice, and two writers racing on one - # scratch name is worse than wasteful. + # If the path exists, someone else is already working return scratch = artifact_path.with_name( f"{artifact_path.stem}.{uuid.uuid4().hex}.tmp{artifact_path.suffix}" @@ -191,18 +163,12 @@ def _write(dataset, artifact_path, writer): def _read_dataset(artifact_path): - """Reads back a cached ``xr.Dataset``. - - ``mask_and_scale=False`` is load-bearing: the connectivity variables carry - ``_FillValue``, which xarray would otherwise consume, handing back float64 - where uxarray's njit kernels require int64 -- they fail to type rather than - returning something wrong, but they do fail. - """ + """Reads back a cached ``xr.Dataset``.""" return xr.open_dataset(artifact_path, mask_and_scale=False).load() def _build(source): - """Reads ``source`` and writes every flavour of it, from the one read.""" + """Reads ``source`` and writes every flavor of it, from the one read.""" grid_path = source[0] if len(source) == 1: uxgrid = ux.open_grid(grid_path) @@ -213,12 +179,11 @@ def _build(source): uxds.load() uxgrid = uxds.uxgrid uxgrid._ds.load() - # A ``UxDataset`` is an ``xr.Dataset`` subclass, so it writes itself; the - # grid half is cached separately just below. + # A ``UxDataset`` grid is cached separately. data_ds = uxds _write( - {name: getattr(uxgrid, name).data for name in _TOPOLOGY_ARRAYS}, + {name: getattr(uxgrid, name).data for name in ["node_lon", "node_lat", "face_node_connectivity"]}, _artifact(source[:1], "topology", ".npz"), # Uncompressed: written once, read by every benchmark process after. lambda arrays, path: np.savez(path, **arrays), @@ -228,11 +193,11 @@ def _build(source): _write(data_ds, _artifact(source, "data", ".nc"), lambda ds, path: ds.to_netcdf(path)) -def _ensure(source, flavour, suffix): +def _ensure(source, flavor, suffix): """Path to a cached artifact, building the source's artifacts if need be.""" - artifact_path = _artifact(source, flavour, suffix) + artifact_path = _artifact(source, flavor, suffix) if not artifact_path.exists(): - _build(source if flavour == "data" else source[:1]) + _build(source if flavor == "data" else source[:1]) return artifact_path @@ -247,7 +212,7 @@ def cached_topology(grid_path): artifact_path = _ensure((Path(grid_path),), "topology", ".npz") if artifact_path not in _loaded: with np.load(artifact_path) as cached: - _loaded[artifact_path] = tuple(cached[name] for name in _TOPOLOGY_ARRAYS) + _loaded[artifact_path] = tuple(cached[name] for name in ["node_lon", "node_lat", "face_node_connectivity"]) return _loaded[artifact_path] @@ -260,13 +225,7 @@ def _cached_grid_ds(grid_path): def cached_grid(grid_path): - """A ``Grid`` carrying everything the reader found in ``grid_path``. - - A fresh ``Grid`` over a shallow copy each call, so a benchmark that - populates or normalizes something does not hand its leftovers to the next - repeat: uxarray assigns new variables into ``_ds`` rather than writing - through the arrays, so the copy isolates that while the data stays shared. - """ + """A fresh ``Grid`` carrying everything the reader found in ``grid_path`` via shallow copy.""" return ux.Grid(_cached_grid_ds(grid_path).copy()) @@ -285,18 +244,12 @@ def prime(include_dyamond=None, workers=1): Returns the sources it had to read. Idempotent, and once warm costs a ``stat`` per file, so it is cheap to call ahead of every run. - The dyamond grids are left out unless ``UXARRAY_BENCH_PRIME=all`` (or - ``include_dyamond``) asks for them, because a filtered run should not pay - for reading four grids off campaign storage that it will never touch. On a - machine that does have them, prime from the CLI before ``asv run``. - ``workers`` reads that many sources at once, which is worth having when the sources differ wildly in size and live somewhere slow: the small ones finish - while a dyamond grid is still being read, instead of queueing behind it. It - costs an interpreter per worker, so it only pays when a read is slower than - a process start -- the default stays sequential for that reason. Processes - rather than threads because the netCDF/HDF5 stack here is not thread-safe - for concurrent opens: threads produce HDF5 errors, measured, not assumed. + while a larger grid is still being read, instead of queueing behind all the + file fetches. Only pays off when a read is slower than a process start due to + the new interpreter startup. Processes rather than threads because the standard + netCDF/HDF5 stack is not generally thread-safe for concurrent opens. """ if include_dyamond is None: include_dyamond = os.environ.get(PRIME_VAR, "").lower() == "all" @@ -309,13 +262,11 @@ def prime(include_dyamond=None, workers=1): missing = [] for source in sources: - flavour, suffix = ("data", ".nc") if len(source) == 2 else ("grid", ".nc") - if not _artifact(source, flavour, suffix).exists(): + flavor, suffix = ("data", ".nc") if len(source) == 2 else ("grid", ".nc") + if not _artifact(source, flavor, suffix).exists(): missing.append(source) - # Reading a (grid, data) pair produces that grid's artifacts too, so a - # grid-only source covered by a pair here would just read the grid a second - # time to write nothing. + # Reading a (grid, data) pair produces that grid's artifacts too paired_grids = {source[0] for source in missing if len(source) == 2} missing = [ source for source in missing if len(source) == 2 or source[0] not in paired_grids @@ -325,10 +276,9 @@ def prime(include_dyamond=None, workers=1): # Largest first: with a pool, the longest read should start earliest, or # it lands last and everything waits on it. missing.sort(key=lambda source: -sum(os.path.getsize(path) for path in source)) - # Spawned, not forked: this process has the netCDF/HDF5 library loaded by - # the time it primes, and a fresh interpreter per worker keeps that state - # out of the children. The extra second of startup is nothing against a - # read this is worth parallelizing. + + # Spawned. This process has the netCDF/HDF5 library loaded by the time it primes, + # and a fresh interpreter per worker keeps that state out of the children. with ProcessPoolExecutor( max_workers=min(workers, len(missing)), mp_context=multiprocessing.get_context("spawn"), @@ -346,15 +296,7 @@ def preload_topologies(grid_paths): Reading an artifact caches it on disk, not in the next process: under ``launch_method: forkserver`` each benchmark is forked from the interpreter that imported the suite, so it starts with whatever *that* process holds and - reads its own copy of everything else. Which is why a benchmark's memory - appears and then vanishes when it exits -- expected, but at 3.75km it means - re-reading gigabytes for every benchmark in the module. - - Loading them in the parent instead means one read per resolution per run, - shared copy-on-write by every child. Off unless ``UXARRAY_BENCH_PRELOAD`` - asks for it, because the parent then holds every resolution at once and asv's - discovery import pays for it too -- a poor trade unless the reads are slow, - which on campaign storage they are. + reads its own copy of everything else. Safe to call at import: reading arrays starts no numba thread pool, which is what a forked child cannot inherit (see :mod:`benchmarks.helpers._warmup`). @@ -371,11 +313,10 @@ def preload_topologies(grid_paths): class CachedFixtures: """Mixin for benchmarks whose subject is not reading a file. - Holds the one ``setup_cache`` the suite shares. asv keys ``setup_cache`` on - where it is defined and groups benchmarks by that key, so this single - definition -- inherited by every such class in every module -- runs once per - ``asv run`` rather than once per class. It returns ``None``, which asv reads - as "no cache argument", so the benchmark signatures stay as they are. + Holds the one canonical ``setup_cache`` the suite shares. ASV keys ``setup_cache`` + on where it is defined and groups benchmarks by that key, so this single + definition runs once per ``asv run`` rather than once per class. It returns + ``None``, which asv reads as "no cache argument". The accessors are re-exported as methods so a ``setup`` reads as ``self.cached_grid(...)`` instead of importing the module's functions @@ -399,11 +340,10 @@ def setup_cache(self): # ``setup_cache`` -- pays for reading a source grid. Worth a line in a batch # script whenever the dyamond grids are in play. print(f"fixture cache: {cache_dir()}", flush=True) + # Four at a time only where the dyamond grids are readable, since those are # the reads worth overlapping: on the oQU pair alone, priming in parallel is # slower than doing it sequentially (1.69s against 0.52s), because starting # an interpreter costs more than reading a small local file. for source in prime(include_dyamond=True, workers=4 if DYAMOND_AVAILABLE else 1) or [None]: - # Unbuffered and one line per source, so a batch log shows how far the - # reading has got. print(f" read {' + '.join(Path(p).name for p in source)}" if source else " nothing to do", flush=True) diff --git a/benchmarks/helpers/_warmup.py b/benchmarks/helpers/_warmup.py index dbab5692b..3ad8a9c1e 100644 --- a/benchmarks/helpers/_warmup.py +++ b/benchmarks/helpers/_warmup.py @@ -1,17 +1,13 @@ """Warming benchmark state in the interpreter every benchmark is forked from. -Under ``launch_method: forkserver`` asv imports the suite once and forks each +Under ``launch_method: forkserver`` ASV imports the suite once and forks each benchmark from that interpreter, so whatever a module prepares at import is -inherited copy-on-write instead of being paid for again by every benchmark -process. On this suite that is 0.5s of JIT and cache loading for the -connectivity kernels and 4.1s of case generation for the gca-gca drivers, each -of which was being repeated per benchmark. +inherited copy-on-write. -Only work that leaves no numba thread pool behind may be warmed this way. -Running a ``parallel=True`` kernel -- or merely calling ``.compile()`` on one -- -launches the pool, and numba's OpenMP layer is not fork-safe, with no at-fork -handler to rebuild it in the child. So this checks rather than trusts: the day a -warmed kernel goes parallel becomes a loud import error rather than a benchmark +Only tasks that leaves no numba thread pool behind may be warmed this way. +Running a ``parallel=True`` kernel or calling ``.compile()`` on one +launches the pool, and numba's OpenMP layer is not fork-safe. So this checks when a +warmed kernel goes parallel, it becomes an import error rather than a benchmark that hangs on a cluster. """ diff --git a/benchmarks/mpas_dyamond.py b/benchmarks/mpas_dyamond.py index a91db9f8a..8c9454a14 100644 --- a/benchmarks/mpas_dyamond.py +++ b/benchmarks/mpas_dyamond.py @@ -16,9 +16,7 @@ class BaseGridBenchmark(CachedFixtures): params = [list(DYAMOND_GRIDS), ] def setup(self, resolution, **kwargs): - # The cached grid, not a fresh read: what these benchmarks measure is - # ``bounds`` and ``to_geodataframe``, not the MPAS reader. ``OpenGrid`` - # below is the one that measures reading, and it opens the real file. + # The cached grid, not a fresh read self.uxgrid = self.cached_grid(grid_path_dict[resolution]) def teardown(self, resolution, **kwargs): diff --git a/benchmarks/mpas_ocean.py b/benchmarks/mpas_ocean.py index cb3a178bf..749a3ab88 100644 --- a/benchmarks/mpas_ocean.py +++ b/benchmarks/mpas_ocean.py @@ -27,12 +27,6 @@ class DatasetBenchmark(CachedFixtures): """Class used as a template for benchmarks requiring a ``UxDataset`` in this module across both resolutions. - - The dataset comes from the fixture cache rather than a fresh - ``open_dataset``: every benchmark below measures an algorithm over the mesh, - not the reader that produced it. The fixture is what the reader produced, - connectivity and ``face_areas`` included, so nothing here silently starts - measuring construction that used to come off disk. """ param_names = ['resolution', ] params = [OQU_RESOLUTIONS, ] @@ -65,9 +59,9 @@ def setup(self, resolution, *args, **kwargs): # The coarsest grid, purely to compile the njit kernel _ = self.cached_grid(OQU_GRIDS[OQU_RESOLUTIONS[0]]).face_areas super().setup(resolution, *args, **kwargs) - # MPAS meshes carry ``face_areas`` on disk and the fixture keeps it, so - # it is dropped here to leave the computation to be measured. Safe to do - # to a fixture: each handout is a fresh ``Grid`` over a shallow copy. + + # MPAS meshes carry ``face_areas`` on disk and computation requires it to + # be dropped. Still safe, because each fixture ``Grid`` is a shallow copy. self.uxgrid._ds = self.uxgrid._ds.drop_vars("face_areas", errors="ignore") def time_face_areas(self, resolution): @@ -136,11 +130,7 @@ class GradientColdStartRss: magnitude lower. Measured in a subprocess of its own rather than through asv's ``peakmem_*``, - which reports ``ru_maxrss`` for the benchmark process. Under - ``launch_method: forkserver`` that process is forked from an interpreter - that has already imported the suite, so ``peakmem_*`` would report a warm - start plus whatever the parent held. A fresh interpreter is the only way to - keep measuring the thing this benchmark is named for. + which reports ``ru_maxrss`` for the benchmark process. """ param_names = ["resolution"] @@ -336,8 +326,7 @@ def time_zonal_average(self, resolution): class ZonalAveragePeakMem: """Peak memory of a cold-start non-conservative zonal-mean sweep. - A fresh interpreter per sample, for the reason spelled out in - :class:`GradientColdStartRss`: the cold start is the subject, and a forked + A fresh interpreter per sample. The cold start is the subject, and a forked benchmark process no longer has one. """ @@ -369,8 +358,7 @@ def track_peakmem_zonal_average(self, resolution): class CrossSectionsPeakMem: """Peak memory of a cold-start constant-latitude cross-section sweep. - A fresh interpreter per sample, for the reason spelled out in - :class:`GradientColdStartRss`. + A fresh interpreter per sample, """ param_names = ["resolution", "lat_step"] From b1a6be60ec13982c985ed25f8c08f12374c62e92 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 24 Aug 2026 11:41:58 -0500 Subject: [PATCH 09/52] cached IO deslop global variables --- benchmarks/helpers/_fixtures.py | 36 ++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/benchmarks/helpers/_fixtures.py b/benchmarks/helpers/_fixtures.py index b2c32df0b..e1da4d1db 100644 --- a/benchmarks/helpers/_fixtures.py +++ b/benchmarks/helpers/_fixtures.py @@ -19,6 +19,18 @@ fresh read per commit. ``prime`` therefore leaves the dyamond grids out unless asked, and there is a CLI (``python -m benchmarks.helpers._fixtures``) to fill the cache from a batch script instead of from inside a benchmark. + +Three environment variables tune all of this: + +``UXARRAY_BENCH_CACHE_DIR`` + root the ``_io_cache`` directory is created under, in place of + ``benchmarks/`` -- worth pointing at local scratch when the checkout itself + lives on a shared filesystem +``UXARRAY_BENCH_PRIME`` + ``all`` primes the dyamond grids as well, which ``prime`` skips by default +``UXARRAY_BENCH_PRELOAD`` + any non-empty value has ``preload_topologies`` do its work; unset, it is a + no-op, since preloading only pays off under ``launch_method: forkserver`` """ import hashlib @@ -56,27 +68,27 @@ BENCHMARK_DIR = Path(__file__).resolve().parents[1] REPO_DIR = BENCHMARK_DIR.parent -_COOKBOOK_URL = ( +_COOKBOOK_MESH_URL = ( "https://github.com/ProjectPythia/unstructured-grid-viz-cookbook/raw/main/meshfiles" ) -def _cookbook(filename): +def _cookbook_mesh(filename): """Path to a Cookbook mesh, fetched once if this checkout lacks it.""" path = BENCHMARK_DIR / filename if not path.is_file(): - urllib.request.urlretrieve(f"{_COOKBOOK_URL}/{filename}", filename=path) + urllib.request.urlretrieve(f"{_COOKBOOK_MESH_URL}/{filename}", filename=path) return path # Grids and grid/data pairs, by mesh resolution. OQU_GRIDS = { - "480km": _cookbook("oQU480.grid.nc"), - "120km": _cookbook("oQU120.grid.nc"), + "480km": _cookbook_mesh("oQU480.grid.nc"), + "120km": _cookbook_mesh("oQU120.grid.nc"), } OQU_DATASETS = { - "480km": (OQU_GRIDS["480km"], _cookbook("oQU480.data.nc")), - "120km": (OQU_GRIDS["120km"], _cookbook("oQU120.data.nc")), + "480km": (OQU_GRIDS["480km"], _cookbook_mesh("oQU480.data.nc")), + "120km": (OQU_GRIDS["120km"], _cookbook_mesh("oQU120.data.nc")), } DYAMOND_GRIDS = { @@ -111,17 +123,13 @@ def _cookbook(filename): REPO_DIR / "test" / "meshfiles" / "ugrid" / "quad-hexagon" / "data.nc", ) -CACHE_DIR_VAR = "UXARRAY_BENCH_CACHE_DIR" -PRIME_VAR = "UXARRAY_BENCH_PRIME" -PRELOAD_VAR = "UXARRAY_BENCH_PRELOAD" - # Per path, which artifacts are actually loaded _loaded = {} def cache_dir(): """Directory the cached artifacts live in: ``benchmarks/_io_cache``.""" - root = Path(os.environ.get(CACHE_DIR_VAR) or BENCHMARK_DIR) + root = Path(os.environ.get("UXARRAY_BENCH_CACHE_DIR") or BENCHMARK_DIR) cached = root / "_io_cache" cached.mkdir(parents=True, exist_ok=True) return cached @@ -252,7 +260,7 @@ def prime(include_dyamond=None, workers=1): netCDF/HDF5 stack is not generally thread-safe for concurrent opens. """ if include_dyamond is None: - include_dyamond = os.environ.get(PRIME_VAR, "").lower() == "all" + include_dyamond = os.environ.get("UXARRAY_BENCH_PRIME", "").lower() == "all" sources = [(path,) for path in OQU_GRIDS.values()] sources += [(path,) for path in GRIDS_BY_FORMAT.values()] @@ -301,7 +309,7 @@ def preload_topologies(grid_paths): Safe to call at import: reading arrays starts no numba thread pool, which is what a forked child cannot inherit (see :mod:`benchmarks.helpers._warmup`). """ - if not os.environ.get(PRELOAD_VAR): + if not os.environ.get("UXARRAY_BENCH_PRELOAD"): return 0 loaded = 0 for grid_path in grid_paths: From 41f68b21eba1d37f81926f9ca80c8ebdeb515733 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 24 Aug 2026 13:06:43 -0500 Subject: [PATCH 10/52] bench cached IO: more global variable deslopping --- benchmarks/bench_connectivity.py | 2 +- benchmarks/helpers/_fixtures.py | 29 ++++++----------------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/benchmarks/bench_connectivity.py b/benchmarks/bench_connectivity.py index 4796c1224..85ca61cb7 100644 --- a/benchmarks/bench_connectivity.py +++ b/benchmarks/bench_connectivity.py @@ -114,7 +114,7 @@ def time_node_face(self, resolution): # of them. Only safe while the connectivity kernels are serial def _warm_parent(): _warmup() - # And, if asked, the topologies themselves... + # And the topologies themselves... preload_topologies(GRIDS_BY_RESOLUTION[res] for res in ALL_RESOLUTIONS) diff --git a/benchmarks/helpers/_fixtures.py b/benchmarks/helpers/_fixtures.py index e1da4d1db..2996be33a 100644 --- a/benchmarks/helpers/_fixtures.py +++ b/benchmarks/helpers/_fixtures.py @@ -16,21 +16,9 @@ Artifacts are keyed on both the uxarray build and the files, because an artifact is one version's reader output and ASV diffs commits. Likewise, there's a -fresh read per commit. ``prime`` therefore leaves the dyamond grids out unless -asked, and there is a CLI (``python -m benchmarks.helpers._fixtures``) to fill -the cache from a batch script instead of from inside a benchmark. - -Three environment variables tune all of this: - -``UXARRAY_BENCH_CACHE_DIR`` - root the ``_io_cache`` directory is created under, in place of - ``benchmarks/`` -- worth pointing at local scratch when the checkout itself - lives on a shared filesystem -``UXARRAY_BENCH_PRIME`` - ``all`` primes the dyamond grids as well, which ``prime`` skips by default -``UXARRAY_BENCH_PRELOAD`` - any non-empty value has ``preload_topologies`` do its work; unset, it is a - no-op, since preloading only pays off under ``launch_method: forkserver`` +fresh read per commit. ``prime`` covers every source that is readable here, +and there is a CLI (``python -m benchmarks.helpers._fixtures``) to fill the +cache from a batch script instead of from inside a benchmark. """ import hashlib @@ -246,7 +234,7 @@ def cached_dataset(grid_path, data_path): return ux.UxDataset(_loaded[artifact_path].copy(), uxgrid=cached_grid(grid_path)) -def prime(include_dyamond=None, workers=1): +def prime(workers=1): """Fills the cache for every source the fixtures can serve. Returns the sources it had to read. Idempotent, and once warm costs a @@ -259,13 +247,10 @@ def prime(include_dyamond=None, workers=1): the new interpreter startup. Processes rather than threads because the standard netCDF/HDF5 stack is not generally thread-safe for concurrent opens. """ - if include_dyamond is None: - include_dyamond = os.environ.get("UXARRAY_BENCH_PRIME", "").lower() == "all" - sources = [(path,) for path in OQU_GRIDS.values()] sources += [(path,) for path in GRIDS_BY_FORMAT.values()] sources += list(OQU_DATASETS.values()) - if include_dyamond and DYAMOND_AVAILABLE: + if DYAMOND_AVAILABLE: sources += [(path,) for path in DYAMOND_GRIDS.values()] missing = [] @@ -309,8 +294,6 @@ def preload_topologies(grid_paths): Safe to call at import: reading arrays starts no numba thread pool, which is what a forked child cannot inherit (see :mod:`benchmarks.helpers._warmup`). """ - if not os.environ.get("UXARRAY_BENCH_PRELOAD"): - return 0 loaded = 0 for grid_path in grid_paths: cached_topology(grid_path) # held by the process-level memo from here on @@ -353,5 +336,5 @@ def setup_cache(self): # the reads worth overlapping: on the oQU pair alone, priming in parallel is # slower than doing it sequentially (1.69s against 0.52s), because starting # an interpreter costs more than reading a small local file. - for source in prime(include_dyamond=True, workers=4 if DYAMOND_AVAILABLE else 1) or [None]: + for source in prime(workers=4 if DYAMOND_AVAILABLE else 1) or [None]: print(f" read {' + '.join(Path(p).name for p in source)}" if source else " nothing to do", flush=True) From 4dad004743e6405d77f114550ab7637ab3914e63 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 24 Aug 2026 18:39:23 -0500 Subject: [PATCH 11/52] Prime the fixture cache before asv run, not during it asv preimports the benchmark suite before it runs any setup_cache (asv/runner.py, spawner.preimport() ahead of the run loop), so on a cold cache bench_connectivity's import-time preload_topologies is what fills it -- serially, in the forkserver parent, before a single benchmark starts. CachedFixtures.setup_cache then finds everything already built, and prime(workers=...) never runs on the path it was written for. Filling it from the CLI first puts those reads back in the parallel prime. Worth a second or two on the GitHub runners, which only see the oQU grids; worth rather more on a machine that can reach the four dyamond grids on campaign storage. Co-Authored-By: Claude Opus 5 --- .github/workflows/asv-benchmarking-pr.yml | 2 ++ .github/workflows/asv-benchmarking.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index cae4aa6f0..49ec2cc27 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -48,6 +48,8 @@ jobs: id: benchmark run: | set -x + # Fill the fixture cache before asv preimports the suite, which would otherwise build it serially in the forkserver parent + (cd .. && python -m benchmarks.helpers._fixtures) # ID this runner asv machine --yes echo "Baseline: ${{ github.event.pull_request.base.sha }} (${{ github.event.pull_request.base.label }})" diff --git a/.github/workflows/asv-benchmarking.yml b/.github/workflows/asv-benchmarking.yml index aa16fc78a..f5f751cf0 100644 --- a/.github/workflows/asv-benchmarking.yml +++ b/.github/workflows/asv-benchmarking.yml @@ -77,6 +77,8 @@ jobs: shell: bash -l {0} id: benchmark run: | + # Fill the fixture cache before asv preimports the suite, which would otherwise build it serially in the forkserver parent + python -m benchmarks.helpers._fixtures cd benchmarks asv machine --machine GH-Actions --os ubuntu-latest --arch x64 --cpu "2-core unknown" --ram 7GB asv run v2024.02.0..main --skip-existing --parallel || true From ec83767c568e9c96cda504b75ca9748c5da26599 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Tue, 25 Aug 2026 15:30:30 -0500 Subject: [PATCH 12/52] Lazy neighborhood filter kernel compilation --- test/test_dependencies.py | 30 +++++++++++++++++++ uxarray/grid/neighbors.py | 62 +++++++++++++++++++++++++++++++++------ 2 files changed, 83 insertions(+), 9 deletions(-) diff --git a/test/test_dependencies.py b/test/test_dependencies.py index fb27bfbce..2be243d01 100644 --- a/test/test_dependencies.py +++ b/test/test_dependencies.py @@ -38,4 +38,34 @@ def test_hvplot_optional(): _assert_not_imported_after_import_uxarray("hvplot") +def test_no_numba_kernels_built_on_import(): + """Test that `import uxarray` does not build any numba kernel. + + ``guvectorize`` compiles at decoration time when it is given explicit + signatures, so a kernel assigned at module scope is built during the + import. This compilation can dominate the uxarray import, and building a + ``target="parallel"`` kernel starts numba's threading layer, which + leaves a thread pool running, making forks unsafe. + """ + code = ( + "import numba, uxarray\n" + "try:\n" + " layer = numba.threading_layer()\n" + "except ValueError:\n" + " pass\n" + "else:\n" + " raise AssertionError(\n" + " f'`import uxarray` started numba threading layer {layer!r}. '\n" + " 'Something it imports builds a parallel kernel at module '\n" + " 'scope; build it on first use instead.'\n" + " )\n" + ) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + # TODO: similar tests for cartopy, holoviews, and other optional deps. diff --git a/uxarray/grid/neighbors.py b/uxarray/grid/neighbors.py index 03a4c7cd5..a0c301b18 100644 --- a/uxarray/grid/neighbors.py +++ b/uxarray/grid/neighbors.py @@ -1,3 +1,4 @@ +import threading import warnings from typing import Callable @@ -1246,6 +1247,43 @@ def kernel(data, flat, starts, counts, param, out): return kernel +class _LazyKernel: + """A kernel that builds itself the first time it is called. + + ``guvectorize`` compiles at decoration time when it is given explicit + signatures, so calling :func:`_make_kernel` at module scope would compile + every reduction during ``import uxarray`` + """ + + __slots__ = ("_reduce_fn", "_kernel", "_lock") + + def __init__(self, reduce_fn): + self._reduce_fn = reduce_fn + self._kernel = None + self._lock = threading.Lock() + + def __call__(self, *args): + kernel = self._kernel + if kernel is None: + # Checked again under the lock: dask's threaded scheduler can call + # one reduction from several workers at once + with self._lock: + kernel = self._kernel + if kernel is None: + kernel = self._kernel = _make_kernel(self._reduce_fn) + return kernel(*args) + + def __getstate__(self): + # Only the reducer travels. A compiled gufunc is a ``numpy.ufunc``, + # which pickle cannot address by name and so refuses outright. + return self._reduce_fn + + def __setstate__(self, reduce_fn): + self._reduce_fn = reduce_fn + self._kernel = None + self._lock = threading.Lock() + + # Reducers take ``(window, param)``; those without a parameter ignore the # second argument. Numba keys its cache by code object rather than qualified # name, so the identically-named lambdas below do not collide. @@ -1284,17 +1322,23 @@ def _median(window, _): # names them -- the data-bound classes reach a kernel by naming the # ``Neighborhood`` method for it, so there is one place per reduction where its # kernel and parameter are chosen. -_MEAN_KERNEL = _make_kernel(lambda window, _: np.mean(window)) -_SUM_KERNEL = _make_kernel(lambda window, _: np.sum(window)) -_MIN_KERNEL = _make_kernel(lambda window, _: np.min(window)) -_MAX_KERNEL = _make_kernel(lambda window, _: np.max(window)) -_PTP_KERNEL = _make_kernel(lambda window, _: np.max(window) - np.min(window)) -_MEDIAN_KERNEL = _make_kernel(_median) -_VAR_KERNEL = _make_kernel(_variance) -_STD_KERNEL = _make_kernel(lambda window, ddof: np.sqrt(_variance(window, ddof))) +# +# Naming a kernel is what binds a reduction to it; building it is deferred to +# the first call, for the reasons in :class:`_LazyKernel`. The two are +# separate on purpose -- each name below still resolves to its own object when +# this module is read, so a reduction that names no kernel is a ``NameError`` +# here rather than a lookup that fails once someone runs it. +_MEAN_KERNEL = _LazyKernel(lambda window, _: np.mean(window)) +_SUM_KERNEL = _LazyKernel(lambda window, _: np.sum(window)) +_MIN_KERNEL = _LazyKernel(lambda window, _: np.min(window)) +_MAX_KERNEL = _LazyKernel(lambda window, _: np.max(window)) +_PTP_KERNEL = _LazyKernel(lambda window, _: np.max(window) - np.min(window)) +_MEDIAN_KERNEL = _LazyKernel(_median) +_VAR_KERNEL = _LazyKernel(_variance) +_STD_KERNEL = _LazyKernel(lambda window, ddof: np.sqrt(_variance(window, ddof))) # ``percentile`` is ``quantile`` on a 0-100 scale, so both methods rescale onto # this one kernel rather than compiling a near-duplicate. -_QUANTILE_KERNEL = _make_kernel(lambda window, q: np.quantile(window, q)) +_QUANTILE_KERNEL = _LazyKernel(lambda window, q: np.quantile(window, q)) def _as_quantile(q, scale: float): From 856441007de9366f08df5d64286302c32e94c136 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Tue, 25 Aug 2026 15:58:47 -0500 Subject: [PATCH 13/52] Lazy nb kernels with functools cache instead --- uxarray/grid/neighbors.py | 135 ++++++++++++++++++-------------------- 1 file changed, 65 insertions(+), 70 deletions(-) diff --git a/uxarray/grid/neighbors.py b/uxarray/grid/neighbors.py index a0c301b18..d796f6d78 100644 --- a/uxarray/grid/neighbors.py +++ b/uxarray/grid/neighbors.py @@ -1,4 +1,4 @@ -import threading +import functools import warnings from typing import Callable @@ -1247,43 +1247,6 @@ def kernel(data, flat, starts, counts, param, out): return kernel -class _LazyKernel: - """A kernel that builds itself the first time it is called. - - ``guvectorize`` compiles at decoration time when it is given explicit - signatures, so calling :func:`_make_kernel` at module scope would compile - every reduction during ``import uxarray`` - """ - - __slots__ = ("_reduce_fn", "_kernel", "_lock") - - def __init__(self, reduce_fn): - self._reduce_fn = reduce_fn - self._kernel = None - self._lock = threading.Lock() - - def __call__(self, *args): - kernel = self._kernel - if kernel is None: - # Checked again under the lock: dask's threaded scheduler can call - # one reduction from several workers at once - with self._lock: - kernel = self._kernel - if kernel is None: - kernel = self._kernel = _make_kernel(self._reduce_fn) - return kernel(*args) - - def __getstate__(self): - # Only the reducer travels. A compiled gufunc is a ``numpy.ufunc``, - # which pickle cannot address by name and so refuses outright. - return self._reduce_fn - - def __setstate__(self, reduce_fn): - self._reduce_fn = reduce_fn - self._kernel = None - self._lock = threading.Lock() - - # Reducers take ``(window, param)``; those without a parameter ignore the # second argument. Numba keys its cache by code object rather than qualified # name, so the identically-named lambdas below do not collide. @@ -1315,30 +1278,62 @@ def _median(window, _): return np.median(window) -# One compiled kernel per reduction. The methods on ``Neighborhood`` below name -# these directly, so there is no dispatch table between the public API and the -# gufuncs: a reduction is reachable only if a method exists for it, and a method -# can only reach the kernel it names. ``Neighborhood`` is the only class that -# names them -- the data-bound classes reach a kernel by naming the -# ``Neighborhood`` method for it, so there is one place per reduction where its -# kernel and parameter are chosen. +# One compiled kernel per reduction. The methods on ``Neighborhood`` below call +# into these directly. Non-compiled functions are only provided hooks through +# ``Neighborhood.reduce``. If new compiled reductions are desired, they should +# follow this pattern. # -# Naming a kernel is what binds a reduction to it; building it is deferred to -# the first call, for the reasons in :class:`_LazyKernel`. The two are -# separate on purpose -- each name below still resolves to its own object when -# this module is read, so a reduction that names no kernel is a ``NameError`` -# here rather than a lookup that fails once someone runs it. -_MEAN_KERNEL = _LazyKernel(lambda window, _: np.mean(window)) -_SUM_KERNEL = _LazyKernel(lambda window, _: np.sum(window)) -_MIN_KERNEL = _LazyKernel(lambda window, _: np.min(window)) -_MAX_KERNEL = _LazyKernel(lambda window, _: np.max(window)) -_PTP_KERNEL = _LazyKernel(lambda window, _: np.max(window) - np.min(window)) -_MEDIAN_KERNEL = _LazyKernel(_median) -_VAR_KERNEL = _LazyKernel(_variance) -_STD_KERNEL = _LazyKernel(lambda window, ddof: np.sqrt(_variance(window, ddof))) +# ``functools.cache`` defers each build to the first call. The deferred compilation +# ensures that these kernels will only be compiled individually and lazily. Further, +# the lazy compilation prevents gufuncs from spawning threadpools eagerly and +# disrupting threading and forking in other contexts. + + +@functools.cache +def _mean_kernel(): + return _make_kernel(lambda window, _: np.mean(window)) + + +@functools.cache +def _sum_kernel(): + return _make_kernel(lambda window, _: np.sum(window)) + + +@functools.cache +def _min_kernel(): + return _make_kernel(lambda window, _: np.min(window)) + + +@functools.cache +def _max_kernel(): + return _make_kernel(lambda window, _: np.max(window)) + + +@functools.cache +def _ptp_kernel(): + return _make_kernel(lambda window, _: np.max(window) - np.min(window)) + + +@functools.cache +def _median_kernel(): + return _make_kernel(_median) + + +@functools.cache +def _var_kernel(): + return _make_kernel(_variance) + + +@functools.cache +def _std_kernel(): + return _make_kernel(lambda window, ddof: np.sqrt(_variance(window, ddof))) + + # ``percentile`` is ``quantile`` on a 0-100 scale, so both methods rescale onto # this one kernel rather than compiling a near-duplicate. -_QUANTILE_KERNEL = _LazyKernel(lambda window, q: np.quantile(window, q)) +@functools.cache +def _quantile_kernel(): + return _make_kernel(lambda window, q: np.quantile(window, q)) def _as_quantile(q, scale: float): @@ -1553,45 +1548,45 @@ def __repr__(self) -> str: def mean(self, uxda): """Mean of each neighborhood.""" - return self._apply_kernel(uxda, _MEAN_KERNEL, 0.0) + return self._apply_kernel(uxda, _mean_kernel, 0.0) def sum(self, uxda): """Sum of each neighborhood.""" - return self._apply_kernel(uxda, _SUM_KERNEL, 0.0) + return self._apply_kernel(uxda, _sum_kernel, 0.0) def min(self, uxda): """Smallest value in each neighborhood.""" - return self._apply_kernel(uxda, _MIN_KERNEL, 0.0) + return self._apply_kernel(uxda, _min_kernel, 0.0) def max(self, uxda): """Largest value in each neighborhood.""" - return self._apply_kernel(uxda, _MAX_KERNEL, 0.0) + return self._apply_kernel(uxda, _max_kernel, 0.0) def ptp(self, uxda): """Peak-to-peak spread (``max - min``) of each neighborhood.""" - return self._apply_kernel(uxda, _PTP_KERNEL, 0.0) + return self._apply_kernel(uxda, _ptp_kernel, 0.0) def median(self, uxda): """Median of each neighborhood.""" - return self._apply_kernel(uxda, _MEDIAN_KERNEL, 0.0) + return self._apply_kernel(uxda, _median_kernel, 0.0) def var(self, uxda, ddof: int = 0): """Variance of each neighborhood, with ``ddof`` delta degrees of freedom.""" - return self._apply_kernel(uxda, _VAR_KERNEL, float(ddof)) + return self._apply_kernel(uxda, _var_kernel, float(ddof)) def std(self, uxda, ddof: int = 0): """Standard deviation of each neighborhood, with ``ddof`` delta degrees of freedom.""" - return self._apply_kernel(uxda, _STD_KERNEL, float(ddof)) + return self._apply_kernel(uxda, _std_kernel, float(ddof)) def quantile(self, uxda, q: float): """Quantile ``q`` (between 0 and 1) of each neighborhood.""" - return self._apply_kernel(uxda, _QUANTILE_KERNEL, _as_quantile(q, 1.0)) + return self._apply_kernel(uxda, _quantile_kernel, _as_quantile(q, 1.0)) def percentile(self, uxda, q: float): """Percentile ``q`` (between 0 and 100) of each neighborhood.""" - return self._apply_kernel(uxda, _QUANTILE_KERNEL, _as_quantile(q, 100.0)) + return self._apply_kernel(uxda, _quantile_kernel, _as_quantile(q, 100.0)) def reduce(self, uxda, func: Callable): """Reduces each neighborhood with an arbitrary callable. @@ -1636,7 +1631,7 @@ def run(block, arrays): # path does too by writing into a float64 output. if block.dtype not in (np.float64, np.float32): block = block.astype(np.float64) - return kernel(block, *arrays, param) + return kernel()(block, *arrays, param) return self._apply(uxda, run) From 7127726beb6ec7a36740730b409df1ba23a62fd7 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Wed, 26 Aug 2026 12:34:36 -0500 Subject: [PATCH 14/52] Move kernels inside --- benchmarks/mpas_ocean.py | 3 +- uxarray/grid/neighbors.py | 138 +++++++++++++++++++------------------- 2 files changed, 72 insertions(+), 69 deletions(-) diff --git a/benchmarks/mpas_ocean.py b/benchmarks/mpas_ocean.py index adf679a3a..04325563f 100644 --- a/benchmarks/mpas_ocean.py +++ b/benchmarks/mpas_ocean.py @@ -107,7 +107,8 @@ def track_nbytes_gradient(self, resolution): def track_peakmem_gradient(self, resolution): """Transient high-water allocation of taking a gradient.""" - return peak_allocated(lambda: self.uxds[data_var].gradient()) + with numba_threads(1): + return peak_allocated(lambda: self.uxds[data_var].gradient()) track_peakmem_gradient.unit = "bytes" diff --git a/uxarray/grid/neighbors.py b/uxarray/grid/neighbors.py index d796f6d78..753092f18 100644 --- a/uxarray/grid/neighbors.py +++ b/uxarray/grid/neighbors.py @@ -1278,64 +1278,6 @@ def _median(window, _): return np.median(window) -# One compiled kernel per reduction. The methods on ``Neighborhood`` below call -# into these directly. Non-compiled functions are only provided hooks through -# ``Neighborhood.reduce``. If new compiled reductions are desired, they should -# follow this pattern. -# -# ``functools.cache`` defers each build to the first call. The deferred compilation -# ensures that these kernels will only be compiled individually and lazily. Further, -# the lazy compilation prevents gufuncs from spawning threadpools eagerly and -# disrupting threading and forking in other contexts. - - -@functools.cache -def _mean_kernel(): - return _make_kernel(lambda window, _: np.mean(window)) - - -@functools.cache -def _sum_kernel(): - return _make_kernel(lambda window, _: np.sum(window)) - - -@functools.cache -def _min_kernel(): - return _make_kernel(lambda window, _: np.min(window)) - - -@functools.cache -def _max_kernel(): - return _make_kernel(lambda window, _: np.max(window)) - - -@functools.cache -def _ptp_kernel(): - return _make_kernel(lambda window, _: np.max(window) - np.min(window)) - - -@functools.cache -def _median_kernel(): - return _make_kernel(_median) - - -@functools.cache -def _var_kernel(): - return _make_kernel(_variance) - - -@functools.cache -def _std_kernel(): - return _make_kernel(lambda window, ddof: np.sqrt(_variance(window, ddof))) - - -# ``percentile`` is ``quantile`` on a 0-100 scale, so both methods rescale onto -# this one kernel rather than compiling a near-duplicate. -@functools.cache -def _quantile_kernel(): - return _make_kernel(lambda window, q: np.quantile(window, q)) - - def _as_quantile(q, scale: float): """Validates ``q`` on a 0-``scale`` scale and returns it as a 0-1 fraction.""" value = float(q) @@ -1546,47 +1488,107 @@ def __repr__(self) -> str: f"neighbors_per_element=[{self._counts.min()}, {self._counts.max()}]>" ) + # One compiled kernel per reduction. The methods below call into these + # directly. Non-compiled functions are only provided hooks through + # ``reduce``. If new compiled reductions are desired, they should follow + # this pattern. + # + # ``functools.cache`` defers each build to the first call. The deferred + # compilation ensures that these kernels will only be compiled individually + # and lazily. Further, the lazy compilation prevents gufuncs from spawning + # threadpools eagerly and disrupting threading and forking in other + # contexts. They are ``staticmethod``s rather than attributes for the same + # reason: a class body runs at import, so assigning them there would + # compile all nine during ``import uxarray``. + + @staticmethod + @functools.cache + def _mean_kernel(): + return _make_kernel(lambda window, _: np.mean(window)) + + @staticmethod + @functools.cache + def _sum_kernel(): + return _make_kernel(lambda window, _: np.sum(window)) + + @staticmethod + @functools.cache + def _min_kernel(): + return _make_kernel(lambda window, _: np.min(window)) + + @staticmethod + @functools.cache + def _max_kernel(): + return _make_kernel(lambda window, _: np.max(window)) + + @staticmethod + @functools.cache + def _ptp_kernel(): + return _make_kernel(lambda window, _: np.max(window) - np.min(window)) + + @staticmethod + @functools.cache + def _median_kernel(): + return _make_kernel(_median) + + @staticmethod + @functools.cache + def _var_kernel(): + return _make_kernel(_variance) + + @staticmethod + @functools.cache + def _std_kernel(): + return _make_kernel(lambda window, ddof: np.sqrt(_variance(window, ddof))) + + # ``percentile`` is ``quantile`` on a 0-100 scale, so both methods + # rescale onto this one kernel rather than compiling a near-duplicate. + @staticmethod + @functools.cache + def _quantile_kernel(): + return _make_kernel(lambda window, q: np.quantile(window, q)) + def mean(self, uxda): """Mean of each neighborhood.""" - return self._apply_kernel(uxda, _mean_kernel, 0.0) + return self._apply_kernel(uxda, self._mean_kernel, 0.0) def sum(self, uxda): """Sum of each neighborhood.""" - return self._apply_kernel(uxda, _sum_kernel, 0.0) + return self._apply_kernel(uxda, self._sum_kernel, 0.0) def min(self, uxda): """Smallest value in each neighborhood.""" - return self._apply_kernel(uxda, _min_kernel, 0.0) + return self._apply_kernel(uxda, self._min_kernel, 0.0) def max(self, uxda): """Largest value in each neighborhood.""" - return self._apply_kernel(uxda, _max_kernel, 0.0) + return self._apply_kernel(uxda, self._max_kernel, 0.0) def ptp(self, uxda): """Peak-to-peak spread (``max - min``) of each neighborhood.""" - return self._apply_kernel(uxda, _ptp_kernel, 0.0) + return self._apply_kernel(uxda, self._ptp_kernel, 0.0) def median(self, uxda): """Median of each neighborhood.""" - return self._apply_kernel(uxda, _median_kernel, 0.0) + return self._apply_kernel(uxda, self._median_kernel, 0.0) def var(self, uxda, ddof: int = 0): """Variance of each neighborhood, with ``ddof`` delta degrees of freedom.""" - return self._apply_kernel(uxda, _var_kernel, float(ddof)) + return self._apply_kernel(uxda, self._var_kernel, float(ddof)) def std(self, uxda, ddof: int = 0): """Standard deviation of each neighborhood, with ``ddof`` delta degrees of freedom.""" - return self._apply_kernel(uxda, _std_kernel, float(ddof)) + return self._apply_kernel(uxda, self._std_kernel, float(ddof)) def quantile(self, uxda, q: float): """Quantile ``q`` (between 0 and 1) of each neighborhood.""" - return self._apply_kernel(uxda, _quantile_kernel, _as_quantile(q, 1.0)) + return self._apply_kernel(uxda, self._quantile_kernel, _as_quantile(q, 1.0)) def percentile(self, uxda, q: float): """Percentile ``q`` (between 0 and 100) of each neighborhood.""" - return self._apply_kernel(uxda, _quantile_kernel, _as_quantile(q, 100.0)) + return self._apply_kernel(uxda, self._quantile_kernel, _as_quantile(q, 100.0)) def reduce(self, uxda, func: Callable): """Reduces each neighborhood with an arbitrary callable. From c00fd1d67d4e08c1ef41310990e7fd77665cbbba Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Wed, 26 Aug 2026 13:32:30 -0500 Subject: [PATCH 15/52] Cached IO: try to impose more CI thread safety --- benchmarks/asv.conf.json | 25 ++++++++++++--- benchmarks/helpers/_warmup.py | 57 +++++++++++++++++++++++++++-------- 2 files changed, 65 insertions(+), 17 deletions(-) diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 535ec6b2a..770ea641d 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -102,10 +102,27 @@ // will not be set for the current combination. // "matrix": { - "setuptools_scm": [""], - "xarray": [""], - "netcdf4": [""], - "pip+pyfma": [""] + "req": { + "setuptools_scm": [""], + "xarray": [""], + "netcdf4": [""], + "pip+pyfma": [""], + + // numba picks its threading layer by availability, and only ``tbb`` + // and ``workqueue`` install fork handlers. Left to itself on a runner + // without TBB it picks ``omp``, which on Linux is libgomp: a benchmark + // forked from a parent that holds a pool then dies with "Terminating: + // fork() called from a process already using GNU OpenMP". numba + // *prefers* TBB, so a developer machine that has it cannot reproduce + // that -- hence pinning it here rather than inheriting whatever the + // environment happens to offer. + "tbb": [""] + }, + + // Belt to that brace: if TBB is ever unavailable, pick the other + // fork-safe layer rather than quietly falling back to the one that + // breaks. ``forksafe`` raises if no fork-safe layer exists at all. + "env_nobuild": {"NUMBA_THREADING_LAYER": ["forksafe"]} }, diff --git a/benchmarks/helpers/_warmup.py b/benchmarks/helpers/_warmup.py index 3ad8a9c1e..5479794de 100644 --- a/benchmarks/helpers/_warmup.py +++ b/benchmarks/helpers/_warmup.py @@ -4,30 +4,61 @@ benchmark from that interpreter, so whatever a module prepares at import is inherited copy-on-write. -Only tasks that leaves no numba thread pool behind may be warmed this way. -Running a ``parallel=True`` kernel or calling ``.compile()`` on one -launches the pool, and numba's OpenMP layer is not fork-safe. So this checks when a -warmed kernel goes parallel, it becomes an import error rather than a benchmark -that hangs on a cluster. +Warming can leave a numba thread pool behind. *Compiling* a ``parallel=True`` +kernel initializes the threading layer -- before the kernel is ever run -- and +``cache=True`` only avoids that while the on-disk cache is warm, which it is +not on a fresh runner. Whether an inherited pool is safe then depends entirely +on which layer numba picked: + +* ``tbb`` and ``workqueue`` install fork handlers and come through it intact. +* ``omp`` does not. Where that is libgomp -- Linux, unless TBB is installed -- + every forked child that touches a parallel kernel is killed with + ``Terminating: fork() called from a process already using GNU OpenMP``. + +numba prefers TBB, so a machine that has it cannot reproduce the failure at +all; the benchmark environment pins it instead (see ``tbb`` and +``NUMBA_THREADING_LAYER`` in ``asv.conf.json``). This reports when that has not +taken effect rather than raising. Raising fails the module's import in the +parent *and* again in every child that re-imports it, so it converts "these +benchmarks are slower than they could be" into "this commit produced no +results" -- and the pool exists either way, and asv forks either way. """ +import sys + import numba __all__ = ["warm_in_parent"] +# Numba installs fork handlers for these two; ``omp`` is on its own. +_FORK_SAFE = frozenset({"tbb", "workqueue"}) + +_reported = False + def warm_in_parent(warm, what): - """Runs ``warm``, then fails if it left a numba thread pool behind. + """Runs ``warm``, then checks any pool it leaves behind survives a fork. - ``what`` names the thing being warmed, for the error message. + ``what`` names the thing being warmed, for the report. """ warm() + try: - numba.threading_layer() + layer = numba.threading_layer() except ValueError: - return # nothing launched a pool, which is what makes this inheritable - raise RuntimeError( - f"warming {what} started numba's thread pool, which a forked benchmark " - "cannot safely inherit -- warm it from setup() instead, now that these " - "kernels run in parallel" + return # nothing initialized a pool, so there is nothing to inherit + + if layer in _FORK_SAFE: + return + + global _reported + if _reported: + return + _reported = True + print( + f"asv: warming {what} left numba's {layer!r} thread pool behind, and " + f"{layer!r} does not survive fork(). Forked benchmarks that run a " + "parallel kernel will be killed by the OpenMP runtime. Install tbb in " + "the benchmark environment, or set NUMBA_THREADING_LAYER=forksafe.", + file=sys.stderr, ) From 526e3206a6f45cd64880b1cdf940f80e905a00f2 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Wed, 26 Aug 2026 14:29:27 -0500 Subject: [PATCH 16/52] Cached IO: lazy interpreter warming; NetCDF warming --- benchmarks/helpers/_fixtures.py | 38 +++++++++++++++++++++++++++++++-- benchmarks/helpers/_warmup.py | 29 +++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/benchmarks/helpers/_fixtures.py b/benchmarks/helpers/_fixtures.py index 2996be33a..9ab574197 100644 --- a/benchmarks/helpers/_fixtures.py +++ b/benchmarks/helpers/_fixtures.py @@ -53,6 +53,8 @@ "prime", ] +from ._warmup import warm_in_parent + BENCHMARK_DIR = Path(__file__).resolve().parents[1] REPO_DIR = BENCHMARK_DIR.parent @@ -221,8 +223,18 @@ def _cached_grid_ds(grid_path): def cached_grid(grid_path): - """A fresh ``Grid`` carrying everything the reader found in ``grid_path`` via shallow copy.""" - return ux.Grid(_cached_grid_ds(grid_path).copy()) + """A fresh ``Grid`` carrying everything the reader found in ``grid_path`` via shallow copy. + + The spec is handed back rather than left to default. ``Grid.__init__`` records + it in ``_ds.attrs``, and the artifact is that ``_ds``, so the original label + survives the round trip -- an MPAS grid stays MPAS. Omitting it warns on every + construction ("Attempting to construct a Grid without passing in + source_grid_spec"), once per benchmark process, which buried the actual + benchmark output. + """ + grid_ds = _cached_grid_ds(grid_path).copy() + spec = grid_ds.attrs.get("source_grid_spec", "UGRID") # uxarray's documented default + return ux.Grid(grid_ds, source_grid_spec=spec) def cached_dataset(grid_path, data_path): @@ -338,3 +350,25 @@ def setup_cache(self): # an interpreter costs more than reading a small local file. for source in prime(workers=4 if DYAMOND_AVAILABLE else 1) or [None]: print(f" read {' + '.join(Path(p).name for p in source)}" if source else " nothing to do", flush=True) + + +def warm_netcdf(): + """Brings the netCDF/HDF5 stack up here, so the forks inherit it loaded. + + The first ``open_dataset`` in a process spends ~65ms initializing the + library before it reads a byte, and nothing else in the parent pays that: + the topology fixtures go through numpy. So today every forked child that + touches a cached ``.nc`` artifact pays it again, one at a time. + + The file is closed again. What the children want is the loaded library, not + a handle -- an inherited HDF5 handle is shared rather than copied, and + parent and child would then read through one file offset. + """ + artifacts = sorted(cache_dir().glob("*.nc"), key=lambda path: path.stat().st_size) + if not artifacts: + return # cache not primed yet, so the first child pays it as before + with xr.open_dataset(artifacts[0], mask_and_scale=False): + pass + + +warm_in_parent(warm_netcdf, "the netCDF backend") diff --git a/benchmarks/helpers/_warmup.py b/benchmarks/helpers/_warmup.py index 5479794de..ae84c10a9 100644 --- a/benchmarks/helpers/_warmup.py +++ b/benchmarks/helpers/_warmup.py @@ -24,11 +24,18 @@ results" -- and the pool exists either way, and asv forks either way. """ +import os import sys import numba -__all__ = ["warm_in_parent"] +__all__ = ["warm_in_parent", "will_run_benchmarks"] + +# ``benchmark.py ``: asv runs discovery, setup_cache and check as their own +# processes, and none of them goes on to run a benchmark, so warming in them is +# time bought for nobody. ``run_server`` -- the forkserver parent, the one whose +# state every benchmark inherits -- is deliberately absent. +_NON_RUNNING_MODES = frozenset({"discover", "setup_cache", "check"}) # Numba installs fork handlers for these two; ``omp`` is on its own. _FORK_SAFE = frozenset({"tbb", "workqueue"}) @@ -36,11 +43,29 @@ _reported = False +def will_run_benchmarks(): + """Whether this interpreter is going to run a benchmark. + + Conservative by construction: an argv this does not recognize is assumed to + be a benchmark run, so a change in how asv invokes itself costs the warm + twice over rather than losing it where it counts. + """ + return not ( + os.path.basename(sys.argv[0]) == "benchmark.py" + and len(sys.argv) > 1 + and sys.argv[1] in _NON_RUNNING_MODES + ) + + def warm_in_parent(warm, what): """Runs ``warm``, then checks any pool it leaves behind survives a fork. - ``what`` names the thing being warmed, for the report. + ``what`` names the thing being warmed, for the report. A no-op in an + interpreter that will not run a benchmark, per :func:`will_run_benchmarks`. """ + if not will_run_benchmarks(): + return + warm() try: From a064b3046d180eebe8b08b7bbf49480bf5823237 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Wed, 26 Aug 2026 15:12:54 -0500 Subject: [PATCH 17/52] Cached IO: configuration tweaks --- .github/workflows/asv-benchmarking-pr.yml | 9 +++++++++ benchmarks/asv.conf.json | 7 ++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 49ec2cc27..c23abe74d 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -43,6 +43,15 @@ jobs: python-build mamba + # asv builds its own conda environment under ``benchmarks/env``, separate + # from the one above, and without this it solves and installs that from + # scratch on every run before a single benchmark starts. + - name: Cache asv's benchmark environment + uses: actions/cache@v6 + with: + path: ${{ env.ASV_DIR }}/env + key: asv-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('benchmarks/asv.conf.json', 'ci/environment.yml') }} + - name: Run Benchmarks shell: bash -l {0} id: benchmark diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 770ea641d..9b3852f53 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -193,7 +193,12 @@ // `asv` will cache results of the recent builds in each // environment, making them faster to install next time. This is // the number of builds to keep, per environment. - // "build_cache_size": 2, + // + // Raised from the default of 2, which is exactly the number of commits + // ``asv continuous`` builds and so leaves no headroom: with rounds + // interleaved it revisits each commit, and a wheel that has been evicted + // in between is built again from scratch. + "build_cache_size": 4, // The commits after which the regression search in `asv publish` // should start looking for regressions. Dictionary whose keys are From a0f4a80adb13e5c7399c85c8bc08e615db9d2693 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Wed, 26 Aug 2026 16:36:04 -0500 Subject: [PATCH 18/52] Partially revert a064b30, add cpu info dump --- .github/workflows/asv-benchmarking-pr.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index c23abe74d..66eca2401 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -31,6 +31,11 @@ jobs: fetch-depth: 0 + - name: Record CPU topology + run: | + # A diagnostic; never fail the job over it. + lscpu | grep -E 'Model name|^CPU\(s\):|Thread\(s\) per core|Core\(s\) per socket|Socket\(s\)|CPU max MHz' || lscpu || true + - name: Set up Conda environment uses: mamba-org/setup-micromamba@v3 with: @@ -43,15 +48,6 @@ jobs: python-build mamba - # asv builds its own conda environment under ``benchmarks/env``, separate - # from the one above, and without this it solves and installs that from - # scratch on every run before a single benchmark starts. - - name: Cache asv's benchmark environment - uses: actions/cache@v6 - with: - path: ${{ env.ASV_DIR }}/env - key: asv-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('benchmarks/asv.conf.json', 'ci/environment.yml') }} - - name: Run Benchmarks shell: bash -l {0} id: benchmark From f54d63435d95df44d89a623756acebd49a1c5b21 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Thu, 27 Aug 2026 12:21:01 -0500 Subject: [PATCH 19/52] Cache the asv environment and fixtures, and report durations Co-Authored-By: Claude Opus 5 --- .github/workflows/asv-benchmarking-pr.yml | 55 ++++++++++++++++++++++- benchmarks/asv.conf.json | 4 +- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 66eca2401..86b7e69fd 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -42,12 +42,31 @@ jobs: environment-file: ${{env.CONDA_ENV_FILE}} cache-environment: true environment-name: uxarray_build - cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}-benchmark" + cache-environment-key: "${{runner.os}}-${{runner.arch}}-${{hashFiles(env.CONDA_ENV_FILE)}}-benchmark" create-args: >- asv python-build mamba + # asv builds its own conda environment under ``benchmarks/env` + - name: Cache asv's benchmark environment + uses: actions/cache@v6 + with: + path: ${{ env.ASV_DIR }}/env + key: asv-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('benchmarks/asv.conf.json', 'ci/environment.yml') }} + restore-keys: | + asv-env-${{ runner.os }}-${{ runner.arch }}- + + - name: Cache the benchmark fixtures + uses: actions/cache@v6 + with: + path: | + ${{ env.ASV_DIR }}/oQU*.nc + ${{ env.ASV_DIR }}/_io_cache + key: asv-fixtures-${{ runner.os }}-${{ hashFiles('benchmarks/helpers/_fixtures.py') }} + restore-keys: | + asv-fixtures-${{ runner.os }}- + - name: Run Benchmarks shell: bash -l {0} id: benchmark @@ -66,6 +85,40 @@ jobs: asv compare --split ${{ github.event.pull_request.base.sha }} ${GITHUB_SHA} > asv_compare_results.txt working-directory: ${{ env.ASV_DIR }} + # asv records a duration per benchmark, plus ```` and + # ```` entries, in the results file it writes. Printing + # them is what tells us where a run's wall clock actually went. + - name: Report where the time went + if: always() + shell: bash -l {0} + run: | + python - <<'PY' || true + import glob, json, os + + for path in sorted(glob.glob("benchmarks/results/*/*.json")): + if os.path.basename(path) in ("machine.json", "benchmarks.json"): + continue + data = json.load(open(path)) + columns = data.get("result_columns") or [] + if "duration" not in columns: + continue + index = columns.index("duration") + rows = sorted( + (float(row[index]), name) + for name, row in data["results"].items() + if len(row) > index and row[index] is not None + ) + total = sum(duration for duration, _ in rows) or 1.0 + params = data.get("params", {}) + print(f"\n=== {data['commit_hash'][:8]} on {params.get('cpu', '?')} " + f"({params.get('num_cpu', '?')} cpu) ===") + for key, value in sorted(data.get("durations", {}).items()): + print(f" {value:8.1f}s {key}") + print(f" {total:8.1f}s all {len(rows)} benchmarks ({total / 60:.1f} min)") + for duration, name in reversed(rows[-15:]): + print(f" {duration:8.1f}s {100 * duration / total:5.1f}% {name}") + PY + - name: Save PR number if: always() run: echo "${{ github.event.pull_request.number }}" > ${{ env.ASV_DIR }}/pr_number.txt diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 9b3852f53..59b9954c3 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -126,8 +126,10 @@ }, + // Just the one command: ``python -m build`` put an sdist and a wheel in + // {build_dir}/dist, which asv never reads, and then this rebuilt the wheel + // into the directory asv actually installs from. "build_command": [ - "python -m build", "python -mpip wheel --no-deps --no-build-isolation --no-index -w {build_cache_dir} {build_dir}" ], From 8f66e30921f81acb3241a488a48c186d33592a81 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Thu, 27 Aug 2026 14:12:25 -0500 Subject: [PATCH 20/52] Fix 'run benchmark' action for manual runs --- .github/workflows/asv-benchmarking-pr.yml | 30 ++++++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 86b7e69fd..939677132 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -70,19 +70,41 @@ jobs: - name: Run Benchmarks shell: bash -l {0} id: benchmark + # Only a pull_request event carries a pull_request payload, so on a + # manual run every one of these expressions is the empty string. Left + # unhandled, ``asv continuous`` quietly reads its one remaining + # argument as a revision and benchmarks it against its own parent, + # while ``asv compare`` exits 2 for want of a second revision -- which + # fails the job, so the caches never save and the comment workflow, + # gated on success, never runs. run: | set -x + BASE="${{ github.event.pull_request.base.sha }}" + LABEL="${PR_HEAD_LABEL:-$GITHUB_REF_NAME}" + if [ -z "$BASE" ]; then + # The merge-base rather than main's tip: a manual run benchmarks the + # branch as it stands, without main's later commits merged in, so + # comparing against main's tip would charge this branch for them. + git rev-parse --verify -q origin/main >/dev/null || git fetch -q origin main:refs/remotes/origin/main + BASE=$(git merge-base "$GITHUB_SHA" origin/main) + fi + if [ "$BASE" = "$(git rev-parse "$GITHUB_SHA")" ]; then + # Dispatched from main itself: compare it against the commit before. + BASE=$(git rev-parse "$GITHUB_SHA^") + fi + # Fail loudly rather than leaving asv to infer a baseline of its own. + test -n "$BASE" || { echo "could not determine a baseline commit" >&2; exit 1; } # Fill the fixture cache before asv preimports the suite, which would otherwise build it serially in the forkserver parent (cd .. && python -m benchmarks.helpers._fixtures) # ID this runner asv machine --yes - echo "Baseline: ${{ github.event.pull_request.base.sha }} (${{ github.event.pull_request.base.label }})" - echo "Contender: ${GITHUB_SHA} ($PR_HEAD_LABEL)" + echo "Baseline: $BASE" + echo "Contender: ${GITHUB_SHA} ($LABEL)" # Run benchmarks for current commit against base ASV_OPTIONS="--split --show-stderr" - asv continuous $ASV_OPTIONS ${{ github.event.pull_request.base.sha }} ${GITHUB_SHA} + asv continuous $ASV_OPTIONS "$BASE" "${GITHUB_SHA}" # Save compare results - asv compare --split ${{ github.event.pull_request.base.sha }} ${GITHUB_SHA} > asv_compare_results.txt + asv compare --split "$BASE" "${GITHUB_SHA}" > asv_compare_results.txt working-directory: ${{ env.ASV_DIR }} # asv records a duration per benchmark, plus ```` and From 3b46b081130bcb974587935e3de5ba9d23bc28a7 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Thu, 27 Aug 2026 14:31:39 -0500 Subject: [PATCH 21/52] Cache the asv environment and fixtures on main too Co-Authored-By: Claude Opus 5 --- .github/workflows/asv-benchmarking-pr.yml | 2 +- .github/workflows/asv-benchmarking.yml | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 939677132..3217b241f 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -53,7 +53,7 @@ jobs: uses: actions/cache@v6 with: path: ${{ env.ASV_DIR }}/env - key: asv-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('benchmarks/asv.conf.json', 'ci/environment.yml') }} + key: asv-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('benchmarks/asv.conf.json') }} restore-keys: | asv-env-${{ runner.os }}-${{ runner.arch }}- diff --git a/.github/workflows/asv-benchmarking.yml b/.github/workflows/asv-benchmarking.yml index f5f751cf0..d4be460f0 100644 --- a/.github/workflows/asv-benchmarking.yml +++ b/.github/workflows/asv-benchmarking.yml @@ -73,6 +73,24 @@ jobs: cp -r uxarray-asv/results benchmarks/ fi + - name: Cache asv's benchmark environment + uses: actions/cache@v6 + with: + path: ${{ env.ASV_DIR }}/env + key: asv-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('benchmarks/asv.conf.json') }} + restore-keys: | + asv-env-${{ runner.os }}-${{ runner.arch }}- + + - name: Cache the benchmark fixtures + uses: actions/cache@v6 + with: + path: | + ${{ env.ASV_DIR }}/oQU*.nc + ${{ env.ASV_DIR }}/_io_cache + key: asv-fixtures-${{ runner.os }}-${{ hashFiles('benchmarks/helpers/_fixtures.py') }} + restore-keys: | + asv-fixtures-${{ runner.os }}- + - name: Run benchmarks shell: bash -l {0} id: benchmark From 04f96f2d03dbb77a40b74f738c0d0be67f5d5d14 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Thu, 27 Aug 2026 14:36:34 -0500 Subject: [PATCH 22/52] Split the suite into shards of roughly equal cost Co-Authored-By: Claude Opus 5 --- benchmarks/helpers/_partition.py | 213 +++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 benchmarks/helpers/_partition.py diff --git a/benchmarks/helpers/_partition.py b/benchmarks/helpers/_partition.py new file mode 100644 index 000000000..28fa6f58a --- /dev/null +++ b/benchmarks/helpers/_partition.py @@ -0,0 +1,213 @@ +"""Splitting the suite into shards of roughly equal cost. + +asv runs one benchmark at a time. ``--parallel`` builds environments in +parallel and nothing else -- its own help text is "Build (but don't benchmark) +in parallel" -- so cutting the wall clock means running several ``asv`` +processes, and a ``time_*`` result is only worth having if nothing else is +competing for the machine while it is measured. That points at one shard per +runner rather than several per runner, and at this module, whose whole job is to +decide which benchmarks each of those runners should claim. + +The split is by whole benchmark, not by parameter combination. asv matches +``--bench`` against the expanded ``name(param0, param1)`` for parameterized +benchmarks, so a finer cut is available -- but measured durations say it is not +needed. The suite is flat: the heaviest single benchmark is about 5% of the +total, and greedy longest-first packing lands within ~1% of a perfect split +even at eight shards. Splitting inside a benchmark would buy nothing and would +put every shard's results in the same row of the same results file, which then +has to be merged element-wise. + +Weights come from the ``duration`` asv records per benchmark in the results file +it writes (``Results.save``), so a partition improves as results accumulate +rather than needing a cost model. Benchmarks with no recorded duration -- new +ones, mostly -- get the median of the ones that have, which is a better guess +than either zero or the mean of a long-tailed distribution. + +Usage:: + + python -m benchmarks.helpers._partition --shards 4 + python -m benchmarks.helpers._partition --shards 4 --shard 0 --bench-args +""" + +import argparse +import json +import os +import re +import statistics +import sys +from pathlib import Path + +__all__ = ["load_benchmarks", "load_weights", "plan", "bench_regexes"] + +BENCHMARK_DIR = Path(__file__).resolve().parents[1] + +# ``setup_cache`` groups whose members may be split across shards. Anything else +# is paid once per shard that holds any of its benchmarks, so those move +# together. ``CachedFixtures.setup_cache`` is just ``prime()``, a stat per file +# once the cache is warm (1.2s in CI), and ``None`` is no setup_cache at all. +SPLITTABLE_PREFIX = "helpers._fixtures:" + +_SKIP_FILES = frozenset({"machine.json", "benchmarks.json"}) + + +def _splittable(setup_cache_key): + """Whether benchmarks sharing this ``setup_cache_key`` may land in different shards.""" + return setup_cache_key is None or str(setup_cache_key).startswith(SPLITTABLE_PREFIX) + + +def load_benchmarks(results_dir): + """The discovered benchmarks, as asv wrote them to ``benchmarks.json``.""" + path = Path(results_dir) / "benchmarks.json" + with open(path) as handle: + discovered = json.load(handle) + # asv stores its own format version alongside the benchmarks. + return {name: value for name, value in discovered.items() if name != "version"} + + +def load_weights(results_dirs): + """Mean recorded duration per benchmark, in seconds, over the files on disk. + + The mean rather than the latest: a benchmark's first run on a cold numba + cache can cost hundreds of times its warm cost (a 9.6s bounds compile + against 13ms warm, in one observed run), and a partition built from one + such outlier sends a whole shard chasing work that is not there. + """ + samples = {} + for results_dir in results_dirs: + root = Path(results_dir) + if not root.is_dir(): + continue + for path in sorted(root.glob("*/*.json")): + if path.name in _SKIP_FILES: + continue + try: + with open(path) as handle: + data = json.load(handle) + except (OSError, ValueError): + continue + columns = data.get("result_columns") or [] + if "duration" not in columns: + continue + index = columns.index("duration") + for name, row in (data.get("results") or {}).items(): + if len(row) <= index or row[index] is None: + continue + samples.setdefault(name, []).append(float(row[index])) + return {name: statistics.fmean(values) for name, values in samples.items()} + + +def plan(benchmarks, n_shards, weights=None): + """Partitions ``benchmarks`` into ``n_shards`` lists of names. + + Greedy longest-first onto the lightest shard so far -- the standard LPT + heuristic, which on a distribution this flat is within about 1% of optimal + and, unlike anything smarter, is obvious enough to debug from the report. + + Deterministic: equal weights are broken by name, so the same inputs always + give the same shards and a shard can compute its own membership without + being told. + """ + if n_shards < 1: + raise ValueError(f"n_shards must be at least 1, got {n_shards}") + weights = dict(weights or {}) + known = [value for value in weights.values() if value > 0] + default = statistics.median(known) if known else 1.0 + + # Group anything sharing an expensive setup_cache, so it is paid once. + units = {} + for name, benchmark in benchmarks.items(): + key = benchmark.get("setup_cache_key") + unit = name if _splittable(key) else f"setup_cache:{key}" + units.setdefault(unit, []).append(name) + + costs = { + unit: sum(weights.get(name, default) for name in names) + for unit, names in units.items() + } + + shards = [[] for _ in range(n_shards)] + loads = [0.0] * n_shards + for unit in sorted(units, key=lambda u: (-costs[u], u)): + target = min(range(n_shards), key=lambda i: (loads[i], i)) + shards[target].extend(sorted(units[unit])) + loads[target] += costs[unit] + return shards + + +def bench_regexes(names): + """``--bench`` patterns selecting exactly ``names`` and nothing else. + + asv filters a parameterized benchmark on ``name(param0, param1)`` and an + unparameterized one on ``name`` (``Benchmarks.__init__``), so the trailing + group has to admit both an open parenthesis and end-of-string. Without it + ``^name$`` silently matches none of a parameterized benchmark's + combinations, and the shard runs nothing. + """ + return [f"^{re.escape(name)}($|\\()" for name in names] + + +def _report(benchmarks, shards, weights): + known = [value for value in weights.values() if value > 0] + default = statistics.median(known) if known else 1.0 + total = sum(weights.get(name, default) for name in benchmarks) + print( + f"{len(benchmarks)} benchmarks, {len(weights)} with recorded durations, " + f"{total / 60:.1f} min of work; median fallback {default:.1f}s" + ) + loads = [sum(weights.get(name, default) for name in shard) for shard in shards] + ideal = total / len(shards) if shards else 0.0 + for index, (shard, load) in enumerate(zip(shards, loads)): + drift = 100 * (load - ideal) / ideal if ideal else 0.0 + print(f" shard {index}: {len(shard):3} benchmarks {load / 60:5.1f} min {drift:+5.1f}%") + if loads and ideal: + print( + f" slowest shard {max(loads) / 60:.1f} min against an ideal " + f"{ideal / 60:.1f}; speedup {total / max(loads):.2f}x of a possible {len(shards)}x" + ) + + +def main(argv=None): + parser = argparse.ArgumentParser( + prog="python -m benchmarks.helpers._partition", + description="Split the benchmark suite into shards of roughly equal cost.", + ) + parser.add_argument("--shards", type=int, default=4, help="Number of shards (default 4).") + parser.add_argument( + "--shard", type=int, default=None, help="Report only this shard, by index." + ) + parser.add_argument( + "--results", + action="append", + default=None, + help="Results directory to read durations and benchmarks.json from. " + "Repeatable; defaults to benchmarks/results.", + ) + parser.add_argument( + "--bench-args", + action="store_true", + help="Print the shard's --bench arguments for asv, rather than a report.", + ) + args = parser.parse_args(argv) + + results_dirs = args.results or [str(BENCHMARK_DIR / "results")] + benchmarks = load_benchmarks(results_dirs[0]) + weights = load_weights(results_dirs) + shards = plan(benchmarks, args.shards, weights) + + if args.bench_args: + if args.shard is None: + parser.error("--bench-args needs --shard") + for pattern in bench_regexes(shards[args.shard]): + print("--bench", pattern) + return 0 + + if args.shard is None: + _report(benchmarks, shards, weights) + else: + for name in shards[args.shard]: + print(name) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From f9e55462dbebdd2f94750311d79b5f68b52fc698 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Thu, 27 Aug 2026 16:29:22 -0500 Subject: [PATCH 23/52] ASV workflow branch for pre-caching IO --- .github/workflows/asv-benchmarking.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/asv-benchmarking.yml b/.github/workflows/asv-benchmarking.yml index d4be460f0..a7e212ed3 100644 --- a/.github/workflows/asv-benchmarking.yml +++ b/.github/workflows/asv-benchmarking.yml @@ -95,8 +95,14 @@ jobs: shell: bash -l {0} id: benchmark run: | - # Fill the fixture cache before asv preimports the suite, which would otherwise build it serially in the forkserver parent - python -m benchmarks.helpers._fixtures + # Fill the fixture cache before asv preimports the suite, which would + # otherwise build it serially in the forkserver parent. Guarded because + # this workflow checks out main whatever ref it was dispatched from + if [ -f benchmarks/helpers/_fixtures.py ]; then + python -m benchmarks.helpers._fixtures + else + echo "no benchmarks/helpers/_fixtures.py on this ref; nothing to prime" + fi cd benchmarks asv machine --machine GH-Actions --os ubuntu-latest --arch x64 --cpu "2-core unknown" --ram 7GB asv run v2024.02.0..main --skip-existing --parallel || true From 966e5a8ac90d1ae795f839e255d1173b6c8bc519 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Thu, 27 Aug 2026 16:51:50 -0500 Subject: [PATCH 24/52] ASV benchmarks skip existing commits --- .github/workflows/asv-benchmarking.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/asv-benchmarking.yml b/.github/workflows/asv-benchmarking.yml index a7e212ed3..b968f9ac3 100644 --- a/.github/workflows/asv-benchmarking.yml +++ b/.github/workflows/asv-benchmarking.yml @@ -9,6 +9,9 @@ on: jobs: benchmark: runs-on: ubuntu-latest + # Without this the platform's 6h cap is the only limit, which is how a + # mis-skipped range quietly turns into a 453-commit build. + timeout-minutes: 180 defaults: run: shell: bash -el {0} @@ -105,7 +108,11 @@ jobs: fi cd benchmarks asv machine --machine GH-Actions --os ubuntu-latest --arch x64 --cpu "2-core unknown" --ram 7GB - asv run v2024.02.0..main --skip-existing --parallel || true + # ``--skip-existing-commits``, not ``--skip-existing``: the latter keys + # its skip set by (commit, env), so asv's per-commit check never + # matches and it builds and installs every commit in the range before + # skipping the benchmarks it already has. + asv run v2024.02.0..main --skip-existing-commits --parallel || true - name: Commit and push benchmark results run: | From 6ad7a7f8f56941540e0c8c289dee30989efb905b Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Thu, 27 Aug 2026 19:39:43 -0500 Subject: [PATCH 25/52] ASV diff against merge base rather than main --- .github/workflows/asv-benchmarking-pr.yml | 19 ++++++++++++++----- .github/workflows/asv-benchmarking.yml | 2 +- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 3217b241f..97936e169 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -12,6 +12,14 @@ on: pull_request: types: [opened, reopened, synchronize, labeled] workflow_dispatch: + inputs: + base_ref: + description: >- + Branch, tag or commit to benchmark against. The comparison is against + its merge-base with the dispatched ref, so a topic branch should name + the branch it will merge into rather than main. + required: false + default: main env: PR_HEAD_LABEL: ${{ github.event.pull_request.head.label }} @@ -82,11 +90,12 @@ jobs: BASE="${{ github.event.pull_request.base.sha }}" LABEL="${PR_HEAD_LABEL:-$GITHUB_REF_NAME}" if [ -z "$BASE" ]; then - # The merge-base rather than main's tip: a manual run benchmarks the - # branch as it stands, without main's later commits merged in, so - # comparing against main's tip would charge this branch for them. - git rev-parse --verify -q origin/main >/dev/null || git fetch -q origin main:refs/remotes/origin/main - BASE=$(git merge-base "$GITHUB_SHA" origin/main) + # The merge-base rather than the base branch's tip + BASE_REF="${{ github.event.inputs.base_ref }}" + BASE_REF="${BASE_REF:-main}" + git rev-parse --verify -q "origin/$BASE_REF" >/dev/null \ + || git fetch -q origin "$BASE_REF:refs/remotes/origin/$BASE_REF" + BASE=$(git merge-base "$GITHUB_SHA" "origin/$BASE_REF") fi if [ "$BASE" = "$(git rev-parse "$GITHUB_SHA")" ]; then # Dispatched from main itself: compare it against the commit before. diff --git a/.github/workflows/asv-benchmarking.yml b/.github/workflows/asv-benchmarking.yml index b968f9ac3..bc9a8a3a0 100644 --- a/.github/workflows/asv-benchmarking.yml +++ b/.github/workflows/asv-benchmarking.yml @@ -112,7 +112,7 @@ jobs: # its skip set by (commit, env), so asv's per-commit check never # matches and it builds and installs every commit in the range before # skipping the benchmarks it already has. - asv run v2024.02.0..main --skip-existing-commits --parallel || true + asv run main^! --skip-existing-commits --parallel || true - name: Commit and push benchmark results run: | From d472982f2b8b239bd4e144347f840f64e8819a0b Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Thu, 27 Aug 2026 19:44:06 -0500 Subject: [PATCH 26/52] ASV multithreading --- .github/workflows/asv-benchmarking-pr.yml | 2 +- benchmarks/asv.conf.hpc.json | 68 +++++++++++++ benchmarks/helpers/_threads.py | 117 ++++++++++++++++++++++ 3 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 benchmarks/asv.conf.hpc.json create mode 100644 benchmarks/helpers/_threads.py diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 97936e169..3c424534d 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -40,8 +40,8 @@ jobs: - name: Record CPU topology + # A diagnostic; never fail the job over it. run: | - # A diagnostic; never fail the job over it. lscpu | grep -E 'Model name|^CPU\(s\):|Thread\(s\) per core|Core\(s\) per socket|Socket\(s\)|CPU max MHz' || lscpu || true - name: Set up Conda environment diff --git a/benchmarks/asv.conf.hpc.json b/benchmarks/asv.conf.hpc.json new file mode 100644 index 000000000..ef14c05a7 --- /dev/null +++ b/benchmarks/asv.conf.hpc.json @@ -0,0 +1,68 @@ +{ + // Thread-scaling variant of ``asv.conf.json``, for a machine with cores to + // spare. Run it explicitly: + // + // asv run --config benchmarks/asv.conf.hpc.json + // asv compare --config benchmarks/asv.conf.hpc.json -E conda:3.11 A B + // + // Why a second file rather than an environment variable: asv records the + // ``env_nobuild`` matrix into every results file it writes and folds it + // into the environment name, so each thread count gets its own result set + // and ``asv compare`` keeps them apart. A count exported in the shell is + // invisible to asv -- runs at 1 and 64 threads land in the *same* file for + // a commit and compare as though they measured the same thing. See + // ``benchmarks/helpers/_threads.py`` for the shell route, which is the + // right one for "just run this faster" and the wrong one for a study. + // + // The two mechanisms are mutually exclusive: asv layers ``env_nobuild`` + // over the inherited environment, so ``NUMBA_NUM_THREADS`` below wins over + // any export. + // + // KEEP IN SYNC with asv.conf.json: ``pythons``, ``environment_type``, + // ``conda_channels``, ``matrix.req`` and ``build_command`` decide what gets + // installed. Let them drift and these numbers stop being comparable to the + // ones the normal config produces, silently. + + "version": 1, + "project": "uxarray", + "project_url": "https://github.com/UXARRAY/uxarray", + "repo": "..", + "branches": ["main"], + "dvcs": "git", + "environment_type": "conda", + "conda_channels": ["conda-forge"], + "install_timeout": 600, + "launch_method": "forkserver", + "show_commit_url": "https://github.com/UXARRAY/uxarray/commit/", + "pythons": ["3.11"], + "benchmark_dir": ".", + "build_command": [ + "python -mpip wheel --no-deps --no-build-isolation --no-index -w {build_cache_dir} {build_dir}" + ], + "build_cache_size": 4, + + // A parallel kernel held at one thread takes roughly as long as the core + // count it would otherwise have used, so the 360s the normal config allows + // is not enough at the bottom of a scaling sweep. + "default_benchmark_timeout": 1800, + + "matrix": { + "req": { + "setuptools_scm": [""], + "xarray": [""], + "netcdf4": [""], + "pip+pyfma": [""], + "tbb": [""] + }, + + // One result set per value, so the run costs its length: four values is + // four passes over the suite. Edit for the node -- powers of two up to + // its physical core count is the usual shape, and + // ``python -m benchmarks.helpers._threads`` reports that count. + // Values above the core count are allowed and simply oversubscribe. + "env_nobuild": { + "NUMBA_THREADING_LAYER": ["forksafe"], + "NUMBA_NUM_THREADS": ["1", "2", "4", "8"] + } + } +} diff --git a/benchmarks/helpers/_threads.py b/benchmarks/helpers/_threads.py new file mode 100644 index 000000000..14ce8315c --- /dev/null +++ b/benchmarks/helpers/_threads.py @@ -0,0 +1,117 @@ +"""Choosing how many threads a benchmark run may use. + +numba reads ``NUMBA_NUM_THREADS`` once, when it brings up its threading layer, +and treats it as a ceiling rather than a setting: ``set_num_threads`` can hold +the pool lower for a block -- which is what :func:`~benchmarks.helpers._peakmem.numba_threads` +does while tracing -- but asking for more than the ceiling raises +``ValueError``. So a run's thread count has to be decided before the benchmark +process imports numba, which means the environment, which is what this module +resolves. + +asv copies ``os.environ`` into the processes it launches +(``Environment.run_executable``), so exporting the variable ahead of ``asv run`` +is enough:: + + export NUMBA_NUM_THREADS=$(python -m benchmarks.helpers._threads) + asv run ... + +One asymmetry to know about: asv layers the ``env_nobuild`` matrix *over* the +inherited environment, not under it, so a variable named in ``asv.conf.json`` +overrides the shell. ``NUMBA_THREADING_LAYER`` is named there because +fork-safety is not negotiable. The thread count deliberately is not, so a node +with more cores than a CI runner can decide for itself. + +The default is physical cores rather than ``os.cpu_count()``. These kernels are +floating-point and memory-bound, and a second hardware thread per core tends to +cost more in contention than it recovers in latency hiding: CI runners with 2 +physical cores plus SMT measured about 1.28x slower per thread on the same +scalar kernels than runners with 4 real cores. + +``UXARRAY_BENCH_THREADS`` overrides the default -- an integer, or ``physical`` +or ``logical`` to name a rule rather than a number. +""" + +import os +import subprocess +import sys + +__all__ = ["logical_cores", "physical_cores", "resolve"] + +_ENV_VAR = "UXARRAY_BENCH_THREADS" + + +def logical_cores(): + """Schedulable CPUs, honouring any affinity mask this process was given. + + ``os.cpu_count()`` reports the machine; ``os.sched_getaffinity`` reports + what this process may actually use, which is the smaller and more useful + number under a batch scheduler or a ``taskset``. + """ + if hasattr(os, "sched_getaffinity"): + return len(os.sched_getaffinity(0)) + return os.cpu_count() or 1 + + +def physical_cores(): + """Cores rather than hardware threads, or the logical count if unknown. + + Deliberately shells out rather than adding a dependency on ``psutil``: the + benchmark environment asv builds is defined by ``asv.conf.json``'s matrix, + and a helper that has to run in it is not worth an entry there. + """ + try: + if sys.platform == "darwin": + out = subprocess.run( + ["sysctl", "-n", "hw.physicalcpu"], + capture_output=True, text=True, timeout=5, check=True, + ).stdout + return max(1, int(out.strip())) + if sys.platform.startswith("linux"): + # One line per logical CPU, ",,,..."; distinct + # (socket, core) pairs are the physical cores. Counting distinct + # core ids alone would collapse two sockets into one. + out = subprocess.run( + ["lscpu", "-p=core,socket"], + capture_output=True, text=True, timeout=5, check=True, + ).stdout + pairs = { + line for line in (l.strip() for l in out.splitlines()) + if line and not line.startswith("#") + } + if pairs: + return max(1, len(pairs)) + except (OSError, ValueError, subprocess.SubprocessError): + pass + return logical_cores() + + +def resolve(spec=None): + """The thread count to run with. + + ``spec`` defaults to ``$UXARRAY_BENCH_THREADS``, and that to ``physical``. + Anything unrecognised falls back to the physical core count rather than + failing: this sits in front of a benchmark run, and refusing to start + because a variable is misspelt costs more than quietly doing the sensible + thing. The resolved number is echoed to stderr so it appears in the log. + """ + if spec is None: + spec = os.environ.get(_ENV_VAR, "").strip() + spec = (spec or "physical").lower() + + if spec == "logical": + return logical_cores() + if spec != "physical": + try: + return max(1, int(spec)) + except ValueError: + print( + f"{_ENV_VAR}={spec!r} is not an integer, 'physical' or 'logical'; " + "using the physical core count", + file=sys.stderr, + ) + # Never hand back more than this process may schedule on. + return min(physical_cores(), logical_cores()) + + +if __name__ == "__main__": + print(resolve()) From 5114500c5ba4af05fe7f749905ac853d513dfe89 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 28 Aug 2026 10:41:08 -0500 Subject: [PATCH 27/52] Generalize caching across machine names --- .github/workflows/asv-benchmarking-pr.yml | 31 ++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 3c424534d..412e7ad5d 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -78,6 +78,8 @@ jobs: - name: Run Benchmarks shell: bash -l {0} id: benchmark + env: + ASV_MACHINE: gh-${{ runner.os }}-${{ runner.arch }} # Only a pull_request event carries a pull_request payload, so on a # manual run every one of these expressions is the empty string. Left # unhandled, ``asv continuous`` quietly reads its one remaining @@ -105,15 +107,38 @@ jobs: test -n "$BASE" || { echo "could not determine a baseline commit" >&2; exit 1; } # Fill the fixture cache before asv preimports the suite, which would otherwise build it serially in the forkserver parent (cd .. && python -m benchmarks.helpers._fixtures) - # ID this runner + # ID this runner, then rename the entry to the pinned name. Passing + # ``--machine`` to ``asv machine`` instead would record the name and + # nothing else: it stores only the fields that differ from the ones it + # detects, and skips filling the rest in, which would empty the cpu + # and num_cpu the step below reports. Detecting first and renaming + # after keeps the real os/arch/cpu/num_cpu/ram, so the results carry + # the hardware that actually ran. asv machine --yes + python - <<'PY' + import json, os, pathlib + + name = os.environ["ASV_MACHINE"] + path = pathlib.Path.home() / ".asv-machine.json" + stored = json.loads(path.read_text()) + version = stored.pop("version") + # Exactly one entry, the one ``asv machine`` just wrote. Unpacked + # rather than indexed so a runner that arrives with a machine file + # already on it fails here instead of quietly keeping the other name. + (detected,) = stored.values() + detected["machine"] = name + path.write_text(json.dumps({name: detected, "version": version}, indent=4)) + PY echo "Baseline: $BASE" echo "Contender: ${GITHUB_SHA} ($LABEL)" # Run benchmarks for current commit against base ASV_OPTIONS="--split --show-stderr" - asv continuous $ASV_OPTIONS "$BASE" "${GITHUB_SHA}" + # ``-m`` is redundant while the machine file holds a single entry -- + # asv falls back to it whatever its name -- and is passed anyway so + # the results stop depending on that fallback. + asv continuous $ASV_OPTIONS -m "$ASV_MACHINE" "$BASE" "${GITHUB_SHA}" # Save compare results - asv compare --split "$BASE" "${GITHUB_SHA}" > asv_compare_results.txt + asv compare --split --machine "$ASV_MACHINE" "$BASE" "${GITHUB_SHA}" > asv_compare_results.txt working-directory: ${{ env.ASV_DIR }} # asv records a duration per benchmark, plus ```` and From 4f044ff8bfd4745df1101b3a75596f54945d0441 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 28 Aug 2026 11:46:44 -0500 Subject: [PATCH 28/52] Fully async sharding (?) --- .github/workflows/asv-benchmarking-pr.yml | 276 +++++++++++++++++----- .gitignore | 3 + benchmarks/helpers/_partition.py | 80 ++++++- 3 files changed, 288 insertions(+), 71 deletions(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 412e7ad5d..3fac938c5 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -20,25 +20,36 @@ on: the branch it will merge into rather than main. required: false default: main + shards: + description: >- + How many runners to spread the suite over. Each shard pays the conda + restore, the wheel install and the ~57s the suite's numba warmups + cost at import, so past about four the added runners mostly pay that + floor again. + required: false + default: "4" env: PR_HEAD_LABEL: ${{ github.event.pull_request.head.label }} + ASV_DIR: "./benchmarks" + CONDA_ENV_FILE: ci/environment.yml + # One machine name for every shard. + ASV_MACHINE: gh-linux-x64 jobs: - benchmark: + setup: + name: Setup if: ${{ contains(github.event.pull_request.labels.*.name, 'run-benchmark') && github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} - name: Linux runs-on: ubuntu-latest - env: - ASV_DIR: "./benchmarks" - CONDA_ENV_FILE: ci/environment.yml - + outputs: + base: ${{ steps.base.outputs.sha }} + shards: ${{ steps.plan.outputs.shards }} + count: ${{ steps.plan.outputs.count }} steps: - uses: actions/checkout@v7 with: fetch-depth: 0 - - name: Record CPU topology # A diagnostic; never fail the job over it. run: | @@ -56,6 +67,29 @@ jobs: python-build mamba + - name: Resolve the baseline commit + id: base + # Only a pull_request event carries a pull_request payload, so on a + # manual run every one of these expressions is the empty string. + run: | + set -x + BASE="${{ github.event.pull_request.base.sha }}" + if [ -z "$BASE" ]; then + # The merge-base rather than the base branch's tip + BASE_REF="${{ github.event.inputs.base_ref }}" + BASE_REF="${BASE_REF:-main}" + git rev-parse --verify -q "origin/$BASE_REF" >/dev/null \ + || git fetch -q origin "$BASE_REF:refs/remotes/origin/$BASE_REF" + BASE=$(git merge-base "$GITHUB_SHA" "origin/$BASE_REF") + fi + if [ "$BASE" = "$(git rev-parse "$GITHUB_SHA")" ]; then + # Dispatched from main itself: compare it against the commit before. + BASE=$(git rev-parse "$GITHUB_SHA^") + fi + # Fail loudly rather than leaving asv to infer a baseline of its own. + test -n "$BASE" || { echo "could not determine a baseline commit" >&2; exit 1; } + echo "sha=$BASE" >> "$GITHUB_OUTPUT" + # asv builds its own conda environment under ``benchmarks/env` - name: Cache asv's benchmark environment uses: actions/cache@v6 @@ -75,75 +109,187 @@ jobs: restore-keys: | asv-fixtures-${{ runner.os }}- - - name: Run Benchmarks + - name: Pre-build and discover shell: bash -l {0} - id: benchmark + working-directory: ${{ env.ASV_DIR }} env: - ASV_MACHINE: gh-${{ runner.os }}-${{ runner.arch }} - # Only a pull_request event carries a pull_request payload, so on a - # manual run every one of these expressions is the empty string. Left - # unhandled, ``asv continuous`` quietly reads its one remaining - # argument as a revision and benchmarks it against its own parent, - # while ``asv compare`` exits 2 for want of a second revision -- which - # fails the job, so the caches never save and the comment workflow, - # gated on success, never runs. + BASE: ${{ steps.base.outputs.sha }} + # ``--bench just-discover`` is asv's own discovery-only mode + # (``commands/run.py``): it creates the environments, builds the project + # for the commit it discovers from, writes ``results/benchmarks.json`` + # and returns 0 without running a benchmark. Done once here rather than + # once per shard, a cold cache costs one conda solve instead of four, + # and the shards restore an environment whose build cache already holds + # both commits' wheels. On a shared filesystem this is also what stops + # concurrent shards racing to install into one ``env/``; separate + # runners have no such race, but they do have the cost. run: | set -x - BASE="${{ github.event.pull_request.base.sha }}" - LABEL="${PR_HEAD_LABEL:-$GITHUB_REF_NAME}" - if [ -z "$BASE" ]; then - # The merge-base rather than the base branch's tip - BASE_REF="${{ github.event.inputs.base_ref }}" - BASE_REF="${BASE_REF:-main}" - git rev-parse --verify -q "origin/$BASE_REF" >/dev/null \ - || git fetch -q origin "$BASE_REF:refs/remotes/origin/$BASE_REF" - BASE=$(git merge-base "$GITHUB_SHA" "origin/$BASE_REF") - fi - if [ "$BASE" = "$(git rev-parse "$GITHUB_SHA")" ]; then - # Dispatched from main itself: compare it against the commit before. - BASE=$(git rev-parse "$GITHUB_SHA^") - fi - # Fail loudly rather than leaving asv to infer a baseline of its own. - test -n "$BASE" || { echo "could not determine a baseline commit" >&2; exit 1; } # Fill the fixture cache before asv preimports the suite, which would otherwise build it serially in the forkserver parent (cd .. && python -m benchmarks.helpers._fixtures) - # ID this runner, then rename the entry to the pinned name. Passing - # ``--machine`` to ``asv machine`` instead would record the name and - # nothing else: it stores only the fields that differ from the ones it - # detects, and skips filling the rest in, which would empty the cpu - # and num_cpu the step below reports. Detecting first and renaming - # after keeps the real os/arch/cpu/num_cpu/ram, so the results carry - # the hardware that actually ran. asv machine --yes - python - <<'PY' - import json, os, pathlib - - name = os.environ["ASV_MACHINE"] - path = pathlib.Path.home() / ".asv-machine.json" - stored = json.loads(path.read_text()) - version = stored.pop("version") - # Exactly one entry, the one ``asv machine`` just wrote. Unpacked - # rather than indexed so a runner that arrives with a machine file - # already on it fails here instead of quietly keeping the other name. - (detected,) = stored.values() - detected["machine"] = name - path.write_text(json.dumps({name: detected, "version": version}, indent=4)) - PY + (cd .. && python -m benchmarks.helpers._machine --name "$ASV_MACHINE") + asv run --bench just-discover "${BASE}^!" + asv run --bench just-discover "${GITHUB_SHA}^!" + + - name: Plan the shards + id: plan + shell: bash -l {0} + working-directory: ${{ env.ASV_DIR }} + env: + SHARDS: ${{ github.event.inputs.shards || '4' }} + run: | + set -x + # The report goes in the log so a lopsided split is visible without + # opening four shard jobs to find which one ran long. + PYTHONPATH=.. python -m benchmarks.helpers._partition --shards "$SHARDS" + echo "count=$SHARDS" >> "$GITHUB_OUTPUT" + python -c "import json, os; print('shards=' + json.dumps(list(range(int(os.environ['SHARDS'])))))" \ + >> "$GITHUB_OUTPUT" + + - name: Upload the discovered suite + uses: actions/upload-artifact@v7 + with: + name: asv-plan + path: ${{ env.ASV_DIR }}/results/benchmarks.json + + benchmark: + name: Shard ${{ matrix.shard }} + needs: setup + runs-on: ubuntu-latest + strategy: + # Every shard is independent, and a shard that dies still leaves the rest + # worth merging. + fail-fast: false + matrix: + shard: ${{ fromJSON(needs.setup.outputs.shards) }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Set up Conda environment + uses: mamba-org/setup-micromamba@v3 + with: + environment-file: ${{env.CONDA_ENV_FILE}} + cache-environment: true + environment-name: uxarray_build + cache-environment-key: "${{runner.os}}-${{runner.arch}}-${{hashFiles(env.CONDA_ENV_FILE)}}-benchmark" + create-args: >- + asv + python-build + mamba + + - name: Cache asv's benchmark environment + uses: actions/cache@v6 + with: + path: ${{ env.ASV_DIR }}/env + key: asv-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('benchmarks/asv.conf.json') }} + restore-keys: | + asv-env-${{ runner.os }}-${{ runner.arch }}- + + - name: Cache the benchmark fixtures + uses: actions/cache@v6 + with: + path: | + ${{ env.ASV_DIR }}/oQU*.nc + ${{ env.ASV_DIR }}/_io_cache + key: asv-fixtures-${{ runner.os }}-${{ hashFiles('benchmarks/helpers/_fixtures.py') }} + restore-keys: | + asv-fixtures-${{ runner.os }}- + + - name: Download the discovered suite + uses: actions/download-artifact@v8 + with: + name: asv-plan + path: plan + + - name: Run shard + shell: bash -l {0} + working-directory: ${{ env.ASV_DIR }} + env: + BASE: ${{ needs.setup.outputs.base }} + SHARDS: ${{ needs.setup.outputs.count }} + SHARD: ${{ matrix.shard }} + # Partitioned from setup's ``benchmarks.json`` rather than from whatever + # this runner's cache happens to hold, so every shard splits the same + # suite the same way. Nothing checks that the shards tile it: were they + # to disagree, benchmarks would simply go unrun and no step would say so. + run: | + set -x + (cd .. && python -m benchmarks.helpers._fixtures) + asv machine --yes + (cd .. && python -m benchmarks.helpers._machine --name "$ASV_MACHINE") + ASV_ARGS=$(PYTHONPATH=.. python -m benchmarks.helpers._partition \ + --shards "$SHARDS" --shard "$SHARD" --results ../plan \ + --config asv.conf.json --asv-args) echo "Baseline: $BASE" - echo "Contender: ${GITHUB_SHA} ($LABEL)" - # Run benchmarks for current commit against base - ASV_OPTIONS="--split --show-stderr" - # ``-m`` is redundant while the machine file holds a single entry -- - # asv falls back to it whatever its name -- and is passed anyway so - # the results stop depending on that fallback. - asv continuous $ASV_OPTIONS -m "$ASV_MACHINE" "$BASE" "${GITHUB_SHA}" - # Save compare results - asv compare --split --machine "$ASV_MACHINE" "$BASE" "${GITHUB_SHA}" > asv_compare_results.txt + echo "Contender: ${GITHUB_SHA} (${PR_HEAD_LABEL:-$GITHUB_REF_NAME})" + asv continuous --split --show-stderr -m "$ASV_MACHINE" $ASV_ARGS \ + "$BASE" "${GITHUB_SHA}" + + - name: Upload shard results + if: always() + uses: actions/upload-artifact@v7 + with: + name: asv-shard-${{ matrix.shard }} + path: ${{ env.ASV_DIR }}/results.shard${{ matrix.shard }} + if-no-files-found: warn + + merge: + name: Merge and compare + needs: [setup, benchmark] + # Runs on a partial fan-out too: a merged tree missing one shard's rows is + # still worth reading, and the artifact is the only place a failed shard's + # half of the run can be seen. + if: ${{ always() && needs.setup.result == 'success' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Set up Conda environment + uses: mamba-org/setup-micromamba@v3 + with: + environment-file: ${{env.CONDA_ENV_FILE}} + cache-environment: true + environment-name: uxarray_build + cache-environment-key: "${{runner.os}}-${{runner.arch}}-${{hashFiles(env.CONDA_ENV_FILE)}}-benchmark" + create-args: >- + asv + python-build + mamba + + # Without merge-multiple each artifact lands in its own + # ``shards/asv-shard-N/``, which is what the merge needs: every shard + # named its results file identically, and telling them apart is the point. + - name: Download the shards + uses: actions/download-artifact@v8 + with: + pattern: asv-shard-* + path: shards + + - name: Merge and compare + shell: bash -l {0} working-directory: ${{ env.ASV_DIR }} + env: + BASE: ${{ needs.setup.outputs.base }} + run: | + set -x + asv machine --yes + (cd .. && python -m benchmarks.helpers._machine --name "$ASV_MACHINE") + (cd .. && python -m benchmarks.helpers._merge --out benchmarks/results shards/asv-shard-*) + asv compare --split --machine "$ASV_MACHINE" "$BASE" "${GITHUB_SHA}" \ + > asv_compare_results.txt + cat asv_compare_results.txt # asv records a duration per benchmark, plus ```` and # ```` entries, in the results file it writes. Printing - # them is what tells us where a run's wall clock actually went. + # them is what tells us where a run's wall clock actually went. Two things + # to read them with: a per-benchmark duration is the final round's only, + # since asv assigns rather than accumulates it, and ```` and + # ```` are the slowest shard's, since every shard paid them. - name: Report where the time went if: always() shell: bash -l {0} @@ -182,7 +328,7 @@ jobs: - uses: actions/upload-artifact@v7 if: always() with: - name: asv-benchmark-results-${{ runner.os }} + name: asv-benchmark-results-Linux path: | ${{ env.ASV_DIR }}/results ${{ env.ASV_DIR }}/asv_compare_results.txt diff --git a/.gitignore b/.gitignore index 5c52cd586..724dfa80b 100644 --- a/.gitignore +++ b/.gitignore @@ -164,3 +164,6 @@ benchmarks/env benchmarks/results benchmarks/html benchmarks/_io_cache +# Generated per shard by benchmarks/helpers/_partition.py --asv-args +benchmarks/asv.conf*.shard*.json +benchmarks/results.shard* diff --git a/benchmarks/helpers/_partition.py b/benchmarks/helpers/_partition.py index 28fa6f58a..a6f22dc54 100644 --- a/benchmarks/helpers/_partition.py +++ b/benchmarks/helpers/_partition.py @@ -14,8 +14,15 @@ needed. The suite is flat: the heaviest single benchmark is about 5% of the total, and greedy longest-first packing lands within ~1% of a perfect split even at eight shards. Splitting inside a benchmark would buy nothing and would -put every shard's results in the same row of the same results file, which then -has to be merged element-wise. +put every shard's results in the same row of the same results file; keeping +whole benchmarks leaves each row owned by one shard, so +:mod:`benchmarks.helpers._merge` can put the tree back together by union. + +Shards still need somewhere separate to write, because asv names its results +file per commit and environment rather than per run and rewrites the whole thing +at the end of a set. ``--config-out`` emits a copy of the config with +``results_dir`` pointed at this shard's own directory, which is what the merge +then reads. Weights come from the ``duration`` asv records per benchmark in the results file it writes (``Results.save``), so a partition improves as results accumulate @@ -26,7 +33,8 @@ Usage:: python -m benchmarks.helpers._partition --shards 4 - python -m benchmarks.helpers._partition --shards 4 --shard 0 --bench-args + asv run $(python -m benchmarks.helpers._partition --shards 4 --shard 0 \ + --config asv.conf.hpc.json --asv-args) """ import argparse @@ -37,7 +45,14 @@ import sys from pathlib import Path -__all__ = ["load_benchmarks", "load_weights", "plan", "bench_regexes"] +__all__ = [ + "bench_regexes", + "load_benchmarks", + "load_weights", + "plan", + "shard_results_dir", + "write_shard_config", +] BENCHMARK_DIR = Path(__file__).resolve().parents[1] @@ -146,6 +161,35 @@ def bench_regexes(names): return [f"^{re.escape(name)}($|\\()" for name in names] +def shard_config_path(base_config, shard): + """Where shard ``shard``'s generated config goes, beside ``base_config``.""" + base = Path(base_config) + return base.with_name(f"{base.stem}.shard{shard}{base.suffix}") + + +def shard_results_dir(results_dir, shard): + """Where shard ``shard`` writes, given the run's ordinary ``results_dir``.""" + return f"{results_dir}.shard{shard}" + + +def write_shard_config(base_config, out_path, shard): + """Writes a copy of ``base_config`` that writes results where ``shard`` should. + + Returns the shard's results directory. + """ + # asv's loader, because an asv config is JSON with javascript comments and + # ``json`` cannot read one. Imported here so the rest of the module stays + # runnable without asv installed. + from asv import util + + config = util.load_json(str(base_config), js_comments=True) + results_dir = shard_results_dir(config.get("results_dir", "results"), shard) + config["results_dir"] = results_dir + with open(out_path, "w") as handle: + json.dump(config, handle, indent=4) + return results_dir + + def _report(benchmarks, shards, weights): known = [value for value in weights.values() if value > 0] default = statistics.median(known) if known else 1.0 @@ -187,6 +231,22 @@ def main(argv=None): action="store_true", help="Print the shard's --bench arguments for asv, rather than a report.", ) + parser.add_argument( + "--config", + default=str(BENCHMARK_DIR / "asv.conf.json"), + help="Base asv config for --config-out (default benchmarks/asv.conf.json).", + ) + parser.add_argument( + "--config-out", + default=None, + help="Where --asv-args writes the shard config (default: beside --config).", + ) + parser.add_argument( + "--asv-args", + action="store_true", + help="Write this shard's config and print every argument its asv run " + "needs, so launching a shard is one substitution. Needs --shard.", + ) args = parser.parse_args(argv) results_dirs = args.results or [str(BENCHMARK_DIR / "results")] @@ -194,9 +254,17 @@ def main(argv=None): weights = load_weights(results_dirs) shards = plan(benchmarks, args.shards, weights) - if args.bench_args: + if args.bench_args or args.asv_args: if args.shard is None: - parser.error("--bench-args needs --shard") + parser.error("--bench-args and --asv-args need --shard") + if args.asv_args: + # Written here rather than by a call of its own: a shard that is + # told which benchmarks to run has to be told where to put them, + # and splitting that across two commands is two chances to pass + # one shard's benchmarks with another's results directory. + config_out = args.config_out or shard_config_path(args.config, args.shard) + write_shard_config(args.config, config_out, args.shard) + print("--config", config_out) for pattern in bench_regexes(shards[args.shard]): print("--bench", pattern) return 0 From 8da3f542b7157026fc48fa322176b665e2d01959 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 28 Aug 2026 11:55:26 -0500 Subject: [PATCH 29/52] Benchmark sharding fix --- .github/workflows/asv-benchmarking-pr.yml | 10 +- benchmarks/helpers/_fixtures.py | 12 +- benchmarks/helpers/_machine.py | 90 ++++++++ benchmarks/helpers/_merge.py | 252 ++++++++++++++++++++++ 4 files changed, 354 insertions(+), 10 deletions(-) create mode 100644 benchmarks/helpers/_machine.py create mode 100644 benchmarks/helpers/_merge.py diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 3fac938c5..c5590cd0b 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -72,7 +72,7 @@ jobs: # Only a pull_request event carries a pull_request payload, so on a # manual run every one of these expressions is the empty string. run: | - set -x + set -ex BASE="${{ github.event.pull_request.base.sha }}" if [ -z "$BASE" ]; then # The merge-base rather than the base branch's tip @@ -124,7 +124,7 @@ jobs: # concurrent shards racing to install into one ``env/``; separate # runners have no such race, but they do have the cost. run: | - set -x + set -ex # Fill the fixture cache before asv preimports the suite, which would otherwise build it serially in the forkserver parent (cd .. && python -m benchmarks.helpers._fixtures) asv machine --yes @@ -139,7 +139,7 @@ jobs: env: SHARDS: ${{ github.event.inputs.shards || '4' }} run: | - set -x + set -ex # The report goes in the log so a lopsided split is visible without # opening four shard jobs to find which one ran long. PYTHONPATH=.. python -m benchmarks.helpers._partition --shards "$SHARDS" @@ -216,7 +216,7 @@ jobs: # suite the same way. Nothing checks that the shards tile it: were they # to disagree, benchmarks would simply go unrun and no step would say so. run: | - set -x + set -ex (cd .. && python -m benchmarks.helpers._fixtures) asv machine --yes (cd .. && python -m benchmarks.helpers._machine --name "$ASV_MACHINE") @@ -276,7 +276,7 @@ jobs: env: BASE: ${{ needs.setup.outputs.base }} run: | - set -x + set -ex asv machine --yes (cd .. && python -m benchmarks.helpers._machine --name "$ASV_MACHINE") (cd .. && python -m benchmarks.helpers._merge --out benchmarks/results shards/asv-shard-*) diff --git a/benchmarks/helpers/_fixtures.py b/benchmarks/helpers/_fixtures.py index 9ab574197..df3e875ab 100644 --- a/benchmarks/helpers/_fixtures.py +++ b/benchmarks/helpers/_fixtures.py @@ -14,11 +14,13 @@ everything the reader produced from ``Grid.open_grid`` and ``Grid.open_dataset`` -Artifacts are keyed on both the uxarray build and the files, because an -artifact is one version's reader output and ASV diffs commits. Likewise, there's a -fresh read per commit. ``prime`` covers every source that is readable here, -and there is a CLI (``python -m benchmarks.helpers._fixtures``) to fill the -cache from a batch script instead of from inside a benchmark. +Artifacts are keyed on the source files and nothing else, so they persist +across runs -- and across the commits ASV diffs, which means a benchmark on the +cached flavors measures its own subject against one fixed reader output rather +than against a per-commit re-read. A source replaced in place misses rather than +being served something stale. ``prime`` covers every source that is readable +here, and there is a CLI (``python -m benchmarks.helpers._fixtures``) to fill +the cache from a batch script instead of from inside a benchmark. """ import hashlib diff --git a/benchmarks/helpers/_machine.py b/benchmarks/helpers/_machine.py new file mode 100644 index 000000000..5d377fffc --- /dev/null +++ b/benchmarks/helpers/_machine.py @@ -0,0 +1,90 @@ +"""Pinning the machine name asv records results under. + +asv keys results on a machine name and defaults it to the hostname +(``Machine.get_defaults``), which on a hosted runner is fresh for every job -- +``runnervmgx7h7`` on one run, something else on the next. A name that never +repeats cannot be compared across runs, and once the suite is sharded it cannot +even be merged within one run: the file asv writes is +``results//-.json``, so every shard has to agree on +```` or there is nothing for :mod:`_merge` to line up. + +``asv machine --machine NAME`` will not do it on its own. That command stores +only the fields that differ from the ones it detected and then skips filling the +rest in (``commands/machine.py``), so naming the machine is precisely what drops +``cpu``, ``num_cpu`` and ``ram`` -- the fields that say what the timings were +measured on, and the ones the duration report prints. Detect first with ``asv +machine --yes``, rename after, which is what this does. + +Idempotent, so a job that runs it twice, or a machine file that arrives already +pinned, is fine. + +Usage:: + + asv machine --yes + python -m benchmarks.helpers._machine --name gh-Linux-X64 +""" + +import argparse +import json +import sys +from pathlib import Path + +__all__ = ["pin"] + +_VERSION_KEY = "version" + + +def default_path(): + """Where asv keeps its machine file (``MachineCollection.get_machine_file_path``).""" + return Path.home() / ".asv-machine.json" + + +def pin(name, path=None): + """Renames the machine file's single entry to ``name``. Returns its details. + + Raises if there is more than one entry and none of them is ``name`` already: + with several to choose from there is no way to tell which one describes the + machine this is running on, and picking wrong would label the results with + another machine's hardware. + """ + path = Path(path) if path is not None else default_path() + stored = json.loads(path.read_text()) + version = stored.pop(_VERSION_KEY, None) + + if name in stored: + detected = stored[name] + elif len(stored) == 1: + (detected,) = stored.values() + else: + raise ValueError( + f"{path} holds {len(stored)} machines ({', '.join(sorted(stored))}) and none " + f"is {name!r}; cannot tell which describes this machine" + ) + + detected["machine"] = name + merged = {name: detected} + if version is not None: + merged[_VERSION_KEY] = version + path.write_text(json.dumps(merged, indent=4)) + return detected + + +def main(argv=None): + parser = argparse.ArgumentParser( + prog="python -m benchmarks.helpers._machine", + description="Rename asv's detected machine entry to a fixed name.", + ) + parser.add_argument("--name", required=True, help="Machine name to pin to.") + parser.add_argument("--path", default=None, help="Machine file (default ~/.asv-machine.json).") + args = parser.parse_args(argv) + + detected = pin(args.name, args.path) + print( + f"{args.name}: {detected.get('cpu', '?')} " + f"({detected.get('num_cpu', '?')} cpu, {detected.get('os', '?')})" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/helpers/_merge.py b/benchmarks/helpers/_merge.py new file mode 100644 index 000000000..638751d8e --- /dev/null +++ b/benchmarks/helpers/_merge.py @@ -0,0 +1,252 @@ +"""Merging a sharded run's results back into one tree. + +asv reads its results file once before running a benchmark set and writes it +once after (``Results.load_data`` then ``Results.save`` in ``commands/run.py``), +and the name it writes is ``results//-.json`` -- one file +per commit and environment, whatever subset of benchmarks the run measured. So +shards sharing a results directory each rewrite that whole file from what they +alone measured, and the last one to finish wins. Each shard therefore gets its +own directory (``_partition --config-out``) and they are combined here. + +The combination is a union rather than an element-wise reconciliation, which is +what the by-whole-benchmark split buys: a row is keyed on the benchmark name and +carries its whole parameter sweep inside, so every row is owned by exactly one +shard. Splitting inside a benchmark would have put two shards in one row. + +Order is restored rather than preserved. ``results`` is a JSON object, asv writes +it with ``compact=True`` -- which disables sorting, so key order is the order asv +appended to it -- and for an unsharded run that order is ``sorted(benchmarks)`` +grouped by ``setup_cache_key`` (``runner.py``, ``iter_run_items``). Shards finish +in whatever order the queue hands back, so :func:`canonical_order` recovers the +order the same suite would have produced serially and every merged file is +written in it. + +Idempotent, and indifferent to shards that have not landed: merging the three +directories that exist gives a valid tree, and merging again when the fourth +arrives puts its rows in their proper place. That is what makes it safe to run +from a polling loop as jobs come back rather than only after a barrier. + +Usage:: + + python -m benchmarks.helpers._merge --out results results.shard* + python -m benchmarks.helpers._merge --out results --quiet results.shard* +""" + +import argparse +import json +import shutil +import sys +from pathlib import Path + +__all__ = ["canonical_order", "merge", "merge_benchmarks", "merge_result_files"] + +BENCHMARK_DIR = Path(__file__).resolve().parents[1] + +MACHINE_FILE = "machine.json" +BENCHMARKS_FILE = "benchmarks.json" +_SPECIAL_FILES = frozenset({MACHINE_FILE, BENCHMARKS_FILE}) + +# asv stores its own format version alongside the data in both files. +_VERSION_KEY = "version" + + +def _load(path): + with open(path) as handle: + return json.load(handle) + + +def _dump(path, data): + """Writes ``data`` the way asv writes a results file. + + ``util.write_json(..., compact=True)`` disables both sorting and + indentation; the sorting is the part that matters, because key order is the + only place a results file records what ran when. + """ + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + json.dump(data, handle) + + +def canonical_order(benchmarks): + """Benchmark names in the order an unsharded ``asv run`` would produce them. + + Mirrors ``runner.run_benchmarks``: it walks ``sorted(benchmarks.items())`` + building ``benchmark_order``, a dict keyed on ``setup_cache_key``, then runs + each of those groups in turn. So the order is by name within a group, and + groups in the order their first member is reached by name. + """ + groups = {} + for name in sorted(benchmarks): + key = benchmarks[name].get("setup_cache_key") + groups.setdefault(key, []).append(name) + return [name for group in groups.values() for name in group] + + +def merge_benchmarks(shard_dirs): + """Union of the shards' ``benchmarks.json``. + + A shard discovers under its own ``--bench`` patterns, so each file holds + only that shard's benchmarks and the full set exists nowhere until here. + ``_partition.load_benchmarks`` needs that full set to plan the next run. + """ + merged, version = {}, None + for shard_dir in shard_dirs: + path = Path(shard_dir) / BENCHMARKS_FILE + if not path.is_file(): + continue + data = _load(path) + version = data.get(_VERSION_KEY, version) + for name, value in data.items(): + if name != _VERSION_KEY: + merged[name] = value + if version is not None: + merged[_VERSION_KEY] = version + return merged + + +def _pick(name, existing, candidate, report): + """Which of two rows for one benchmark to keep. + + Only reachable when a name landed in more than one shard, which the + partition does not do -- so it means the plan the shards ran was not the one + that produced them. Preferring a row that has a result over one that does + not, then the later ``started_at``, keeps a re-run over the run it replaced + instead of picking on file order. + """ + if existing == candidate: + return existing + + def rank(row): + return (row.get("result") is not None, row.get("started_at") or 0) + + keep, drop = (candidate, existing) if rank(candidate) > rank(existing) else (existing, candidate) + report( + f"{name}: found in more than one shard with different data; keeping the " + f"row started at {keep.get('started_at')} over {drop.get('started_at')}" + ) + return keep + + +def merge_result_files(datas, order, report): + """One results file from several shards' versions of it. + + ``datas`` are the parsed files, in shard order; ``order`` is the name order + to write. Every field outside ``results`` and ``durations`` describes the + commit and environment rather than the run, and is identical across shards + by construction, so the first shard's copy carries over untouched. + """ + merged = dict(datas[0]) + columns = list(merged.get("result_columns") or []) + + rows, durations = {}, {} + for data in datas: + # Read each row against its own file's columns. Identical in practice -- + # one asv builds every shard -- but a row is a bare list, so aligning it + # to the wrong header would silently shift every value. + shard_columns = data.get("result_columns") or columns + for name, row in (data.get("results") or {}).items(): + values = dict(zip(shard_columns, row)) + rows[name] = ( + _pick(name, rows[name], values, report) if name in rows else values + ) + # ``durations`` holds only the ```` and ```` + # entries; a benchmark's own duration lives in its row. Every shard pays + # both, so the max is the one a single run would have reported, and the + # sum would describe work no single wall clock ever saw. + for key, value in (data.get("durations") or {}).items(): + durations[key] = max(durations.get(key, 0.0), float(value)) + + known = [name for name in order if name in rows] + extra = sorted(name for name in rows if name not in set(order)) + if extra: + report(f"{len(extra)} row(s) not in benchmarks.json, appended: {', '.join(extra[:3])}...") + + results = {} + for name in known + extra: + row = [rows[name].get(column) for column in columns] + # asv drops trailing nulls when it writes a row; keeping that keeps the + # merged file the same size as the one a serial run would have written. + while row and row[-1] is None: + row.pop() + results[name] = row + + merged["results"] = results + merged["durations"] = durations + return merged + + +def merge(shard_dirs, out_dir, report=lambda message: None): + """Merges ``shard_dirs`` into ``out_dir``. Returns a per-file row count.""" + shard_dirs = [Path(d) for d in shard_dirs] + out_dir = Path(out_dir) + resolved_out = out_dir.resolve() + if any(d.resolve() == resolved_out for d in shard_dirs): + raise ValueError(f"--out {out_dir} is also a shard directory; refusing to merge in place") + + present = [d for d in shard_dirs if d.is_dir()] + for missing in [d for d in shard_dirs if not d.is_dir()]: + report(f"{missing}: not there yet, skipped") + if not present: + raise ValueError("no shard directories to merge") + + benchmarks = merge_benchmarks(present) + order = canonical_order({k: v for k, v in benchmarks.items() if k != _VERSION_KEY}) + out_dir.mkdir(parents=True, exist_ok=True) + if len(benchmarks) > (1 if _VERSION_KEY in benchmarks else 0): + _dump(out_dir / BENCHMARKS_FILE, benchmarks) + + # One group per (machine, results file): a shard writes the same file name as + # every other shard of its commit and environment, which is the collision + # this module exists to undo. + groups = {} + for shard_dir in present: + for path in sorted(shard_dir.glob("*/*.json")): + if path.name in _SPECIAL_FILES: + continue + groups.setdefault((path.parent.name, path.name), []).append(path) + for machine_path in sorted(shard_dir.glob(f"*/{MACHINE_FILE}")): + target = out_dir / machine_path.parent.name / MACHINE_FILE + if target.is_file() and _load(target) != _load(machine_path): + report(f"{machine_path}: disagrees with the machine.json already merged") + else: + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(machine_path, target) + + counts = {} + for (machine, filename), paths in sorted(groups.items()): + merged = merge_result_files( + [_load(p) for p in paths], + order, + lambda message, f=filename: report(f"{f}: {message}"), + ) + _dump(out_dir / machine / filename, merged) + counts[f"{machine}/{filename}"] = (len(merged["results"]), len(paths)) + return counts + + +def main(argv=None): + parser = argparse.ArgumentParser( + prog="python -m benchmarks.helpers._merge", + description="Merge a sharded run's results directories into one tree.", + ) + parser.add_argument("shards", nargs="+", help="Shard results directories to merge.") + parser.add_argument( + "--out", + default=str(BENCHMARK_DIR / "results"), + help="Directory to write the merged tree to (default benchmarks/results).", + ) + parser.add_argument("--quiet", action="store_true", help="Suppress per-file notes.") + args = parser.parse_args(argv) + + def report(message): + if not args.quiet: + print(f" {message}", file=sys.stderr) + + counts = merge(args.shards, args.out, report) + for name, (rows, shards) in sorted(counts.items()): + print(f"{name}: {rows} benchmarks from {shards} shard(s)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From f7d8118ae0e2f1427b59c71a14722b186222b83d Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 28 Aug 2026 12:44:24 -0500 Subject: [PATCH 30/52] Sharding fix 2 --- .github/workflows/asv-benchmarking-pr.yml | 67 ++++++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index c5590cd0b..3c1af3ca0 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -109,6 +109,32 @@ jobs: restore-keys: | asv-fixtures-${{ runner.os }}- + # ``_partition`` weights each benchmark by the duration asv recorded for + # it, and falls back to the median of the ones it knows when it knows + # none -- which, with no results on a fresh runner, is every one of them, + # making the split by count. Restoring the last run's tree is what turns + # the packing back into a cost-balanced one. Restore-only here; the merge + # job saves the updated tree under a key of its own. + # The environment cache key has no commit in it -- deliberately, since the + # conda environment does not depend on one -- so on a hit ``actions/cache`` + # saves nothing at the end of the job and the wheels built above would go + # with it, leaving every shard to build both commits again. The wheels get + # their own cache, keyed on the pair, and it is a couple of MB against the + # environment's 568, so keying it per run costs the cache budget little. + - name: Cache the built wheels + uses: actions/cache@v6 + with: + path: ${{ env.ASV_DIR }}/env/*/asv-build-cache + key: asv-wheels-${{ runner.os }}-${{ steps.base.outputs.sha }}-${{ github.sha }} + + - name: Restore recorded durations + uses: actions/cache/restore@v6 + with: + path: ${{ env.ASV_DIR }}/results + key: asv-results-${{ runner.os }}-${{ github.run_id }} + restore-keys: | + asv-results-${{ runner.os }}- + - name: Pre-build and discover shell: bash -l {0} working-directory: ${{ env.ASV_DIR }} @@ -147,11 +173,14 @@ jobs: python -c "import json, os; print('shards=' + json.dumps(list(range(int(os.environ['SHARDS'])))))" \ >> "$GITHUB_OUTPUT" + # The whole tree rather than ``benchmarks.json`` alone: the shards + # partition from this, and a partition is only reproducible if they weigh + # the same durations setup weighed. - name: Upload the discovered suite uses: actions/upload-artifact@v7 with: name: asv-plan - path: ${{ env.ASV_DIR }}/results/benchmarks.json + path: ${{ env.ASV_DIR }}/results benchmark: name: Shard ${{ matrix.shard }} @@ -198,6 +227,18 @@ jobs: restore-keys: | asv-fixtures-${{ runner.os }}- + # The environment cache key has no commit in it -- deliberately, since the + # conda environment does not depend on one -- so on a hit ``actions/cache`` + # saves nothing at the end of the job and the wheels built above would go + # with it, leaving every shard to build both commits again. The wheels get + # their own cache, keyed on the pair, and it is a couple of MB against the + # environment's 568, so keying it per run costs the cache budget little. + - name: Cache the built wheels + uses: actions/cache@v6 + with: + path: ${{ env.ASV_DIR }}/env/*/asv-build-cache + key: asv-wheels-${{ runner.os }}-${{ needs.setup.outputs.base }}-${{ github.sha }} + - name: Download the discovered suite uses: actions/download-artifact@v8 with: @@ -225,8 +266,21 @@ jobs: --config asv.conf.json --asv-args) echo "Baseline: $BASE" echo "Contender: ${GITHUB_SHA} (${PR_HEAD_LABEL:-$GITHUB_REF_NAME})" + # asv returns 2 when a benchmark failed rather than when the run + # broke (``commands/run.py``: ``if failures: return 2``), and this + # shard's other results are complete and worth merging. The old + # single-job workflow tolerated it by accident -- ``asv compare`` ran + # last in the same step and its status was the step's -- so make it + # deliberate here, loudly, rather than letting one long-broken + # benchmark redden every shard that happens to hold it. + status=0 asv continuous --split --show-stderr -m "$ASV_MACHINE" $ASV_ARGS \ - "$BASE" "${GITHUB_SHA}" + "$BASE" "${GITHUB_SHA}" || status=$? + if [ "$status" -eq 2 ]; then + echo "::warning title=Benchmark failures in shard ${SHARD}::asv exited 2; see the failed entries above" + elif [ "$status" -ne 0 ]; then + exit "$status" + fi - name: Upload shard results if: always() @@ -321,6 +375,15 @@ jobs: print(f" {duration:8.1f}s {100 * duration / total:5.1f}% {name}") PY + # Keyed on the run id so every run stores its own entry and setup's + # prefixed restore picks up the newest; there is nothing to restore here. + - name: Save recorded durations for the next run + if: always() + uses: actions/cache/save@v6 + with: + path: ${{ env.ASV_DIR }}/results + key: asv-results-${{ runner.os }}-${{ github.run_id }} + - name: Save PR number if: always() run: echo "${{ github.event.pull_request.number }}" > ${{ env.ASV_DIR }}/pr_number.txt From 99c49b528ab8fff923eb54a02cca131a0a1ce109 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 28 Aug 2026 12:55:10 -0500 Subject: [PATCH 31/52] Rework shard partitioning --- benchmarks/helpers/_partition.py | 85 ++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 15 deletions(-) diff --git a/benchmarks/helpers/_partition.py b/benchmarks/helpers/_partition.py index a6f22dc54..57c3a7dd0 100644 --- a/benchmarks/helpers/_partition.py +++ b/benchmarks/helpers/_partition.py @@ -8,15 +8,23 @@ runner rather than several per runner, and at this module, whose whole job is to decide which benchmarks each of those runners should claim. -The split is by whole benchmark, not by parameter combination. asv matches -``--bench`` against the expanded ``name(param0, param1)`` for parameterized -benchmarks, so a finer cut is available -- but measured durations say it is not -needed. The suite is flat: the heaviest single benchmark is about 5% of the -total, and greedy longest-first packing lands within ~1% of a perfect split -even at eight shards. Splitting inside a benchmark would buy nothing and would -put every shard's results in the same row of the same results file; keeping -whole benchmarks leaves each row owned by one shard, so -:mod:`benchmarks.helpers._merge` can put the tree back together by union. +The split is by whole class, not by benchmark and not by parameter +combination. asv matches ``--bench`` against the expanded +``name(param0, param1)``, so a much finer cut is available, but a class's +benchmarks share the kernels its first one compiles and splitting them makes +every shard pay that compile again. Measured: ``face_bounds.FaceBounds``'s four +benchmarks cost ~59s together in one process, where ``time_face_bounds`` paid +the bounds compile and the three ``track_*`` variants rode on it warm at under +7s each; scattered one per shard across four runners they cost 221s, every one +of them paying the compile alone. ``cache=True`` does not save this -- asv +reinstalls the wheel for each commit and the on-disk cache goes with it. The +suite stays flat enough at class granularity for greedy longest-first packing +to land close to a perfect split. + +Going finer than a whole benchmark would also put two shards in one row of one +results file, with no sane way to reconcile them. Whole benchmarks leave every +row owned by exactly one shard, which is what lets +:mod:`benchmarks.helpers._merge` rebuild the tree by union. Shards still need somewhere separate to write, because asv names its results file per commit and environment rather than per run and rewrites the whole thing @@ -70,6 +78,58 @@ def _splittable(setup_cache_key): return setup_cache_key is None or str(setup_cache_key).startswith(SPLITTABLE_PREFIX) +def _owner(name): + """The class -- or the module, for a bare function -- a benchmark belongs to.""" + return name.rsplit(".", 1)[0] + + +def _units(benchmarks): + """Benchmarks that have to ride together, as ``{root name: [names]}``. + + Two constraints, unioned so a ``setup_cache`` group spanning classes pulls + those classes together rather than contradicting them: + + ``class`` + Its benchmarks share whatever its first one compiles (see the module + docstring for what splitting one measured). + ``setup_cache`` + Anything sharing a ``setup_cache`` expensive enough to matter, which asv + would otherwise run once in every shard holding a member. + + Roots are the lexicographically smallest member, so the grouping is + deterministic and a shard can still work out its own membership. + """ + parent = {} + + def find(x): + parent.setdefault(x, x) + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + + def union(a, b): + ra, rb = find(a), find(b) + if ra != rb: + parent[max(ra, rb)] = min(ra, rb) + + groups = {} + for name, benchmark in benchmarks.items(): + find(name) + groups.setdefault(("class", _owner(name)), []).append(name) + key = benchmark.get("setup_cache_key") + if not _splittable(key): + groups.setdefault(("setup_cache", key), []).append(name) + for members in groups.values(): + for other in members[1:]: + union(members[0], other) + + units = {} + for name in benchmarks: + units.setdefault(find(name), []).append(name) + return units + + def load_benchmarks(results_dir): """The discovered benchmarks, as asv wrote them to ``benchmarks.json``.""" path = Path(results_dir) / "benchmarks.json" @@ -128,12 +188,7 @@ def plan(benchmarks, n_shards, weights=None): known = [value for value in weights.values() if value > 0] default = statistics.median(known) if known else 1.0 - # Group anything sharing an expensive setup_cache, so it is paid once. - units = {} - for name, benchmark in benchmarks.items(): - key = benchmark.get("setup_cache_key") - unit = name if _splittable(key) else f"setup_cache:{key}" - units.setdefault(unit, []).append(name) + units = _units(benchmarks) costs = { unit: sum(weights.get(name, default) for name in names) From 8428e1d68168b8aea3b95d3c397c35a5e554d9d6 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 28 Aug 2026 13:46:25 -0500 Subject: [PATCH 32/52] Load balancing and removing thread sweeps --- benchmarks/asv.conf.hpc.json | 17 +++++++++++++++- benchmarks/asv.conf.json | 15 +++++++++++++- benchmarks/helpers/_partition.py | 35 ++++++++++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/benchmarks/asv.conf.hpc.json b/benchmarks/asv.conf.hpc.json index ef14c05a7..62f3d9bcb 100644 --- a/benchmarks/asv.conf.hpc.json +++ b/benchmarks/asv.conf.hpc.json @@ -60,9 +60,24 @@ // its physical core count is the usual shape, and // ``python -m benchmarks.helpers._threads`` reports that count. // Values above the core count are allowed and simply oversubscribe. + // One value each: a matrix entry with several values is a separate + // environment per value, and asv runs the whole suite in every one of + // them. Only 24 of the 86 benchmarks reach a ``parallel=True`` kernel, + // so a four-value sweep ran the other 62 four times over to print the + // same number four times. ``NUMBA_NUM_THREADS`` is the one knob here; + // to sweep it, see ``--env`` in ``benchmarks/helpers/_partition.py``. + // + // The BLAS variables are pinned because they are what made the old + // sweep unreadable: numpy would take the whole node in every + // environment, so the ``NUMBA_NUM_THREADS=1`` column was never a + // single-threaded baseline and the scaling curve it implied was + // measuring contention. Pinned, numba's threads are the only ones. "env_nobuild": { "NUMBA_THREADING_LAYER": ["forksafe"], - "NUMBA_NUM_THREADS": ["1", "2", "4", "8"] + "NUMBA_NUM_THREADS": ["8"], + "OMP_NUM_THREADS": ["1"], + "MKL_NUM_THREADS": ["1"], + "OPENBLAS_NUM_THREADS": ["1"] } } } diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 59b9954c3..57114b6ad 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -122,7 +122,20 @@ // Belt to that brace: if TBB is ever unavailable, pick the other // fork-safe layer rather than quietly falling back to the one that // breaks. ``forksafe`` raises if no fork-safe layer exists at all. - "env_nobuild": {"NUMBA_THREADING_LAYER": ["forksafe"]} + // + // The BLAS variables are pinned so numpy cannot take the runner's four + // vCPUs out from under whatever is being measured. Unpinned it competes + // with numba for the same cores, which shows up as run-to-run spread in + // every benchmark that touches an array, not just the threaded ones. + // ``NUMBA_NUM_THREADS`` is deliberately left alone: numba's default is + // the whole machine, and with BLAS out of the way that is now an + // unambiguous four rather than four contended with numpy's. + "env_nobuild": { + "NUMBA_THREADING_LAYER": ["forksafe"], + "OMP_NUM_THREADS": ["1"], + "MKL_NUM_THREADS": ["1"], + "OPENBLAS_NUM_THREADS": ["1"] + } }, diff --git a/benchmarks/helpers/_partition.py b/benchmarks/helpers/_partition.py index 57c3a7dd0..f24e75d3e 100644 --- a/benchmarks/helpers/_partition.py +++ b/benchmarks/helpers/_partition.py @@ -43,6 +43,13 @@ python -m benchmarks.helpers._partition --shards 4 asv run $(python -m benchmarks.helpers._partition --shards 4 --shard 0 \ --config asv.conf.hpc.json --asv-args) + + # A thread sweep, for whoever wants one. ``--shards 1`` is the whole suite; + # add ``--bench`` to hold it to the benchmarks that can actually respond. + for n in 1 2 4 8; do + asv run $(python -m benchmarks.helpers._partition --shards 1 --shard 0 \ + --config asv.conf.hpc.json --env NUMBA_NUM_THREADS=$n --asv-args) + done """ import argparse @@ -227,9 +234,13 @@ def shard_results_dir(results_dir, shard): return f"{results_dir}.shard{shard}" -def write_shard_config(base_config, out_path, shard): +def write_shard_config(base_config, out_path, shard, env=None): """Writes a copy of ``base_config`` that writes results where ``shard`` should. + ``env`` overrides ``env_nobuild`` variables. An override lands in the + environment's name, so asv files those results under a name of their own and + a sweep's runs do not overwrite one another. + Returns the shard's results directory. """ # asv's loader, because an asv config is JSON with javascript comments and @@ -240,6 +251,11 @@ def write_shard_config(base_config, out_path, shard): config = util.load_json(str(base_config), js_comments=True) results_dir = shard_results_dir(config.get("results_dir", "results"), shard) config["results_dir"] = results_dir + if env: + matrix = config.setdefault("matrix", {}).setdefault("env_nobuild", {}) + # One value per variable: a list of several is a separate environment + # per value, and asv would run the whole suite in each of them. + matrix.update({key: [value] for key, value in env.items()}) with open(out_path, "w") as handle: json.dump(config, handle, indent=4) return results_dir @@ -296,6 +312,15 @@ def main(argv=None): default=None, help="Where --asv-args writes the shard config (default: beside --config).", ) + parser.add_argument( + "--env", + action="append", + default=[], + metavar="KEY=VALUE", + help="Override an env_nobuild variable in the generated config. Repeatable; " + "use it to sweep a variable the config pins to one value, e.g. " + "--env NUMBA_NUM_THREADS=4.", + ) parser.add_argument( "--asv-args", action="store_true", @@ -317,8 +342,14 @@ def main(argv=None): # told which benchmarks to run has to be told where to put them, # and splitting that across two commands is two chances to pass # one shard's benchmarks with another's results directory. + env = {} + for entry in args.env: + key, sep, value = entry.partition("=") + if not sep: + parser.error(f"--env wants KEY=VALUE, got {entry!r}") + env[key] = value config_out = args.config_out or shard_config_path(args.config, args.shard) - write_shard_config(args.config, config_out, args.shard) + write_shard_config(args.config, config_out, args.shard, env) print("--config", config_out) for pattern in bench_regexes(shards[args.shard]): print("--bench", pattern) From 6bcf1e7124e4ab1d7829df6bc4c8fa26625fdc69 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 28 Aug 2026 14:03:50 -0500 Subject: [PATCH 33/52] HPC benchmark scipts --- benchmarks/helpers/_machine.py | 28 +++++++---- benchmarks/hpc/local.sh | 86 ++++++++++++++++++++++++++++++++++ benchmarks/hpc/stage.pbs | 82 ++++++++++++++++++++++++++++++++ benchmarks/hpc/submit.sh | 84 +++++++++++++++++++++++++++++++++ 4 files changed, 270 insertions(+), 10 deletions(-) create mode 100755 benchmarks/hpc/local.sh create mode 100644 benchmarks/hpc/stage.pbs create mode 100755 benchmarks/hpc/submit.sh diff --git a/benchmarks/helpers/_machine.py b/benchmarks/helpers/_machine.py index 5d377fffc..def418b29 100644 --- a/benchmarks/helpers/_machine.py +++ b/benchmarks/helpers/_machine.py @@ -40,12 +40,16 @@ def default_path(): def pin(name, path=None): - """Renames the machine file's single entry to ``name``. Returns its details. + """Renames the machine file's freshly detected entry to ``name``. - Raises if there is more than one entry and none of them is ``name`` already: - with several to choose from there is no way to tell which one describes the - machine this is running on, and picking wrong would label the results with - another machine's hardware. + Returns its details. Entries for other machines are left alone -- a runner + has only the one, but a laptop that has recorded a couple should not lose + them to a benchmark run. + + Raises if there is more than one entry and none is ``name`` already: with + several to choose from there is no telling which describes the machine this + is running on, and picking wrong would label the results with another + machine's hardware. """ path = Path(path) if path is not None else default_path() stored = json.loads(path.read_text()) @@ -54,18 +58,22 @@ def pin(name, path=None): if name in stored: detected = stored[name] elif len(stored) == 1: - (detected,) = stored.values() + # Rename it: the old key was the hostname, which is what we are here to + # stop the results being filed under. + (old_name,) = stored + detected = stored.pop(old_name) else: raise ValueError( f"{path} holds {len(stored)} machines ({', '.join(sorted(stored))}) and none " - f"is {name!r}; cannot tell which describes this machine" + f"is {name!r}; cannot tell which describes this machine. Pass --name " + f"one of them, or delete the file and let ``asv machine --yes`` rebuild it" ) detected["machine"] = name - merged = {name: detected} + stored[name] = detected if version is not None: - merged[_VERSION_KEY] = version - path.write_text(json.dumps(merged, indent=4)) + stored[_VERSION_KEY] = version + path.write_text(json.dumps(stored, indent=4)) return detected diff --git a/benchmarks/hpc/local.sh b/benchmarks/hpc/local.sh new file mode 100755 index 000000000..96cb410dd --- /dev/null +++ b/benchmarks/hpc/local.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Runs the sharded suite on the node you are already sitting on -- an +# interactive PBS session, typically -- with the shards concurrent rather than +# queued as separate jobs. +# +# This only makes sense because a derecho CPU node has 128 cores and a shard at +# NUMBA_NUM_THREADS=8 wants nine of them. Each shard is pinned to its own slice +# so they cannot land on each other's cores; what they do still share is memory +# bandwidth and last-level cache, and for the grid operations in this suite that +# is not nothing. So: the BASE-vs-HEAD ratios ``asv compare`` reports stay +# usable, since both sides of a comparison run inside the same shard under the +# same contention, but absolute timings come out noisier than a run that had a +# node to itself. Take those from one-shard-per-node (``submit.sh``). +# +# Usage, from the repository root: +# +# ./benchmarks/hpc/local.sh +# SHARDS=8 THREADS=4 ./benchmarks/hpc/local.sh +# REV=main^! ./benchmarks/hpc/local.sh +# +set -euo pipefail + +REPO="${REPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" +SHARDS="${SHARDS:-4}" +THREADS="${THREADS:-8}" +export REPO SHARDS THREADS +export REV="${REV:-HEAD^!}" +export CONFIG="${CONFIG:-asv.conf.hpc.json}" +export ASV_MACHINE="${ASV_MACHINE:-derecho}" +export ASV_ACTIVATE="${ASV_ACTIVATE:-true}" + +STAGE_SCRIPT="$REPO/benchmarks/hpc/stage.pbs" +LOGS="${LOGS:-$REPO/benchmarks/hpc/logs}" +mkdir -p "$LOGS" + +# NCPUS is what PBS sets for the resources this session actually holds; nproc +# would count the whole node even where the job was given a slice of it. +CORES="${NCPUS:-$(nproc)}" +PER=$((CORES / SHARDS)) +if [ "$PER" -lt "$((THREADS + 1))" ]; then + echo "warning: $CORES cores over $SHARDS shards is $PER each, under the" >&2 + echo " $THREADS threads a shard wants; they will oversubscribe" >&2 +fi + +PIN="" +if command -v taskset >/dev/null; then + PIN="taskset" +else + echo "warning: no taskset, shards will not be pinned and will drift across cores" >&2 +fi + +echo "== setup ==" +STAGE=setup bash "$STAGE_SCRIPT" 2>&1 | tee "$LOGS/setup.log" + +echo "== $SHARDS shards, $PER cores each, $THREADS threads each ==" +pids=() +for S in $(seq 0 $((SHARDS - 1))); do + lo=$((S * PER)) + hi=$((lo + PER - 1)) + if [ -n "$PIN" ]; then + SHARD=$S STAGE=shard taskset -c "$lo-$hi" bash "$STAGE_SCRIPT" \ + >"$LOGS/shard$S.log" 2>&1 & + else + SHARD=$S STAGE=shard bash "$STAGE_SCRIPT" >"$LOGS/shard$S.log" 2>&1 & + fi + pid=$! + pids+=("$pid") + echo " shard $S -> cores $lo-$hi, pid $pid, log $LOGS/shard$S.log" +done + +# Every shard is waited on and its status reported, but a failure does not stop +# the merge: a tree missing one shard's rows is still worth having, same as the +# ``afteranyarray`` dependency the PBS path uses. +failed=0 +for S in $(seq 0 $((SHARDS - 1))); do + if wait "${pids[$S]}"; then + echo " shard $S ok" + else + echo " shard $S FAILED (see $LOGS/shard$S.log)" >&2 + failed=$((failed + 1)) + fi +done + +echo "== merge ==" +STAGE=merge bash "$STAGE_SCRIPT" 2>&1 | tee "$LOGS/merge.log" +[ "$failed" -eq 0 ] || echo "$failed shard(s) failed; merged what landed" >&2 diff --git a/benchmarks/hpc/stage.pbs b/benchmarks/hpc/stage.pbs new file mode 100644 index 000000000..6ed6fc47d --- /dev/null +++ b/benchmarks/hpc/stage.pbs @@ -0,0 +1,82 @@ +#!/bin/bash +# One script, three stages of a sharded asv run. Submitted by ``submit.sh``, +# which sets STAGE and the rest of the environment; not meant to be qsub'd by +# hand. Defaults here are only so a stage is runnable outside PBS for debugging. +# +#PBS -N asv +#PBS -j oe +#PBS -l select=1:ncpus=128 +#PBS -l walltime=12:00:00 + +set -euo pipefail + +STAGE="${STAGE:?STAGE must be setup, shard or merge}" +SHARDS="${SHARDS:-4}" +REV="${REV:-HEAD^!}" +REPO="${REPO:?REPO must be the repository root}" +CONFIG="${CONFIG:-asv.conf.hpc.json}" +# One name for every shard: asv files results under +# ``results//-.json``, so shards that disagree leave +# nothing for the merge to line up. +ASV_MACHINE="${ASV_MACHINE:-derecho}" +# Shared, so the shards read the dyamond grids off campaign storage once +# between them rather than once each; the setup stage fills it. Optional, +# because running the stages locally has nothing to share and +# ``_fixtures.cache_dir`` already defaults to the checkout. +if [ -n "${UXARRAY_BENCH_CACHE_DIR:-}" ]; then + export UXARRAY_BENCH_CACHE_DIR +fi + +eval "${ASV_ACTIVATE:-true}" +cd "$REPO/benchmarks" + +case "$STAGE" in +setup) + # Everything that must happen exactly once, because every shard shares the + # filesystem it happens on: the fixture cache, the machine file, and the + # asv environments plus the wheel. Four shards building into one + # ``env/`` concurrently is the race this stage exists to prevent. + (cd .. && python -m benchmarks.helpers._fixtures) + asv machine --yes + (cd .. && python -m benchmarks.helpers._machine --name "$ASV_MACHINE") + # asv's own discovery-only mode: creates the environments, builds the + # project, writes results/benchmarks.json, runs no benchmark. The four + # NUMBA_NUM_THREADS environments share one build directory -- ``env_nobuild`` + # variables are omitted from the name ``dir_name`` hashes -- so this one + # build serves all of them. + asv run --bench just-discover "$REV" + PYTHONPATH=.. python -m benchmarks.helpers._partition --shards "$SHARDS" + ;; +shard) + SHARD="${PBS_ARRAY_INDEX:-${SHARD:?}}" + # ``--asv-args`` writes this shard's config -- a results_dir of its own, + # plus any --env override -- and prints it with the --bench patterns. + # THREADS is optional: unset, the config's own NUMBA_NUM_THREADS stands. + # Set, it overrides it, and lands in the environment name, so a run at one + # thread count files its results separately from a run at another. + THREAD_ARG="" + if [ -n "${THREADS:-}" ]; then + THREAD_ARG="--env NUMBA_NUM_THREADS=$THREADS" + fi + ASV_ARGS=$(PYTHONPATH=.. python -m benchmarks.helpers._partition \ + --shards "$SHARDS" --shard "$SHARD" --config "$CONFIG" $THREAD_ARG --asv-args) + status=0 + asv run --show-stderr -m "$ASV_MACHINE" $ASV_ARGS "$REV" || status=$? + # 2 is "a benchmark failed", not "the shard broke": its other results are + # complete and the merge should still get them. Anything else is real. + if [ "$status" -eq 2 ]; then + echo "asv exited 2: some benchmarks failed, see above" >&2 + elif [ "$status" -ne 0 ]; then + exit "$status" + fi + ;; +merge) + (cd .. && python -m benchmarks.helpers._merge \ + --out benchmarks/results benchmarks/results.shard*) + asv show -m "$ASV_MACHINE" || true + ;; +*) + echo "unknown STAGE $STAGE" >&2 + exit 2 + ;; +esac diff --git a/benchmarks/hpc/submit.sh b/benchmarks/hpc/submit.sh new file mode 100755 index 000000000..8c7bd7699 --- /dev/null +++ b/benchmarks/hpc/submit.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Submits a sharded asv run on derecho as three chained PBS jobs. +# +# setup one node, once: fixture cache, machine file, asv environments and +# the wheel, then the shard plan. Everything the shards would +# otherwise race each other to create on the filesystem they share. +# shards a job array, one node each, exclusive. A ``time_*`` result is only +# worth having if nothing else is competing for the machine, so one +# shard per node rather than several. +# merge one node, once: combines the shards' results directories and shows +# the run. +# +# The merge depends on ``afteranyarray`` rather than ``afterokarray`` on +# purpose: a shard that fails still leaves results worth merging, and a suite +# with one long-broken benchmark should not cost you the other eighty. +# +# Usage: +# +# PBS_ACCOUNT=UXXX0001 ./benchmarks/hpc/submit.sh +# PBS_ACCOUNT=UXXX0001 THREADS=8 ./benchmarks/hpc/submit.sh +# PBS_ACCOUNT=UXXX0001 SHARDS=8 REV=main^! ./benchmarks/hpc/submit.sh +# +# THREADS overrides the config's NUMBA_NUM_THREADS and is recorded in the +# environment name, so runs at different thread counts do not overwrite each +# other's results. Leave it unset to take the config's value. +# +set -euo pipefail + +: "${PBS_ACCOUNT:?set PBS_ACCOUNT to your project code, e.g. PBS_ACCOUNT=UXXX0001}" + +REPO="${REPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" +SHARDS="${SHARDS:-4}" +# A single commit by default. ``asv run`` takes a range, so ``main^!`` is main's +# tip alone and ``base..head`` is every commit between. +REV="${REV:-HEAD^!}" +QUEUE="${QUEUE:-main}" +CONFIG="${CONFIG:-asv.conf.hpc.json}" +ASV_MACHINE="${ASV_MACHINE:-derecho}" +WALLTIME="${WALLTIME:-12:00:00}" +SETUP_WALLTIME="${SETUP_WALLTIME:-02:00:00}" +# Off /glade/derecho/scratch so every shard shares one fixture cache. The +# default in ``_fixtures.cache_dir`` follows the checkout, which would give a +# second working tree its own empty cache and re-read every source. +CACHE_DIR="${UXARRAY_BENCH_CACHE_DIR:-/glade/derecho/scratch/$USER/uxarray-bench}" +# Submit from a shell that already has ``asv`` on PATH -- ``-V`` below carries +# that environment into all three jobs, which is both simpler and less brittle +# than reactivating inside them. Set ASV_ACTIVATE only if you would rather the +# jobs do it themselves; it is eval'd once per stage. +export ASV_ACTIVATE="${ASV_ACTIVATE:-true}" +command -v asv >/dev/null || [ "$ASV_ACTIVATE" != "true" ] || { + echo "asv is not on PATH; activate your environment first, or set ASV_ACTIVATE" >&2 + exit 1 +} + +STAGE_SCRIPT="$REPO/benchmarks/hpc/stage.pbs" +# Passed through the environment rather than in ``-v``, whose value list is +# comma-separated and so cannot hold an activation command or a path with a +# comma in it. +export REPO SHARDS REV CONFIG ASV_MACHINE +export THREADS="${THREADS:-}" +export UXARRAY_BENCH_CACHE_DIR="$CACHE_DIR" + +mkdir -p "$CACHE_DIR" + +setup=$(qsub -A "$PBS_ACCOUNT" -q "$QUEUE" -N asv-setup \ + -l select=1:ncpus=128 -l walltime="$SETUP_WALLTIME" \ + -V -v "STAGE=setup" "$STAGE_SCRIPT") +echo "setup $setup" + +shards=$(qsub -A "$PBS_ACCOUNT" -q "$QUEUE" -N asv-shard \ + -J "0-$((SHARDS - 1))" \ + -l select=1:ncpus=128 -l walltime="$WALLTIME" \ + -W "depend=afterok:$setup" \ + -V -v "STAGE=shard" "$STAGE_SCRIPT") +echo "shards $shards ($SHARDS of them)" + +merge=$(qsub -A "$PBS_ACCOUNT" -q "$QUEUE" -N asv-merge \ + -l select=1:ncpus=1 -l walltime=00:30:00 \ + -W "depend=afteranyarray:$shards" \ + -V -v "STAGE=merge" "$STAGE_SCRIPT") +echo "merge $merge" +echo +echo "watch with: qstat -u $USER -t" +echo "results in: $REPO/benchmarks/results/$ASV_MACHINE/" From 63dfedbdd56ea0e0dca37874a711b64eeb4f08cc Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 28 Aug 2026 14:12:55 -0500 Subject: [PATCH 34/52] HPC bugs --- benchmarks/helpers/_machine.py | 42 ++++++++++++++++++++-------------- benchmarks/hpc/local.sh | 9 +++++--- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/benchmarks/helpers/_machine.py b/benchmarks/helpers/_machine.py index def418b29..1ecc1f6a6 100644 --- a/benchmarks/helpers/_machine.py +++ b/benchmarks/helpers/_machine.py @@ -26,6 +26,7 @@ import argparse import json +import platform import sys from pathlib import Path @@ -39,34 +40,38 @@ def default_path(): return Path.home() / ".asv-machine.json" -def pin(name, path=None): +def pin(name, path=None, hostname=None): """Renames the machine file's freshly detected entry to ``name``. Returns its details. Entries for other machines are left alone -- a runner - has only the one, but a laptop that has recorded a couple should not lose - them to a benchmark run. - - Raises if there is more than one entry and none is ``name`` already: with - several to choose from there is no telling which describes the machine this - is running on, and picking wrong would label the results with another - machine's hardware. + has only the one, but a login node that has recorded every compute node it + ever landed on should not lose them to a benchmark run. + + The fresh entry is the one keyed by this host's name, since that is what + ``asv machine --yes`` writes (``Machine.get_defaults`` takes it from + ``platform.uname``). Renaming it is the whole point: several nodes of one + cluster should file their results under one machine, or a sharded run has + nothing to merge. Falls back to a lone entry whatever its name, for a runner + whose hostname has already been renamed away by an earlier call. """ path = Path(path) if path is not None else default_path() + hostname = hostname if hostname is not None else platform.node() stored = json.loads(path.read_text()) version = stored.pop(_VERSION_KEY, None) - if name in stored: + if hostname in stored: + detected = stored.pop(hostname) + elif name in stored: detected = stored[name] elif len(stored) == 1: - # Rename it: the old key was the hostname, which is what we are here to - # stop the results being filed under. - (old_name,) = stored - detected = stored.pop(old_name) + (only,) = stored + detected = stored.pop(only) else: raise ValueError( - f"{path} holds {len(stored)} machines ({', '.join(sorted(stored))}) and none " - f"is {name!r}; cannot tell which describes this machine. Pass --name " - f"one of them, or delete the file and let ``asv machine --yes`` rebuild it" + f"{path} holds {len(stored)} machines ({', '.join(sorted(stored))}), none of " + f"them this host ({hostname!r}) and none of them {name!r}; cannot tell which " + f"describes the machine this is running on. Run ``asv machine --yes`` first, " + f"or pass --name one of the recorded machines" ) detected["machine"] = name @@ -84,9 +89,12 @@ def main(argv=None): ) parser.add_argument("--name", required=True, help="Machine name to pin to.") parser.add_argument("--path", default=None, help="Machine file (default ~/.asv-machine.json).") + parser.add_argument( + "--hostname", default=None, help="Host whose entry to rename (default this one)." + ) args = parser.parse_args(argv) - detected = pin(args.name, args.path) + detected = pin(args.name, args.path, args.hostname) print( f"{args.name}: {detected.get('cpu', '?')} " f"({detected.get('num_cpu', '?')} cpu, {detected.get('os', '?')})" diff --git a/benchmarks/hpc/local.sh b/benchmarks/hpc/local.sh index 96cb410dd..02a43fc99 100755 --- a/benchmarks/hpc/local.sh +++ b/benchmarks/hpc/local.sh @@ -33,9 +33,12 @@ STAGE_SCRIPT="$REPO/benchmarks/hpc/stage.pbs" LOGS="${LOGS:-$REPO/benchmarks/hpc/logs}" mkdir -p "$LOGS" -# NCPUS is what PBS sets for the resources this session actually holds; nproc -# would count the whole node even where the job was given a slice of it. -CORES="${NCPUS:-$(nproc)}" +# ``nproc`` rather than PBS's NCPUS, which is the ncpus *requested per chunk* -- +# 1 for a plain ``qsub -I`` -- and says nothing about what the node will let this +# session use. ``nproc`` reports the CPUs actually available to this process, +# honouring any affinity mask or cpuset the job was given. Override with CORES +# to hold the run to fewer. +CORES="${CORES:-$(nproc)}" PER=$((CORES / SHARDS)) if [ "$PER" -lt "$((THREADS + 1))" ]; then echo "warning: $CORES cores over $SHARDS shards is $PER each, under the" >&2 From 48038946e9cdda1d3b89823e678add06e3bb6595 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 28 Aug 2026 15:45:00 -0500 Subject: [PATCH 35/52] Benchmark machine names --- benchmarks/helpers/_machine.py | 68 ++++++++++++++++++++++++++++++++-- benchmarks/hpc/local.sh | 4 +- benchmarks/hpc/stage.pbs | 6 ++- benchmarks/hpc/submit.sh | 3 +- 4 files changed, 73 insertions(+), 8 deletions(-) diff --git a/benchmarks/helpers/_machine.py b/benchmarks/helpers/_machine.py index 1ecc1f6a6..439ae0205 100644 --- a/benchmarks/helpers/_machine.py +++ b/benchmarks/helpers/_machine.py @@ -26,7 +26,9 @@ import argparse import json +import os import platform +import re import sys from pathlib import Path @@ -35,12 +37,34 @@ _VERSION_KEY = "version" +def default_name(): + """A machine name that survives landing on a different node next time. + + The scheduler puts you on ``derecho3`` one day and ``crhtc70`` the next, and + asv keys results on ``platform.uname``'s node name, so left alone it records + a new machine every login and the results scatter across all of them. + + ``NCAR_HOST`` is the reliable answer where it is set -- it names the cluster + rather than the node, which is the granularity results want. Failing that, + the node name with its trailing digits removed, which folds ``derecho3`` and + ``derecho5`` together but *not* ``derecho3`` and ``crhtc70``: login and + compute nodes of one cluster do not share a stem. Set ``ASV_MACHINE`` + yourself if you move between them without ``NCAR_HOST``. + """ + for variable in ("ASV_MACHINE", "NCAR_HOST"): + value = os.environ.get(variable) + if value: + return value, variable + node = platform.node().split(".")[0] + return re.sub(r"[-_]?\d+$", "", node) or node, None + + def default_path(): """Where asv keeps its machine file (``MachineCollection.get_machine_file_path``).""" return Path.home() / ".asv-machine.json" -def pin(name, path=None, hostname=None): +def pin(name, path=None, hostname=None, sole=False): """Renames the machine file's freshly detected entry to ``name``. Returns its details. Entries for other machines are left alone -- a runner @@ -75,6 +99,8 @@ def pin(name, path=None, hostname=None): ) detected["machine"] = name + if sole: + stored = {} stored[name] = detected if version is not None: stored[_VERSION_KEY] = version @@ -87,18 +113,52 @@ def main(argv=None): prog="python -m benchmarks.helpers._machine", description="Rename asv's detected machine entry to a fixed name.", ) - parser.add_argument("--name", required=True, help="Machine name to pin to.") + parser.add_argument( + "--name", + default=None, + help="Machine name to pin to. Defaults to $ASV_MACHINE, then $NCAR_HOST, " + "then this host's name with trailing digits removed.", + ) parser.add_argument("--path", default=None, help="Machine file (default ~/.asv-machine.json).") parser.add_argument( "--hostname", default=None, help="Host whose entry to rename (default this one)." ) + parser.add_argument( + "--sole", + action="store_true", + help="Drop every other machine from the file. asv falls back to a lone entry " + "whatever the hostname, so this makes bare ``asv run``/``asv show`` work from " + "any node without -m. Use it where you only ever benchmark one machine.", + ) + parser.add_argument( + "--quiet", action="store_true", help="Print only the pinned name, for capturing." + ) + parser.add_argument( + "--print", + dest="print_only", + action="store_true", + help="Print the name that would be pinned and change nothing.", + ) args = parser.parse_args(argv) - detected = pin(args.name, args.path, args.hostname) + name, source = (args.name, "--name") if args.name else default_name() + if args.print_only: + print(name) + return 0 + detected = pin(name, args.path, args.hostname, sole=args.sole) + if args.quiet: + print(name) + return 0 print( - f"{args.name}: {detected.get('cpu', '?')} " + f"{name}: {detected.get('cpu', '?')} " f"({detected.get('num_cpu', '?')} cpu, {detected.get('os', '?')})" ) + if source is None: + print( + f" note: {name!r} came from this host's name. A cluster's login and compute " + f"nodes do not share a stem, so set ASV_MACHINE (or rely on NCAR_HOST) if you " + f"benchmark from both, or the results will still split in two.", + ) return 0 diff --git a/benchmarks/hpc/local.sh b/benchmarks/hpc/local.sh index 02a43fc99..b50263340 100755 --- a/benchmarks/hpc/local.sh +++ b/benchmarks/hpc/local.sh @@ -26,7 +26,9 @@ THREADS="${THREADS:-8}" export REPO SHARDS THREADS export REV="${REV:-HEAD^!}" export CONFIG="${CONFIG:-asv.conf.hpc.json}" -export ASV_MACHINE="${ASV_MACHINE:-derecho}" +# Left empty on purpose: stage.pbs derives it, so this works unchanged on +# derecho and casper both. +export ASV_MACHINE="${ASV_MACHINE:-}" export ASV_ACTIVATE="${ASV_ACTIVATE:-true}" STAGE_SCRIPT="$REPO/benchmarks/hpc/stage.pbs" diff --git a/benchmarks/hpc/stage.pbs b/benchmarks/hpc/stage.pbs index 6ed6fc47d..13b8569dc 100644 --- a/benchmarks/hpc/stage.pbs +++ b/benchmarks/hpc/stage.pbs @@ -17,8 +17,10 @@ REPO="${REPO:?REPO must be the repository root}" CONFIG="${CONFIG:-asv.conf.hpc.json}" # One name for every shard: asv files results under # ``results//-.json``, so shards that disagree leave -# nothing for the merge to line up. -ASV_MACHINE="${ASV_MACHINE:-derecho}" +# nothing for the merge to line up. Derived rather than assumed, because the +# scheduler hands you a different node each login and asv would otherwise +# record each one as a machine of its own -- see ``_machine.default_name``. +ASV_MACHINE="${ASV_MACHINE:-$(cd "$REPO" && python -m benchmarks.helpers._machine --print)}" # Shared, so the shards read the dyamond grids off campaign storage once # between them rather than once each; the setup stage fills it. Optional, # because running the stages locally has nothing to share and diff --git a/benchmarks/hpc/submit.sh b/benchmarks/hpc/submit.sh index 8c7bd7699..801c71f59 100755 --- a/benchmarks/hpc/submit.sh +++ b/benchmarks/hpc/submit.sh @@ -35,7 +35,8 @@ SHARDS="${SHARDS:-4}" REV="${REV:-HEAD^!}" QUEUE="${QUEUE:-main}" CONFIG="${CONFIG:-asv.conf.hpc.json}" -ASV_MACHINE="${ASV_MACHINE:-derecho}" +# Empty means stage.pbs derives it from NCAR_HOST or the node name. +ASV_MACHINE="${ASV_MACHINE:-}" WALLTIME="${WALLTIME:-12:00:00}" SETUP_WALLTIME="${SETUP_WALLTIME:-02:00:00}" # Off /glade/derecho/scratch so every shard shares one fixture cache. The From 6a4882dedea16fb78f064ee19cfd239097643867 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 28 Aug 2026 15:49:00 -0500 Subject: [PATCH 36/52] Thread/core numbers for benchmarks --- benchmarks/hpc/local.sh | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/benchmarks/hpc/local.sh b/benchmarks/hpc/local.sh index b50263340..52a62878b 100755 --- a/benchmarks/hpc/local.sh +++ b/benchmarks/hpc/local.sh @@ -30,17 +30,34 @@ export CONFIG="${CONFIG:-asv.conf.hpc.json}" # derecho and casper both. export ASV_MACHINE="${ASV_MACHINE:-}" export ASV_ACTIVATE="${ASV_ACTIVATE:-true}" +# Also evaluated here, not just in the stages, so the core count below can be +# read with python rather than with a shell tool that lies about it. +eval "$ASV_ACTIVATE" STAGE_SCRIPT="$REPO/benchmarks/hpc/stage.pbs" LOGS="${LOGS:-$REPO/benchmarks/hpc/logs}" mkdir -p "$LOGS" -# ``nproc`` rather than PBS's NCPUS, which is the ncpus *requested per chunk* -- -# 1 for a plain ``qsub -I`` -- and says nothing about what the node will let this -# session use. ``nproc`` reports the CPUs actually available to this process, -# honouring any affinity mask or cpuset the job was given. Override with CORES -# to hold the run to fewer. -CORES="${CORES:-$(nproc)}" +# Neither PBS's NCPUS nor ``nproc`` can be trusted for this. NCPUS is the ncpus +# *requested per chunk*, 1 for a plain ``qsub -I``, and says nothing about the +# node. And GNU ``nproc`` honours OMP_NUM_THREADS and OMP_THREAD_LIMIT, so in a +# session that sets either it reports the OpenMP thread limit -- 1, or 2 -- and +# not the machine's cores at all. +# +# The affinity mask is the real answer: the CPUs this process may actually run +# on. It respects a cpuset the scheduler imposed and ignores OpenMP entirely. +# Override with CORES to hold the run to fewer than the mask allows. +CORES="${CORES:-$(python -c ' +import os +try: + print(len(os.sched_getaffinity(0))) +except AttributeError: # not Linux + print(os.cpu_count() or 1) +')}" +if ! [ "$CORES" -ge 1 ] 2>/dev/null; then + echo "could not work out a core count (got ${CORES:-empty}); set CORES" >&2 + exit 1 +fi PER=$((CORES / SHARDS)) if [ "$PER" -lt "$((THREADS + 1))" ]; then echo "warning: $CORES cores over $SHARDS shards is $PER each, under the" >&2 From cd8d5ed4233ae326b3479ea9221f9ef303a65def Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 28 Aug 2026 16:04:28 -0500 Subject: [PATCH 37/52] ASV machine bug --- benchmarks/hpc/stage.pbs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/benchmarks/hpc/stage.pbs b/benchmarks/hpc/stage.pbs index 13b8569dc..305e1db15 100644 --- a/benchmarks/hpc/stage.pbs +++ b/benchmarks/hpc/stage.pbs @@ -46,7 +46,13 @@ setup) # NUMBA_NUM_THREADS environments share one build directory -- ``env_nobuild`` # variables are omitted from the name ``dir_name`` hashes -- so this one # build serves all of them. - asv run --bench just-discover "$REV" + # ``-m`` on every asv call from here on. The pin above renamed this host's + # entry to $ASV_MACHINE, so the hostname asv would otherwise look itself up + # under no longer exists in the machine file -- and once that file holds + # more than one machine, asv's fall-back to a lone entry does not apply + # either. Without it this fails with "No information stored about machine + # ''" immediately after the pin reports success. + asv run --bench just-discover -m "$ASV_MACHINE" "$REV" PYTHONPATH=.. python -m benchmarks.helpers._partition --shards "$SHARDS" ;; shard) From 98e9c9dea577e67ae058b98b4100c786ca90582a Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 28 Aug 2026 16:30:11 -0500 Subject: [PATCH 38/52] benchmark shard bug --- benchmarks/hpc/stage.pbs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/benchmarks/hpc/stage.pbs b/benchmarks/hpc/stage.pbs index 305e1db15..1d2c1074d 100644 --- a/benchmarks/hpc/stage.pbs +++ b/benchmarks/hpc/stage.pbs @@ -39,20 +39,28 @@ setup) # asv environments plus the wheel. Four shards building into one # ``env/`` concurrently is the race this stage exists to prevent. (cd .. && python -m benchmarks.helpers._fixtures) - asv machine --yes + asv machine --yes --config "$CONFIG" (cd .. && python -m benchmarks.helpers._machine --name "$ASV_MACHINE") # asv's own discovery-only mode: creates the environments, builds the # project, writes results/benchmarks.json, runs no benchmark. The four # NUMBA_NUM_THREADS environments share one build directory -- ``env_nobuild`` # variables are omitted from the name ``dir_name`` hashes -- so this one # build serves all of them. + # ``--config`` on every asv call, and ``-m`` on every one after the pin. + # Without ``--config`` asv falls back to ``asv.conf.json`` in the working + # directory -- the CI config -- so this stage would discover, create the + # environment and build against a different matrix from the one the shards + # then run under. The two happen to share a ``req`` matrix, and so an + # environment directory, which is why it worked at all rather than failing; + # it just meant the pre-build was warming the wrong config's environment. + # # ``-m`` on every asv call from here on. The pin above renamed this host's # entry to $ASV_MACHINE, so the hostname asv would otherwise look itself up # under no longer exists in the machine file -- and once that file holds # more than one machine, asv's fall-back to a lone entry does not apply # either. Without it this fails with "No information stored about machine # ''" immediately after the pin reports success. - asv run --bench just-discover -m "$ASV_MACHINE" "$REV" + asv run --bench just-discover --config "$CONFIG" -m "$ASV_MACHINE" "$REV" PYTHONPATH=.. python -m benchmarks.helpers._partition --shards "$SHARDS" ;; shard) @@ -81,7 +89,7 @@ shard) merge) (cd .. && python -m benchmarks.helpers._merge \ --out benchmarks/results benchmarks/results.shard*) - asv show -m "$ASV_MACHINE" || true + asv show --config "$CONFIG" -m "$ASV_MACHINE" || true ;; *) echo "unknown STAGE $STAGE" >&2 From f417df896990425b263fab2c47b8ffee589987b6 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Thu, 17 Sep 2026 15:19:19 -0500 Subject: [PATCH 39/52] Cached benchmarks IO deslop p1 --- benchmarks/helpers/_fixtures.py | 76 +++++++++------------------------ 1 file changed, 21 insertions(+), 55 deletions(-) diff --git a/benchmarks/helpers/_fixtures.py b/benchmarks/helpers/_fixtures.py index 9ab574197..5dabed51d 100644 --- a/benchmarks/helpers/_fixtures.py +++ b/benchmarks/helpers/_fixtures.py @@ -3,22 +3,6 @@ ASV gives every benchmark its own process, so a grid opened in ``setup`` is opened once per benchmark rather than once per run. This module provides flexible access to files across benchmark runs by caching the needed files. - -Benchmarks that do measure opening a file behave as before. - -Two flavors: - -``topology`` - the three arrays ``Grid.from_topology`` needs and nothing else -``grid`` / ``dataset`` - everything the reader produced from ``Grid.open_grid`` and - ``Grid.open_dataset`` - -Artifacts are keyed on both the uxarray build and the files, because an -artifact is one version's reader output and ASV diffs commits. Likewise, there's a -fresh read per commit. ``prime`` covers every source that is readable here, -and there is a CLI (``python -m benchmarks.helpers._fixtures``) to fill the -cache from a batch script instead of from inside a benchmark. """ import hashlib @@ -58,27 +42,27 @@ BENCHMARK_DIR = Path(__file__).resolve().parents[1] REPO_DIR = BENCHMARK_DIR.parent -_COOKBOOK_MESH_URL = ( +_OQU_MESH_URL = ( "https://github.com/ProjectPythia/unstructured-grid-viz-cookbook/raw/main/meshfiles" ) -def _cookbook_mesh(filename): +def _oqu_mesh(filename): """Path to a Cookbook mesh, fetched once if this checkout lacks it.""" path = BENCHMARK_DIR / filename if not path.is_file(): - urllib.request.urlretrieve(f"{_COOKBOOK_MESH_URL}/{filename}", filename=path) + urllib.request.urlretrieve(f"{_OQU_MESH_URL}/{filename}", filename=path) return path # Grids and grid/data pairs, by mesh resolution. OQU_GRIDS = { - "480km": _cookbook_mesh("oQU480.grid.nc"), - "120km": _cookbook_mesh("oQU120.grid.nc"), + "480km": _oqu_mesh("oQU480.grid.nc"), + "120km": _oqu_mesh("oQU120.grid.nc"), } OQU_DATASETS = { - "480km": (OQU_GRIDS["480km"], _cookbook_mesh("oQU480.data.nc")), - "120km": (OQU_GRIDS["120km"], _cookbook_mesh("oQU120.data.nc")), + "480km": (OQU_GRIDS["480km"], _oqu_mesh("oQU480.data.nc")), + "120km": (OQU_GRIDS["120km"], _oqu_mesh("oQU120.data.nc")), } DYAMOND_GRIDS = { @@ -160,7 +144,7 @@ def _write(dataset, artifact_path, writer): os.replace(scratch, artifact_path) -def _read_dataset(artifact_path): +def _read_cached_dataset(artifact_path): """Reads back a cached ``xr.Dataset``.""" return xr.open_dataset(artifact_path, mask_and_scale=False).load() @@ -218,7 +202,7 @@ def _cached_grid_ds(grid_path): """The cached internal dataset of ``grid_path``, held for this process.""" artifact_path = _ensure((Path(grid_path),), "grid", ".nc") if artifact_path not in _loaded: - _loaded[artifact_path] = _read_dataset(artifact_path) + _loaded[artifact_path] = _read_cached_dataset(artifact_path) return _loaded[artifact_path] @@ -242,22 +226,24 @@ def cached_dataset(grid_path, data_path): source = (Path(grid_path), Path(data_path)) artifact_path = _ensure(source, "data", ".nc") if artifact_path not in _loaded: - _loaded[artifact_path] = _read_dataset(artifact_path) + _loaded[artifact_path] = _read_cached_dataset(artifact_path) return ux.UxDataset(_loaded[artifact_path].copy(), uxgrid=cached_grid(grid_path)) def prime(workers=1): """Fills the cache for every source the fixtures can serve. - Returns the sources it had to read. Idempotent, and once warm costs a - ``stat`` per file, so it is cheap to call ahead of every run. + Returns the sources it had to read. Idempotent, and once warm it only costs a + ``stat`` per file. ``workers`` reads that many sources at once, which is worth having when the sources differ wildly in size and live somewhere slow: the small ones finish while a larger grid is still being read, instead of queueing behind all the file fetches. Only pays off when a read is slower than a process start due to - the new interpreter startup. Processes rather than threads because the standard - netCDF/HDF5 stack is not generally thread-safe for concurrent opens. + the new interpreter overhead. + + Processes rather than threads here, because the standard netCDF/HDF5 stack is not + generally thread-safe for concurrent opens. """ sources = [(path,) for path in OQU_GRIDS.values()] sources += [(path,) for path in GRIDS_BY_FORMAT.values()] @@ -296,16 +282,8 @@ def prime(workers=1): def preload_topologies(grid_paths): - """Loads topologies here so forked benchmarks inherit them. + """Loads topologies here so forked benchmarks inherit them.""" - Reading an artifact caches it on disk, not in the next process: under - ``launch_method: forkserver`` each benchmark is forked from the interpreter - that imported the suite, so it starts with whatever *that* process holds and - reads its own copy of everything else. - - Safe to call at import: reading arrays starts no numba thread pool, which is - what a forked child cannot inherit (see :mod:`benchmarks.helpers._warmup`). - """ loaded = 0 for grid_path in grid_paths: cached_topology(grid_path) # held by the process-level memo from here on @@ -339,31 +317,19 @@ def setup_cache(self): if __name__ == "__main__": - # Fills the cache ahead of ``asv run``, so no benchmark -- and not even - # ``setup_cache`` -- pays for reading a source grid. Worth a line in a batch - # script whenever the dyamond grids are in play. + # Fills the cache ahead of ``asv run`` print(f"fixture cache: {cache_dir()}", flush=True) # Four at a time only where the dyamond grids are readable, since those are - # the reads worth overlapping: on the oQU pair alone, priming in parallel is - # slower than doing it sequentially (1.69s against 0.52s), because starting - # an interpreter costs more than reading a small local file. + # the reads worth overlapping. Priming in parallel with small files is + # slower than doing it sequentially due to overhead. for source in prime(workers=4 if DYAMOND_AVAILABLE else 1) or [None]: print(f" read {' + '.join(Path(p).name for p in source)}" if source else " nothing to do", flush=True) def warm_netcdf(): - """Brings the netCDF/HDF5 stack up here, so the forks inherit it loaded. + """Brings the netCDF/HDF5 stack up here, so the forks inherit it loaded.""" - The first ``open_dataset`` in a process spends ~65ms initializing the - library before it reads a byte, and nothing else in the parent pays that: - the topology fixtures go through numpy. So today every forked child that - touches a cached ``.nc`` artifact pays it again, one at a time. - - The file is closed again. What the children want is the loaded library, not - a handle -- an inherited HDF5 handle is shared rather than copied, and - parent and child would then read through one file offset. - """ artifacts = sorted(cache_dir().glob("*.nc"), key=lambda path: path.stat().st_size) if not artifacts: return # cache not primed yet, so the first child pays it as before From 7bb949a222801d9875dd84610e87b4d176c63aa6 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Thu, 17 Sep 2026 15:54:11 -0500 Subject: [PATCH 40/52] Cached benchmark IO deslop p2 --- benchmarks/asv.conf.json | 20 ++++++----------- benchmarks/geometry_samebody_gcagca.py | 6 ++--- benchmarks/helpers/_warmup.py | 31 ++------------------------ 3 files changed, 11 insertions(+), 46 deletions(-) diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 9b3852f53..e68aa2e74 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -62,12 +62,7 @@ "install_timeout": 600, // Fork each benchmark from one interpreter that has already imported the - // suite, instead of starting a fresh one per benchmark. Saves the ~0.9s - // uxarray import and the numba kernel load on every one of ~160 benchmark - // processes, and lets a fixture loaded at import be inherited rather than - // re-read. asv leaves this at "spawn" by default because fork and threads - // mix badly; nothing in this suite runs a parallel kernel at import time, - // which is what makes it safe here -- keep it that way. + // suite, instead of starting a fresh one per benchmark. "launch_method": "forkserver", // ``benchmark_timeout`` is not a key asv reads -- the one that sets the @@ -114,14 +109,13 @@ // forked from a parent that holds a pool then dies with "Terminating: // fork() called from a process already using GNU OpenMP". numba // *prefers* TBB, so a developer machine that has it cannot reproduce - // that -- hence pinning it here rather than inheriting whatever the - // environment happens to offer. + // that, hence pinning it here. "tbb": [""] }, // Belt to that brace: if TBB is ever unavailable, pick the other - // fork-safe layer rather than quietly falling back to the one that - // breaks. ``forksafe`` raises if no fork-safe layer exists at all. + // fork-safe layer rather. ``forksafe`` raises if no fork-safe layer + // exists at all. "env_nobuild": {"NUMBA_THREADING_LAYER": ["forksafe"]} }, @@ -195,9 +189,9 @@ // the number of builds to keep, per environment. // // Raised from the default of 2, which is exactly the number of commits - // ``asv continuous`` builds and so leaves no headroom: with rounds - // interleaved it revisits each commit, and a wheel that has been evicted - // in between is built again from scratch. + // ``asv continuous`` builds and so it leaves no headroom: with forking, + // we need an extra round per commit, so 4 is the baseline for 2 commits + // with correct caching. "build_cache_size": 4, // The commits after which the regression search in `asv publish` diff --git a/benchmarks/geometry_samebody_gcagca.py b/benchmarks/geometry_samebody_gcagca.py index adddc7b65..11364ffed 100644 --- a/benchmarks/geometry_samebody_gcagca.py +++ b/benchmarks/geometry_samebody_gcagca.py @@ -318,10 +318,8 @@ def ns(t): def _prepare(): """The packed cases, with the batched drivers warmed, once per process. - Building the cases is 4.06s of the 4.27s this used to spend in every - ``setup``; compiling the drivers is 0.08s, since they are all - ``@njit(cache=True)``. This method allows for reuse of cases in forked - benchmarks to reduce time spent on case generation. + Rebuilding cases in every asv benchmark that needs them was previously the + bulk of walltime spent on these benchmarks. """ global _prepared if _prepared is None: diff --git a/benchmarks/helpers/_warmup.py b/benchmarks/helpers/_warmup.py index ae84c10a9..c597ca9bb 100644 --- a/benchmarks/helpers/_warmup.py +++ b/benchmarks/helpers/_warmup.py @@ -3,25 +3,6 @@ Under ``launch_method: forkserver`` ASV imports the suite once and forks each benchmark from that interpreter, so whatever a module prepares at import is inherited copy-on-write. - -Warming can leave a numba thread pool behind. *Compiling* a ``parallel=True`` -kernel initializes the threading layer -- before the kernel is ever run -- and -``cache=True`` only avoids that while the on-disk cache is warm, which it is -not on a fresh runner. Whether an inherited pool is safe then depends entirely -on which layer numba picked: - -* ``tbb`` and ``workqueue`` install fork handlers and come through it intact. -* ``omp`` does not. Where that is libgomp -- Linux, unless TBB is installed -- - every forked child that touches a parallel kernel is killed with - ``Terminating: fork() called from a process already using GNU OpenMP``. - -numba prefers TBB, so a machine that has it cannot reproduce the failure at -all; the benchmark environment pins it instead (see ``tbb`` and -``NUMBA_THREADING_LAYER`` in ``asv.conf.json``). This reports when that has not -taken effect rather than raising. Raising fails the module's import in the -parent *and* again in every child that re-imports it, so it converts "these -benchmarks are slower than they could be" into "this commit produced no -results" -- and the pool exists either way, and asv forks either way. """ import os @@ -31,10 +12,7 @@ __all__ = ["warm_in_parent", "will_run_benchmarks"] -# ``benchmark.py ``: asv runs discovery, setup_cache and check as their own -# processes, and none of them goes on to run a benchmark, so warming in them is -# time bought for nobody. ``run_server`` -- the forkserver parent, the one whose -# state every benchmark inherits -- is deliberately absent. +# ``benchmark.py ``: asv runs discovery, setup_cache and check as their own processes _NON_RUNNING_MODES = frozenset({"discover", "setup_cache", "check"}) # Numba installs fork handlers for these two; ``omp`` is on its own. @@ -44,12 +22,7 @@ def will_run_benchmarks(): - """Whether this interpreter is going to run a benchmark. - - Conservative by construction: an argv this does not recognize is assumed to - be a benchmark run, so a change in how asv invokes itself costs the warm - twice over rather than losing it where it counts. - """ + """Whether this interpreter is going to run a benchmark.""" return not ( os.path.basename(sys.argv[0]) == "benchmark.py" and len(sys.argv) > 1 From 353805c2b0be892db1c4f48787b17fef4a4b655d Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Thu, 17 Sep 2026 17:28:29 -0500 Subject: [PATCH 41/52] Cached benchmark IO deslop p3 --- benchmarks/mpas_ocean.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/benchmarks/mpas_ocean.py b/benchmarks/mpas_ocean.py index cec790073..7364451a6 100644 --- a/benchmarks/mpas_ocean.py +++ b/benchmarks/mpas_ocean.py @@ -20,20 +20,14 @@ data_var = 'bottomDepth' # Sample budget for the benchmarks with long single call runtimes. -# -# Only the classes whose slowest parameter clears ~0.25s carry this; on the rest -# the cap would never bind and would cost samples for nothing. SLOW_CALL_REPEAT = (2, 3, 8.0) -# Paths, and fetching the files in the first place, both live in -# ``helpers._fixtures`` now -- ``bench_connectivity`` draws the same grids from it. file_path_dict = OQU_DATASETS class DatasetBenchmark(CachedFixtures): """Class used as a template for benchmarks requiring a ``UxDataset`` in - this module across both resolutions. - """ + this module across both resolutions.""" param_names = ['resolution', ] params = [OQU_RESOLUTIONS, ] @@ -334,10 +328,7 @@ def time_zonal_average(self, resolution): class ZonalAveragePeakMem: """Peak memory of a cold-start non-conservative zonal-mean sweep. - - A fresh interpreter per sample. The cold start is the subject, and a forked - benchmark process no longer has one. - """ + Uses a fresh interpreter per sample.""" param_names = ["resolution"] params = [OQU_RESOLUTIONS] @@ -366,9 +357,7 @@ def track_peakmem_zonal_average(self, resolution): class CrossSectionsPeakMem: """Peak memory of a cold-start constant-latitude cross-section sweep. - - A fresh interpreter per sample, - """ + Uses a fresh interpreter per sample.""" param_names = ["resolution", "lat_step"] params = [OQU_RESOLUTIONS, [1, 2, 4]] From 2739e6fd98a6d01eb46a5e2103ec7c4af87d4dd8 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Tue, 6 Oct 2026 15:45:39 -0500 Subject: [PATCH 42/52] Benchmark shards: asv run instead of asv continuous asv continuous is the same interleaved run plus a comparison, and exits 1 when the comparison finds a regression -- the status it also uses for a broken run -- so any slower benchmark failed its shard (shards 0 and 1 in run 37510066776). Shards now only measure; the merge job compares. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/asv-benchmarking-pr.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 976db9a70..315978508 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -266,6 +266,12 @@ jobs: --config asv.conf.json --asv-args) echo "Baseline: $BASE" echo "Contender: ${GITHUB_SHA} (${PR_HEAD_LABEL:-$GITHUB_REF_NAME})" + # ``asv run`` over both commits rather than ``asv continuous``: + # continuous is this same interleaved run followed by a comparison, + # and it exits 1 when that comparison finds a regression -- the same + # status it uses for a broken run -- so a slower PR would fail its + # shard. The merge job compares the whole suite; a shard only measures. + printf '%s\n' "${GITHUB_SHA}" "$BASE" > "$RUNNER_TEMP/commits.txt" # asv returns 2 when a benchmark failed rather than when the run # broke (``commands/run.py``: ``if failures: return 2``), and this # shard's other results are complete and worth merging. The old @@ -274,8 +280,8 @@ jobs: # deliberate here, loudly, rather than letting one long-broken # benchmark redden every shard that happens to hold it. status=0 - asv continuous --split --show-stderr -m "$ASV_MACHINE" $ASV_ARGS \ - "$BASE" "${GITHUB_SHA}" || status=$? + asv run --show-stderr -m "$ASV_MACHINE" $ASV_ARGS \ + "HASHFILE:$RUNNER_TEMP/commits.txt" || status=$? if [ "$status" -eq 2 ]; then echo "::warning title=Benchmark failures in shard ${SHARD}::asv exited 2; see the failed entries above" elif [ "$status" -ne 0 ]; then From cc1d85098ea65302595f859e2677d1add6699804 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Tue, 6 Oct 2026 16:44:30 -0500 Subject: [PATCH 43/52] Drop merge leftovers in gcagca and mpas_ocean benchmarks The main merge left a duplicate warm_in_parent import and a docstring-only diff against main; both files now match main. Co-Authored-By: Claude Opus 5.5 --- benchmarks/geometry_samebody_gcagca.py | 2 -- benchmarks/mpas_ocean.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/benchmarks/geometry_samebody_gcagca.py b/benchmarks/geometry_samebody_gcagca.py index c899f51ed..11364ffed 100644 --- a/benchmarks/geometry_samebody_gcagca.py +++ b/benchmarks/geometry_samebody_gcagca.py @@ -32,8 +32,6 @@ from .helpers._warmup import warm_in_parent -from .helpers._warmup import warm_in_parent - @njit(cache=True, inline="always", error_model="numpy") def _fp64_gca(w0, w1, v0, v1): diff --git a/benchmarks/mpas_ocean.py b/benchmarks/mpas_ocean.py index 442242ac3..7364451a6 100644 --- a/benchmarks/mpas_ocean.py +++ b/benchmarks/mpas_ocean.py @@ -27,8 +27,7 @@ class DatasetBenchmark(CachedFixtures): """Class used as a template for benchmarks requiring a ``UxDataset`` in - this module across both resolutions. - """ + this module across both resolutions.""" param_names = ['resolution', ] params = [OQU_RESOLUTIONS, ] From aa5a2678191efca4f7925b345d9375e29c26f9e3 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Tue, 6 Oct 2026 16:44:30 -0500 Subject: [PATCH 44/52] Condense comments and docstrings added on the benchmark sharding branch Text only: module docstrings cut to purpose, key constraints and usage; comments to one or two lines of why. Fixes two that had drifted from the code (a stale stderr claim in _threads.resolve, a contradictory matrix note in asv.conf.hpc.json) and a comment above the wheels cache that described the durations cache. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/asv-benchmarking-pr.yml | 113 +++++------------ .github/workflows/asv-benchmarking.yml | 15 +-- benchmarks/asv.conf.hpc.json | 51 ++------ benchmarks/asv.conf.json | 10 +- benchmarks/helpers/_machine.py | 67 +++------- benchmarks/helpers/_merge.py | 86 ++++--------- benchmarks/helpers/_partition.py | 141 +++++++--------------- benchmarks/helpers/_threads.py | 55 ++------- benchmarks/hpc/local.sh | 37 ++---- benchmarks/hpc/stage.pbs | 59 +++------ benchmarks/hpc/submit.sh | 38 ++---- 11 files changed, 184 insertions(+), 488 deletions(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 315978508..9dd3a4458 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -15,17 +15,14 @@ on: inputs: base_ref: description: >- - Branch, tag or commit to benchmark against. The comparison is against - its merge-base with the dispatched ref, so a topic branch should name - the branch it will merge into rather than main. + Branch, tag or commit to compare against, via its merge-base with the + dispatched ref (so name the branch a topic branch will merge into). required: false default: main shards: description: >- - How many runners to spread the suite over. Each shard pays the conda - restore, the wheel install and the ~57s the suite's numba warmups - cost at import, so past about four the added runners mostly pay that - floor again. + Runners to split the suite over. Each pays a fixed ~1 min (env + restore, install, numba warmup), so past about four gains little. required: false default: "4" @@ -33,7 +30,7 @@ env: PR_HEAD_LABEL: ${{ github.event.pull_request.head.label }} ASV_DIR: "./benchmarks" CONDA_ENV_FILE: ci/environment.yml - # One machine name for every shard. + # Shared by every shard so their results merge under one machine. ASV_MACHINE: gh-linux-x64 jobs: @@ -69,13 +66,11 @@ jobs: - name: Resolve the baseline commit id: base - # Only a pull_request event carries a pull_request payload, so on a - # manual run every one of these expressions is the empty string. + # pull_request.* expressions are empty on workflow_dispatch. run: | set -ex BASE="${{ github.event.pull_request.base.sha }}" if [ -z "$BASE" ]; then - # The merge-base rather than the base branch's tip BASE_REF="${{ github.event.inputs.base_ref }}" BASE_REF="${BASE_REF:-main}" git rev-parse --verify -q "origin/$BASE_REF" >/dev/null \ @@ -86,13 +81,12 @@ jobs: # Dispatched from main itself: compare it against the commit before. BASE=$(git rev-parse "$GITHUB_SHA^") fi - # Fail loudly rather than leaving asv to infer a baseline of its own. + # Fail rather than let asv infer a baseline of its own. test -n "$BASE" || { echo "could not determine a baseline commit" >&2; exit 1; } echo "sha=$BASE" >> "$GITHUB_OUTPUT" - # asv builds its own conda environment under ``benchmarks/env``. - # No restore-keys: asv reuses a restored env without checking it, so a - # prefix match would hand it one built from an older environment file. + # asv rebuilds its own env under benchmarks/env/ each run; caching skips the + # solve. No restore-keys: asv reuses a restored env without checking it. - name: Cache asv's benchmark environment uses: actions/cache@v6 with: @@ -109,24 +103,16 @@ jobs: restore-keys: | asv-fixtures-${{ runner.os }}- - # ``_partition`` weights each benchmark by the duration asv recorded for - # it, and falls back to the median of the ones it knows when it knows - # none -- which, with no results on a fresh runner, is every one of them, - # making the split by count. Restoring the last run's tree is what turns - # the packing back into a cost-balanced one. Restore-only here; the merge - # job saves the updated tree under a key of its own. - # The environment cache key has no commit in it -- deliberately, since the - # conda environment does not depend on one -- so on a hit ``actions/cache`` - # saves nothing at the end of the job and the wheels built above would go - # with it, leaving every shard to build both commits again. The wheels get - # their own cache, keyed on the pair, and it is a couple of MB against the - # environment's 568, so keying it per run costs the cache budget little. + # The env cache key has no commit in it, so on a hit nothing is saved and the + # wheels built below would be lost; cache them (a few MB) per commit pair. - name: Cache the built wheels uses: actions/cache@v6 with: path: ${{ env.ASV_DIR }}/env/*/asv-build-cache key: asv-wheels-${{ runner.os }}-${{ steps.base.outputs.sha }}-${{ github.sha }} + # _partition balances shards by asv's recorded durations; with none it + # splits by count. Restore-only: the merge job saves the updated tree. - name: Restore recorded durations uses: actions/cache/restore@v6 with: @@ -140,15 +126,9 @@ jobs: working-directory: ${{ env.ASV_DIR }} env: BASE: ${{ steps.base.outputs.sha }} - # ``--bench just-discover`` is asv's own discovery-only mode - # (``commands/run.py``): it creates the environments, builds the project - # for the commit it discovers from, writes ``results/benchmarks.json`` - # and returns 0 without running a benchmark. Done once here rather than - # once per shard, a cold cache costs one conda solve instead of four, - # and the shards restore an environment whose build cache already holds - # both commits' wheels. On a shared filesystem this is also what stops - # concurrent shards racing to install into one ``env/``; separate - # runners have no such race, but they do have the cost. + # --bench just-discover builds the env and the commit's wheel and writes + # results/benchmarks.json without running anything. Done once here so a + # cold cache costs one conda solve, not one per shard. run: | set -ex # Fill the fixture cache before asv preimports the suite, which would otherwise build it serially in the forkserver parent @@ -166,16 +146,13 @@ jobs: SHARDS: ${{ github.event.inputs.shards || '4' }} run: | set -ex - # The report goes in the log so a lopsided split is visible without - # opening four shard jobs to find which one ran long. + # Logs the split so a lopsided one is visible without opening each shard. PYTHONPATH=.. python -m benchmarks.helpers._partition --shards "$SHARDS" echo "count=$SHARDS" >> "$GITHUB_OUTPUT" python -c "import json, os; print('shards=' + json.dumps(list(range(int(os.environ['SHARDS'])))))" \ >> "$GITHUB_OUTPUT" - # The whole tree rather than ``benchmarks.json`` alone: the shards - # partition from this, and a partition is only reproducible if they weigh - # the same durations setup weighed. + # The whole tree, not just benchmarks.json, so shards weigh the same durations. - name: Upload the discovered suite uses: actions/upload-artifact@v7 with: @@ -187,8 +164,7 @@ jobs: needs: setup runs-on: ubuntu-latest strategy: - # Every shard is independent, and a shard that dies still leaves the rest - # worth merging. + # A failed shard still leaves the rest worth merging. fail-fast: false matrix: shard: ${{ fromJSON(needs.setup.outputs.shards) }} @@ -209,8 +185,7 @@ jobs: python-build mamba - # No restore-keys: asv reuses a restored env without checking it, so a - # prefix match would hand it one built from an older environment file. + # No restore-keys: asv reuses a restored env without checking it. - name: Cache asv's benchmark environment uses: actions/cache@v6 with: @@ -227,12 +202,7 @@ jobs: restore-keys: | asv-fixtures-${{ runner.os }}- - # The environment cache key has no commit in it -- deliberately, since the - # conda environment does not depend on one -- so on a hit ``actions/cache`` - # saves nothing at the end of the job and the wheels built above would go - # with it, leaving every shard to build both commits again. The wheels get - # their own cache, keyed on the pair, and it is a couple of MB against the - # environment's 568, so keying it per run costs the cache budget little. + # Setup's wheels; see the note there. - name: Cache the built wheels uses: actions/cache@v6 with: @@ -252,10 +222,8 @@ jobs: BASE: ${{ needs.setup.outputs.base }} SHARDS: ${{ needs.setup.outputs.count }} SHARD: ${{ matrix.shard }} - # Partitioned from setup's ``benchmarks.json`` rather than from whatever - # this runner's cache happens to hold, so every shard splits the same - # suite the same way. Nothing checks that the shards tile it: were they - # to disagree, benchmarks would simply go unrun and no step would say so. + # Partition from setup's plan, not this runner's cache, so all shards split the + # same suite. Nothing checks they tile it; a mismatch silently skips benchmarks. run: | set -ex (cd .. && python -m benchmarks.helpers._fixtures) @@ -266,19 +234,11 @@ jobs: --config asv.conf.json --asv-args) echo "Baseline: $BASE" echo "Contender: ${GITHUB_SHA} (${PR_HEAD_LABEL:-$GITHUB_REF_NAME})" - # ``asv run`` over both commits rather than ``asv continuous``: - # continuous is this same interleaved run followed by a comparison, - # and it exits 1 when that comparison finds a regression -- the same - # status it uses for a broken run -- so a slower PR would fail its - # shard. The merge job compares the whole suite; a shard only measures. + # asv run, not continuous: continuous exits 1 on a regression, same as a + # broken run. The merge job does the comparison. printf '%s\n' "${GITHUB_SHA}" "$BASE" > "$RUNNER_TEMP/commits.txt" - # asv returns 2 when a benchmark failed rather than when the run - # broke (``commands/run.py``: ``if failures: return 2``), and this - # shard's other results are complete and worth merging. The old - # single-job workflow tolerated it by accident -- ``asv compare`` ran - # last in the same step and its status was the step's -- so make it - # deliberate here, loudly, rather than letting one long-broken - # benchmark redden every shard that happens to hold it. + # Exit 2 means a benchmark failed, not the run; keep the shard's other + # results and warn rather than fail. status=0 asv run --show-stderr -m "$ASV_MACHINE" $ASV_ARGS \ "HASHFILE:$RUNNER_TEMP/commits.txt" || status=$? @@ -299,9 +259,7 @@ jobs: merge: name: Merge and compare needs: [setup, benchmark] - # Runs on a partial fan-out too: a merged tree missing one shard's rows is - # still worth reading, and the artifact is the only place a failed shard's - # half of the run can be seen. + # Run even if a shard failed: the surviving shards' results are still worth merging. if: ${{ always() && needs.setup.result == 'success' }} runs-on: ubuntu-latest steps: @@ -321,9 +279,7 @@ jobs: python-build mamba - # Without merge-multiple each artifact lands in its own - # ``shards/asv-shard-N/``, which is what the merge needs: every shard - # named its results file identically, and telling them apart is the point. + # No merge-multiple: the shards' results files share names. - name: Download the shards uses: actions/download-artifact@v8 with: @@ -344,12 +300,8 @@ jobs: > asv_compare_results.txt cat asv_compare_results.txt - # asv records a duration per benchmark, plus ```` and - # ```` entries, in the results file it writes. Printing - # them is what tells us where a run's wall clock actually went. Two things - # to read them with: a per-benchmark duration is the final round's only, - # since asv assigns rather than accumulates it, and ```` and - # ```` are the slowest shard's, since every shard paid them. + # Prints asv's recorded durations. A benchmark's is its final round only; + # and are the slowest shard's, since every shard paid them. - name: Report where the time went if: always() shell: bash -l {0} @@ -381,8 +333,7 @@ jobs: print(f" {duration:8.1f}s {100 * duration / total:5.1f}% {name}") PY - # Keyed on the run id so every run stores its own entry and setup's - # prefixed restore picks up the newest; there is nothing to restore here. + # Keyed per run so setup's prefix restore picks up the newest. - name: Save recorded durations for the next run if: always() uses: actions/cache/save@v6 diff --git a/.github/workflows/asv-benchmarking.yml b/.github/workflows/asv-benchmarking.yml index 99e9a18e3..c98aecea6 100644 --- a/.github/workflows/asv-benchmarking.yml +++ b/.github/workflows/asv-benchmarking.yml @@ -9,8 +9,7 @@ on: jobs: benchmark: runs-on: ubuntu-latest - # Without this the platform's 6h cap is the only limit, which is how a - # mis-skipped range quietly turns into a 453-commit build. + # Otherwise only the 6h platform cap stops a runaway range build. timeout-minutes: 180 defaults: run: @@ -76,10 +75,8 @@ jobs: cp -r uxarray-asv/results benchmarks/ fi - # asv builds its own conda environment under ``benchmarks/env`` from - # ci/environment.yml (see asv.conf.json). No restore-keys: asv reuses a - # restored env without checking it, so a prefix match would hand it one - # built from an older environment file. + # asv builds benchmarks/env from ci/environment.yml (per asv.conf.json), not + # ci/asv.yml. No restore-keys: asv reuses a restored env without checking it. - name: Cache asv's benchmark environment uses: actions/cache@v6 with: @@ -104,10 +101,8 @@ jobs: python -m benchmarks.helpers._fixtures cd benchmarks asv machine --machine GH-Actions --os ubuntu-latest --arch x64 --cpu "2-core unknown" --ram 7GB - # ``--skip-existing-commits``, not ``--skip-existing``: the latter keys - # its skip set by (commit, env), so asv's per-commit check never - # matches and it builds and installs every commit in the range before - # skipping the benchmarks it already has. + # --skip-existing-commits, not --skip-existing: the latter still builds + # every commit in the range before skipping benchmarks it already has. asv run main^! --skip-existing-commits --parallel || true - name: Commit and push benchmark results diff --git a/benchmarks/asv.conf.hpc.json b/benchmarks/asv.conf.hpc.json index 62f3d9bcb..31e9d4b8e 100644 --- a/benchmarks/asv.conf.hpc.json +++ b/benchmarks/asv.conf.hpc.json @@ -1,27 +1,15 @@ { - // Thread-scaling variant of ``asv.conf.json``, for a machine with cores to - // spare. Run it explicitly: + // Thread-scaling variant of ``asv.conf.json``: // // asv run --config benchmarks/asv.conf.hpc.json - // asv compare --config benchmarks/asv.conf.hpc.json -E conda:3.11 A B // - // Why a second file rather than an environment variable: asv records the - // ``env_nobuild`` matrix into every results file it writes and folds it - // into the environment name, so each thread count gets its own result set - // and ``asv compare`` keeps them apart. A count exported in the shell is - // invisible to asv -- runs at 1 and 64 threads land in the *same* file for - // a commit and compare as though they measured the same thing. See - // ``benchmarks/helpers/_threads.py`` for the shell route, which is the - // right one for "just run this faster" and the wrong one for a study. + // A separate file, not a shell export, because asv folds ``env_nobuild`` + // into the environment name, keeping each thread count's results apart; + // it also overrides any exported ``NUMBA_NUM_THREADS``. // - // The two mechanisms are mutually exclusive: asv layers ``env_nobuild`` - // over the inherited environment, so ``NUMBA_NUM_THREADS`` below wins over - // any export. - // - // KEEP IN SYNC with asv.conf.json: ``pythons``, ``environment_type``, - // ``conda_channels``, ``matrix.req`` and ``build_command`` decide what gets - // installed. Let them drift and these numbers stop being comparable to the - // ones the normal config produces, silently. + // KEEP IN SYNC with asv.conf.json (``pythons``, ``environment_type``, + // ``conda_channels``, ``matrix.req``, ``build_command``) or results + // silently stop being comparable. "version": 1, "project": "uxarray", @@ -41,9 +29,7 @@ ], "build_cache_size": 4, - // A parallel kernel held at one thread takes roughly as long as the core - // count it would otherwise have used, so the 360s the normal config allows - // is not enough at the bottom of a scaling sweep. + // A parallel kernel at one thread runs ~core-count times longer. "default_benchmark_timeout": 1800, "matrix": { @@ -55,23 +41,10 @@ "tbb": [""] }, - // One result set per value, so the run costs its length: four values is - // four passes over the suite. Edit for the node -- powers of two up to - // its physical core count is the usual shape, and - // ``python -m benchmarks.helpers._threads`` reports that count. - // Values above the core count are allowed and simply oversubscribe. - // One value each: a matrix entry with several values is a separate - // environment per value, and asv runs the whole suite in every one of - // them. Only 24 of the 86 benchmarks reach a ``parallel=True`` kernel, - // so a four-value sweep ran the other 62 four times over to print the - // same number four times. ``NUMBA_NUM_THREADS`` is the one knob here; - // to sweep it, see ``--env`` in ``benchmarks/helpers/_partition.py``. - // - // The BLAS variables are pinned because they are what made the old - // sweep unreadable: numpy would take the whole node in every - // environment, so the ``NUMBA_NUM_THREADS=1`` column was never a - // single-threaded baseline and the scaling curve it implied was - // measuring contention. Pinned, numba's threads are the only ones. + // One value each: every value is a full pass over the suite, though + // few benchmarks reach a ``parallel=True`` kernel. To sweep + // ``NUMBA_NUM_THREADS``, see ``--env`` in ``helpers/_partition.py``. + // BLAS is pinned so numba's threads are the only ones measured. "env_nobuild": { "NUMBA_THREADING_LAYER": ["forksafe"], "NUMBA_NUM_THREADS": ["8"], diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 449cf263f..18133e54e 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -128,11 +128,7 @@ // Belt to that brace: if TBB is ever unavailable, pick the other // fork-safe layer rather. ``forksafe`` raises if no fork-safe layer // exists at all. - // - // The BLAS variables are pinned so numpy does not compete with numba - // for the runner's cores, which shows up as run-to-run spread in every - // benchmark that touches an array. ``NUMBA_NUM_THREADS`` is left at - // numba's default of the whole machine. + // BLAS is pinned so numpy does not compete with numba for cores. "env_nobuild": { "NUMBA_THREADING_LAYER": ["forksafe"], "OMP_NUM_THREADS": ["1"], @@ -142,9 +138,7 @@ }, - // Just the one command: ``python -m build`` put an sdist and a wheel in - // {build_dir}/dist, which asv never reads, and then this rebuilt the wheel - // into the directory asv actually installs from. + // No ``python -m build``: its output went to {build_dir}/dist, unused. "build_command": [ "python -mpip wheel --no-deps --no-build-isolation --no-index -w {build_cache_dir} {build_dir}" ], diff --git a/benchmarks/helpers/_machine.py b/benchmarks/helpers/_machine.py index 439ae0205..9a8de5c31 100644 --- a/benchmarks/helpers/_machine.py +++ b/benchmarks/helpers/_machine.py @@ -1,22 +1,9 @@ -"""Pinning the machine name asv records results under. - -asv keys results on a machine name and defaults it to the hostname -(``Machine.get_defaults``), which on a hosted runner is fresh for every job -- -``runnervmgx7h7`` on one run, something else on the next. A name that never -repeats cannot be compared across runs, and once the suite is sharded it cannot -even be merged within one run: the file asv writes is -``results//-.json``, so every shard has to agree on -```` or there is nothing for :mod:`_merge` to line up. - -``asv machine --machine NAME`` will not do it on its own. That command stores -only the fields that differ from the ones it detected and then skips filling the -rest in (``commands/machine.py``), so naming the machine is precisely what drops -``cpu``, ``num_cpu`` and ``ram`` -- the fields that say what the timings were -measured on, and the ones the duration report prints. Detect first with ``asv -machine --yes``, rename after, which is what this does. - -Idempotent, so a job that runs it twice, or a machine file that arrives already -pinned, is fine. +"""Pins the machine name asv files results under. + +asv defaults it to the hostname, which changes every hosted-runner job and +every cluster node, so results cannot be compared across runs or merged across +shards (``results//...``). ``asv machine --machine NAME`` alone drops +``cpu``/``num_cpu``/``ram``, hence detect first, rename after. Idempotent. Usage:: @@ -38,18 +25,11 @@ def default_name(): - """A machine name that survives landing on a different node next time. - - The scheduler puts you on ``derecho3`` one day and ``crhtc70`` the next, and - asv keys results on ``platform.uname``'s node name, so left alone it records - a new machine every login and the results scatter across all of them. - - ``NCAR_HOST`` is the reliable answer where it is set -- it names the cluster - rather than the node, which is the granularity results want. Failing that, - the node name with its trailing digits removed, which folds ``derecho3`` and - ``derecho5`` together but *not* ``derecho3`` and ``crhtc70``: login and - compute nodes of one cluster do not share a stem. Set ``ASV_MACHINE`` - yourself if you move between them without ``NCAR_HOST``. + """Returns ``(name, source_variable)`` stable across nodes of one cluster. + + ``$ASV_MACHINE``, then ``$NCAR_HOST`` (names the cluster, not the node), + then the node name minus trailing digits. Login and compute nodes do not + share a stem, so set ``ASV_MACHINE`` if you use both without ``NCAR_HOST``. """ for variable in ("ASV_MACHINE", "NCAR_HOST"): value = os.environ.get(variable) @@ -60,23 +40,15 @@ def default_name(): def default_path(): - """Where asv keeps its machine file (``MachineCollection.get_machine_file_path``).""" + """asv's machine file (``MachineCollection.get_machine_file_path``).""" return Path.home() / ".asv-machine.json" def pin(name, path=None, hostname=None, sole=False): - """Renames the machine file's freshly detected entry to ``name``. - - Returns its details. Entries for other machines are left alone -- a runner - has only the one, but a login node that has recorded every compute node it - ever landed on should not lose them to a benchmark run. - - The fresh entry is the one keyed by this host's name, since that is what - ``asv machine --yes`` writes (``Machine.get_defaults`` takes it from - ``platform.uname``). Renaming it is the whole point: several nodes of one - cluster should file their results under one machine, or a sharded run has - nothing to merge. Falls back to a lone entry whatever its name, for a runner - whose hostname has already been renamed away by an earlier call. + """Renames the machine file's freshly detected entry to ``name``; returns it. + + The fresh entry is the one keyed by this hostname, else one already named + ``name``, else a lone entry. Other entries are kept unless ``sole``. """ path = Path(path) if path is not None else default_path() hostname = hostname if hostname is not None else platform.node() @@ -116,8 +88,7 @@ def main(argv=None): parser.add_argument( "--name", default=None, - help="Machine name to pin to. Defaults to $ASV_MACHINE, then $NCAR_HOST, " - "then this host's name with trailing digits removed.", + help="Default $ASV_MACHINE, $NCAR_HOST, then hostname minus trailing digits.", ) parser.add_argument("--path", default=None, help="Machine file (default ~/.asv-machine.json).") parser.add_argument( @@ -126,9 +97,7 @@ def main(argv=None): parser.add_argument( "--sole", action="store_true", - help="Drop every other machine from the file. asv falls back to a lone entry " - "whatever the hostname, so this makes bare ``asv run``/``asv show`` work from " - "any node without -m. Use it where you only ever benchmark one machine.", + help="Drop other machines, so bare asv run/show work on any node without -m.", ) parser.add_argument( "--quiet", action="store_true", help="Print only the pinned name, for capturing." diff --git a/benchmarks/helpers/_merge.py b/benchmarks/helpers/_merge.py index 638751d8e..ebb6c1a98 100644 --- a/benchmarks/helpers/_merge.py +++ b/benchmarks/helpers/_merge.py @@ -1,30 +1,13 @@ -"""Merging a sharded run's results back into one tree. - -asv reads its results file once before running a benchmark set and writes it -once after (``Results.load_data`` then ``Results.save`` in ``commands/run.py``), -and the name it writes is ``results//-.json`` -- one file -per commit and environment, whatever subset of benchmarks the run measured. So -shards sharing a results directory each rewrite that whole file from what they -alone measured, and the last one to finish wins. Each shard therefore gets its -own directory (``_partition --config-out``) and they are combined here. - -The combination is a union rather than an element-wise reconciliation, which is -what the by-whole-benchmark split buys: a row is keyed on the benchmark name and -carries its whole parameter sweep inside, so every row is owned by exactly one -shard. Splitting inside a benchmark would have put two shards in one row. - -Order is restored rather than preserved. ``results`` is a JSON object, asv writes -it with ``compact=True`` -- which disables sorting, so key order is the order asv -appended to it -- and for an unsharded run that order is ``sorted(benchmarks)`` -grouped by ``setup_cache_key`` (``runner.py``, ``iter_run_items``). Shards finish -in whatever order the queue hands back, so :func:`canonical_order` recovers the -order the same suite would have produced serially and every merged file is -written in it. - -Idempotent, and indifferent to shards that have not landed: merging the three -directories that exist gives a valid tree, and merging again when the fourth -arrives puts its rows in their proper place. That is what makes it safe to run -from a polling loop as jobs come back rather than only after a barrier. +"""Merges a sharded run's results directories back into one tree. + +asv rewrites the whole ``/-.json`` at the end of a run, +so shards sharing a directory would overwrite one another; each writes its own +and they are combined here. Shards split by whole benchmark, so every row +belongs to one shard and the merge is a union. + +Rows are written in the order a serial run would produce +(:func:`canonical_order`). Idempotent and tolerant of shards not yet landed, so +it can run as jobs come back. Usage:: @@ -56,11 +39,9 @@ def _load(path): def _dump(path, data): - """Writes ``data`` the way asv writes a results file. + """Writes ``data`` unsorted, like asv's ``write_json(..., compact=True)``. - ``util.write_json(..., compact=True)`` disables both sorting and - indentation; the sorting is the part that matters, because key order is the - only place a results file records what ran when. + Key order is the only record of what ran when. """ path.parent.mkdir(parents=True, exist_ok=True) with open(path, "w", encoding="utf-8") as handle: @@ -70,10 +51,8 @@ def _dump(path, data): def canonical_order(benchmarks): """Benchmark names in the order an unsharded ``asv run`` would produce them. - Mirrors ``runner.run_benchmarks``: it walks ``sorted(benchmarks.items())`` - building ``benchmark_order``, a dict keyed on ``setup_cache_key``, then runs - each of those groups in turn. So the order is by name within a group, and - groups in the order their first member is reached by name. + Mirrors ``runner.run_benchmarks``: by name within each ``setup_cache_key`` + group, groups ordered by their first member's name. """ groups = {} for name in sorted(benchmarks): @@ -85,9 +64,8 @@ def canonical_order(benchmarks): def merge_benchmarks(shard_dirs): """Union of the shards' ``benchmarks.json``. - A shard discovers under its own ``--bench`` patterns, so each file holds - only that shard's benchmarks and the full set exists nowhere until here. - ``_partition.load_benchmarks`` needs that full set to plan the next run. + Each shard only lists its own benchmarks; ``_partition`` needs the full set + to plan the next run. """ merged, version = {}, None for shard_dir in shard_dirs: @@ -107,11 +85,8 @@ def merge_benchmarks(shard_dirs): def _pick(name, existing, candidate, report): """Which of two rows for one benchmark to keep. - Only reachable when a name landed in more than one shard, which the - partition does not do -- so it means the plan the shards ran was not the one - that produced them. Preferring a row that has a result over one that does - not, then the later ``started_at``, keeps a re-run over the run it replaced - instead of picking on file order. + Only reached if shards ran different plans. Prefers a row with a result, + then the later ``started_at``, so a re-run beats the run it replaced. """ if existing == candidate: return existing @@ -128,31 +103,25 @@ def rank(row): def merge_result_files(datas, order, report): - """One results file from several shards' versions of it. + """One results file from several shards' copies, rows written in ``order``. - ``datas`` are the parsed files, in shard order; ``order`` is the name order - to write. Every field outside ``results`` and ``durations`` describes the - commit and environment rather than the run, and is identical across shards - by construction, so the first shard's copy carries over untouched. + Fields other than ``results`` and ``durations`` are identical across shards, + so the first shard's are kept. """ merged = dict(datas[0]) columns = list(merged.get("result_columns") or []) rows, durations = {}, {} for data in datas: - # Read each row against its own file's columns. Identical in practice -- - # one asv builds every shard -- but a row is a bare list, so aligning it - # to the wrong header would silently shift every value. + # Rows are bare lists, so align each to its own file's columns. shard_columns = data.get("result_columns") or columns for name, row in (data.get("results") or {}).items(): values = dict(zip(shard_columns, row)) rows[name] = ( _pick(name, rows[name], values, report) if name in rows else values ) - # ``durations`` holds only the ```` and ```` - # entries; a benchmark's own duration lives in its row. Every shard pays - # both, so the max is the one a single run would have reported, and the - # sum would describe work no single wall clock ever saw. + # Only ```` and ```` entries, which every shard + # pays; the max is what a single run would report. for key, value in (data.get("durations") or {}).items(): durations[key] = max(durations.get(key, 0.0), float(value)) @@ -164,8 +133,7 @@ def merge_result_files(datas, order, report): results = {} for name in known + extra: row = [rows[name].get(column) for column in columns] - # asv drops trailing nulls when it writes a row; keeping that keeps the - # merged file the same size as the one a serial run would have written. + # Matches asv, which drops trailing nulls. while row and row[-1] is None: row.pop() results[name] = row @@ -195,9 +163,7 @@ def merge(shard_dirs, out_dir, report=lambda message: None): if len(benchmarks) > (1 if _VERSION_KEY in benchmarks else 0): _dump(out_dir / BENCHMARKS_FILE, benchmarks) - # One group per (machine, results file): a shard writes the same file name as - # every other shard of its commit and environment, which is the collision - # this module exists to undo. + # Shards of one commit and environment write the same file name. groups = {} for shard_dir in present: for path in sorted(shard_dir.glob("*/*.json")): diff --git a/benchmarks/helpers/_partition.py b/benchmarks/helpers/_partition.py index f24e75d3e..e59c443e9 100644 --- a/benchmarks/helpers/_partition.py +++ b/benchmarks/helpers/_partition.py @@ -1,42 +1,17 @@ -"""Splitting the suite into shards of roughly equal cost. - -asv runs one benchmark at a time. ``--parallel`` builds environments in -parallel and nothing else -- its own help text is "Build (but don't benchmark) -in parallel" -- so cutting the wall clock means running several ``asv`` -processes, and a ``time_*`` result is only worth having if nothing else is -competing for the machine while it is measured. That points at one shard per -runner rather than several per runner, and at this module, whose whole job is to -decide which benchmarks each of those runners should claim. - -The split is by whole class, not by benchmark and not by parameter -combination. asv matches ``--bench`` against the expanded -``name(param0, param1)``, so a much finer cut is available, but a class's -benchmarks share the kernels its first one compiles and splitting them makes -every shard pay that compile again. Measured: ``face_bounds.FaceBounds``'s four -benchmarks cost ~59s together in one process, where ``time_face_bounds`` paid -the bounds compile and the three ``track_*`` variants rode on it warm at under -7s each; scattered one per shard across four runners they cost 221s, every one -of them paying the compile alone. ``cache=True`` does not save this -- asv -reinstalls the wheel for each commit and the on-disk cache goes with it. The -suite stays flat enough at class granularity for greedy longest-first packing -to land close to a perfect split. - -Going finer than a whole benchmark would also put two shards in one row of one -results file, with no sane way to reconcile them. Whole benchmarks leave every -row owned by exactly one shard, which is what lets -:mod:`benchmarks.helpers._merge` rebuild the tree by union. - -Shards still need somewhere separate to write, because asv names its results -file per commit and environment rather than per run and rewrites the whole thing -at the end of a set. ``--config-out`` emits a copy of the config with -``results_dir`` pointed at this shard's own directory, which is what the merge -then reads. - -Weights come from the ``duration`` asv records per benchmark in the results file -it writes (``Results.save``), so a partition improves as results accumulate -rather than needing a cost model. Benchmarks with no recorded duration -- new -ones, mostly -- get the median of the ones that have, which is a better guess -than either zero or the mean of a long-tailed distribution. +"""Splits the suite into shards of roughly equal cost, one per runner. + +asv's ``--parallel`` only parallelizes environment builds, and ``time_*`` +results need an uncontended machine, so each shard gets its own runner. + +Shards split by whole class: a class's benchmarks share the kernels its first +one compiles, so splitting it makes every shard recompile (``FaceBounds``: 59s +in one shard, 221s across four). Numba's ``cache=True`` does not help, as asv +reinstalls the wheel per commit. Whole benchmarks also leave every results row +owned by one shard, so :mod:`benchmarks.helpers._merge` can merge by union. + +Each shard writes to its own ``results_dir``, since asv rewrites the whole +per-commit results file at the end of a run. Weights are the ``duration`` asv +records per benchmark; benchmarks without one get the median. Usage:: @@ -44,8 +19,7 @@ asv run $(python -m benchmarks.helpers._partition --shards 4 --shard 0 \ --config asv.conf.hpc.json --asv-args) - # A thread sweep, for whoever wants one. ``--shards 1`` is the whole suite; - # add ``--bench`` to hold it to the benchmarks that can actually respond. + # Thread sweep over the whole suite (add ``--bench`` to narrow it). for n in 1 2 4 8; do asv run $(python -m benchmarks.helpers._partition --shards 1 --shard 0 \ --config asv.conf.hpc.json --env NUMBA_NUM_THREADS=$n --asv-args) @@ -71,17 +45,15 @@ BENCHMARK_DIR = Path(__file__).resolve().parents[1] -# ``setup_cache`` groups whose members may be split across shards. Anything else -# is paid once per shard that holds any of its benchmarks, so those move -# together. ``CachedFixtures.setup_cache`` is just ``prime()``, a stat per file -# once the cache is warm (1.2s in CI), and ``None`` is no setup_cache at all. +# ``setup_cache`` keys cheap enough to repeat per shard (``prime()`` is a stat +# per file once warm). Other groups would rerun in every shard, so stay together. SPLITTABLE_PREFIX = "helpers._fixtures:" _SKIP_FILES = frozenset({"machine.json", "benchmarks.json"}) def _splittable(setup_cache_key): - """Whether benchmarks sharing this ``setup_cache_key`` may land in different shards.""" + """Whether benchmarks sharing this ``setup_cache_key`` may span shards.""" return setup_cache_key is None or str(setup_cache_key).startswith(SPLITTABLE_PREFIX) @@ -91,20 +63,11 @@ def _owner(name): def _units(benchmarks): - """Benchmarks that have to ride together, as ``{root name: [names]}``. - - Two constraints, unioned so a ``setup_cache`` group spanning classes pulls - those classes together rather than contradicting them: + """Benchmarks that must share a shard, as ``{root name: [names]}``. - ``class`` - Its benchmarks share whatever its first one compiles (see the module - docstring for what splitting one measured). - ``setup_cache`` - Anything sharing a ``setup_cache`` expensive enough to matter, which asv - would otherwise run once in every shard holding a member. - - Roots are the lexicographically smallest member, so the grouping is - deterministic and a shard can still work out its own membership. + Unions same-class benchmarks with those sharing a non-splittable + ``setup_cache``. Roots are the smallest member name, so grouping is + deterministic. """ parent = {} @@ -147,12 +110,10 @@ def load_benchmarks(results_dir): def load_weights(results_dirs): - """Mean recorded duration per benchmark, in seconds, over the files on disk. + """Mean recorded duration per benchmark, in seconds. - The mean rather than the latest: a benchmark's first run on a cold numba - cache can cost hundreds of times its warm cost (a 9.6s bounds compile - against 13ms warm, in one observed run), and a partition built from one - such outlier sends a whole shard chasing work that is not there. + Mean, not latest: a cold numba cache can make one run hundreds of times + slower than warm, and a single outlier would skew the plan. """ samples = {} for results_dir in results_dirs: @@ -181,13 +142,8 @@ def load_weights(results_dirs): def plan(benchmarks, n_shards, weights=None): """Partitions ``benchmarks`` into ``n_shards`` lists of names. - Greedy longest-first onto the lightest shard so far -- the standard LPT - heuristic, which on a distribution this flat is within about 1% of optimal - and, unlike anything smarter, is obvious enough to debug from the report. - - Deterministic: equal weights are broken by name, so the same inputs always - give the same shards and a shard can compute its own membership without - being told. + Greedy longest-first (LPT): within about 1% of optimal on this suite and + easy to debug. Ties break by name, so a shard can compute its own membership. """ if n_shards < 1: raise ValueError(f"n_shards must be at least 1, got {n_shards}") @@ -212,13 +168,10 @@ def plan(benchmarks, n_shards, weights=None): def bench_regexes(names): - """``--bench`` patterns selecting exactly ``names`` and nothing else. + """``--bench`` patterns selecting exactly ``names``. - asv filters a parameterized benchmark on ``name(param0, param1)`` and an - unparameterized one on ``name`` (``Benchmarks.__init__``), so the trailing - group has to admit both an open parenthesis and end-of-string. Without it - ``^name$`` silently matches none of a parameterized benchmark's - combinations, and the shard runs nothing. + asv matches parameterized benchmarks as ``name(p0, p1)``, so the trailing + group admits ``(`` as well as end-of-string; ``^name$`` would match none. """ return [f"^{re.escape(name)}($|\\()" for name in names] @@ -235,17 +188,12 @@ def shard_results_dir(results_dir, shard): def write_shard_config(base_config, out_path, shard, env=None): - """Writes a copy of ``base_config`` that writes results where ``shard`` should. - - ``env`` overrides ``env_nobuild`` variables. An override lands in the - environment's name, so asv files those results under a name of their own and - a sweep's runs do not overwrite one another. + """Copies ``base_config`` with the shard's ``results_dir``; returns that dir. - Returns the shard's results directory. + ``env`` overrides ``env_nobuild`` variables. Overrides land in the + environment name, so a sweep's runs do not overwrite one another. """ - # asv's loader, because an asv config is JSON with javascript comments and - # ``json`` cannot read one. Imported here so the rest of the module stays - # runnable without asv installed. + # asv configs are JSON with JS comments; lazy so the module runs without asv. from asv import util config = util.load_json(str(base_config), js_comments=True) @@ -253,8 +201,7 @@ def write_shard_config(base_config, out_path, shard, env=None): config["results_dir"] = results_dir if env: matrix = config.setdefault("matrix", {}).setdefault("env_nobuild", {}) - # One value per variable: a list of several is a separate environment - # per value, and asv would run the whole suite in each of them. + # Several values would each be an environment running the whole suite. matrix.update({key: [value] for key, value in env.items()}) with open(out_path, "w") as handle: json.dump(config, handle, indent=4) @@ -294,13 +241,12 @@ def main(argv=None): "--results", action="append", default=None, - help="Results directory to read durations and benchmarks.json from. " - "Repeatable; defaults to benchmarks/results.", + help="Results directory to read (repeatable; default benchmarks/results).", ) parser.add_argument( "--bench-args", action="store_true", - help="Print the shard's --bench arguments for asv, rather than a report.", + help="Print the shard's --bench arguments instead of a report.", ) parser.add_argument( "--config", @@ -317,15 +263,12 @@ def main(argv=None): action="append", default=[], metavar="KEY=VALUE", - help="Override an env_nobuild variable in the generated config. Repeatable; " - "use it to sweep a variable the config pins to one value, e.g. " - "--env NUMBA_NUM_THREADS=4.", + help="Override an env_nobuild variable, e.g. NUMBA_NUM_THREADS=4 (repeatable).", ) parser.add_argument( "--asv-args", action="store_true", - help="Write this shard's config and print every argument its asv run " - "needs, so launching a shard is one substitution. Needs --shard.", + help="Write this shard's config and print its asv run arguments. Needs --shard.", ) args = parser.parse_args(argv) @@ -338,10 +281,8 @@ def main(argv=None): if args.shard is None: parser.error("--bench-args and --asv-args need --shard") if args.asv_args: - # Written here rather than by a call of its own: a shard that is - # told which benchmarks to run has to be told where to put them, - # and splitting that across two commands is two chances to pass - # one shard's benchmarks with another's results directory. + # Emitted with ``--bench`` so a shard can't pair its benchmarks with + # another shard's results dir. env = {} for entry in args.env: key, sep, value = entry.partition("=") diff --git a/benchmarks/helpers/_threads.py b/benchmarks/helpers/_threads.py index 14ce8315c..a4dacf57f 100644 --- a/benchmarks/helpers/_threads.py +++ b/benchmarks/helpers/_threads.py @@ -1,34 +1,15 @@ -"""Choosing how many threads a benchmark run may use. +"""Resolves how many threads a benchmark run may use. -numba reads ``NUMBA_NUM_THREADS`` once, when it brings up its threading layer, -and treats it as a ceiling rather than a setting: ``set_num_threads`` can hold -the pool lower for a block -- which is what :func:`~benchmarks.helpers._peakmem.numba_threads` -does while tracing -- but asking for more than the ceiling raises -``ValueError``. So a run's thread count has to be decided before the benchmark -process imports numba, which means the environment, which is what this module -resolves. - -asv copies ``os.environ`` into the processes it launches -(``Environment.run_executable``), so exporting the variable ahead of ``asv run`` -is enough:: +numba treats ``NUMBA_NUM_THREADS`` as a ceiling fixed at import, so the count +must be in the environment before ``asv run`` (asv passes ``os.environ`` on). +It is deliberately absent from ``env_nobuild``, which would override the shell:: export NUMBA_NUM_THREADS=$(python -m benchmarks.helpers._threads) asv run ... -One asymmetry to know about: asv layers the ``env_nobuild`` matrix *over* the -inherited environment, not under it, so a variable named in ``asv.conf.json`` -overrides the shell. ``NUMBA_THREADING_LAYER`` is named there because -fork-safety is not negotiable. The thread count deliberately is not, so a node -with more cores than a CI runner can decide for itself. - -The default is physical cores rather than ``os.cpu_count()``. These kernels are -floating-point and memory-bound, and a second hardware thread per core tends to -cost more in contention than it recovers in latency hiding: CI runners with 2 -physical cores plus SMT measured about 1.28x slower per thread on the same -scalar kernels than runners with 4 real cores. - -``UXARRAY_BENCH_THREADS`` overrides the default -- an integer, or ``physical`` -or ``logical`` to name a rule rather than a number. +Defaults to physical cores: SMT siblings measured ~1.28x slower per thread on +these memory-bound kernels. ``UXARRAY_BENCH_THREADS`` overrides it with an +integer, ``physical`` or ``logical``. """ import os @@ -41,12 +22,7 @@ def logical_cores(): - """Schedulable CPUs, honouring any affinity mask this process was given. - - ``os.cpu_count()`` reports the machine; ``os.sched_getaffinity`` reports - what this process may actually use, which is the smaller and more useful - number under a batch scheduler or a ``taskset``. - """ + """Schedulable CPUs, honouring the affinity mask (batch scheduler, ``taskset``).""" if hasattr(os, "sched_getaffinity"): return len(os.sched_getaffinity(0)) return os.cpu_count() or 1 @@ -55,9 +31,7 @@ def logical_cores(): def physical_cores(): """Cores rather than hardware threads, or the logical count if unknown. - Deliberately shells out rather than adding a dependency on ``psutil``: the - benchmark environment asv builds is defined by ``asv.conf.json``'s matrix, - and a helper that has to run in it is not worth an entry there. + Shells out rather than adding ``psutil`` to the asv environment matrix. """ try: if sys.platform == "darwin": @@ -67,9 +41,7 @@ def physical_cores(): ).stdout return max(1, int(out.strip())) if sys.platform.startswith("linux"): - # One line per logical CPU, ",,,..."; distinct - # (socket, core) pairs are the physical cores. Counting distinct - # core ids alone would collapse two sockets into one. + # Distinct (core, socket) pairs; core ids alone collapse sockets. out = subprocess.run( ["lscpu", "-p=core,socket"], capture_output=True, text=True, timeout=5, check=True, @@ -88,11 +60,8 @@ def physical_cores(): def resolve(spec=None): """The thread count to run with. - ``spec`` defaults to ``$UXARRAY_BENCH_THREADS``, and that to ``physical``. - Anything unrecognised falls back to the physical core count rather than - failing: this sits in front of a benchmark run, and refusing to start - because a variable is misspelt costs more than quietly doing the sensible - thing. The resolved number is echoed to stderr so it appears in the log. + ``spec`` defaults to ``$UXARRAY_BENCH_THREADS``, then ``physical``. Invalid + values warn and fall back to physical rather than blocking the run. """ if spec is None: spec = os.environ.get(_ENV_VAR, "").strip() diff --git a/benchmarks/hpc/local.sh b/benchmarks/hpc/local.sh index 52a62878b..a7caff07a 100755 --- a/benchmarks/hpc/local.sh +++ b/benchmarks/hpc/local.sh @@ -1,16 +1,8 @@ #!/usr/bin/env bash -# Runs the sharded suite on the node you are already sitting on -- an -# interactive PBS session, typically -- with the shards concurrent rather than -# queued as separate jobs. -# -# This only makes sense because a derecho CPU node has 128 cores and a shard at -# NUMBA_NUM_THREADS=8 wants nine of them. Each shard is pinned to its own slice -# so they cannot land on each other's cores; what they do still share is memory -# bandwidth and last-level cache, and for the grid operations in this suite that -# is not nothing. So: the BASE-vs-HEAD ratios ``asv compare`` reports stay -# usable, since both sides of a comparison run inside the same shard under the -# same contention, but absolute timings come out noisier than a run that had a -# node to itself. Take those from one-shard-per-node (``submit.sh``). +# Runs the sharded suite concurrently on the current node (e.g. a ``qsub -I`` +# session), each shard pinned to its own core slice. Shared memory bandwidth +# and cache add noise: ``asv compare`` ratios stay usable, but take absolute +# timings from one-shard-per-node (``submit.sh``). # # Usage, from the repository root: # @@ -26,27 +18,18 @@ THREADS="${THREADS:-8}" export REPO SHARDS THREADS export REV="${REV:-HEAD^!}" export CONFIG="${CONFIG:-asv.conf.hpc.json}" -# Left empty on purpose: stage.pbs derives it, so this works unchanged on -# derecho and casper both. +# Empty on purpose: stage.pbs derives it per cluster. export ASV_MACHINE="${ASV_MACHINE:-}" export ASV_ACTIVATE="${ASV_ACTIVATE:-true}" -# Also evaluated here, not just in the stages, so the core count below can be -# read with python rather than with a shell tool that lies about it. +# Activated here too so the core count below can come from python. eval "$ASV_ACTIVATE" STAGE_SCRIPT="$REPO/benchmarks/hpc/stage.pbs" LOGS="${LOGS:-$REPO/benchmarks/hpc/logs}" mkdir -p "$LOGS" -# Neither PBS's NCPUS nor ``nproc`` can be trusted for this. NCPUS is the ncpus -# *requested per chunk*, 1 for a plain ``qsub -I``, and says nothing about the -# node. And GNU ``nproc`` honours OMP_NUM_THREADS and OMP_THREAD_LIMIT, so in a -# session that sets either it reports the OpenMP thread limit -- 1, or 2 -- and -# not the machine's cores at all. -# -# The affinity mask is the real answer: the CPUs this process may actually run -# on. It respects a cpuset the scheduler imposed and ignores OpenMP entirely. -# Override with CORES to hold the run to fewer than the mask allows. +# The affinity mask, not PBS's NCPUS (per-chunk request) or ``nproc`` (honours +# OMP_NUM_THREADS). Override with CORES. CORES="${CORES:-$(python -c ' import os try: @@ -90,9 +73,7 @@ for S in $(seq 0 $((SHARDS - 1))); do echo " shard $S -> cores $lo-$hi, pid $pid, log $LOGS/shard$S.log" done -# Every shard is waited on and its status reported, but a failure does not stop -# the merge: a tree missing one shard's rows is still worth having, same as the -# ``afteranyarray`` dependency the PBS path uses. +# A failed shard does not stop the merge (cf. ``afteranyarray`` in submit.sh). failed=0 for S in $(seq 0 $((SHARDS - 1))); do if wait "${pids[$S]}"; then diff --git a/benchmarks/hpc/stage.pbs b/benchmarks/hpc/stage.pbs index 1d2c1074d..04b7af2e2 100644 --- a/benchmarks/hpc/stage.pbs +++ b/benchmarks/hpc/stage.pbs @@ -1,7 +1,6 @@ #!/bin/bash -# One script, three stages of a sharded asv run. Submitted by ``submit.sh``, -# which sets STAGE and the rest of the environment; not meant to be qsub'd by -# hand. Defaults here are only so a stage is runnable outside PBS for debugging. +# The setup, shard and merge stages of a sharded asv run, selected by STAGE. +# Submitted by ``submit.sh`` (or run by ``local.sh``), not qsub'd by hand. # #PBS -N asv #PBS -j oe @@ -15,16 +14,10 @@ SHARDS="${SHARDS:-4}" REV="${REV:-HEAD^!}" REPO="${REPO:?REPO must be the repository root}" CONFIG="${CONFIG:-asv.conf.hpc.json}" -# One name for every shard: asv files results under -# ``results//-.json``, so shards that disagree leave -# nothing for the merge to line up. Derived rather than assumed, because the -# scheduler hands you a different node each login and asv would otherwise -# record each one as a machine of its own -- see ``_machine.default_name``. +# One machine name for all shards, or the merge has nothing to line up; see +# ``_machine.default_name``. ASV_MACHINE="${ASV_MACHINE:-$(cd "$REPO" && python -m benchmarks.helpers._machine --print)}" -# Shared, so the shards read the dyamond grids off campaign storage once -# between them rather than once each; the setup stage fills it. Optional, -# because running the stages locally has nothing to share and -# ``_fixtures.cache_dir`` already defaults to the checkout. +# Optional shared cache so shards read the source grids once between them. if [ -n "${UXARRAY_BENCH_CACHE_DIR:-}" ]; then export UXARRAY_BENCH_CACHE_DIR fi @@ -34,42 +27,25 @@ cd "$REPO/benchmarks" case "$STAGE" in setup) - # Everything that must happen exactly once, because every shard shares the - # filesystem it happens on: the fixture cache, the machine file, and the - # asv environments plus the wheel. Four shards building into one - # ``env/`` concurrently is the race this stage exists to prevent. + # Done once so concurrent shards do not race to build the shared + # fixture cache, machine file and ``env/``. (cd .. && python -m benchmarks.helpers._fixtures) asv machine --yes --config "$CONFIG" (cd .. && python -m benchmarks.helpers._machine --name "$ASV_MACHINE") - # asv's own discovery-only mode: creates the environments, builds the - # project, writes results/benchmarks.json, runs no benchmark. The four - # NUMBA_NUM_THREADS environments share one build directory -- ``env_nobuild`` - # variables are omitted from the name ``dir_name`` hashes -- so this one - # build serves all of them. - # ``--config`` on every asv call, and ``-m`` on every one after the pin. - # Without ``--config`` asv falls back to ``asv.conf.json`` in the working - # directory -- the CI config -- so this stage would discover, create the - # environment and build against a different matrix from the one the shards - # then run under. The two happen to share a ``req`` matrix, and so an - # environment directory, which is why it worked at all rather than failing; - # it just meant the pre-build was warming the wrong config's environment. - # - # ``-m`` on every asv call from here on. The pin above renamed this host's - # entry to $ASV_MACHINE, so the hostname asv would otherwise look itself up - # under no longer exists in the machine file -- and once that file holds - # more than one machine, asv's fall-back to a lone entry does not apply - # either. Without it this fails with "No information stored about machine - # ''" immediately after the pin reports success. + # Discovery only: builds the environments and benchmarks.json, runs + # nothing. ``env_nobuild`` is not hashed into ``env/``, so one build + # serves every thread count. + # ``--config`` everywhere, else asv picks up the CI asv.conf.json. + # ``-m`` after the pin, which renamed this host's entry: asv would fail + # with "No information stored about machine ''". asv run --bench just-discover --config "$CONFIG" -m "$ASV_MACHINE" "$REV" PYTHONPATH=.. python -m benchmarks.helpers._partition --shards "$SHARDS" ;; shard) SHARD="${PBS_ARRAY_INDEX:-${SHARD:?}}" - # ``--asv-args`` writes this shard's config -- a results_dir of its own, - # plus any --env override -- and prints it with the --bench patterns. - # THREADS is optional: unset, the config's own NUMBA_NUM_THREADS stands. - # Set, it overrides it, and lands in the environment name, so a run at one - # thread count files its results separately from a run at another. + # ``--asv-args`` writes this shard's config (own results_dir, --env + # override) and prints the asv arguments. THREADS, if set, overrides the + # config's NUMBA_NUM_THREADS and lands in the environment name. THREAD_ARG="" if [ -n "${THREADS:-}" ]; then THREAD_ARG="--env NUMBA_NUM_THREADS=$THREADS" @@ -78,8 +54,7 @@ shard) --shards "$SHARDS" --shard "$SHARD" --config "$CONFIG" $THREAD_ARG --asv-args) status=0 asv run --show-stderr -m "$ASV_MACHINE" $ASV_ARGS "$REV" || status=$? - # 2 is "a benchmark failed", not "the shard broke": its other results are - # complete and the merge should still get them. Anything else is real. + # 2 means some benchmarks failed; the rest are still worth merging. if [ "$status" -eq 2 ]; then echo "asv exited 2: some benchmarks failed, see above" >&2 elif [ "$status" -ne 0 ]; then diff --git a/benchmarks/hpc/submit.sh b/benchmarks/hpc/submit.sh index 801c71f59..e8e41d42a 100755 --- a/benchmarks/hpc/submit.sh +++ b/benchmarks/hpc/submit.sh @@ -1,18 +1,8 @@ #!/usr/bin/env bash -# Submits a sharded asv run on derecho as three chained PBS jobs. -# -# setup one node, once: fixture cache, machine file, asv environments and -# the wheel, then the shard plan. Everything the shards would -# otherwise race each other to create on the filesystem they share. -# shards a job array, one node each, exclusive. A ``time_*`` result is only -# worth having if nothing else is competing for the machine, so one -# shard per node rather than several. -# merge one node, once: combines the shards' results directories and shows -# the run. -# -# The merge depends on ``afteranyarray`` rather than ``afterokarray`` on -# purpose: a shard that fails still leaves results worth merging, and a suite -# with one long-broken benchmark should not cost you the other eighty. +# Submits a sharded asv run on derecho as three chained PBS jobs: setup (once, +# so shards do not race on the shared filesystem), a job array of shards (one +# node each, so ``time_*`` results are uncontended), and merge. The merge uses +# ``afteranyarray`` so one failed shard does not cost the others' results. # # Usage: # @@ -21,8 +11,7 @@ # PBS_ACCOUNT=UXXX0001 SHARDS=8 REV=main^! ./benchmarks/hpc/submit.sh # # THREADS overrides the config's NUMBA_NUM_THREADS and is recorded in the -# environment name, so runs at different thread counts do not overwrite each -# other's results. Leave it unset to take the config's value. +# environment name, so thread counts do not overwrite each other's results. # set -euo pipefail @@ -30,8 +19,7 @@ set -euo pipefail REPO="${REPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" SHARDS="${SHARDS:-4}" -# A single commit by default. ``asv run`` takes a range, so ``main^!`` is main's -# tip alone and ``base..head`` is every commit between. +# An asv range: ``main^!`` is one commit, ``base..head`` every commit between. REV="${REV:-HEAD^!}" QUEUE="${QUEUE:-main}" CONFIG="${CONFIG:-asv.conf.hpc.json}" @@ -39,14 +27,10 @@ CONFIG="${CONFIG:-asv.conf.hpc.json}" ASV_MACHINE="${ASV_MACHINE:-}" WALLTIME="${WALLTIME:-12:00:00}" SETUP_WALLTIME="${SETUP_WALLTIME:-02:00:00}" -# Off /glade/derecho/scratch so every shard shares one fixture cache. The -# default in ``_fixtures.cache_dir`` follows the checkout, which would give a -# second working tree its own empty cache and re-read every source. +# On scratch so every shard and worktree shares one fixture cache. CACHE_DIR="${UXARRAY_BENCH_CACHE_DIR:-/glade/derecho/scratch/$USER/uxarray-bench}" -# Submit from a shell that already has ``asv`` on PATH -- ``-V`` below carries -# that environment into all three jobs, which is both simpler and less brittle -# than reactivating inside them. Set ASV_ACTIVATE only if you would rather the -# jobs do it themselves; it is eval'd once per stage. +# ``-V`` carries this shell's ``asv`` into the jobs. ASV_ACTIVATE, if set, is +# eval'd once per stage instead. export ASV_ACTIVATE="${ASV_ACTIVATE:-true}" command -v asv >/dev/null || [ "$ASV_ACTIVATE" != "true" ] || { echo "asv is not on PATH; activate your environment first, or set ASV_ACTIVATE" >&2 @@ -54,9 +38,7 @@ command -v asv >/dev/null || [ "$ASV_ACTIVATE" != "true" ] || { } STAGE_SCRIPT="$REPO/benchmarks/hpc/stage.pbs" -# Passed through the environment rather than in ``-v``, whose value list is -# comma-separated and so cannot hold an activation command or a path with a -# comma in it. +# Via the environment, not ``-v``, whose comma-separated list cannot hold commas. export REPO SHARDS REV CONFIG ASV_MACHINE export THREADS="${THREADS:-}" export UXARRAY_BENCH_CACHE_DIR="$CACHE_DIR" From 189e78f69a229baff4bc8c2e6537eb2cb7ff4809 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Tue, 6 Oct 2026 16:49:53 -0500 Subject: [PATCH 45/52] Trim neighborhood benchmark radii NeighborhoodReduce ran at r=15, where a 120km face has ~600 neighbors and time_dataset_reduce alone took over a minute per round -- the class that bounded the slowest shard. It now runs at 5, shared with NeighborhoodDask as NEIGHBORHOOD_RADIUS. NeighborhoodBuild swept r over 1, 5 and 15 for every benchmark. The timings now skip 15 (1.3s per call at 120km) and the track_* benchmarks skip 1, where peakmem and nbytes barely move (362k vs 384k at 480km). asv reads params from the method first, so this stays one class and the kernels still compile once. Co-Authored-By: Claude Opus 5.5 --- benchmarks/mpas_ocean.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/benchmarks/mpas_ocean.py b/benchmarks/mpas_ocean.py index 7364451a6..a75393286 100644 --- a/benchmarks/mpas_ocean.py +++ b/benchmarks/mpas_ocean.py @@ -386,17 +386,24 @@ def track_peakmem_const_lat(self, resolution, lat_step): track_peakmem_const_lat.unit = "bytes" +# Great-circle radii in degrees. Mean element spacing is roughly 4.3 degrees at +# 480km and 1.1 at 120km, so 5.0 is a few neighbors per face on the coarse mesh +# and dozens on the fine one. +NEIGHBORHOOD_RADIUS = 5.0 + + class NeighborhoodBuild(DatasetBenchmark): """Construction cost of a ``Neighborhood``, split into its three stages. ``Neighborhood`` claims the neighbor query costs more than any reduction run - on it. ``r`` is a great-circle radius in degrees, against - a mean element spacing of roughly 4.3 degrees at 480km and 1.1 at 120km, so - the smallest radius here is near self-only on the coarser mesh. + on it. The timings skip r=15 (over a second per call at 120km); the + ``track_*`` benchmarks skip r=1, where they barely move. """ param_names = DatasetBenchmark.param_names + ['r'] - params = DatasetBenchmark.params + [[1.0, 5.0, 15.0]] + params = DatasetBenchmark.params + [[1.0, NEIGHBORHOOD_RADIUS]] + # asv reads ``params`` from the method before the class. + _track_params = DatasetBenchmark.params + [[NEIGHBORHOOD_RADIUS, 15.0]] def setup(self, resolution, r): super().setup(resolution) @@ -420,6 +427,7 @@ def track_nbytes_neighbors(self, resolution, r): nb = Neighborhood(self.uxgrid, r=r, on="face centers") return nb._flat.nbytes + nb._starts.nbytes + nb._counts.nbytes + track_nbytes_neighbors.params = _track_params track_nbytes_neighbors.unit = "bytes" def track_peakmem_build(self, resolution, r): @@ -427,6 +435,7 @@ def track_peakmem_build(self, resolution, r): return peak_allocated( lambda: Neighborhood(self.uxgrid, r=r, on="face centers")) + track_peakmem_build.params = _track_params track_peakmem_build.unit = "bytes" def track_mean_neighbors(self, resolution, r): @@ -434,6 +443,7 @@ def track_mean_neighbors(self, resolution, r): nb = Neighborhood(self.uxgrid, r=r, on="face centers") return round(float(nb.n_neighbors.mean()), 2) + track_mean_neighbors.params = _track_params track_mean_neighbors.unit = "elements" @@ -453,7 +463,7 @@ class NeighborhoodReduce(DatasetBenchmark): param_names = DatasetBenchmark.param_names + ['reduction'] params = DatasetBenchmark.params + [['mean', 'median']] - radius = 15.0 + radius = NEIGHBORHOOD_RADIUS @staticmethod def _run(neighborhood, reduction): @@ -521,7 +531,7 @@ class NeighborhoodDask(DatasetBenchmark): params = DatasetBenchmark.params + [['numpy', 'time_chunks', 'grid_chunks']] n_time = 12 - radius = 5.0 + radius = NEIGHBORHOOD_RADIUS def setup(self, resolution, chunking): super().setup(resolution) From dc3df00b43ed8fc1a0a0d6910a428493a0ff8f35 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Tue, 6 Oct 2026 17:38:43 -0500 Subject: [PATCH 46/52] Simplify the benchmark sharding helpers, workflow and HPC scripts Same shards, caches and pre-builds; about 630 fewer lines. - _merge: union rows and max durations. Every shard's benchmarks.json already lists the whole suite (asv saves all it discovers), and the conflict, ordering and column-realignment paths were unreachable. - _partition: group by class. No setup_cache outside _fixtures spans classes, so the union-find never merged more. --shard now implies --asv-args; the uncalled --bench-args and --config-out are gone, and the report lists the heaviest benchmarks in place of the workflow's inline script. - _machine: write asv's detected specs under the pinned name directly, replacing `asv machine --yes` plus a rename. - Drop the unused _threads.py and asv.conf.hpc.json. The latter had drifted from asv.conf.json (no conda_environment_file), so HPC runs built a different env than CI. stage.pbs now always sets NUMBA_NUM_THREADS (default 8; numba's default counts SMT siblings) and passes -a timeout=1800. - local.sh pins each shard to whole physical cores in socket order, so concurrent shards no longer share SMT siblings on nodes that number them N and N + cores. - PR workflow: YAML anchors for the steps repeated across jobs, one SHARDS setting, and a shorter baseline resolution. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/asv-benchmarking-pr.yml | 164 +++----------- .gitignore | 2 +- benchmarks/asv.conf.hpc.json | 56 ----- benchmarks/helpers/_machine.py | 137 ++---------- benchmarks/helpers/_merge.py | 217 +++--------------- benchmarks/helpers/_partition.py | 258 ++++++---------------- benchmarks/helpers/_threads.py | 86 -------- benchmarks/hpc/local.sh | 66 +++--- benchmarks/hpc/stage.pbs | 43 ++-- benchmarks/hpc/submit.sh | 8 +- 10 files changed, 205 insertions(+), 832 deletions(-) delete mode 100644 benchmarks/asv.conf.hpc.json delete mode 100644 benchmarks/helpers/_threads.py diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 9dd3a4458..26b20a473 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -32,6 +32,7 @@ env: CONDA_ENV_FILE: ci/environment.yml # Shared by every shard so their results merge under one machine. ASV_MACHINE: gh-linux-x64 + SHARDS: ${{ github.event.inputs.shards || '4' }} jobs: setup: @@ -41,9 +42,9 @@ jobs: outputs: base: ${{ steps.base.outputs.sha }} shards: ${{ steps.plan.outputs.shards }} - count: ${{ steps.plan.outputs.count }} steps: - - uses: actions/checkout@v7 + - &checkout + uses: actions/checkout@v7 with: fetch-depth: 0 @@ -52,7 +53,8 @@ jobs: run: | lscpu | grep -E 'Model name|^CPU\(s\):|Thread\(s\) per core|Core\(s\) per socket|Socket\(s\)|CPU max MHz' || lscpu || true - - name: Set up Conda environment + - &conda + name: Set up Conda environment uses: mamba-org/setup-micromamba@v3 with: environment-file: ${{env.CONDA_ENV_FILE}} @@ -67,33 +69,29 @@ jobs: - name: Resolve the baseline commit id: base # pull_request.* expressions are empty on workflow_dispatch. + env: + BASE: ${{ github.event.pull_request.base.sha }} + BASE_REF: ${{ github.event.inputs.base_ref }} run: | set -ex - BASE="${{ github.event.pull_request.base.sha }}" if [ -z "$BASE" ]; then - BASE_REF="${{ github.event.inputs.base_ref }}" - BASE_REF="${BASE_REF:-main}" - git rev-parse --verify -q "origin/$BASE_REF" >/dev/null \ - || git fetch -q origin "$BASE_REF:refs/remotes/origin/$BASE_REF" - BASE=$(git merge-base "$GITHUB_SHA" "origin/$BASE_REF") - fi - if [ "$BASE" = "$(git rev-parse "$GITHUB_SHA")" ]; then - # Dispatched from main itself: compare it against the commit before. - BASE=$(git rev-parse "$GITHUB_SHA^") + BASE=$(git merge-base HEAD "origin/$BASE_REF" 2>/dev/null || git merge-base HEAD "$BASE_REF") fi - # Fail rather than let asv infer a baseline of its own. - test -n "$BASE" || { echo "could not determine a baseline commit" >&2; exit 1; } + # Dispatched from the base itself: compare against the commit before. + [ "$BASE" != "$GITHUB_SHA" ] || BASE=$(git rev-parse HEAD^) echo "sha=$BASE" >> "$GITHUB_OUTPUT" # asv rebuilds its own env under benchmarks/env/ each run; caching skips the # solve. No restore-keys: asv reuses a restored env without checking it. - - name: Cache asv's benchmark environment + - &asv-env + name: Cache asv's benchmark environment uses: actions/cache@v6 with: path: ${{ env.ASV_DIR }}/env key: "asv-env-${{runner.os}}-${{runner.arch}}-${{hashFiles(env.CONDA_ENV_FILE, 'benchmarks/asv.conf.json')}}" - - name: Cache the benchmark fixtures + - &fixtures + name: Cache the benchmark fixtures uses: actions/cache@v6 with: path: | @@ -104,12 +102,13 @@ jobs: asv-fixtures-${{ runner.os }}- # The env cache key has no commit in it, so on a hit nothing is saved and the - # wheels built below would be lost; cache them (a few MB) per commit pair. - - name: Cache the built wheels + # wheels setup builds would be lost; cache them (a few MB) for the shards. + - &wheels + name: Cache the built wheels uses: actions/cache@v6 with: path: ${{ env.ASV_DIR }}/env/*/asv-build-cache - key: asv-wheels-${{ runner.os }}-${{ steps.base.outputs.sha }}-${{ github.sha }} + key: asv-wheels-${{ runner.os }}-${{ github.run_id }} # _partition balances shards by asv's recorded durations; with none it # splits by count. Restore-only: the merge job saves the updated tree. @@ -121,7 +120,8 @@ jobs: restore-keys: | asv-results-${{ runner.os }}- - - name: Pre-build and discover + - name: Pre-build, discover and plan + id: plan shell: bash -l {0} working-directory: ${{ env.ASV_DIR }} env: @@ -133,24 +133,12 @@ jobs: set -ex # Fill the fixture cache before asv preimports the suite, which would otherwise build it serially in the forkserver parent (cd .. && python -m benchmarks.helpers._fixtures) - asv machine --yes - (cd .. && python -m benchmarks.helpers._machine --name "$ASV_MACHINE") + (cd .. && python -m benchmarks.helpers._machine) asv run --bench just-discover "${BASE}^!" asv run --bench just-discover "${GITHUB_SHA}^!" - - - name: Plan the shards - id: plan - shell: bash -l {0} - working-directory: ${{ env.ASV_DIR }} - env: - SHARDS: ${{ github.event.inputs.shards || '4' }} - run: | - set -ex # Logs the split so a lopsided one is visible without opening each shard. PYTHONPATH=.. python -m benchmarks.helpers._partition --shards "$SHARDS" - echo "count=$SHARDS" >> "$GITHUB_OUTPUT" - python -c "import json, os; print('shards=' + json.dumps(list(range(int(os.environ['SHARDS'])))))" \ - >> "$GITHUB_OUTPUT" + echo "shards=[$(seq -s, 0 $((SHARDS - 1)))]" >> "$GITHUB_OUTPUT" # The whole tree, not just benchmarks.json, so shards weigh the same durations. - name: Upload the discovered suite @@ -169,45 +157,11 @@ jobs: matrix: shard: ${{ fromJSON(needs.setup.outputs.shards) }} steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - - name: Set up Conda environment - uses: mamba-org/setup-micromamba@v3 - with: - environment-file: ${{env.CONDA_ENV_FILE}} - cache-environment: true - environment-name: uxarray_build - cache-environment-key: "${{runner.os}}-${{runner.arch}}-${{hashFiles(env.CONDA_ENV_FILE)}}-benchmark" - create-args: >- - asv - python-build - mamba - - # No restore-keys: asv reuses a restored env without checking it. - - name: Cache asv's benchmark environment - uses: actions/cache@v6 - with: - path: ${{ env.ASV_DIR }}/env - key: "asv-env-${{runner.os}}-${{runner.arch}}-${{hashFiles(env.CONDA_ENV_FILE, 'benchmarks/asv.conf.json')}}" - - - name: Cache the benchmark fixtures - uses: actions/cache@v6 - with: - path: | - ${{ env.ASV_DIR }}/oQU*.nc - ${{ env.ASV_DIR }}/_io_cache - key: asv-fixtures-${{ runner.os }}-${{ hashFiles('benchmarks/helpers/_fixtures.py') }} - restore-keys: | - asv-fixtures-${{ runner.os }}- - - # Setup's wheels; see the note there. - - name: Cache the built wheels - uses: actions/cache@v6 - with: - path: ${{ env.ASV_DIR }}/env/*/asv-build-cache - key: asv-wheels-${{ runner.os }}-${{ needs.setup.outputs.base }}-${{ github.sha }} + - *checkout + - *conda + - *asv-env + - *fixtures + - *wheels - name: Download the discovered suite uses: actions/download-artifact@v8 @@ -220,18 +174,15 @@ jobs: working-directory: ${{ env.ASV_DIR }} env: BASE: ${{ needs.setup.outputs.base }} - SHARDS: ${{ needs.setup.outputs.count }} SHARD: ${{ matrix.shard }} # Partition from setup's plan, not this runner's cache, so all shards split the # same suite. Nothing checks they tile it; a mismatch silently skips benchmarks. run: | set -ex (cd .. && python -m benchmarks.helpers._fixtures) - asv machine --yes - (cd .. && python -m benchmarks.helpers._machine --name "$ASV_MACHINE") + (cd .. && python -m benchmarks.helpers._machine) ASV_ARGS=$(PYTHONPATH=.. python -m benchmarks.helpers._partition \ - --shards "$SHARDS" --shard "$SHARD" --results ../plan \ - --config asv.conf.json --asv-args) + --shards "$SHARDS" --shard "$SHARD" --results ../plan --config asv.conf.json) echo "Baseline: $BASE" echo "Contender: ${GITHUB_SHA} (${PR_HEAD_LABEL:-$GITHUB_REF_NAME})" # asv run, not continuous: continuous exits 1 on a regression, same as a @@ -263,21 +214,8 @@ jobs: if: ${{ always() && needs.setup.result == 'success' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - - name: Set up Conda environment - uses: mamba-org/setup-micromamba@v3 - with: - environment-file: ${{env.CONDA_ENV_FILE}} - cache-environment: true - environment-name: uxarray_build - cache-environment-key: "${{runner.os}}-${{runner.arch}}-${{hashFiles(env.CONDA_ENV_FILE)}}-benchmark" - create-args: >- - asv - python-build - mamba + - *checkout + - *conda # No merge-multiple: the shards' results files share names. - name: Download the shards @@ -293,45 +231,13 @@ jobs: BASE: ${{ needs.setup.outputs.base }} run: | set -ex - asv machine --yes - (cd .. && python -m benchmarks.helpers._machine --name "$ASV_MACHINE") + (cd .. && python -m benchmarks.helpers._machine) (cd .. && python -m benchmarks.helpers._merge --out benchmarks/results shards/asv-shard-*) asv compare --split --machine "$ASV_MACHINE" "$BASE" "${GITHUB_SHA}" \ > asv_compare_results.txt cat asv_compare_results.txt - - # Prints asv's recorded durations. A benchmark's is its final round only; - # and are the slowest shard's, since every shard paid them. - - name: Report where the time went - if: always() - shell: bash -l {0} - run: | - python - <<'PY' || true - import glob, json, os - - for path in sorted(glob.glob("benchmarks/results/*/*.json")): - if os.path.basename(path) in ("machine.json", "benchmarks.json"): - continue - data = json.load(open(path)) - columns = data.get("result_columns") or [] - if "duration" not in columns: - continue - index = columns.index("duration") - rows = sorted( - (float(row[index]), name) - for name, row in data["results"].items() - if len(row) > index and row[index] is not None - ) - total = sum(duration for duration, _ in rows) or 1.0 - params = data.get("params", {}) - print(f"\n=== {data['commit_hash'][:8]} on {params.get('cpu', '?')} " - f"({params.get('num_cpu', '?')} cpu) ===") - for key, value in sorted(data.get("durations", {}).items()): - print(f" {value:8.1f}s {key}") - print(f" {total:8.1f}s all {len(rows)} benchmarks ({total / 60:.1f} min)") - for duration, name in reversed(rows[-15:]): - print(f" {duration:8.1f}s {100 * duration / total:5.1f}% {name}") - PY + # Where the time went, and how the next run will split it. + PYTHONPATH=.. python -m benchmarks.helpers._partition --shards "$SHARDS" || true # Keyed per run so setup's prefix restore picks up the newest. - name: Save recorded durations for the next run diff --git a/.gitignore b/.gitignore index 724dfa80b..54c92b40f 100644 --- a/.gitignore +++ b/.gitignore @@ -164,6 +164,6 @@ benchmarks/env benchmarks/results benchmarks/html benchmarks/_io_cache -# Generated per shard by benchmarks/helpers/_partition.py --asv-args +# Generated per shard by benchmarks/helpers/_partition.py --shard benchmarks/asv.conf*.shard*.json benchmarks/results.shard* diff --git a/benchmarks/asv.conf.hpc.json b/benchmarks/asv.conf.hpc.json deleted file mode 100644 index 31e9d4b8e..000000000 --- a/benchmarks/asv.conf.hpc.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - // Thread-scaling variant of ``asv.conf.json``: - // - // asv run --config benchmarks/asv.conf.hpc.json - // - // A separate file, not a shell export, because asv folds ``env_nobuild`` - // into the environment name, keeping each thread count's results apart; - // it also overrides any exported ``NUMBA_NUM_THREADS``. - // - // KEEP IN SYNC with asv.conf.json (``pythons``, ``environment_type``, - // ``conda_channels``, ``matrix.req``, ``build_command``) or results - // silently stop being comparable. - - "version": 1, - "project": "uxarray", - "project_url": "https://github.com/UXARRAY/uxarray", - "repo": "..", - "branches": ["main"], - "dvcs": "git", - "environment_type": "conda", - "conda_channels": ["conda-forge"], - "install_timeout": 600, - "launch_method": "forkserver", - "show_commit_url": "https://github.com/UXARRAY/uxarray/commit/", - "pythons": ["3.11"], - "benchmark_dir": ".", - "build_command": [ - "python -mpip wheel --no-deps --no-build-isolation --no-index -w {build_cache_dir} {build_dir}" - ], - "build_cache_size": 4, - - // A parallel kernel at one thread runs ~core-count times longer. - "default_benchmark_timeout": 1800, - - "matrix": { - "req": { - "setuptools_scm": [""], - "xarray": [""], - "netcdf4": [""], - "pip+pyfma": [""], - "tbb": [""] - }, - - // One value each: every value is a full pass over the suite, though - // few benchmarks reach a ``parallel=True`` kernel. To sweep - // ``NUMBA_NUM_THREADS``, see ``--env`` in ``helpers/_partition.py``. - // BLAS is pinned so numba's threads are the only ones measured. - "env_nobuild": { - "NUMBA_THREADING_LAYER": ["forksafe"], - "NUMBA_NUM_THREADS": ["8"], - "OMP_NUM_THREADS": ["1"], - "MKL_NUM_THREADS": ["1"], - "OPENBLAS_NUM_THREADS": ["1"] - } - } -} diff --git a/benchmarks/helpers/_machine.py b/benchmarks/helpers/_machine.py index 9a8de5c31..85a707c22 100644 --- a/benchmarks/helpers/_machine.py +++ b/benchmarks/helpers/_machine.py @@ -1,135 +1,38 @@ -"""Pins the machine name asv files results under. +"""Records this host in asv's machine file under a stable name, and prints it. -asv defaults it to the hostname, which changes every hosted-runner job and -every cluster node, so results cannot be compared across runs or merged across -shards (``results//...``). ``asv machine --machine NAME`` alone drops -``cpu``/``num_cpu``/``ram``, hence detect first, rename after. Idempotent. +asv names the machine after the hostname, which changes every hosted-runner +job and every cluster node, so results cannot be compared across runs or +merged across shards (``results//...``). ``asv machine --machine +NAME`` alone drops the detected ``cpu``/``num_cpu``/``ram``, hence this. + +The name is ``$ASV_MACHINE``, then ``$NCAR_HOST`` (the cluster, not the node), +then the node name minus trailing digits. Login and compute nodes do not share +a stem, so set ``ASV_MACHINE`` if you use both without ``NCAR_HOST``. Usage:: - asv machine --yes - python -m benchmarks.helpers._machine --name gh-Linux-X64 + python -m benchmarks.helpers._machine # record, print the name + python -m benchmarks.helpers._machine --print # just print it """ -import argparse -import json import os import platform import re import sys -from pathlib import Path - -__all__ = ["pin"] - -_VERSION_KEY = "version" def default_name(): - """Returns ``(name, source_variable)`` stable across nodes of one cluster. - - ``$ASV_MACHINE``, then ``$NCAR_HOST`` (names the cluster, not the node), - then the node name minus trailing digits. Login and compute nodes do not - share a stem, so set ``ASV_MACHINE`` if you use both without ``NCAR_HOST``. - """ for variable in ("ASV_MACHINE", "NCAR_HOST"): - value = os.environ.get(variable) - if value: - return value, variable + if os.environ.get(variable): + return os.environ[variable] node = platform.node().split(".")[0] - return re.sub(r"[-_]?\d+$", "", node) or node, None - - -def default_path(): - """asv's machine file (``MachineCollection.get_machine_file_path``).""" - return Path.home() / ".asv-machine.json" - - -def pin(name, path=None, hostname=None, sole=False): - """Renames the machine file's freshly detected entry to ``name``; returns it. - - The fresh entry is the one keyed by this hostname, else one already named - ``name``, else a lone entry. Other entries are kept unless ``sole``. - """ - path = Path(path) if path is not None else default_path() - hostname = hostname if hostname is not None else platform.node() - stored = json.loads(path.read_text()) - version = stored.pop(_VERSION_KEY, None) - - if hostname in stored: - detected = stored.pop(hostname) - elif name in stored: - detected = stored[name] - elif len(stored) == 1: - (only,) = stored - detected = stored.pop(only) - else: - raise ValueError( - f"{path} holds {len(stored)} machines ({', '.join(sorted(stored))}), none of " - f"them this host ({hostname!r}) and none of them {name!r}; cannot tell which " - f"describes the machine this is running on. Run ``asv machine --yes`` first, " - f"or pass --name one of the recorded machines" - ) - - detected["machine"] = name - if sole: - stored = {} - stored[name] = detected - if version is not None: - stored[_VERSION_KEY] = version - path.write_text(json.dumps(stored, indent=4)) - return detected - - -def main(argv=None): - parser = argparse.ArgumentParser( - prog="python -m benchmarks.helpers._machine", - description="Rename asv's detected machine entry to a fixed name.", - ) - parser.add_argument( - "--name", - default=None, - help="Default $ASV_MACHINE, $NCAR_HOST, then hostname minus trailing digits.", - ) - parser.add_argument("--path", default=None, help="Machine file (default ~/.asv-machine.json).") - parser.add_argument( - "--hostname", default=None, help="Host whose entry to rename (default this one)." - ) - parser.add_argument( - "--sole", - action="store_true", - help="Drop other machines, so bare asv run/show work on any node without -m.", - ) - parser.add_argument( - "--quiet", action="store_true", help="Print only the pinned name, for capturing." - ) - parser.add_argument( - "--print", - dest="print_only", - action="store_true", - help="Print the name that would be pinned and change nothing.", - ) - args = parser.parse_args(argv) - - name, source = (args.name, "--name") if args.name else default_name() - if args.print_only: - print(name) - return 0 - detected = pin(name, args.path, args.hostname, sole=args.sole) - if args.quiet: - print(name) - return 0 - print( - f"{name}: {detected.get('cpu', '?')} " - f"({detected.get('num_cpu', '?')} cpu, {detected.get('os', '?')})" - ) - if source is None: - print( - f" note: {name!r} came from this host's name. A cluster's login and compute " - f"nodes do not share a stem, so set ASV_MACHINE (or rely on NCAR_HOST) if you " - f"benchmark from both, or the results will still split in two.", - ) - return 0 + return re.sub(r"[-_]?\d+$", "", node) or node if __name__ == "__main__": - sys.exit(main()) + name = default_name() + if "--print" not in sys.argv[1:]: + from asv.machine import Machine, MachineCollection + + MachineCollection.save(name, {**Machine.get_defaults(), "machine": name}) + print(name) diff --git a/benchmarks/helpers/_merge.py b/benchmarks/helpers/_merge.py index ebb6c1a98..7f2a11f30 100644 --- a/benchmarks/helpers/_merge.py +++ b/benchmarks/helpers/_merge.py @@ -1,193 +1,49 @@ """Merges a sharded run's results directories back into one tree. asv rewrites the whole ``/-.json`` at the end of a run, -so shards sharing a directory would overwrite one another; each writes its own -and they are combined here. Shards split by whole benchmark, so every row -belongs to one shard and the merge is a union. - -Rows are written in the order a serial run would produce -(:func:`canonical_order`). Idempotent and tolerant of shards not yet landed, so -it can run as jobs come back. +so each shard writes its own directory and they are combined here. Shards +split by whole benchmark, so every row belongs to one shard and the merge is a +union. Usage:: python -m benchmarks.helpers._merge --out results results.shard* - python -m benchmarks.helpers._merge --out results --quiet results.shard* """ import argparse import json -import shutil import sys from pathlib import Path -__all__ = ["canonical_order", "merge", "merge_benchmarks", "merge_result_files"] - BENCHMARK_DIR = Path(__file__).resolve().parents[1] -MACHINE_FILE = "machine.json" -BENCHMARKS_FILE = "benchmarks.json" -_SPECIAL_FILES = frozenset({MACHINE_FILE, BENCHMARKS_FILE}) - -# asv stores its own format version alongside the data in both files. -_VERSION_KEY = "version" - - -def _load(path): - with open(path) as handle: - return json.load(handle) - - -def _dump(path, data): - """Writes ``data`` unsorted, like asv's ``write_json(..., compact=True)``. - - Key order is the only record of what ran when. - """ - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w", encoding="utf-8") as handle: - json.dump(data, handle) - - -def canonical_order(benchmarks): - """Benchmark names in the order an unsharded ``asv run`` would produce them. - - Mirrors ``runner.run_benchmarks``: by name within each ``setup_cache_key`` - group, groups ordered by their first member's name. - """ - groups = {} - for name in sorted(benchmarks): - key = benchmarks[name].get("setup_cache_key") - groups.setdefault(key, []).append(name) - return [name for group in groups.values() for name in group] - - -def merge_benchmarks(shard_dirs): - """Union of the shards' ``benchmarks.json``. - - Each shard only lists its own benchmarks; ``_partition`` needs the full set - to plan the next run. - """ - merged, version = {}, None - for shard_dir in shard_dirs: - path = Path(shard_dir) / BENCHMARKS_FILE - if not path.is_file(): - continue - data = _load(path) - version = data.get(_VERSION_KEY, version) - for name, value in data.items(): - if name != _VERSION_KEY: - merged[name] = value - if version is not None: - merged[_VERSION_KEY] = version - return merged - - -def _pick(name, existing, candidate, report): - """Which of two rows for one benchmark to keep. - - Only reached if shards ran different plans. Prefers a row with a result, - then the later ``started_at``, so a re-run beats the run it replaced. - """ - if existing == candidate: - return existing - - def rank(row): - return (row.get("result") is not None, row.get("started_at") or 0) - - keep, drop = (candidate, existing) if rank(candidate) > rank(existing) else (existing, candidate) - report( - f"{name}: found in more than one shard with different data; keeping the " - f"row started at {keep.get('started_at')} over {drop.get('started_at')}" - ) - return keep - - -def merge_result_files(datas, order, report): - """One results file from several shards' copies, rows written in ``order``. - - Fields other than ``results`` and ``durations`` are identical across shards, - so the first shard's are kept. - """ - merged = dict(datas[0]) - columns = list(merged.get("result_columns") or []) - - rows, durations = {}, {} - for data in datas: - # Rows are bare lists, so align each to its own file's columns. - shard_columns = data.get("result_columns") or columns - for name, row in (data.get("results") or {}).items(): - values = dict(zip(shard_columns, row)) - rows[name] = ( - _pick(name, rows[name], values, report) if name in rows else values - ) - # Only ```` and ```` entries, which every shard - # pays; the max is what a single run would report. - for key, value in (data.get("durations") or {}).items(): - durations[key] = max(durations.get(key, 0.0), float(value)) - - known = [name for name in order if name in rows] - extra = sorted(name for name in rows if name not in set(order)) - if extra: - report(f"{len(extra)} row(s) not in benchmarks.json, appended: {', '.join(extra[:3])}...") - - results = {} - for name in known + extra: - row = [rows[name].get(column) for column in columns] - # Matches asv, which drops trailing nulls. - while row and row[-1] is None: - row.pop() - results[name] = row - - merged["results"] = results - merged["durations"] = durations - return merged - - -def merge(shard_dirs, out_dir, report=lambda message: None): - """Merges ``shard_dirs`` into ``out_dir``. Returns a per-file row count.""" - shard_dirs = [Path(d) for d in shard_dirs] - out_dir = Path(out_dir) - resolved_out = out_dir.resolve() - if any(d.resolve() == resolved_out for d in shard_dirs): - raise ValueError(f"--out {out_dir} is also a shard directory; refusing to merge in place") - - present = [d for d in shard_dirs if d.is_dir()] - for missing in [d for d in shard_dirs if not d.is_dir()]: - report(f"{missing}: not there yet, skipped") - if not present: - raise ValueError("no shard directories to merge") - - benchmarks = merge_benchmarks(present) - order = canonical_order({k: v for k, v in benchmarks.items() if k != _VERSION_KEY}) - out_dir.mkdir(parents=True, exist_ok=True) - if len(benchmarks) > (1 if _VERSION_KEY in benchmarks else 0): - _dump(out_dir / BENCHMARKS_FILE, benchmarks) - - # Shards of one commit and environment write the same file name. - groups = {} - for shard_dir in present: - for path in sorted(shard_dir.glob("*/*.json")): - if path.name in _SPECIAL_FILES: +# Every shard's benchmarks.json lists the whole suite (asv saves all it +# discovers, not just what ``--bench`` selects), and machine.json is pinned to +# one name, so any shard's copy will do. +_WHOLE_FILES = frozenset({"benchmarks.json", "machine.json"}) + + +def merge(shard_dirs, out_dir): + """Merges ``shard_dirs`` into ``out_dir``; returns ``{relative path: data}``.""" + merged = {} + for shard_dir in map(Path, shard_dirs): + for path in sorted(shard_dir.rglob("*.json")): + rel = path.relative_to(shard_dir) + data = json.loads(path.read_text()) + if rel not in merged or path.name in _WHOLE_FILES: + merged[rel] = data continue - groups.setdefault((path.parent.name, path.name), []).append(path) - for machine_path in sorted(shard_dir.glob(f"*/{MACHINE_FILE}")): - target = out_dir / machine_path.parent.name / MACHINE_FILE - if target.is_file() and _load(target) != _load(machine_path): - report(f"{machine_path}: disagrees with the machine.json already merged") - else: - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(machine_path, target) - - counts = {} - for (machine, filename), paths in sorted(groups.items()): - merged = merge_result_files( - [_load(p) for p in paths], - order, - lambda message, f=filename: report(f"{f}: {message}"), - ) - _dump(out_dir / machine / filename, merged) - counts[f"{machine}/{filename}"] = (len(merged["results"]), len(paths)) - return counts + into = merged[rel] + into["results"].update(data["results"]) + # ```` and ````, which every shard pays; the + # max is what a single run would report. + for key, value in data.get("durations", {}).items(): + into["durations"][key] = max(into["durations"].get(key, 0.0), value) + for rel, data in merged.items(): + target = Path(out_dir) / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(data)) + return merged def main(argv=None): @@ -201,16 +57,13 @@ def main(argv=None): default=str(BENCHMARK_DIR / "results"), help="Directory to write the merged tree to (default benchmarks/results).", ) - parser.add_argument("--quiet", action="store_true", help="Suppress per-file notes.") args = parser.parse_args(argv) - - def report(message): - if not args.quiet: - print(f" {message}", file=sys.stderr) - - counts = merge(args.shards, args.out, report) - for name, (rows, shards) in sorted(counts.items()): - print(f"{name}: {rows} benchmarks from {shards} shard(s)") + merged = merge(args.shards, args.out) + if not merged: + parser.error(f"no results under {', '.join(args.shards)}") + for rel, data in sorted(merged.items()): + if "results" in data: + print(f"{rel}: {len(data['results'])} benchmarks") return 0 diff --git a/benchmarks/helpers/_partition.py b/benchmarks/helpers/_partition.py index e59c443e9..99f4e81ad 100644 --- a/benchmarks/helpers/_partition.py +++ b/benchmarks/helpers/_partition.py @@ -6,8 +6,8 @@ Shards split by whole class: a class's benchmarks share the kernels its first one compiles, so splitting it makes every shard recompile (``FaceBounds``: 59s in one shard, 221s across four). Numba's ``cache=True`` does not help, as asv -reinstalls the wheel per commit. Whole benchmarks also leave every results row -owned by one shard, so :mod:`benchmarks.helpers._merge` can merge by union. +reinstalls the wheel per commit. Every ``setup_cache`` but the suite-wide one in +``_fixtures`` (cheap to repeat) lives on a single class, so stays in one shard. Each shard writes to its own ``results_dir``, since asv rewrites the whole per-commit results file at the end of a run. Weights are the ``duration`` asv @@ -16,97 +16,31 @@ Usage:: python -m benchmarks.helpers._partition --shards 4 - asv run $(python -m benchmarks.helpers._partition --shards 4 --shard 0 \ - --config asv.conf.hpc.json --asv-args) + asv run $(python -m benchmarks.helpers._partition --shards 4 --shard 0) # Thread sweep over the whole suite (add ``--bench`` to narrow it). for n in 1 2 4 8; do asv run $(python -m benchmarks.helpers._partition --shards 1 --shard 0 \ - --config asv.conf.hpc.json --env NUMBA_NUM_THREADS=$n --asv-args) + --env NUMBA_NUM_THREADS=$n) done """ import argparse import json -import os import re import statistics import sys from pathlib import Path -__all__ = [ - "bench_regexes", - "load_benchmarks", - "load_weights", - "plan", - "shard_results_dir", - "write_shard_config", -] - BENCHMARK_DIR = Path(__file__).resolve().parents[1] -# ``setup_cache`` keys cheap enough to repeat per shard (``prime()`` is a stat -# per file once warm). Other groups would rerun in every shard, so stay together. -SPLITTABLE_PREFIX = "helpers._fixtures:" - -_SKIP_FILES = frozenset({"machine.json", "benchmarks.json"}) - - -def _splittable(setup_cache_key): - """Whether benchmarks sharing this ``setup_cache_key`` may span shards.""" - return setup_cache_key is None or str(setup_cache_key).startswith(SPLITTABLE_PREFIX) - - -def _owner(name): - """The class -- or the module, for a bare function -- a benchmark belongs to.""" - return name.rsplit(".", 1)[0] - - -def _units(benchmarks): - """Benchmarks that must share a shard, as ``{root name: [names]}``. - - Unions same-class benchmarks with those sharing a non-splittable - ``setup_cache``. Roots are the smallest member name, so grouping is - deterministic. - """ - parent = {} - - def find(x): - parent.setdefault(x, x) - while parent[x] != x: - parent[x] = parent[parent[x]] - x = parent[x] - return x - - def union(a, b): - ra, rb = find(a), find(b) - if ra != rb: - parent[max(ra, rb)] = min(ra, rb) - - groups = {} - for name, benchmark in benchmarks.items(): - find(name) - groups.setdefault(("class", _owner(name)), []).append(name) - key = benchmark.get("setup_cache_key") - if not _splittable(key): - groups.setdefault(("setup_cache", key), []).append(name) - for members in groups.values(): - for other in members[1:]: - union(members[0], other) - - units = {} - for name in benchmarks: - units.setdefault(find(name), []).append(name) - return units - def load_benchmarks(results_dir): """The discovered benchmarks, as asv wrote them to ``benchmarks.json``.""" - path = Path(results_dir) / "benchmarks.json" - with open(path) as handle: - discovered = json.load(handle) + discovered = json.loads((Path(results_dir) / "benchmarks.json").read_text()) # asv stores its own format version alongside the benchmarks. - return {name: value for name, value in discovered.items() if name != "version"} + discovered.pop("version", None) + return discovered def load_weights(results_dirs): @@ -117,115 +51,79 @@ def load_weights(results_dirs): """ samples = {} for results_dir in results_dirs: - root = Path(results_dir) - if not root.is_dir(): - continue - for path in sorted(root.glob("*/*.json")): - if path.name in _SKIP_FILES: - continue - try: - with open(path) as handle: - data = json.load(handle) - except (OSError, ValueError): - continue + for path in sorted(Path(results_dir).glob("*/*.json")): + data = json.loads(path.read_text()) columns = data.get("result_columns") or [] if "duration" not in columns: continue index = columns.index("duration") - for name, row in (data.get("results") or {}).items(): - if len(row) <= index or row[index] is None: - continue - samples.setdefault(name, []).append(float(row[index])) + for name, row in data["results"].items(): + if len(row) > index and row[index] is not None: + samples.setdefault(name, []).append(float(row[index])) return {name: statistics.fmean(values) for name, values in samples.items()} +def costs(benchmarks, weights): + """Each benchmark's weight, the median for those without one.""" + known = [value for value in weights.values() if value > 0] + default = statistics.median(known) if known else 1.0 + return {name: weights.get(name, default) for name in benchmarks} + + def plan(benchmarks, n_shards, weights=None): """Partitions ``benchmarks`` into ``n_shards`` lists of names. Greedy longest-first (LPT): within about 1% of optimal on this suite and - easy to debug. Ties break by name, so a shard can compute its own membership. + easy to debug. Ties break by name, so every shard computes the same plan. """ - if n_shards < 1: - raise ValueError(f"n_shards must be at least 1, got {n_shards}") - weights = dict(weights or {}) - known = [value for value in weights.values() if value > 0] - default = statistics.median(known) if known else 1.0 - - units = _units(benchmarks) - - costs = { - unit: sum(weights.get(name, default) for name in names) - for unit, names in units.items() - } + cost = costs(benchmarks, weights or {}) + classes = {} + for name in sorted(benchmarks): + # The class -- or the module, for a bare function. + classes.setdefault(name.rsplit(".", 1)[0], []).append(name) + class_cost = {owner: sum(cost[name] for name in names) for owner, names in classes.items()} shards = [[] for _ in range(n_shards)] loads = [0.0] * n_shards - for unit in sorted(units, key=lambda u: (-costs[u], u)): - target = min(range(n_shards), key=lambda i: (loads[i], i)) - shards[target].extend(sorted(units[unit])) - loads[target] += costs[unit] + for owner in sorted(classes, key=lambda owner: (-class_cost[owner], owner)): + target = loads.index(min(loads)) + shards[target].extend(classes[owner]) + loads[target] += class_cost[owner] return shards -def bench_regexes(names): - """``--bench`` patterns selecting exactly ``names``. +def write_shard_config(base_config, shard, env): + """Copies ``base_config`` beside itself with the shard's ``results_dir``. - asv matches parameterized benchmarks as ``name(p0, p1)``, so the trailing - group admits ``(`` as well as end-of-string; ``^name$`` would match none. - """ - return [f"^{re.escape(name)}($|\\()" for name in names] - - -def shard_config_path(base_config, shard): - """Where shard ``shard``'s generated config goes, beside ``base_config``.""" - base = Path(base_config) - return base.with_name(f"{base.stem}.shard{shard}{base.suffix}") - - -def shard_results_dir(results_dir, shard): - """Where shard ``shard`` writes, given the run's ordinary ``results_dir``.""" - return f"{results_dir}.shard{shard}" - - -def write_shard_config(base_config, out_path, shard, env=None): - """Copies ``base_config`` with the shard's ``results_dir``; returns that dir. - - ``env`` overrides ``env_nobuild`` variables. Overrides land in the + ``env`` overrides ``env_nobuild`` variables, which asv folds into the environment name, so a sweep's runs do not overwrite one another. """ # asv configs are JSON with JS comments; lazy so the module runs without asv. from asv import util - config = util.load_json(str(base_config), js_comments=True) - results_dir = shard_results_dir(config.get("results_dir", "results"), shard) - config["results_dir"] = results_dir - if env: - matrix = config.setdefault("matrix", {}).setdefault("env_nobuild", {}) - # Several values would each be an environment running the whole suite. - matrix.update({key: [value] for key, value in env.items()}) - with open(out_path, "w") as handle: - json.dump(config, handle, indent=4) - return results_dir + base = Path(base_config) + config = util.load_json(str(base), js_comments=True) + config["results_dir"] = f"{config.get('results_dir', 'results')}.shard{shard}" + # Several values would each be an environment running the whole suite. + config["matrix"]["env_nobuild"].update({key: [value] for key, value in env.items()}) + out = base.with_name(f"{base.stem}.shard{shard}{base.suffix}") + out.write_text(json.dumps(config, indent=4)) + return out def _report(benchmarks, shards, weights): - known = [value for value in weights.values() if value > 0] - default = statistics.median(known) if known else 1.0 - total = sum(weights.get(name, default) for name in benchmarks) - print( - f"{len(benchmarks)} benchmarks, {len(weights)} with recorded durations, " - f"{total / 60:.1f} min of work; median fallback {default:.1f}s" - ) - loads = [sum(weights.get(name, default) for name in shard) for shard in shards] - ideal = total / len(shards) if shards else 0.0 + cost = costs(benchmarks, weights) + total = sum(cost.values()) + loads = [sum(cost[name] for name in shard) for shard in shards] + ideal = total / len(shards) + print(f"{len(benchmarks)} benchmarks, {len(weights)} with recorded durations, " + f"{total / 60:.1f} min of work") for index, (shard, load) in enumerate(zip(shards, loads)): - drift = 100 * (load - ideal) / ideal if ideal else 0.0 - print(f" shard {index}: {len(shard):3} benchmarks {load / 60:5.1f} min {drift:+5.1f}%") - if loads and ideal: - print( - f" slowest shard {max(loads) / 60:.1f} min against an ideal " - f"{ideal / 60:.1f}; speedup {total / max(loads):.2f}x of a possible {len(shards)}x" - ) + print(f" shard {index}: {len(shard):3} benchmarks {load / 60:5.1f} min " + f"{100 * (load - ideal) / ideal:+5.1f}%") + print(f" speedup {total / max(loads):.2f}x of a possible {len(shards)}x") + for name in sorted(cost, key=cost.get, reverse=True)[:15]: + print(f" {cost[name]:8.1f}s {name}") def main(argv=None): @@ -235,28 +133,17 @@ def main(argv=None): ) parser.add_argument("--shards", type=int, default=4, help="Number of shards (default 4).") parser.add_argument( - "--shard", type=int, default=None, help="Report only this shard, by index." + "--shard", type=int, help="Write this shard's config and print its asv run arguments." ) parser.add_argument( "--results", action="append", - default=None, help="Results directory to read (repeatable; default benchmarks/results).", ) - parser.add_argument( - "--bench-args", - action="store_true", - help="Print the shard's --bench arguments instead of a report.", - ) parser.add_argument( "--config", default=str(BENCHMARK_DIR / "asv.conf.json"), - help="Base asv config for --config-out (default benchmarks/asv.conf.json).", - ) - parser.add_argument( - "--config-out", - default=None, - help="Where --asv-args writes the shard config (default: beside --config).", + help="Base asv config for --shard (default benchmarks/asv.conf.json).", ) parser.add_argument( "--env", @@ -265,42 +152,23 @@ def main(argv=None): metavar="KEY=VALUE", help="Override an env_nobuild variable, e.g. NUMBA_NUM_THREADS=4 (repeatable).", ) - parser.add_argument( - "--asv-args", - action="store_true", - help="Write this shard's config and print its asv run arguments. Needs --shard.", - ) args = parser.parse_args(argv) results_dirs = args.results or [str(BENCHMARK_DIR / "results")] benchmarks = load_benchmarks(results_dirs[0]) weights = load_weights(results_dirs) shards = plan(benchmarks, args.shards, weights) - - if args.bench_args or args.asv_args: - if args.shard is None: - parser.error("--bench-args and --asv-args need --shard") - if args.asv_args: - # Emitted with ``--bench`` so a shard can't pair its benchmarks with - # another shard's results dir. - env = {} - for entry in args.env: - key, sep, value = entry.partition("=") - if not sep: - parser.error(f"--env wants KEY=VALUE, got {entry!r}") - env[key] = value - config_out = args.config_out or shard_config_path(args.config, args.shard) - write_shard_config(args.config, config_out, args.shard, env) - print("--config", config_out) - for pattern in bench_regexes(shards[args.shard]): - print("--bench", pattern) - return 0 - if args.shard is None: _report(benchmarks, shards, weights) - else: - for name in shards[args.shard]: - print(name) + return 0 + + env = dict(entry.split("=", 1) for entry in args.env) + # ``--bench`` alongside ``--config``, so a shard can't pair its benchmarks + # with another shard's results dir. asv matches parameterized benchmarks as + # ``name(p0, p1)``, hence ``($|\()`` rather than ``$``. + print("--config", write_shard_config(args.config, args.shard, env)) + for name in shards[args.shard]: + print("--bench", f"^{re.escape(name)}($|\\()") return 0 diff --git a/benchmarks/helpers/_threads.py b/benchmarks/helpers/_threads.py deleted file mode 100644 index a4dacf57f..000000000 --- a/benchmarks/helpers/_threads.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Resolves how many threads a benchmark run may use. - -numba treats ``NUMBA_NUM_THREADS`` as a ceiling fixed at import, so the count -must be in the environment before ``asv run`` (asv passes ``os.environ`` on). -It is deliberately absent from ``env_nobuild``, which would override the shell:: - - export NUMBA_NUM_THREADS=$(python -m benchmarks.helpers._threads) - asv run ... - -Defaults to physical cores: SMT siblings measured ~1.28x slower per thread on -these memory-bound kernels. ``UXARRAY_BENCH_THREADS`` overrides it with an -integer, ``physical`` or ``logical``. -""" - -import os -import subprocess -import sys - -__all__ = ["logical_cores", "physical_cores", "resolve"] - -_ENV_VAR = "UXARRAY_BENCH_THREADS" - - -def logical_cores(): - """Schedulable CPUs, honouring the affinity mask (batch scheduler, ``taskset``).""" - if hasattr(os, "sched_getaffinity"): - return len(os.sched_getaffinity(0)) - return os.cpu_count() or 1 - - -def physical_cores(): - """Cores rather than hardware threads, or the logical count if unknown. - - Shells out rather than adding ``psutil`` to the asv environment matrix. - """ - try: - if sys.platform == "darwin": - out = subprocess.run( - ["sysctl", "-n", "hw.physicalcpu"], - capture_output=True, text=True, timeout=5, check=True, - ).stdout - return max(1, int(out.strip())) - if sys.platform.startswith("linux"): - # Distinct (core, socket) pairs; core ids alone collapse sockets. - out = subprocess.run( - ["lscpu", "-p=core,socket"], - capture_output=True, text=True, timeout=5, check=True, - ).stdout - pairs = { - line for line in (l.strip() for l in out.splitlines()) - if line and not line.startswith("#") - } - if pairs: - return max(1, len(pairs)) - except (OSError, ValueError, subprocess.SubprocessError): - pass - return logical_cores() - - -def resolve(spec=None): - """The thread count to run with. - - ``spec`` defaults to ``$UXARRAY_BENCH_THREADS``, then ``physical``. Invalid - values warn and fall back to physical rather than blocking the run. - """ - if spec is None: - spec = os.environ.get(_ENV_VAR, "").strip() - spec = (spec or "physical").lower() - - if spec == "logical": - return logical_cores() - if spec != "physical": - try: - return max(1, int(spec)) - except ValueError: - print( - f"{_ENV_VAR}={spec!r} is not an integer, 'physical' or 'logical'; " - "using the physical core count", - file=sys.stderr, - ) - # Never hand back more than this process may schedule on. - return min(physical_cores(), logical_cores()) - - -if __name__ == "__main__": - print(resolve()) diff --git a/benchmarks/hpc/local.sh b/benchmarks/hpc/local.sh index a7caff07a..5a8a0f911 100755 --- a/benchmarks/hpc/local.sh +++ b/benchmarks/hpc/local.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Runs the sharded suite concurrently on the current node (e.g. a ``qsub -I`` -# session), each shard pinned to its own core slice. Shared memory bandwidth +# session), each shard pinned to its own physical cores. Shared memory bandwidth # and cache add noise: ``asv compare`` ratios stay usable, but take absolute # timings from one-shard-per-node (``submit.sh``). # @@ -17,60 +17,54 @@ SHARDS="${SHARDS:-4}" THREADS="${THREADS:-8}" export REPO SHARDS THREADS export REV="${REV:-HEAD^!}" -export CONFIG="${CONFIG:-asv.conf.hpc.json}" # Empty on purpose: stage.pbs derives it per cluster. export ASV_MACHINE="${ASV_MACHINE:-}" export ASV_ACTIVATE="${ASV_ACTIVATE:-true}" -# Activated here too so the core count below can come from python. +# Activated here too so the core list below can come from python. eval "$ASV_ACTIVATE" STAGE_SCRIPT="$REPO/benchmarks/hpc/stage.pbs" LOGS="${LOGS:-$REPO/benchmarks/hpc/logs}" mkdir -p "$LOGS" -# The affinity mask, not PBS's NCPUS (per-chunk request) or ``nproc`` (honours -# OMP_NUM_THREADS). Override with CORES. -CORES="${CORES:-$(python -c ' -import os -try: - print(len(os.sched_getaffinity(0))) -except AttributeError: # not Linux - print(os.cpu_count() or 1) -')}" -if ! [ "$CORES" -ge 1 ] 2>/dev/null; then - echo "could not work out a core count (got ${CORES:-empty}); set CORES" >&2 +# One CPU per physical core this job may use, in socket order. Slicing that +# keeps shards off each other's SMT siblings (~1.28x slower per thread) and, +# where a shard fits, on one socket. Contiguous logical CPU ranges do neither +# when siblings are numbered N and N + cores. +CPUS=($(python - <<'PY' +import os, subprocess +allowed = os.sched_getaffinity(0) +lscpu = subprocess.run( + ["lscpu", "-p=CPU,CORE,SOCKET"], capture_output=True, text=True, check=True +).stdout +first = {} +for line in lscpu.splitlines(): + if not line.startswith("#"): + cpu, core, socket = map(int, line.split(",")) + if cpu in allowed: + first.setdefault((socket, core), cpu) +print(*(first[key] for key in sorted(first))) +PY +)) +PER=$((${#CPUS[@]} / SHARDS)) +if [ "$PER" -lt 1 ]; then + echo "found ${#CPUS[@]} physical cores, too few for $SHARDS shards" >&2 exit 1 -fi -PER=$((CORES / SHARDS)) -if [ "$PER" -lt "$((THREADS + 1))" ]; then - echo "warning: $CORES cores over $SHARDS shards is $PER each, under the" >&2 +elif [ "$PER" -lt "$((THREADS + 1))" ]; then + echo "warning: ${#CPUS[@]} cores over $SHARDS shards is $PER each, under the" >&2 echo " $THREADS threads a shard wants; they will oversubscribe" >&2 fi -PIN="" -if command -v taskset >/dev/null; then - PIN="taskset" -else - echo "warning: no taskset, shards will not be pinned and will drift across cores" >&2 -fi - echo "== setup ==" STAGE=setup bash "$STAGE_SCRIPT" 2>&1 | tee "$LOGS/setup.log" echo "== $SHARDS shards, $PER cores each, $THREADS threads each ==" pids=() for S in $(seq 0 $((SHARDS - 1))); do - lo=$((S * PER)) - hi=$((lo + PER - 1)) - if [ -n "$PIN" ]; then - SHARD=$S STAGE=shard taskset -c "$lo-$hi" bash "$STAGE_SCRIPT" \ - >"$LOGS/shard$S.log" 2>&1 & - else - SHARD=$S STAGE=shard bash "$STAGE_SCRIPT" >"$LOGS/shard$S.log" 2>&1 & - fi - pid=$! - pids+=("$pid") - echo " shard $S -> cores $lo-$hi, pid $pid, log $LOGS/shard$S.log" + cpus=$(IFS=,; echo "${CPUS[*]:S * PER:PER}") + SHARD=$S STAGE=shard taskset -c "$cpus" bash "$STAGE_SCRIPT" >"$LOGS/shard$S.log" 2>&1 & + pids+=("$!") + echo " shard $S -> cpus $cpus, pid $!, log $LOGS/shard$S.log" done # A failed shard does not stop the merge (cf. ``afteranyarray`` in submit.sh). diff --git a/benchmarks/hpc/stage.pbs b/benchmarks/hpc/stage.pbs index 04b7af2e2..fab20b690 100644 --- a/benchmarks/hpc/stage.pbs +++ b/benchmarks/hpc/stage.pbs @@ -13,47 +13,40 @@ STAGE="${STAGE:?STAGE must be setup, shard or merge}" SHARDS="${SHARDS:-4}" REV="${REV:-HEAD^!}" REPO="${REPO:?REPO must be the repository root}" -CONFIG="${CONFIG:-asv.conf.hpc.json}" -# One machine name for all shards, or the merge has nothing to line up; see -# ``_machine.default_name``. -ASV_MACHINE="${ASV_MACHINE:-$(cd "$REPO" && python -m benchmarks.helpers._machine --print)}" -# Optional shared cache so shards read the source grids once between them. -if [ -n "${UXARRAY_BENCH_CACHE_DIR:-}" ]; then - export UXARRAY_BENCH_CACHE_DIR -fi +# Always set: numba's own default counts SMT siblings as cores. Applied through +# env_nobuild, so it lands in asv's environment name and thread counts do not +# overwrite each other's results. +THREADS="${THREADS:-8}" eval "${ASV_ACTIVATE:-true}" cd "$REPO/benchmarks" +# One machine name for all shards, or the merge has nothing to line up; see +# ``helpers/_machine.py``. +export ASV_MACHINE="${ASV_MACHINE:-$(cd .. && python -m benchmarks.helpers._machine --print)}" case "$STAGE" in setup) # Done once so concurrent shards do not race to build the shared # fixture cache, machine file and ``env/``. (cd .. && python -m benchmarks.helpers._fixtures) - asv machine --yes --config "$CONFIG" - (cd .. && python -m benchmarks.helpers._machine --name "$ASV_MACHINE") + (cd .. && python -m benchmarks.helpers._machine) # Discovery only: builds the environments and benchmarks.json, runs # nothing. ``env_nobuild`` is not hashed into ``env/``, so one build - # serves every thread count. - # ``--config`` everywhere, else asv picks up the CI asv.conf.json. - # ``-m`` after the pin, which renamed this host's entry: asv would fail - # with "No information stored about machine ''". - asv run --bench just-discover --config "$CONFIG" -m "$ASV_MACHINE" "$REV" + # serves every thread count. ``-m`` because the machine file has the + # pinned name, not this host's. + asv run --bench just-discover -m "$ASV_MACHINE" "$REV" PYTHONPATH=.. python -m benchmarks.helpers._partition --shards "$SHARDS" ;; shard) SHARD="${PBS_ARRAY_INDEX:-${SHARD:?}}" - # ``--asv-args`` writes this shard's config (own results_dir, --env - # override) and prints the asv arguments. THREADS, if set, overrides the - # config's NUMBA_NUM_THREADS and lands in the environment name. - THREAD_ARG="" - if [ -n "${THREADS:-}" ]; then - THREAD_ARG="--env NUMBA_NUM_THREADS=$THREADS" - fi + # Writes this shard's config (own results_dir, thread count) and prints + # the asv arguments. ASV_ARGS=$(PYTHONPATH=.. python -m benchmarks.helpers._partition \ - --shards "$SHARDS" --shard "$SHARD" --config "$CONFIG" $THREAD_ARG --asv-args) + --shards "$SHARDS" --shard "$SHARD" --env NUMBA_NUM_THREADS="$THREADS") status=0 - asv run --show-stderr -m "$ASV_MACHINE" $ASV_ARGS "$REV" || status=$? + # A parallel kernel at one thread runs ~core-count times longer than at + # eight, so allow more than CI's default timeout. + asv run --show-stderr -m "$ASV_MACHINE" -a timeout=1800 $ASV_ARGS "$REV" || status=$? # 2 means some benchmarks failed; the rest are still worth merging. if [ "$status" -eq 2 ]; then echo "asv exited 2: some benchmarks failed, see above" >&2 @@ -64,7 +57,7 @@ shard) merge) (cd .. && python -m benchmarks.helpers._merge \ --out benchmarks/results benchmarks/results.shard*) - asv show --config "$CONFIG" -m "$ASV_MACHINE" || true + asv show -m "$ASV_MACHINE" || true ;; *) echo "unknown STAGE $STAGE" >&2 diff --git a/benchmarks/hpc/submit.sh b/benchmarks/hpc/submit.sh index e8e41d42a..def25ab29 100755 --- a/benchmarks/hpc/submit.sh +++ b/benchmarks/hpc/submit.sh @@ -10,8 +10,8 @@ # PBS_ACCOUNT=UXXX0001 THREADS=8 ./benchmarks/hpc/submit.sh # PBS_ACCOUNT=UXXX0001 SHARDS=8 REV=main^! ./benchmarks/hpc/submit.sh # -# THREADS overrides the config's NUMBA_NUM_THREADS and is recorded in the -# environment name, so thread counts do not overwrite each other's results. +# THREADS (default 8, see stage.pbs) sets NUMBA_NUM_THREADS and is recorded in +# the environment name, so thread counts do not overwrite each other's results. # set -euo pipefail @@ -22,7 +22,6 @@ SHARDS="${SHARDS:-4}" # An asv range: ``main^!`` is one commit, ``base..head`` every commit between. REV="${REV:-HEAD^!}" QUEUE="${QUEUE:-main}" -CONFIG="${CONFIG:-asv.conf.hpc.json}" # Empty means stage.pbs derives it from NCAR_HOST or the node name. ASV_MACHINE="${ASV_MACHINE:-}" WALLTIME="${WALLTIME:-12:00:00}" @@ -39,8 +38,7 @@ command -v asv >/dev/null || [ "$ASV_ACTIVATE" != "true" ] || { STAGE_SCRIPT="$REPO/benchmarks/hpc/stage.pbs" # Via the environment, not ``-v``, whose comma-separated list cannot hold commas. -export REPO SHARDS REV CONFIG ASV_MACHINE -export THREADS="${THREADS:-}" +export REPO SHARDS REV ASV_MACHINE THREADS export UXARRAY_BENCH_CACHE_DIR="$CACHE_DIR" mkdir -p "$CACHE_DIR" From d136d953c96eb8d7b5af3fa553b7f2240db6da15 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Wed, 7 Oct 2026 18:20:48 -0500 Subject: [PATCH 47/52] Remove hpc scripts (maybe re-add on a new branch) --- benchmarks/hpc/local.sh | 83 ---------------------------------------- benchmarks/hpc/stage.pbs | 66 -------------------------------- benchmarks/hpc/submit.sh | 65 ------------------------------- 3 files changed, 214 deletions(-) delete mode 100755 benchmarks/hpc/local.sh delete mode 100644 benchmarks/hpc/stage.pbs delete mode 100755 benchmarks/hpc/submit.sh diff --git a/benchmarks/hpc/local.sh b/benchmarks/hpc/local.sh deleted file mode 100755 index 5a8a0f911..000000000 --- a/benchmarks/hpc/local.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -# Runs the sharded suite concurrently on the current node (e.g. a ``qsub -I`` -# session), each shard pinned to its own physical cores. Shared memory bandwidth -# and cache add noise: ``asv compare`` ratios stay usable, but take absolute -# timings from one-shard-per-node (``submit.sh``). -# -# Usage, from the repository root: -# -# ./benchmarks/hpc/local.sh -# SHARDS=8 THREADS=4 ./benchmarks/hpc/local.sh -# REV=main^! ./benchmarks/hpc/local.sh -# -set -euo pipefail - -REPO="${REPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" -SHARDS="${SHARDS:-4}" -THREADS="${THREADS:-8}" -export REPO SHARDS THREADS -export REV="${REV:-HEAD^!}" -# Empty on purpose: stage.pbs derives it per cluster. -export ASV_MACHINE="${ASV_MACHINE:-}" -export ASV_ACTIVATE="${ASV_ACTIVATE:-true}" -# Activated here too so the core list below can come from python. -eval "$ASV_ACTIVATE" - -STAGE_SCRIPT="$REPO/benchmarks/hpc/stage.pbs" -LOGS="${LOGS:-$REPO/benchmarks/hpc/logs}" -mkdir -p "$LOGS" - -# One CPU per physical core this job may use, in socket order. Slicing that -# keeps shards off each other's SMT siblings (~1.28x slower per thread) and, -# where a shard fits, on one socket. Contiguous logical CPU ranges do neither -# when siblings are numbered N and N + cores. -CPUS=($(python - <<'PY' -import os, subprocess -allowed = os.sched_getaffinity(0) -lscpu = subprocess.run( - ["lscpu", "-p=CPU,CORE,SOCKET"], capture_output=True, text=True, check=True -).stdout -first = {} -for line in lscpu.splitlines(): - if not line.startswith("#"): - cpu, core, socket = map(int, line.split(",")) - if cpu in allowed: - first.setdefault((socket, core), cpu) -print(*(first[key] for key in sorted(first))) -PY -)) -PER=$((${#CPUS[@]} / SHARDS)) -if [ "$PER" -lt 1 ]; then - echo "found ${#CPUS[@]} physical cores, too few for $SHARDS shards" >&2 - exit 1 -elif [ "$PER" -lt "$((THREADS + 1))" ]; then - echo "warning: ${#CPUS[@]} cores over $SHARDS shards is $PER each, under the" >&2 - echo " $THREADS threads a shard wants; they will oversubscribe" >&2 -fi - -echo "== setup ==" -STAGE=setup bash "$STAGE_SCRIPT" 2>&1 | tee "$LOGS/setup.log" - -echo "== $SHARDS shards, $PER cores each, $THREADS threads each ==" -pids=() -for S in $(seq 0 $((SHARDS - 1))); do - cpus=$(IFS=,; echo "${CPUS[*]:S * PER:PER}") - SHARD=$S STAGE=shard taskset -c "$cpus" bash "$STAGE_SCRIPT" >"$LOGS/shard$S.log" 2>&1 & - pids+=("$!") - echo " shard $S -> cpus $cpus, pid $!, log $LOGS/shard$S.log" -done - -# A failed shard does not stop the merge (cf. ``afteranyarray`` in submit.sh). -failed=0 -for S in $(seq 0 $((SHARDS - 1))); do - if wait "${pids[$S]}"; then - echo " shard $S ok" - else - echo " shard $S FAILED (see $LOGS/shard$S.log)" >&2 - failed=$((failed + 1)) - fi -done - -echo "== merge ==" -STAGE=merge bash "$STAGE_SCRIPT" 2>&1 | tee "$LOGS/merge.log" -[ "$failed" -eq 0 ] || echo "$failed shard(s) failed; merged what landed" >&2 diff --git a/benchmarks/hpc/stage.pbs b/benchmarks/hpc/stage.pbs deleted file mode 100644 index fab20b690..000000000 --- a/benchmarks/hpc/stage.pbs +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -# The setup, shard and merge stages of a sharded asv run, selected by STAGE. -# Submitted by ``submit.sh`` (or run by ``local.sh``), not qsub'd by hand. -# -#PBS -N asv -#PBS -j oe -#PBS -l select=1:ncpus=128 -#PBS -l walltime=12:00:00 - -set -euo pipefail - -STAGE="${STAGE:?STAGE must be setup, shard or merge}" -SHARDS="${SHARDS:-4}" -REV="${REV:-HEAD^!}" -REPO="${REPO:?REPO must be the repository root}" -# Always set: numba's own default counts SMT siblings as cores. Applied through -# env_nobuild, so it lands in asv's environment name and thread counts do not -# overwrite each other's results. -THREADS="${THREADS:-8}" - -eval "${ASV_ACTIVATE:-true}" -cd "$REPO/benchmarks" -# One machine name for all shards, or the merge has nothing to line up; see -# ``helpers/_machine.py``. -export ASV_MACHINE="${ASV_MACHINE:-$(cd .. && python -m benchmarks.helpers._machine --print)}" - -case "$STAGE" in -setup) - # Done once so concurrent shards do not race to build the shared - # fixture cache, machine file and ``env/``. - (cd .. && python -m benchmarks.helpers._fixtures) - (cd .. && python -m benchmarks.helpers._machine) - # Discovery only: builds the environments and benchmarks.json, runs - # nothing. ``env_nobuild`` is not hashed into ``env/``, so one build - # serves every thread count. ``-m`` because the machine file has the - # pinned name, not this host's. - asv run --bench just-discover -m "$ASV_MACHINE" "$REV" - PYTHONPATH=.. python -m benchmarks.helpers._partition --shards "$SHARDS" - ;; -shard) - SHARD="${PBS_ARRAY_INDEX:-${SHARD:?}}" - # Writes this shard's config (own results_dir, thread count) and prints - # the asv arguments. - ASV_ARGS=$(PYTHONPATH=.. python -m benchmarks.helpers._partition \ - --shards "$SHARDS" --shard "$SHARD" --env NUMBA_NUM_THREADS="$THREADS") - status=0 - # A parallel kernel at one thread runs ~core-count times longer than at - # eight, so allow more than CI's default timeout. - asv run --show-stderr -m "$ASV_MACHINE" -a timeout=1800 $ASV_ARGS "$REV" || status=$? - # 2 means some benchmarks failed; the rest are still worth merging. - if [ "$status" -eq 2 ]; then - echo "asv exited 2: some benchmarks failed, see above" >&2 - elif [ "$status" -ne 0 ]; then - exit "$status" - fi - ;; -merge) - (cd .. && python -m benchmarks.helpers._merge \ - --out benchmarks/results benchmarks/results.shard*) - asv show -m "$ASV_MACHINE" || true - ;; -*) - echo "unknown STAGE $STAGE" >&2 - exit 2 - ;; -esac diff --git a/benchmarks/hpc/submit.sh b/benchmarks/hpc/submit.sh deleted file mode 100755 index def25ab29..000000000 --- a/benchmarks/hpc/submit.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# Submits a sharded asv run on derecho as three chained PBS jobs: setup (once, -# so shards do not race on the shared filesystem), a job array of shards (one -# node each, so ``time_*`` results are uncontended), and merge. The merge uses -# ``afteranyarray`` so one failed shard does not cost the others' results. -# -# Usage: -# -# PBS_ACCOUNT=UXXX0001 ./benchmarks/hpc/submit.sh -# PBS_ACCOUNT=UXXX0001 THREADS=8 ./benchmarks/hpc/submit.sh -# PBS_ACCOUNT=UXXX0001 SHARDS=8 REV=main^! ./benchmarks/hpc/submit.sh -# -# THREADS (default 8, see stage.pbs) sets NUMBA_NUM_THREADS and is recorded in -# the environment name, so thread counts do not overwrite each other's results. -# -set -euo pipefail - -: "${PBS_ACCOUNT:?set PBS_ACCOUNT to your project code, e.g. PBS_ACCOUNT=UXXX0001}" - -REPO="${REPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" -SHARDS="${SHARDS:-4}" -# An asv range: ``main^!`` is one commit, ``base..head`` every commit between. -REV="${REV:-HEAD^!}" -QUEUE="${QUEUE:-main}" -# Empty means stage.pbs derives it from NCAR_HOST or the node name. -ASV_MACHINE="${ASV_MACHINE:-}" -WALLTIME="${WALLTIME:-12:00:00}" -SETUP_WALLTIME="${SETUP_WALLTIME:-02:00:00}" -# On scratch so every shard and worktree shares one fixture cache. -CACHE_DIR="${UXARRAY_BENCH_CACHE_DIR:-/glade/derecho/scratch/$USER/uxarray-bench}" -# ``-V`` carries this shell's ``asv`` into the jobs. ASV_ACTIVATE, if set, is -# eval'd once per stage instead. -export ASV_ACTIVATE="${ASV_ACTIVATE:-true}" -command -v asv >/dev/null || [ "$ASV_ACTIVATE" != "true" ] || { - echo "asv is not on PATH; activate your environment first, or set ASV_ACTIVATE" >&2 - exit 1 -} - -STAGE_SCRIPT="$REPO/benchmarks/hpc/stage.pbs" -# Via the environment, not ``-v``, whose comma-separated list cannot hold commas. -export REPO SHARDS REV ASV_MACHINE THREADS -export UXARRAY_BENCH_CACHE_DIR="$CACHE_DIR" - -mkdir -p "$CACHE_DIR" - -setup=$(qsub -A "$PBS_ACCOUNT" -q "$QUEUE" -N asv-setup \ - -l select=1:ncpus=128 -l walltime="$SETUP_WALLTIME" \ - -V -v "STAGE=setup" "$STAGE_SCRIPT") -echo "setup $setup" - -shards=$(qsub -A "$PBS_ACCOUNT" -q "$QUEUE" -N asv-shard \ - -J "0-$((SHARDS - 1))" \ - -l select=1:ncpus=128 -l walltime="$WALLTIME" \ - -W "depend=afterok:$setup" \ - -V -v "STAGE=shard" "$STAGE_SCRIPT") -echo "shards $shards ($SHARDS of them)" - -merge=$(qsub -A "$PBS_ACCOUNT" -q "$QUEUE" -N asv-merge \ - -l select=1:ncpus=1 -l walltime=00:30:00 \ - -W "depend=afteranyarray:$shards" \ - -V -v "STAGE=merge" "$STAGE_SCRIPT") -echo "merge $merge" -echo -echo "watch with: qstat -u $USER -t" -echo "results in: $REPO/benchmarks/results/$ASV_MACHINE/" From c88d68d69d11c08c2a81ef7d7e366e922acb7da3 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Wed, 7 Oct 2026 19:04:51 -0500 Subject: [PATCH 48/52] Build neighborhood benchmark setups once per process; seed PR durations from main asv_runner re-runs setup before every sample, about six times per benchmark process. NeighborhoodReduce.time_reduce timed a 5ms kernel at 120km but took 27s, and NeighborhoodDask.time_mean 49s for 1-40ms calls, almost all setup. That made NeighborhoodReduce alone larger than a quarter of the suite, which capped a 4-shard split at 3.2x. - NeighborhoodReduce and NeighborhoodDask build their fixture, warmup and neighborhood once per process. The timed calls only read them. The grid caches a single ball tree and time_dataset_reduce leaves it on edges, so setup restores the face tree each sample, as a fresh setup did. - main's benchmark workflow saves the commit it just ran under the asv-results cache key. PR caches are scoped to their PR, so every PR's first run had no durations and split by count (this branch's first run: shards of 3:11, 4:48, 2:17 and 9:49). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/asv-benchmarking-pr.yml | 3 +- .github/workflows/asv-benchmarking.yml | 25 ++++++++++++++++ benchmarks/mpas_ocean.py | 36 +++++++++++++++++++++-- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 26b20a473..b7968d7b6 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -111,7 +111,8 @@ jobs: key: asv-wheels-${{ runner.os }}-${{ github.run_id }} # _partition balances shards by asv's recorded durations; with none it - # splits by count. Restore-only: the merge job saves the updated tree. + # splits by count. Restore-only: the merge job saves the updated tree. A + # PR's first run falls back to main's, saved by asv-benchmarking.yml. - name: Restore recorded durations uses: actions/cache/restore@v6 with: diff --git a/.github/workflows/asv-benchmarking.yml b/.github/workflows/asv-benchmarking.yml index c98aecea6..aaf367ae6 100644 --- a/.github/workflows/asv-benchmarking.yml +++ b/.github/workflows/asv-benchmarking.yml @@ -126,3 +126,28 @@ jobs: force: true repository: UXARRAY/uxarray-asv directory: uxarray-asv + + # The PR workflow plans its shards from asv's recorded durations, and a + # PR's own caches are invisible to other PRs, so without this every PR's + # first run splits by count. Only this commit's results: _partition + # averages every file it finds, and the history has since-redefined + # benchmarks. After the push, so the pruning never reaches uxarray-asv. + - name: Keep only this commit's durations + id: durations + working-directory: ${{ env.ASV_DIR }} + run: | + sha=$(git rev-parse HEAD) + find results -name '*.json' ! -name benchmarks.json ! -name machine.json \ + ! -name "${sha:0:8}-*" -delete + # A failed run leaves nothing; don't shadow an older cache with that. + if compgen -G "results/*/${sha:0:8}-*.json" > /dev/null; then + echo "found=true" >> "$GITHUB_OUTPUT" + fi + + # Same path and key prefix the PR workflow restores. + - name: Save recorded durations for PR runs + if: steps.durations.outputs.found == 'true' + uses: actions/cache/save@v6 + with: + path: ${{ env.ASV_DIR }}/results + key: asv-results-${{ runner.os }}-${{ github.run_id }} diff --git a/benchmarks/mpas_ocean.py b/benchmarks/mpas_ocean.py index a75393286..8d8687e64 100644 --- a/benchmarks/mpas_ocean.py +++ b/benchmarks/mpas_ocean.py @@ -391,6 +391,19 @@ def track_peakmem_const_lat(self, resolution, lat_step): # and dozens on the fine one. NEIGHBORHOOD_RADIUS = 5.0 +# asv re-runs ``setup`` before every sample, about six times per benchmark +# process, and the neighborhood setups (fixture, warmup reduction, radius +# query) cost far more than the calls they time: ``NeighborhoodReduce. +# time_reduce`` timed a 5ms kernel at 120km but took 27s, almost all setup. The +# timed calls only read what setup builds, so it is built once per process. +_prepared = {} + + +def _once_per_process(key, build): + if key not in _prepared: + _prepared[key] = build() + return _prepared[key] + class NeighborhoodBuild(DatasetBenchmark): """Construction cost of a ``Neighborhood``, split into its three stages. @@ -475,6 +488,17 @@ def _run(neighborhood, reduction): return getattr(neighborhood, reduction)() def setup(self, resolution, reduction): + self.uxds, self.nb = _once_per_process( + ('NeighborhoodReduce', resolution, reduction), + lambda: self._prepare(resolution, reduction)) + # The grid caches one ball tree, and ``time_dataset_reduce`` leaves it + # on edges. Put back the face tree a fresh setup leaves, so every sample + # rebuilds the same trees. + self.uxds.uxgrid.get_ball_tree(coordinates="face centers", + coordinate_system="spherical", + distance_metric="haversine") + + def _prepare(self, resolution, reduction): super().setup(resolution) uxgrid = self.uxds.uxgrid @@ -498,7 +522,7 @@ def setup(self, resolution, reduction): # locations now so the first timed call is not the one that pays. _, _, _ = uxgrid.node_lon, uxgrid.edge_lon, uxgrid.face_lon - self.nb = self.uxds[data_var].neighborhood(r=self.radius) + return self.uxds, self.uxds[data_var].neighborhood(r=self.radius) def time_reduce(self, resolution, reduction): """The kernel alone: the query was paid for in ``setup``.""" @@ -534,6 +558,11 @@ class NeighborhoodDask(DatasetBenchmark): radius = NEIGHBORHOOD_RADIUS def setup(self, resolution, chunking): + self.uxds, self.nb = _once_per_process( + ('NeighborhoodDask', resolution, chunking), + lambda: self._prepare(resolution, chunking)) + + def _prepare(self, resolution, chunking): super().setup(resolution) grid, data = file_path_dict[self.params[0][0]] _ = ux.open_dataset(grid, data)[data_var].neighborhood(r=1.0).mean() @@ -550,15 +579,16 @@ def setup(self, resolution, chunking): # Built here, so these measure the reduction and the graph it runs # through rather than the query. - self.nb = uxda.neighborhood(r=self.radius) + nb = uxda.neighborhood(r=self.radius) # One reduction here too, to warm the dask graph path -- and, for # 'grid_chunks', to let the rechunk warning through exactly once... - _ = self.nb.mean().compute() + _ = nb.mean().compute() # ...then silence the repeats. warnings.filterwarnings('ignore', category=UserWarning, message='Rechunking') + return self.uxds, nb def time_mean(self, resolution, chunking): _ = self.nb.mean().compute() From 00506a19c06a65bb157b25fb314dc779d25a60a4 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 12 Oct 2026 13:45:03 -0500 Subject: [PATCH 49/52] Simplify the shard merge; restore the results artifact's runner.os name - _merge: seed each file from the first shard that has it and fold later shards' results in, telling result files by their "results" key rather than a list of whole-file names. rel is now relpath, since keys keep the machine directory. - The results artifact is named with runner.os again, as on main. The comment workflow still downloads asv-benchmark-results-Linux. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/asv-benchmarking-pr.yml | 2 +- benchmarks/helpers/_merge.py | 29 ++++++++++++----------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index b7968d7b6..ae3d78421 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -255,7 +255,7 @@ jobs: - uses: actions/upload-artifact@v7 if: always() with: - name: asv-benchmark-results-Linux + name: asv-benchmark-results-${{ runner.os }} path: | ${{ env.ASV_DIR }}/results ${{ env.ASV_DIR }}/asv_compare_results.txt diff --git a/benchmarks/helpers/_merge.py b/benchmarks/helpers/_merge.py index 7f2a11f30..5899b2e3b 100644 --- a/benchmarks/helpers/_merge.py +++ b/benchmarks/helpers/_merge.py @@ -17,30 +17,31 @@ BENCHMARK_DIR = Path(__file__).resolve().parents[1] -# Every shard's benchmarks.json lists the whole suite (asv saves all it -# discovers, not just what ``--bench`` selects), and machine.json is pinned to -# one name, so any shard's copy will do. -_WHOLE_FILES = frozenset({"benchmarks.json", "machine.json"}) - def merge(shard_dirs, out_dir): - """Merges ``shard_dirs`` into ``out_dir``; returns ``{relative path: data}``.""" + """Merges ``shard_dirs`` into ``out_dir``; returns ``{relative path: data}``. + + Files without ``results`` keep the first shard's copy: every shard's + benchmarks.json lists the whole suite (asv saves all it discovers, not just + what ``--bench`` selects), and machine.json is pinned to one name. + """ merged = {} for shard_dir in map(Path, shard_dirs): for path in sorted(shard_dir.rglob("*.json")): - rel = path.relative_to(shard_dir) + relpath = path.relative_to(shard_dir) data = json.loads(path.read_text()) - if rel not in merged or path.name in _WHOLE_FILES: - merged[rel] = data + # The first shard with this file seeds it; later shards' results + # fold into that copy. + into = merged.setdefault(relpath, data) + if into is data or "results" not in data: continue - into = merged[rel] into["results"].update(data["results"]) # ```` and ````, which every shard pays; the # max is what a single run would report. for key, value in data.get("durations", {}).items(): into["durations"][key] = max(into["durations"].get(key, 0.0), value) - for rel, data in merged.items(): - target = Path(out_dir) / rel + for relpath, data in merged.items(): + target = Path(out_dir) / relpath target.parent.mkdir(parents=True, exist_ok=True) target.write_text(json.dumps(data)) return merged @@ -61,9 +62,9 @@ def main(argv=None): merged = merge(args.shards, args.out) if not merged: parser.error(f"no results under {', '.join(args.shards)}") - for rel, data in sorted(merged.items()): + for relpath, data in sorted(merged.items()): if "results" in data: - print(f"{rel}: {len(data['results'])} benchmarks") + print(f"{relpath}: {len(data['results'])} benchmarks") return 0 From 41913f832b57b6f46c36c21e18c0eb81ec34b697 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 12 Oct 2026 13:48:53 -0500 Subject: [PATCH 50/52] Weigh shards by the median recorded duration load_weights keeps every sample, so the median costs nothing over the mean and ignores a single cold-numba outlier once there are three or more, as in a local results dir with history. CI restores two samples per benchmark (base and PR), where the two agree. Co-Authored-By: Claude Opus 5.5 --- benchmarks/helpers/_partition.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/benchmarks/helpers/_partition.py b/benchmarks/helpers/_partition.py index 99f4e81ad..33b4b0422 100644 --- a/benchmarks/helpers/_partition.py +++ b/benchmarks/helpers/_partition.py @@ -44,10 +44,11 @@ def load_benchmarks(results_dir): def load_weights(results_dirs): - """Mean recorded duration per benchmark, in seconds. + """Median recorded duration per benchmark, in seconds. - Mean, not latest: a cold numba cache can make one run hundreds of times - slower than warm, and a single outlier would skew the plan. + Median, not latest or mean: a cold numba cache can make one run hundreds of + times slower than warm, and a single outlier would skew the plan. With two + samples, as in CI (the base and PR commits), it is the mean. """ samples = {} for results_dir in results_dirs: @@ -60,7 +61,7 @@ def load_weights(results_dirs): for name, row in data["results"].items(): if len(row) > index and row[index] is not None: samples.setdefault(name, []).append(float(row[index])) - return {name: statistics.fmean(values) for name, values in samples.items()} + return {name: statistics.median(values) for name, values in samples.items()} def costs(benchmarks, weights): From 5abfba4fdf921abaa59d35f3414590f8805c5595 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 12 Oct 2026 13:56:57 -0500 Subject: [PATCH 51/52] Read result files' duration column directly in load_weights Skip machine.json by its missing "results", as _merge does, and index result_columns without a fallback: every v2 results file (asv >= 0.5) has a duration column, and asv itself won't read a v1 file without asv update. The median doesn't depend on order, so the glob needn't be sorted. Co-Authored-By: Claude Opus 5.5 --- benchmarks/helpers/_partition.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/benchmarks/helpers/_partition.py b/benchmarks/helpers/_partition.py index 33b4b0422..2d5eb54ea 100644 --- a/benchmarks/helpers/_partition.py +++ b/benchmarks/helpers/_partition.py @@ -52,12 +52,11 @@ def load_weights(results_dirs): """ samples = {} for results_dir in results_dirs: - for path in sorted(Path(results_dir).glob("*/*.json")): + for path in Path(results_dir).glob("*/*.json"): data = json.loads(path.read_text()) - columns = data.get("result_columns") or [] - if "duration" not in columns: + if "results" not in data: # machine.json continue - index = columns.index("duration") + index = data["result_columns"].index("duration") for name, row in data["results"].items(): if len(row) > index and row[index] is not None: samples.setdefault(name, []).append(float(row[index])) From 63f795c9aad5af9b237a85f7ef0ad308a5dbecc6 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 12 Oct 2026 14:08:27 -0500 Subject: [PATCH 52/52] Name plan's groupings for what they hold benchmark_classes maps each class (or module, for a bare function) to its (benchmark name, cost) pairs, and class_costs to their total, in place of classes, class_cost and owner. Plans are unchanged. Co-Authored-By: Claude Opus 5.5 --- benchmarks/helpers/_partition.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/benchmarks/helpers/_partition.py b/benchmarks/helpers/_partition.py index 2d5eb54ea..533f4c02e 100644 --- a/benchmarks/helpers/_partition.py +++ b/benchmarks/helpers/_partition.py @@ -77,18 +77,22 @@ def plan(benchmarks, n_shards, weights=None): easy to debug. Ties break by name, so every shard computes the same plan. """ cost = costs(benchmarks, weights or {}) - classes = {} + # (benchmark name, cost) pairs by class, or by module for a bare function; + # a shard takes a benchmark class whole. + benchmark_classes = {} for name in sorted(benchmarks): - # The class -- or the module, for a bare function. - classes.setdefault(name.rsplit(".", 1)[0], []).append(name) - class_cost = {owner: sum(cost[name] for name in names) for owner, names in classes.items()} + benchmark_classes.setdefault(name.rsplit(".", 1)[0], []).append((name, cost[name])) + class_costs = { + benchmark_class: sum(benchmark_cost for _, benchmark_cost in members) + for benchmark_class, members in benchmark_classes.items() + } shards = [[] for _ in range(n_shards)] loads = [0.0] * n_shards - for owner in sorted(classes, key=lambda owner: (-class_cost[owner], owner)): + for benchmark_class in sorted(benchmark_classes, key=lambda c: (-class_costs[c], c)): target = loads.index(min(loads)) - shards[target].extend(classes[owner]) - loads[target] += class_cost[owner] + shards[target].extend(name for name, _ in benchmark_classes[benchmark_class]) + loads[target] += class_costs[benchmark_class] return shards