From 74063f79c7bb242c2d430798a8682f612415f435 Mon Sep 17 00:00:00 2001
From: "Eugenio \"Jay\" Zuccarelli"
<11176606+jayzuccarelli@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:48:45 +0000
Subject: [PATCH 1/3] Add io_whole_file benchmark and an opt-in io group
Add an "io" benchmark group, resolved through the "io" tag and excluded
from the default group, so I/O benchmarks only run with "-b io".
io_whole_file reads and writes a whole file in a single call:
open().read()/write() in binary mode plus Path.read_bytes/write_bytes at
4 KiB and 1 MiB, and Path.read_text/write_text at 1 MiB (6 results).
Part of #399.
---
doc/benchmarks.rst | 19 +++
doc/changelog.rst | 4 +
pyperformance/data-files/benchmarks/MANIFEST | 4 +
.../bm_io_whole_file/pyproject.toml | 10 ++
.../bm_io_whole_file/run_benchmark.py | 138 ++++++++++++++++++
5 files changed, 175 insertions(+)
create mode 100644 pyperformance/data-files/benchmarks/bm_io_whole_file/pyproject.toml
create mode 100644 pyperformance/data-files/benchmarks/bm_io_whole_file/run_benchmark.py
diff --git a/doc/benchmarks.rst b/doc/benchmarks.rst
index 7701d512..a02b1f92 100644
--- a/doc/benchmarks.rst
+++ b/doc/benchmarks.rst
@@ -18,6 +18,8 @@ Available benchmark groups:
* ``all``: Group including all benchmarks
* ``apps``: "High-level" applicative benchmarks (2to3, Chameleon, Tornado HTTP)
* ``default``: Group of benchmarks run by default by the ``run`` command
+* ``io``: File and stream I/O benchmarks. Not part of ``default``: run them
+ explicitly with ``-b io``.
* ``math``: Float and integers
* ``regex``: Collection of regular expression benchmarks
* ``serialize``: Benchmarks on ``pickle`` and ``json`` modules
@@ -306,6 +308,23 @@ rendered):
See the `html5lib project `_.
+io
+--
+
+I/O benchmarks (``io`` group, not run by default). They stay at the binary
+(bytes) I/O layer where possible and use a fresh temporary directory per run.
+Files stay in the OS page cache and writes are not fsync'ed, so they measure
+Python's I/O stack and system call overhead rather than the storage device.
+Results still depend on the operating system and file system of the temporary
+directory.
+
+* ``io_whole_file``: read or write a whole file in a single call with
+ ``open().read()`` / ``open().write()`` in binary mode plus
+ ``pathlib.Path.read_bytes()`` / ``write_bytes()``, at 4 KiB and 1 MiB
+ (``io_whole_file_read_bytes_4k``, ``io_whole_file_write_bytes_4k``, ...),
+ and ``Path.read_text()`` / ``write_text()`` at 1 MiB.
+
+
json_dumps, json_loads
----------------------
diff --git a/doc/changelog.rst b/doc/changelog.rst
index 8735b3bb..762a6c13 100644
--- a/doc/changelog.rst
+++ b/doc/changelog.rst
@@ -1,6 +1,10 @@
Changelog
=========
+Version 1.15.0
+--------------
+* Add opt-in ``io`` benchmark group with the ``io_whole_file`` benchmark
+
Version 1.14.0
--------------
* Bump ``pyperf`` to 2.10.0
diff --git a/pyperformance/data-files/benchmarks/MANIFEST b/pyperformance/data-files/benchmarks/MANIFEST
index 35777453..6fee5d6b 100644
--- a/pyperformance/data-files/benchmarks/MANIFEST
+++ b/pyperformance/data-files/benchmarks/MANIFEST
@@ -54,6 +54,7 @@ hexiom
# Unable to get the program 'hg' from the virtual environment
#hg_startup
html5lib
+io_whole_file
json_dumps
json_loads
logging
@@ -107,6 +108,8 @@ xml_etree
[group default]
+# I/O benchmarks are opt-in: run them with "-b io".
+-io_whole_file
[group asyncio]
[group startup]
[group regex]
@@ -114,3 +117,4 @@ xml_etree
[group apps]
[group math]
[group template]
+[group io]
diff --git a/pyperformance/data-files/benchmarks/bm_io_whole_file/pyproject.toml b/pyperformance/data-files/benchmarks/bm_io_whole_file/pyproject.toml
new file mode 100644
index 00000000..84f03a99
--- /dev/null
+++ b/pyperformance/data-files/benchmarks/bm_io_whole_file/pyproject.toml
@@ -0,0 +1,10 @@
+[project]
+name = "pyperformance_bm_io_whole_file"
+requires-python = ">=3.8"
+dependencies = ["pyperf"]
+urls = {repository = "https://github.com/python/pyperformance"}
+dynamic = ["version"]
+
+[tool.pyperformance]
+name = "io_whole_file"
+tags = "io"
diff --git a/pyperformance/data-files/benchmarks/bm_io_whole_file/run_benchmark.py b/pyperformance/data-files/benchmarks/bm_io_whole_file/run_benchmark.py
new file mode 100644
index 00000000..db9e37b6
--- /dev/null
+++ b/pyperformance/data-files/benchmarks/bm_io_whole_file/run_benchmark.py
@@ -0,0 +1,138 @@
+"""
+Benchmark reading and writing a whole file in a single call.
+
+Covers the binary pattern ``open(path, "rb").read()`` /
+``open(path, "wb").write(data)`` together with the ``pathlib.Path``
+helpers that wrap it (``read_bytes``, ``write_bytes``), at 4 KiB, where
+per-call overhead and the number of system calls dominate, and at 1 MiB,
+where buffer allocation and copying dominate. ``Path.read_text`` /
+``Path.write_text`` are measured at 1 MiB.
+
+The file lives in a fresh temporary directory and stays in the OS page cache,
+so this measures Python's I/O stack and system call overhead, not the storage
+device. Writes are not fsync'ed and always create a new file: the file is
+removed outside the timed region, because truncating and rewriting an
+existing file can make the filesystem start writeback on close (ext4
+``auto_da_alloc``), which would measure the disk. Larger sizes are left out
+on purpose: they scale linearly from 1 MiB and get closer to dirty page
+throttling (``vm.dirty_*``), which measures the kernel's writeback policy
+rather than Python.
+
+Before the benchmarks run, an 8 MiB block is allocated and freed once. On
+glibc this raises malloc's dynamic mmap and trim thresholds well above the
+1 MiB buffers used here; otherwise ``read_text_1m``, which frees two 1 MiB
+blocks per call, can sit right at the trim threshold and, depending on
+earlier heap history, give the memory back to the kernel and fault it in
+again on every call (~3x slower).
+"""
+
+import io
+import os
+import pathlib
+import shutil
+import tempfile
+
+import pyperf
+
+
+SIZES = {
+ "4k": 4 * 1024,
+ "1m": 1024 * 1024,
+}
+TEXT_SIZE = "1m"
+
+
+def make_bytes(size):
+ return (bytes(range(256)) * (size // 256 + 1))[:size]
+
+
+def make_text(size):
+ line = "abcdefghijklmnopqrstuvwxyz0123456789" * 2 + "\n"
+ return (line * (size // len(line) + 1))[:size]
+
+
+def bench_read_bytes(loops, path):
+ range_it = range(loops)
+ t0 = pyperf.perf_counter()
+ for _ in range_it:
+ with open(path, "rb") as f:
+ f.read()
+ path.read_bytes()
+ return pyperf.perf_counter() - t0
+
+
+def bench_write_bytes(loops, path, data):
+ other = path.with_suffix(".2")
+ elapsed = 0
+ for _ in range(loops):
+ t0 = pyperf.perf_counter()
+ with open(path, "wb") as f:
+ f.write(data)
+ other.write_bytes(data)
+ elapsed += pyperf.perf_counter() - t0
+ os.unlink(path)
+ os.unlink(other)
+ return elapsed
+
+
+def bench_read_text(loops, path):
+ range_it = range(loops)
+ t0 = pyperf.perf_counter()
+ for _ in range_it:
+ path.read_text(encoding="utf-8")
+ return pyperf.perf_counter() - t0
+
+
+def bench_write_text(loops, path, text):
+ elapsed = 0
+ for _ in range(loops):
+ t0 = pyperf.perf_counter()
+ path.write_text(text, encoding="utf-8")
+ elapsed += pyperf.perf_counter() - t0
+ os.unlink(path)
+ return elapsed
+
+
+if __name__ == "__main__":
+ runner = pyperf.Runner()
+ runner.metadata["description"] = (
+ "Read and write a whole file in a single call (page cache warm)"
+ )
+ runner.metadata["io_default_buffer_size"] = io.DEFAULT_BUFFER_SIZE
+
+ # Pin glibc's dynamic mmap/trim thresholds above the working set, so the
+ # results do not depend on heap history (see the module docstring).
+ bytes(8 * 1024 * 1024)
+
+ tmp_dir = tempfile.mkdtemp()
+ try:
+ tmp_path = pathlib.Path(tmp_dir)
+ for name, size in SIZES.items():
+ data = make_bytes(size)
+ path = tmp_path / f"read_{name}.bin"
+ path.write_bytes(data)
+ runner.bench_time_func(
+ f"io_whole_file_read_bytes_{name}", bench_read_bytes, path
+ )
+ runner.bench_time_func(
+ f"io_whole_file_write_bytes_{name}",
+ bench_write_bytes,
+ tmp_path / f"write_{name}.bin",
+ data,
+ )
+ os.unlink(path)
+
+ text = make_text(SIZES[TEXT_SIZE])
+ path = tmp_path / f"read_{TEXT_SIZE}.txt"
+ path.write_text(text, encoding="utf-8")
+ runner.bench_time_func(
+ f"io_whole_file_read_text_{TEXT_SIZE}", bench_read_text, path
+ )
+ runner.bench_time_func(
+ f"io_whole_file_write_text_{TEXT_SIZE}",
+ bench_write_text,
+ tmp_path / f"write_{TEXT_SIZE}.txt",
+ text,
+ )
+ finally:
+ shutil.rmtree(tmp_dir)
From 488e9a24cdfb3aed77a929ab4fb0330003cd0408 Mon Sep 17 00:00:00 2001
From: "Eugenio \"Jay\" Zuccarelli"
<11176606+jayzuccarelli@users.noreply.github.com>
Date: Wed, 28 Oct 2026 11:52:09 +0000
Subject: [PATCH 2/3] io_whole_file: measure one API per result
---
doc/benchmarks.rst | 10 +++++-----
.../bm_io_whole_file/run_benchmark.py | 17 +++++++----------
2 files changed, 12 insertions(+), 15 deletions(-)
diff --git a/doc/benchmarks.rst b/doc/benchmarks.rst
index a02b1f92..fc698834 100644
--- a/doc/benchmarks.rst
+++ b/doc/benchmarks.rst
@@ -318,11 +318,11 @@ Python's I/O stack and system call overhead rather than the storage device.
Results still depend on the operating system and file system of the temporary
directory.
-* ``io_whole_file``: read or write a whole file in a single call with
- ``open().read()`` / ``open().write()`` in binary mode plus
- ``pathlib.Path.read_bytes()`` / ``write_bytes()``, at 4 KiB and 1 MiB
- (``io_whole_file_read_bytes_4k``, ``io_whole_file_write_bytes_4k``, ...),
- and ``Path.read_text()`` / ``write_text()`` at 1 MiB.
+* ``io_whole_file``: read or write a whole file in a single call, one API
+ per result: ``open().read()`` / ``open().write()`` in binary mode at 4 KiB
+ and 1 MiB (``io_whole_file_read_bytes_4k``,
+ ``io_whole_file_write_bytes_4k``, ...), and ``pathlib.Path.read_text()`` /
+ ``write_text()`` at 1 MiB.
json_dumps, json_loads
diff --git a/pyperformance/data-files/benchmarks/bm_io_whole_file/run_benchmark.py b/pyperformance/data-files/benchmarks/bm_io_whole_file/run_benchmark.py
index db9e37b6..0f67206b 100644
--- a/pyperformance/data-files/benchmarks/bm_io_whole_file/run_benchmark.py
+++ b/pyperformance/data-files/benchmarks/bm_io_whole_file/run_benchmark.py
@@ -1,12 +1,13 @@
"""
Benchmark reading and writing a whole file in a single call.
-Covers the binary pattern ``open(path, "rb").read()`` /
-``open(path, "wb").write(data)`` together with the ``pathlib.Path``
-helpers that wrap it (``read_bytes``, ``write_bytes``), at 4 KiB, where
-per-call overhead and the number of system calls dominate, and at 1 MiB,
-where buffer allocation and copying dominate. ``Path.read_text`` /
-``Path.write_text`` are measured at 1 MiB.
+Each result measures a single API. The binary pattern
+``open(path, "rb").read()`` / ``open(path, "wb").write(data)`` is measured at
+4 KiB, where per-call overhead and the number of system calls dominate, and
+at 1 MiB, where buffer allocation and copying dominate.
+``Path.read_text`` / ``Path.write_text`` are measured at 1 MiB.
+``Path.read_bytes`` / ``Path.write_bytes`` are thin wrappers over the same
+``open()`` calls, so they are not measured separately.
The file lives in a fresh temporary directory and stays in the OS page cache,
so this measures Python's I/O stack and system call overhead, not the storage
@@ -57,21 +58,17 @@ def bench_read_bytes(loops, path):
for _ in range_it:
with open(path, "rb") as f:
f.read()
- path.read_bytes()
return pyperf.perf_counter() - t0
def bench_write_bytes(loops, path, data):
- other = path.with_suffix(".2")
elapsed = 0
for _ in range(loops):
t0 = pyperf.perf_counter()
with open(path, "wb") as f:
f.write(data)
- other.write_bytes(data)
elapsed += pyperf.perf_counter() - t0
os.unlink(path)
- os.unlink(other)
return elapsed
From 827a7c82a3cb02f77060f77160eb4ad6ca07c1bb Mon Sep 17 00:00:00 2001
From: "Eugenio \"Jay\" Zuccarelli"
<11176606+jayzuccarelli@users.noreply.github.com>
Date: Sat, 31 Oct 2026 01:06:51 +0000
Subject: [PATCH 3/3] io_whole_file: measure uncached reads
Add io_whole_file_read_bytes_{4k,1m}_uncached. Before each timed
open().read(), the file's pages are dropped from the page cache with
os.posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED), outside the timed region,
so the read has to wait for the storage device. The read file is fsync'ed
once in setup because DONTNEED only drops clean pages.
The uncached results are skipped where os.posix_fadvise is missing (macOS,
Windows). Writes stay cached-only: an uncached write would need an fsync in
the timed region, which measures the device's flush path, not Python.
---
doc/benchmarks.rst | 13 ++--
doc/changelog.rst | 1 +
.../bm_io_whole_file/run_benchmark.py | 64 +++++++++++++++----
3 files changed, 63 insertions(+), 15 deletions(-)
diff --git a/doc/benchmarks.rst b/doc/benchmarks.rst
index fc698834..c5f8ea08 100644
--- a/doc/benchmarks.rst
+++ b/doc/benchmarks.rst
@@ -314,15 +314,20 @@ io
I/O benchmarks (``io`` group, not run by default). They stay at the binary
(bytes) I/O layer where possible and use a fresh temporary directory per run.
Files stay in the OS page cache and writes are not fsync'ed, so they measure
-Python's I/O stack and system call overhead rather than the storage device.
-Results still depend on the operating system and file system of the temporary
-directory.
+Python's I/O stack and system call overhead rather than the storage device,
+except for results ending in ``_uncached``. Results still depend on the
+operating system and file system of the temporary directory.
* ``io_whole_file``: read or write a whole file in a single call, one API
per result: ``open().read()`` / ``open().write()`` in binary mode at 4 KiB
and 1 MiB (``io_whole_file_read_bytes_4k``,
``io_whole_file_write_bytes_4k``, ...), and ``pathlib.Path.read_text()`` /
- ``write_text()`` at 1 MiB.
+ ``write_text()`` at 1 MiB. The binary reads are also measured with the
+ file's pages dropped from the page cache before each read
+ (``io_whole_file_read_bytes_4k_uncached``,
+ ``io_whole_file_read_bytes_1m_uncached``), so the read waits for the storage
+ device. These need ``os.posix_fadvise()`` and are skipped where it is not
+ available (macOS, Windows).
json_dumps, json_loads
diff --git a/doc/changelog.rst b/doc/changelog.rst
index 762a6c13..fd8ac95c 100644
--- a/doc/changelog.rst
+++ b/doc/changelog.rst
@@ -4,6 +4,7 @@ Changelog
Version 1.15.0
--------------
* Add opt-in ``io`` benchmark group with the ``io_whole_file`` benchmark
+ (cached reads and writes, plus uncached reads)
Version 1.14.0
--------------
diff --git a/pyperformance/data-files/benchmarks/bm_io_whole_file/run_benchmark.py b/pyperformance/data-files/benchmarks/bm_io_whole_file/run_benchmark.py
index 0f67206b..e375f70c 100644
--- a/pyperformance/data-files/benchmarks/bm_io_whole_file/run_benchmark.py
+++ b/pyperformance/data-files/benchmarks/bm_io_whole_file/run_benchmark.py
@@ -9,15 +9,29 @@
``Path.read_bytes`` / ``Path.write_bytes`` are thin wrappers over the same
``open()`` calls, so they are not measured separately.
-The file lives in a fresh temporary directory and stays in the OS page cache,
-so this measures Python's I/O stack and system call overhead, not the storage
-device. Writes are not fsync'ed and always create a new file: the file is
-removed outside the timed region, because truncating and rewriting an
-existing file can make the filesystem start writeback on close (ext4
-``auto_da_alloc``), which would measure the disk. Larger sizes are left out
-on purpose: they scale linearly from 1 MiB and get closer to dirty page
-throttling (``vm.dirty_*``), which measures the kernel's writeback policy
-rather than Python.
+The file lives in a fresh temporary directory. Except for the ``*_uncached``
+results below, it stays in the OS page cache, so this measures Python's I/O
+stack and system call overhead, not the storage device. Writes are not
+fsync'ed and always create a new file: the file is removed outside the timed
+region, because truncating and rewriting an existing file can make the
+filesystem start writeback on close (ext4 ``auto_da_alloc``), which would
+measure the disk. Larger sizes are left out on purpose: they scale linearly
+from 1 MiB and get closer to dirty page throttling (``vm.dirty_*``), which
+measures the kernel's writeback policy rather than Python.
+
+The ``*_uncached`` results measure ``open(path, "rb").read()`` when the data
+is not in the page cache, so the read has to wait for the storage device.
+Before each timed read, the file's pages are dropped with
+``os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)``, outside the timed
+region. This needs no special privileges, but only drops clean pages, so the
+file is fsync'ed once after it is written. These results mostly track the
+device, so they are kept apart from the cached ones by name. They are skipped
+where ``os.posix_fadvise`` is not available (macOS, Windows): macOS's
+``F_NOCACHE`` only affects later I/O on one file descriptor and does not evict
+pages that are already cached. On a RAM-backed temporary directory (tmpfs)
+there is nothing to drop, so they match the cached results. Writes are only
+measured cached: an uncached write would need an fsync in the timed region,
+which measures the device's flush path rather than Python.
Before the benchmarks run, an 8 MiB block is allocated and freed once. On
glibc this raises malloc's dynamic mmap and trim thresholds well above the
@@ -61,6 +75,25 @@ def bench_read_bytes(loops, path):
return pyperf.perf_counter() - t0
+def drop_page_cache(path):
+ fd = os.open(path, os.O_RDONLY)
+ try:
+ os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
+ finally:
+ os.close(fd)
+
+
+def bench_read_bytes_uncached(loops, path):
+ elapsed = 0
+ for _ in range(loops):
+ drop_page_cache(path)
+ t0 = pyperf.perf_counter()
+ with open(path, "rb") as f:
+ f.read()
+ elapsed += pyperf.perf_counter() - t0
+ return elapsed
+
+
def bench_write_bytes(loops, path, data):
elapsed = 0
for _ in range(loops):
@@ -93,9 +126,10 @@ def bench_write_text(loops, path, text):
if __name__ == "__main__":
runner = pyperf.Runner()
runner.metadata["description"] = (
- "Read and write a whole file in a single call (page cache warm)"
+ "Read and write a whole file in a single call"
)
runner.metadata["io_default_buffer_size"] = io.DEFAULT_BUFFER_SIZE
+ uncached = hasattr(os, "posix_fadvise")
# Pin glibc's dynamic mmap/trim thresholds above the working set, so the
# results do not depend on heap history (see the module docstring).
@@ -107,10 +141,18 @@ def bench_write_text(loops, path, text):
for name, size in SIZES.items():
data = make_bytes(size)
path = tmp_path / f"read_{name}.bin"
- path.write_bytes(data)
+ with open(path, "wb") as f:
+ f.write(data)
+ os.fsync(f.fileno())
runner.bench_time_func(
f"io_whole_file_read_bytes_{name}", bench_read_bytes, path
)
+ if uncached:
+ runner.bench_time_func(
+ f"io_whole_file_read_bytes_{name}_uncached",
+ bench_read_bytes_uncached,
+ path,
+ )
runner.bench_time_func(
f"io_whole_file_write_bytes_{name}",
bench_write_bytes,