From 0021eae76c5e50ed34b1db9dcc655b6c3e84f68b Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 9 Sep 2026 19:50:21 +0530 Subject: [PATCH 1/3] Add free-threaded CPython (3.14t) support - Mark all Cython extensions with freethreading_compatible=True so the interpreter no longer re-enables the GIL when numcodecs is imported - Require Cython>=3.1, needed for the freethreading_compatible directive - blosc: replace the lazily created multiprocessing.Lock (and get_mutex()) with a module-level threading.Lock, reset in forked children via os.register_at_fork - blosc: take the lock in _init, _destroy, set_nthreads and the global context path of decompress, so blosc_set_nthreads cannot re-create the global context while another thread is using it - blosc: compute the compressor list once at import; blosc_list_compressors() fills a static buffer without locking and was called on every compress() - CI: add 3.14t to the test matrix, installing without pcodec, zfpy and google_crc32c which do not ship free-threaded wheels yet - Wheels: enable cpython-freethreading in cibuildwheel, build cp314t-* wheels, and assert in the wheel test that the GIL is not re-enabled on free-threaded builds --- .github/workflows/ci.yaml | 8 ++++- .github/workflows/wheel.yaml | 6 ++-- pyproject.toml | 4 +-- src/numcodecs/_shuffle.pyx | 1 + src/numcodecs/blosc.pyx | 60 +++++++++++++++++------------------- src/numcodecs/compat_ext.pyx | 1 + src/numcodecs/fletcher32.pyx | 1 + src/numcodecs/jenkins.pyx | 1 + src/numcodecs/lz4.pyx | 1 + src/numcodecs/vlen.pyx | 1 + src/numcodecs/zstd.pyx | 1 + 11 files changed, 49 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 53f41dc85..ec741c710 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.12", "3.13", "3.14"] + python-version: ["3.12", "3.13", "3.14", "3.14t"] # macos-15-large is an intel runner, macos-14 is an arm64 runner platform: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-15-large, macos-14] @@ -34,8 +34,14 @@ jobs: cache: 'pip' - name: Install numcodecs + if: ${{ !endsWith(matrix.python-version, 't') }} run: python -m pip install -v ".[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]" + - name: Install numcodecs (free-threaded) + # pcodec, zfpy and google_crc32c do not ship free-threaded wheels yet + if: ${{ endsWith(matrix.python-version, 't') }} + run: python -m pip install -v ".[test,test_extras,msgpack,crc32c]" + - name: List installed packages run: python -m pip list diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index 947650e40..d8fa6ab11 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -22,8 +22,10 @@ jobs: # macos-15-large is an intel runner, macos-14 is an arm64 runner os: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-15-large, macos-14] env: - CIBW_TEST_COMMAND: python -c "import numcodecs" - CIBW_BUILD: "cp312-* cp313-* cp314-*" + # on free-threaded builds also check that no extension re-enables the GIL + CIBW_TEST_COMMAND: python -c "import sys, sysconfig, numcodecs; assert not (sysconfig.get_config_var('Py_GIL_DISABLED') and sys._is_gil_enabled())" + CIBW_BUILD: "cp312-* cp313-* cp314-* cp314t-*" + CIBW_ENABLE: cpython-freethreading CIBW_SKIP: "*-musllinux_* *win32 *_i686 *_s390x" # note: cibuildwheel config-settings are set in pyproject.toml diff --git a/pyproject.toml b/pyproject.toml index ce472c983..185ecc3a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "meson-python>=0.17", "meson>=1.6.0", "setuptools-scm>=6.2", - "Cython>=3.0", + "Cython>=3.1", "numpy>=2", ] build-backend = "mesonpy" @@ -84,7 +84,7 @@ dev = [ "meson-python>=0.17", "meson>=1.6.0", "ninja", - "cython>=3.0", + "cython>=3.1", "setuptools-scm>=6.2", "numpy>=2", "pytest==9.0.3", diff --git a/src/numcodecs/_shuffle.pyx b/src/numcodecs/_shuffle.pyx index 0f0dafeb8..667beb5ab 100644 --- a/src/numcodecs/_shuffle.pyx +++ b/src/numcodecs/_shuffle.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True cimport cython diff --git a/src/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx index d61f73588..a7e98ee6b 100644 --- a/src/numcodecs/blosc.pyx +++ b/src/numcodecs/blosc.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True import threading import multiprocessing import os @@ -72,22 +73,20 @@ AUTOSHUFFLE = -1 # automatic block size - let blosc decide AUTOBLOCKS = 0 -# synchronization -_MUTEX = None -_MUTEX_IS_INIT = False - -def get_mutex(): - global _MUTEX_IS_INIT, _MUTEX - if not _MUTEX_IS_INIT: - try: - mutex = multiprocessing.Lock() - except OSError: - mutex = None - except ImportError: - mutex = None - _MUTEX = mutex - _MUTEX_IS_INIT = True - return _MUTEX +# computed once at import; blosc_list_compressors() fills a static buffer without locking +_COMPRESSORS = tuple(blosc_list_compressors().decode('ascii').split(',')) + +# serializes access to blosc's global context +_MUTEX = threading.Lock() + + +def _reset_mutex_after_fork(): + global _MUTEX + _MUTEX = threading.Lock() + + +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_reset_mutex_after_fork) # store ID of process that first loads the module, so we can detect a fork later _importer_pid = os.getpid() @@ -95,19 +94,19 @@ _importer_pid = os.getpid() def _init(): """Initialize the Blosc library environment.""" - blosc_init() + with _MUTEX: + blosc_init() def _destroy(): """Destroy the Blosc library environment.""" - blosc_destroy() + with _MUTEX: + blosc_destroy() def list_compressors(): """Get a list of compressors supported in the current build.""" - s = blosc_list_compressors() - s = s.decode('ascii') - return s.split(',') + return list(_COMPRESSORS) def get_nthreads(): @@ -119,7 +118,9 @@ def get_nthreads(): def set_nthreads(int nthreads): """Set the number of threads that Blosc uses internally for compression and decompression.""" - return blosc_set_nthreads(nthreads) + # blosc_set_nthreads re-creates the global context + with _MUTEX: + return blosc_set_nthreads(nthreads) def _cbuffer_sizes(source): @@ -247,7 +248,7 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, # check valid cname early cname_str = cname.decode('ascii') - if cname_str not in list_compressors(): + if cname_str not in _COMPRESSORS: _err_bad_cname(cname_str) # obtain source memoryview @@ -288,8 +289,8 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, # N.B., we are using blosc's global context, and so we need to use a lock # to ensure no-one else can modify the global context while we're setting it - # up and using it. - with get_mutex(): + # up and using it (including set_nthreads/_destroy re-initializing it). + with _MUTEX: # set compressor compressor_set = blosc_set_compressor(cname) @@ -385,8 +386,9 @@ def decompress(source, dest=None): # perform decompression if _get_use_threads(): # allow blosc to use threads internally - with nogil: - ret = blosc_decompress(source_ptr, dest_ptr, nbytes) + with _MUTEX: + with nogil: + ret = blosc_decompress(source_ptr, dest_ptr, nbytes) else: with nogil: ret = blosc_decompress_ctx(source_ptr, dest_ptr, nbytes, 1) @@ -410,10 +412,6 @@ def _get_use_threads(): global use_threads proc = multiprocessing.current_process() - # check if locks are available, and if not no threads - if not get_mutex(): - return False - # check for fork if proc.pid != _importer_pid: # If this module has been imported in the parent process, and the current process diff --git a/src/numcodecs/compat_ext.pyx b/src/numcodecs/compat_ext.pyx index 05dc2bf42..9671afde0 100644 --- a/src/numcodecs/compat_ext.pyx +++ b/src/numcodecs/compat_ext.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True from cpython.buffer cimport PyBuffer_IsContiguous from cpython.memoryview cimport PyMemoryView_GET_BUFFER diff --git a/src/numcodecs/fletcher32.pyx b/src/numcodecs/fletcher32.pyx index 56a5f793c..1ecbb45d6 100644 --- a/src/numcodecs/fletcher32.pyx +++ b/src/numcodecs/fletcher32.pyx @@ -1,6 +1,7 @@ # cython: language_level=3 # cython: overflowcheck=False # cython: cdivision=True +# cython: freethreading_compatible=True from libc.stdint cimport uint8_t, uint16_t, uint32_t diff --git a/src/numcodecs/jenkins.pyx b/src/numcodecs/jenkins.pyx index 0efaf26ce..acdaa1c1b 100644 --- a/src/numcodecs/jenkins.pyx +++ b/src/numcodecs/jenkins.pyx @@ -1,6 +1,7 @@ # cython: language_level=3 # cython: overflowcheck=False # cython: cdivision=True +# cython: freethreading_compatible=True """ Cython implementation of Bob Jenkin's hashlittle from lookup3.c. diff --git a/src/numcodecs/lz4.pyx b/src/numcodecs/lz4.pyx index 9b1b431c8..6eac246b5 100644 --- a/src/numcodecs/lz4.pyx +++ b/src/numcodecs/lz4.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True from libc.stdint cimport uint8_t, uint32_t diff --git a/src/numcodecs/vlen.pyx b/src/numcodecs/vlen.pyx index 69ae620a8..f51e8cad7 100644 --- a/src/numcodecs/vlen.pyx +++ b/src/numcodecs/vlen.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True cimport cython diff --git a/src/numcodecs/zstd.pyx b/src/numcodecs/zstd.pyx index 7e070a62a..7b6ff3c02 100644 --- a/src/numcodecs/zstd.pyx +++ b/src/numcodecs/zstd.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True from cpython.bytes cimport PyBytes_AS_STRING, PyBytes_FromStringAndSize From d32bf2a825cb5de472bbf77ae2be35ca110abd67 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 9 Sep 2026 21:06:52 +0530 Subject: [PATCH 2/3] try to fix macos CI --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ec741c710..b5bb507cc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,7 +18,7 @@ jobs: defaults: run: - shell: bash -el {0} + shell: bash -e {0} steps: - name: Checkout source @@ -46,7 +46,6 @@ jobs: run: python -m pip list - name: Run tests - shell: "bash -l {0}" run: pytest -v - uses: codecov/codecov-action@v6 From 6296059aad135acbcc8f1a8fe7c6a514d2d52179 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 9 Sep 2026 22:02:12 +0530 Subject: [PATCH 3/3] add locking to get_nthreads --- src/numcodecs/blosc.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx index a7e98ee6b..8c52dd95c 100644 --- a/src/numcodecs/blosc.pyx +++ b/src/numcodecs/blosc.pyx @@ -112,7 +112,8 @@ def list_compressors(): def get_nthreads(): """Get the number of threads that Blosc uses internally for compression and decompression.""" - return blosc_get_nthreads() + with _MUTEX: + return blosc_get_nthreads() def set_nthreads(int nthreads):