diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 53f41dc85..b5bb507cc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,13 +12,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.12", "3.13", "3.14"] + python-version: ["3.12", "3.13", "3.14", "3.14t"] # macos-15-large is an intel runner, macos-14 is an arm64 runner platform: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-15-large, macos-14] defaults: run: - shell: bash -el {0} + shell: bash -e {0} steps: - name: Checkout source @@ -34,13 +34,18 @@ jobs: cache: 'pip' - name: Install numcodecs + if: ${{ !endsWith(matrix.python-version, 't') }} run: python -m pip install -v ".[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]" + - name: Install numcodecs (free-threaded) + # pcodec, zfpy and google_crc32c do not ship free-threaded wheels yet + if: ${{ endsWith(matrix.python-version, 't') }} + run: python -m pip install -v ".[test,test_extras,msgpack,crc32c]" + - name: List installed packages run: python -m pip list - name: Run tests - shell: "bash -l {0}" run: pytest -v - uses: codecov/codecov-action@v6 diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index 947650e40..d8fa6ab11 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -22,8 +22,10 @@ jobs: # macos-15-large is an intel runner, macos-14 is an arm64 runner os: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-15-large, macos-14] env: - CIBW_TEST_COMMAND: python -c "import numcodecs" - CIBW_BUILD: "cp312-* cp313-* cp314-*" + # on free-threaded builds also check that no extension re-enables the GIL + CIBW_TEST_COMMAND: python -c "import sys, sysconfig, numcodecs; assert not (sysconfig.get_config_var('Py_GIL_DISABLED') and sys._is_gil_enabled())" + CIBW_BUILD: "cp312-* cp313-* cp314-* cp314t-*" + CIBW_ENABLE: cpython-freethreading CIBW_SKIP: "*-musllinux_* *win32 *_i686 *_s390x" # note: cibuildwheel config-settings are set in pyproject.toml diff --git a/pyproject.toml b/pyproject.toml index ce472c983..185ecc3a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "meson-python>=0.17", "meson>=1.6.0", "setuptools-scm>=6.2", - "Cython>=3.0", + "Cython>=3.1", "numpy>=2", ] build-backend = "mesonpy" @@ -84,7 +84,7 @@ dev = [ "meson-python>=0.17", "meson>=1.6.0", "ninja", - "cython>=3.0", + "cython>=3.1", "setuptools-scm>=6.2", "numpy>=2", "pytest==9.0.3", diff --git a/src/numcodecs/_shuffle.pyx b/src/numcodecs/_shuffle.pyx index 0f0dafeb8..667beb5ab 100644 --- a/src/numcodecs/_shuffle.pyx +++ b/src/numcodecs/_shuffle.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True cimport cython diff --git a/src/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx index d61f73588..8c52dd95c 100644 --- a/src/numcodecs/blosc.pyx +++ b/src/numcodecs/blosc.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True import threading import multiprocessing import os @@ -72,22 +73,20 @@ AUTOSHUFFLE = -1 # automatic block size - let blosc decide AUTOBLOCKS = 0 -# synchronization -_MUTEX = None -_MUTEX_IS_INIT = False - -def get_mutex(): - global _MUTEX_IS_INIT, _MUTEX - if not _MUTEX_IS_INIT: - try: - mutex = multiprocessing.Lock() - except OSError: - mutex = None - except ImportError: - mutex = None - _MUTEX = mutex - _MUTEX_IS_INIT = True - return _MUTEX +# computed once at import; blosc_list_compressors() fills a static buffer without locking +_COMPRESSORS = tuple(blosc_list_compressors().decode('ascii').split(',')) + +# serializes access to blosc's global context +_MUTEX = threading.Lock() + + +def _reset_mutex_after_fork(): + global _MUTEX + _MUTEX = threading.Lock() + + +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_reset_mutex_after_fork) # store ID of process that first loads the module, so we can detect a fork later _importer_pid = os.getpid() @@ -95,31 +94,34 @@ _importer_pid = os.getpid() def _init(): """Initialize the Blosc library environment.""" - blosc_init() + with _MUTEX: + blosc_init() def _destroy(): """Destroy the Blosc library environment.""" - blosc_destroy() + with _MUTEX: + blosc_destroy() def list_compressors(): """Get a list of compressors supported in the current build.""" - s = blosc_list_compressors() - s = s.decode('ascii') - return s.split(',') + return list(_COMPRESSORS) def get_nthreads(): """Get the number of threads that Blosc uses internally for compression and decompression.""" - return blosc_get_nthreads() + with _MUTEX: + return blosc_get_nthreads() def set_nthreads(int nthreads): """Set the number of threads that Blosc uses internally for compression and decompression.""" - return blosc_set_nthreads(nthreads) + # blosc_set_nthreads re-creates the global context + with _MUTEX: + return blosc_set_nthreads(nthreads) def _cbuffer_sizes(source): @@ -247,7 +249,7 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, # check valid cname early cname_str = cname.decode('ascii') - if cname_str not in list_compressors(): + if cname_str not in _COMPRESSORS: _err_bad_cname(cname_str) # obtain source memoryview @@ -288,8 +290,8 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, # N.B., we are using blosc's global context, and so we need to use a lock # to ensure no-one else can modify the global context while we're setting it - # up and using it. - with get_mutex(): + # up and using it (including set_nthreads/_destroy re-initializing it). + with _MUTEX: # set compressor compressor_set = blosc_set_compressor(cname) @@ -385,8 +387,9 @@ def decompress(source, dest=None): # perform decompression if _get_use_threads(): # allow blosc to use threads internally - with nogil: - ret = blosc_decompress(source_ptr, dest_ptr, nbytes) + with _MUTEX: + with nogil: + ret = blosc_decompress(source_ptr, dest_ptr, nbytes) else: with nogil: ret = blosc_decompress_ctx(source_ptr, dest_ptr, nbytes, 1) @@ -410,10 +413,6 @@ def _get_use_threads(): global use_threads proc = multiprocessing.current_process() - # check if locks are available, and if not no threads - if not get_mutex(): - return False - # check for fork if proc.pid != _importer_pid: # If this module has been imported in the parent process, and the current process diff --git a/src/numcodecs/compat_ext.pyx b/src/numcodecs/compat_ext.pyx index 05dc2bf42..9671afde0 100644 --- a/src/numcodecs/compat_ext.pyx +++ b/src/numcodecs/compat_ext.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True from cpython.buffer cimport PyBuffer_IsContiguous from cpython.memoryview cimport PyMemoryView_GET_BUFFER diff --git a/src/numcodecs/fletcher32.pyx b/src/numcodecs/fletcher32.pyx index 56a5f793c..1ecbb45d6 100644 --- a/src/numcodecs/fletcher32.pyx +++ b/src/numcodecs/fletcher32.pyx @@ -1,6 +1,7 @@ # cython: language_level=3 # cython: overflowcheck=False # cython: cdivision=True +# cython: freethreading_compatible=True from libc.stdint cimport uint8_t, uint16_t, uint32_t diff --git a/src/numcodecs/jenkins.pyx b/src/numcodecs/jenkins.pyx index 0efaf26ce..acdaa1c1b 100644 --- a/src/numcodecs/jenkins.pyx +++ b/src/numcodecs/jenkins.pyx @@ -1,6 +1,7 @@ # cython: language_level=3 # cython: overflowcheck=False # cython: cdivision=True +# cython: freethreading_compatible=True """ Cython implementation of Bob Jenkin's hashlittle from lookup3.c. diff --git a/src/numcodecs/lz4.pyx b/src/numcodecs/lz4.pyx index 9b1b431c8..6eac246b5 100644 --- a/src/numcodecs/lz4.pyx +++ b/src/numcodecs/lz4.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True from libc.stdint cimport uint8_t, uint32_t diff --git a/src/numcodecs/vlen.pyx b/src/numcodecs/vlen.pyx index 69ae620a8..f51e8cad7 100644 --- a/src/numcodecs/vlen.pyx +++ b/src/numcodecs/vlen.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True cimport cython diff --git a/src/numcodecs/zstd.pyx b/src/numcodecs/zstd.pyx index 7e070a62a..7b6ff3c02 100644 --- a/src/numcodecs/zstd.pyx +++ b/src/numcodecs/zstd.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True from cpython.bytes cimport PyBytes_AS_STRING, PyBytes_FromStringAndSize