diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 37c437eba..c1834697b 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -131,6 +131,27 @@ blocks: - ./tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse - tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse - artifact push workflow wheelhouse-linux-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/ + - name: "Wheels: Linux s390x" + run: + when: "tag =~ '.*'" + dependencies: + - "Pre-release Validation" + task: + agent: + machine: + type: s1-ubuntu-24-s390x-4 + env_vars: + - name: OS_NAME + value: linux + jobs: + - name: Build + commands: + - export ARCH=s390x + - sem-version python 3.13 + - pip install uv + - ./tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse + - tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse + - artifact push workflow wheelhouse-linux-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/ - name: "Wheels: Linux x64" run: when: "tag =~ '.*'" @@ -477,6 +498,33 @@ blocks: - sem-version python 3.9 - pip install uv - tools/verify-free-threaded-job.sh + - name: "Wheel Verification: Linux s390x" + run: + when: "tag =~ '.*'" + dependencies: + - "Wheels: Linux s390x" + task: + agent: + machine: + type: s1-ubuntu-24-s390x-4 + env_vars: + - name: OS_NAME + value: linux + jobs: + - name: Verify + commands: + - export ARCH=s390x + - sem-version python 3.9 + - artifact pull workflow artifacts + - cd artifacts && ls *.tgz |xargs -n1 tar -xvf && cd .. + - tools/test-wheels.sh artifacts/wheelhouse + - name: Verify free threaded + commands: + - export ARCH=s390x + # sem-version only provides a pip to bootstrap uv; the wheel runs in the 3.14t venv uv creates. + - sem-version python 3.13 + - pip install uv + - tools/verify-free-threaded-job.sh - name: "Wheel Verification: OSX x64" run: when: "tag =~ '.*'" @@ -627,6 +675,7 @@ blocks: dependencies: - "Wheel Verification: Linux x64" - "Wheel Verification: Linux arm64" + - "Wheel Verification: Linux s390x" - "Wheel Verification: OSX x64" - "Wheel Verification: OSX arm64" - "Wheel Verification: Windows" diff --git a/CHANGELOG.md b/CHANGELOG.md index bb77a1d66..7e60378e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ v2.16.0 is a feature release with the following features, fixes and enhancements ### Enhancements - `confluent_kafka` now declares itself GIL-safe, enabling real multi-core parallelism on free-threaded CPython builds. See the [Multithreading Guide](docs/multithreading-guide.md) for thread-safety details and free-threaded caveats. (#2347) - Add Python 3.14t wheels (#2352) +- Add Linux s390x (IBM Z) wheels, including Python 3.14t (#2360) - Producer `close()` now aborts any open transaction (#2347) - Async IO Consumer's default worker pool size has been increased from 2 to 100 (#2347) - Add support for saving Azure key version with DEK (#2306) diff --git a/requirements/requirements-tests-install-nogil-s390x.txt b/requirements/requirements-tests-install-nogil-s390x.txt new file mode 100644 index 000000000..50ed99bd2 --- /dev/null +++ b/requirements/requirements-tests-install-nogil-s390x.txt @@ -0,0 +1,15 @@ +# Free-threaded (no-GIL) variant of requirements-tests-install-s390x.txt, used by +# the "Verify free threaded" job of the Linux s390x Wheel Verification block +# (tools/verify-free-threaded-job.sh). +# +# Same as requirements-tests-install-nogil.txt but WITHOUT +# requirements-schemaregistry.txt and trivup, for the reason given in +# requirements-tests-install-s390x.txt: they pull cryptography (via authlib and +# jwcrypto), which publishes no s390x wheels. tests/conftest.py skips collecting +# tests/schema_registry on s390x to match. +# Keep in sync with requirements-tests-install-nogil.txt when adding new includes. +-r requirements-tests.txt +-r requirements-avro-nogil.txt +-r requirements-protobuf.txt +-r requirements-json.txt +-r requirements-oauthbearer-aws.txt diff --git a/requirements/requirements-tests-install-s390x.txt b/requirements/requirements-tests-install-s390x.txt new file mode 100644 index 000000000..e3a9a1d2f --- /dev/null +++ b/requirements/requirements-tests-install-s390x.txt @@ -0,0 +1,22 @@ +# Reduced test-install set for s390x (IBM Z). +# +# Same as requirements-tests-install.txt but WITHOUT requirements-schemaregistry.txt +# and requirements-rules.txt, whose transitive deps (cryptography, google-re2, +# tink) publish no s390x wheels and would fall back to slow/failing source builds +# on the s390x agent (e.g. google-re2 needs the abseil C++ headers). +# +# This is only the smoke-test dependency set: tools/smoketest.sh runs the +# top-level unit tests, which do not import the rules/schema-registry crypto +# stack. (It then also installs the [avro], [protobuf] and [json] extras, which +# pull the schema-registry deps, so cryptography is still built from source +# there.) Drop this file and use requirements-tests-install.txt once +# cryptography/google-re2/tink ship s390x wheels. +# +# trivup is also omitted: it is only used by tests/integration (not exercised by +# the smoke test) and it pulls jwcrypto -> cryptography, reintroducing the same +# no-s390x-wheel source build this file exists to avoid. +-r requirements-tests.txt +-r requirements-avro.txt +-r requirements-protobuf.txt +-r requirements-json.txt +-r requirements-oauthbearer-aws.txt diff --git a/tests/conftest.py b/tests/conftest.py index d2abf2d91..8a7d5b109 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,13 +16,26 @@ # limitations under the License. # +import platform import sys import sysconfig +import warnings import pytest FREE_THREADED_BUILD = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) +# s390x: tests/schema_registry needs cryptography (via authlib), which ships no +# s390x wheels, and its conftest.py imports it at load time, so skip it here. +collect_ignore = [] +if platform.machine() == "s390x": + collect_ignore = ["schema_registry"] + warnings.warn( + "s390x: skipping collection of tests/schema_registry, whose " + "schema-registry deps (cryptography via authlib) ship no s390x wheels", + RuntimeWarning, + ) + if FREE_THREADED_BUILD: diff --git a/tools/smoketest.sh b/tools/smoketest.sh index db0a98245..d68e2adb9 100755 --- a/tools/smoketest.sh +++ b/tools/smoketest.sh @@ -60,7 +60,19 @@ for py in 3.9 ; do hash -r uv pip install pkginfo - uv pip install -r requirements/requirements-tests-install.txt + # On s390x, use the reduced test-install set: cryptography, google-re2 and + # tink (pulled by the schemaregistry/rules requirements) have no s390x + # wheels and would fall back to slow/failing source builds. The top-level + # unit tests don't need them. The [avro]/[protobuf]/[json] extras + # installed below do pull the schema-registry deps, so on s390x + # cryptography is built from source there (needs a C compiler and + # OpenSSL >= 3.0 headers). + if [[ "$(uname -m)" == "s390x" ]]; then + tests_install_reqs="requirements/requirements-tests-install-s390x.txt" + else + tests_install_reqs="requirements/requirements-tests-install.txt" + fi + uv pip install -r "$tests_install_reqs" # Get the packages version so we can pin the install # command to this version (which hopefully loads it from the wheeldir diff --git a/tools/test-manylinux.sh b/tools/test-manylinux.sh index df4f93953..b3c648441 100755 --- a/tools/test-manylinux.sh +++ b/tools/test-manylinux.sh @@ -41,6 +41,13 @@ function setup_ubuntu { apt-get install -y -q python3.9 apt-get install -y -q python3.9-distutils apt-get install -y -q curl + # s390x: several test and extra dependencies (fastavro, psutil, librt via + # mypy, and cryptography via the schema-registry deps of the extras the + # smoke test installs) publish no s390x wheels and are built from source, + # which needs a C compiler plus the Python and OpenSSL headers. + if [[ $(uname -m) == "s390x" ]]; then + apt-get install -y -q gcc python3.9-dev libssl-dev pkg-config + fi } @@ -79,8 +86,8 @@ function run_all_with_docker { fi [[ ! -z $DOCKER_IMAGES ]] || \ - # LTS and stable release of popular Linux distros. - DOCKER_IMAGES="ubuntu:20.04 ubuntu:22.04" + # Supported LTS releases of popular Linux distros. + DOCKER_IMAGES="ubuntu:22.04 ubuntu:24.04 ubuntu:26.04" _wheels="$wheelhouse/*manylinux*.whl" diff --git a/tools/verify-free-threaded-job.sh b/tools/verify-free-threaded-job.sh index bd13000a6..4ebe4207a 100755 --- a/tools/verify-free-threaded-job.sh +++ b/tools/verify-free-threaded-job.sh @@ -5,28 +5,38 @@ # run the free-threaded wheel checks (tools/verify-free-threaded-wheel.sh) and then the unit suite. # # Run from the repo root with uv on PATH. Expects OS_NAME (linux|osx) and ARCH -# (x64|arm64) as the Wheel blocks set them, and LIBRDKAFKA_VERSION for the wheel -# checks. +# (x64|arm64, or s390x on linux) as the Wheel blocks set them, and +# LIBRDKAFKA_VERSION for the wheel checks. set -eu : "${OS_NAME:?set OS_NAME to linux or osx}" -: "${ARCH:?set ARCH to x64 or arm64}" +: "${ARCH:?set ARCH to x64, arm64 or s390x}" -# cibuildwheel tags Linux wheels manylinux_*_{x86_64,aarch64} and macOS wheels -# macosx_*_{x86_64,arm64}. +# cibuildwheel tags Linux wheels manylinux_*_{x86_64,aarch64,s390x} and macOS +# wheels macosx_*_{x86_64,arm64}. case "$OS_NAME-$ARCH" in linux-x64) wheel_glob='*-cp314t-manylinux*x86_64.whl' ;; linux-arm64) wheel_glob='*-cp314t-manylinux*aarch64.whl' ;; + linux-s390x) wheel_glob='*-cp314t-manylinux*s390x.whl' ;; osx-x64) wheel_glob='*-cp314t-macosx*x86_64.whl' ;; osx-arm64) wheel_glob='*-cp314t-macosx*arm64.whl' ;; *) echo "$0: unsupported OS_NAME-ARCH '$OS_NAME-$ARCH'" >&2; exit 1 ;; esac echo "Verifying the free-threaded wheel matching $wheel_glob for $OS_NAME-$ARCH" +# s390x: cryptography (via authlib, trivup's jwcrypto) ships no s390x wheels, +# so this installs the reduced requirements-tests-install-nogil-s390x.txt; +# tests/conftest.py skips tests/schema_registry there to match. +if [[ $ARCH == "s390x" ]]; then + tests_install_reqs=requirements/requirements-tests-install-nogil-s390x.txt +else + tests_install_reqs=requirements/requirements-tests-install-nogil.txt +fi + uv venv _venv314t --python 3.14t source _venv314t/bin/activate -uv pip install -r requirements/requirements-tests-install-nogil.txt +uv pip install -r "$tests_install_reqs" artifact pull workflow artifacts # Fail fast, with a clear message, if the pull brought no wheel tarballs at all. diff --git a/tools/wheels/build-wheels.sh b/tools/wheels/build-wheels.sh index af6b3098e..45c2efbd3 100755 --- a/tools/wheels/build-wheels.sh +++ b/tools/wheels/build-wheels.sh @@ -13,6 +13,7 @@ export CIBW_TEST_REQUIRES="pytest" export CIBW_TEST_COMMAND="pytest {project}/tests/test_error.py" export CIBW_MANYLINUX_X86_64_IMAGE="manylinux_2_28" export CIBW_MANYLINUX_AARCH64_IMAGE="manylinux_2_28" +export CIBW_MANYLINUX_S390X_IMAGE="manylinux_2_28" librdkafka_version=$1 wheeldir=$2 diff --git a/tools/wheels/install-librdkafka.sh b/tools/wheels/install-librdkafka.sh index 3a772a2ae..1e4ac33df 100755 --- a/tools/wheels/install-librdkafka.sh +++ b/tools/wheels/install-librdkafka.sh @@ -28,8 +28,10 @@ ARCH=${ARCH:-x64} if [[ $OSTYPE == linux* ]]; then # Linux - # Copy the librdkafka build with least dependencies to librdkafka.so.1 - if [[ $ARCH == arm64* ]]; then + # Copy the librdkafka build with least dependencies to librdkafka.so.1. + # arm64 and s390x redist runtimes ship only the plain librdkafka.so (no + # centos8- variant), so both use it directly; x64 uses the centos8 build. + if [[ $ARCH == arm64* || $ARCH == s390x* ]]; then cp -v runtimes/linux-$ARCH/native/{librdkafka.so,librdkafka.so.1} else cp -v runtimes/linux-$ARCH/native/{centos8-librdkafka.so,librdkafka.so.1}