diff --git a/MODULE.bazel b/MODULE.bazel index 3eaedcc6..4042e925 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -126,6 +126,13 @@ http_archive( urls = ["https://maven.pkg.github.com/NVIDIA/ncore/com/nvidia/ncore/test-data-v4/0.2/test-data-v4-0.2.tar.gz"], ) +# time-trimmed zarr format v2 / v3 serializations of the test-data-v4 sequence +http_archive( + name = "test-data-v4-trimmed", + sha256 = "a4a3457829431e30cdc5bb55cc00a8e6d5dfdc5d52db2221594807b4c9fff81d", + urls = ["https://maven.pkg.github.com/NVIDIA/ncore/com/nvidia/ncore/test-data-v4-trimmed/0.2/test-data-v4-trimmed-0.2.tar.gz"], +) + # documentation binary data http_archive( name = "ncore_docs_data", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 7d07a1da..63996be3 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1502,9 +1502,9 @@ "bzlTransitiveDigest": "zc0QWNGxYYxPvHvT1CA/9+X0h1JuhXagw7k7q1jOhwg=", "usagesDigest": "rf/YOtMUZwRHGFAE+9knWdjNz4HOysfZIlX5MJbhT+E=", "recordedFileInputs": { - "@@//deps/pip/requirements_3_11.txt": "845ddefb3a79c099a05f39880752241d3c30cefb3774c0611f253d5bff3f774f", + "@@//deps/pip/requirements_3_11.txt": "a13d4223fd7b56af9192342a5af35245f7678050a5d3e33fd5ea686da8e79680", "@@//deps/pip/requirements_3_12.txt": "56ddea672a89523c8b93e54bcbcffc28c91aeff4f0e72c8b95306dc287af0068", - "@@//deps/pip/requirements_3_13.txt": "48cd877f84a1ac6277b6cd9d8f5c211f2f2b49e058dc1f1c0717e3b1bfbfa689", + "@@//deps/pip/requirements_3_13.txt": "81e3dc5cc6fd0ad9c8a3245c6645328adf2bbc40040f84f10b3cb9b43278e9fb", "@@//deps/pip/requirements_3_8.txt": "94c254d645dc1d0555f48e66fa499aa2efecfc5e547500cd4ed409aee28e8880", "@@rules_fuzzing+//fuzzing/requirements.txt": "ab04664be026b632a0d2a2446c4f65982b7654f5b6851d2f9d399a19b7242a5b", "@@rules_python+//tools/publish/requirements_darwin.txt": "095d4a4f3d639dce831cd493367631cd51b53665292ab20194bac2c0c6458fa8", @@ -1733,36 +1733,6 @@ ] } }, - "ncore_pip_deps_311_asciitree_sdist_4aa4b9b6": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cu128/torch" - ], - "filename": "asciitree-0.3.3.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "ncore_pip_deps_311", - "requirement": "asciitree==0.3.3", - "sha256": "4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e", - "urls": [ - "https://files.pythonhosted.org/packages/2d/6a/885bc91484e1aa8f618f6f0228d76d0e67000b0fdd6090673b777e311913/asciitree-0.3.3.tar.gz" - ] - } - }, "ncore_pip_deps_311_asttokens_py3_none_any_15a3ebc0": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -3545,6 +3515,60 @@ ] } }, + "ncore_pip_deps_311_donfig_py3_none_any_2a3175ce": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "donfig-0.8.1.post1-py3-none-any.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "donfig==0.8.1.post1", + "sha256": "2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", + "urls": [ + "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl" + ] + } + }, + "ncore_pip_deps_311_donfig_sdist_3bef3413": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cu128/torch" + ], + "filename": "donfig-0.8.1.post1.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "donfig==0.8.1.post1", + "sha256": "3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", + "urls": [ + "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz" + ] + } + }, "ncore_pip_deps_311_dracopy_cp311_cp311_macosx_10_9_x86_64_c4131151": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -3791,60 +3815,6 @@ ] } }, - "ncore_pip_deps_311_fasteners_py3_none_any_758819cb": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "fasteners-0.19-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "ncore_pip_deps_311", - "requirement": "fasteners==0.19", - "sha256": "758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237", - "urls": [ - "https://files.pythonhosted.org/packages/61/bf/fd60001b3abc5222d8eaa4a204cd8c0ae78e75adc688f33ce4bf25b7fafa/fasteners-0.19-py3-none-any.whl" - ] - } - }, - "ncore_pip_deps_311_fasteners_sdist_b4f37c3a": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cu128/torch" - ], - "filename": "fasteners-0.19.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "ncore_pip_deps_311", - "requirement": "fasteners==0.19", - "sha256": "b4f37c3ac52d8a445af3a66bce57b33b5e90b97c696b7b984f530cf8f0ded09c", - "urls": [ - "https://files.pythonhosted.org/packages/5f/d4/e834d929be54bfadb1f3e3b931c38e956aaa3b235a46a3c764c26c774902/fasteners-0.19.tar.gz" - ] - } - }, "ncore_pip_deps_311_fastjsonschema_py3_none_any_c9e5b7e9": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -4361,6 +4331,156 @@ ] } }, + "ncore_pip_deps_311_google_crc32c_cp311_cp311_macosx_12_0_arm64_457d0d9a": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp311-cp311-macosx_12_0_arm64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "google-crc32c==1.9.0", + "sha256": "457d0d9a4718fd52b1494eac5c200ad25beeadbdc91843d550a003910838589f", + "urls": [ + "https://files.pythonhosted.org/packages/0e/55/a2f07f15e624f0de79359b1a6c1deb59ec5061bd3b38744b3b2849400662/google_crc32c-1.9.0-cp311-cp311-macosx_12_0_arm64.whl" + ] + } + }, + "ncore_pip_deps_311_google_crc32c_cp311_cp311_macosx_12_0_x86_64_ccfe4002": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp311-cp311-macosx_12_0_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "google-crc32c==1.9.0", + "sha256": "ccfe40021fd6afe23361175cf7551e3cef5fd34dc1ebe319f14993a83579e0eb", + "urls": [ + "https://files.pythonhosted.org/packages/f8/b3/923743597b774bbcf12a7c3e00e48d745e15fd616ad7489a40a63fff8f2f/google_crc32c-1.9.0-cp311-cp311-macosx_12_0_x86_64.whl" + ] + } + }, + "ncore_pip_deps_311_google_crc32c_cp311_cp311_manylinux_2_17_aarch64_fbef61a3": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "google-crc32c==1.9.0", + "sha256": "fbef61a3794e011c65fb4396a196cf123a7f474fe5a443db8e5dd7d751b9e6d4", + "urls": [ + "https://files.pythonhosted.org/packages/df/a6/4d0352fe889663e0d81cea7fc664ec9158727384de4a44ab10e9967a7682/google_crc32c-1.9.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl" + ] + } + }, + "ncore_pip_deps_311_google_crc32c_cp311_cp311_manylinux_2_17_x86_64_86764b99": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "google-crc32c==1.9.0", + "sha256": "86764b99e7a607830d93cb5b75e0ec3ff6cb06d3c274624418473cee701900d4", + "urls": [ + "https://files.pythonhosted.org/packages/aa/e3/26685384e4b66ff0928d9566ef6110a7df76029175a1842329d7e3515f10/google_crc32c-1.9.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" + ] + } + }, + "ncore_pip_deps_311_google_crc32c_cp311_cp311_win_amd64_43a2dc26": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp311-cp311-win_amd64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "google-crc32c==1.9.0", + "sha256": "43a2dc26f9be213fbe0b4fc4a1088c5d45cbfcb3247420ccc820f0fc3edeea86", + "urls": [ + "https://files.pythonhosted.org/packages/cb/ce/4e90102e84880e97d3cf935f2672ecd29191bdeacf57f01740f92debda00/google_crc32c-1.9.0-cp311-cp311-win_amd64.whl" + ] + } + }, + "ncore_pip_deps_311_google_crc32c_sdist_7b8c84c3": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cu128/torch" + ], + "filename": "google_crc32c-1.9.0.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "google-crc32c==1.9.0", + "sha256": "7b8c84c3d159ab6817fe3f74e6e6cef099c3f95dcec3abc0d8afb1404642efbe", + "urls": [ + "https://files.pythonhosted.org/packages/fa/25/9cb0c1c31c45b893eb8f11ae70b3f4309432d59b5acaebca5dbe791729a4/google_crc32c-1.9.0.tar.gz" + ] + } + }, "ncore_pip_deps_311_google_pasta_py3_none_any_b3248279": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -12269,7 +12389,7 @@ ] } }, - "ncore_pip_deps_311_pyzmq_cp311_cp311_macosx_10_15_universal2_c0598838": { + "ncore_pip_deps_311_pyyaml_cp311_cp311_macosx_10_13_x86_64_44edc647": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -12283,17 +12403,17 @@ "cp311_osx_x86_64", "cp311_windows_x86_64" ], - "filename": "pyzmq-26.2.1-cp311-cp311-macosx_10_15_universal2.whl", + "filename": "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "ncore_pip_deps_311", - "requirement": "pyzmq==26.2.1", - "sha256": "c059883840e634a21c5b31d9b9a0e2b48f991b94d60a811092bc37992715146a", + "requirement": "pyyaml==6.0.3", + "sha256": "44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", "urls": [ - "https://files.pythonhosted.org/packages/b9/03/5ecc46a6ed5971299f5c03e016ca637802d8660e44392bea774fb7797405/pyzmq-26.2.1-cp311-cp311-macosx_10_15_universal2.whl" + "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl" ] } }, - "ncore_pip_deps_311_pyzmq_cp311_cp311_macosx_10_9_x86_64_ed038a92": { + "ncore_pip_deps_311_pyyaml_cp311_cp311_macosx_11_0_arm64_652cb6ed": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -12307,17 +12427,17 @@ "cp311_osx_x86_64", "cp311_windows_x86_64" ], - "filename": "pyzmq-26.2.1-cp311-cp311-macosx_10_9_x86_64.whl", + "filename": "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "ncore_pip_deps_311", - "requirement": "pyzmq==26.2.1", - "sha256": "ed038a921df836d2f538e509a59cb638df3e70ca0fcd70d0bf389dfcdf784d2a", + "requirement": "pyyaml==6.0.3", + "sha256": "652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", "urls": [ - "https://files.pythonhosted.org/packages/40/51/48fec8f990ee644f461ff14c8fe5caa341b0b9b3a0ad7544f8ef17d6f528/pyzmq-26.2.1-cp311-cp311-macosx_10_9_x86_64.whl" + "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl" ] } }, - "ncore_pip_deps_311_pyzmq_cp311_cp311_manylinux_2_17_aarch64_9027a7fc": { + "ncore_pip_deps_311_pyyaml_cp311_cp311_manylinux_2_17_aarch64_10892704": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -12331,17 +12451,17 @@ "cp311_osx_x86_64", "cp311_windows_x86_64" ], - "filename": "pyzmq-26.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "filename": "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "ncore_pip_deps_311", - "requirement": "pyzmq==26.2.1", - "sha256": "9027a7fcf690f1a3635dc9e55e38a0d6602dbbc0548935d08d46d2e7ec91f454", + "requirement": "pyyaml==6.0.3", + "sha256": "10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", "urls": [ - "https://files.pythonhosted.org/packages/c1/f4/f322b389727c687845e38470b48d7a43c18a83f26d4d5084603c6c3f79ca/pyzmq-26.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl" ] } }, - "ncore_pip_deps_311_pyzmq_cp311_cp311_manylinux_2_17_x86_64_f0019cc8": { + "ncore_pip_deps_311_pyyaml_cp311_cp311_manylinux_2_17_s390x_850774a7": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -12355,11 +12475,233 @@ "cp311_osx_x86_64", "cp311_windows_x86_64" ], - "filename": "pyzmq-26.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "filename": "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "ncore_pip_deps_311", - "requirement": "pyzmq==26.2.1", - "sha256": "f0019cc804ac667fb8c8eaecdb66e6d4a68acf2e155d5c7d6381a5645bd93ae4", + "requirement": "pyyaml==6.0.3", + "sha256": "850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", + "urls": [ + "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl" + ] + } + }, + "ncore_pip_deps_311_pyyaml_cp311_cp311_manylinux_2_17_x86_64_b8bb0864": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyyaml==6.0.3", + "sha256": "b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", + "urls": [ + "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" + ] + } + }, + "ncore_pip_deps_311_pyyaml_cp311_cp311_musllinux_1_2_aarch64_1d37d57a": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyyaml==6.0.3", + "sha256": "1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", + "urls": [ + "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl" + ] + } + }, + "ncore_pip_deps_311_pyyaml_cp311_cp311_musllinux_1_2_x86_64_37503bfb": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyyaml==6.0.3", + "sha256": "37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", + "urls": [ + "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl" + ] + } + }, + "ncore_pip_deps_311_pyyaml_cp311_cp311_win_amd64_9f3bfb49": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyyaml==6.0.3", + "sha256": "9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", + "urls": [ + "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl" + ] + } + }, + "ncore_pip_deps_311_pyyaml_sdist_d7662337": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cu128/torch" + ], + "filename": "pyyaml-6.0.3.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyyaml==6.0.3", + "sha256": "d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", + "urls": [ + "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz" + ] + } + }, + "ncore_pip_deps_311_pyzmq_cp311_cp311_macosx_10_15_universal2_c0598838": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyzmq-26.2.1-cp311-cp311-macosx_10_15_universal2.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyzmq==26.2.1", + "sha256": "c059883840e634a21c5b31d9b9a0e2b48f991b94d60a811092bc37992715146a", + "urls": [ + "https://files.pythonhosted.org/packages/b9/03/5ecc46a6ed5971299f5c03e016ca637802d8660e44392bea774fb7797405/pyzmq-26.2.1-cp311-cp311-macosx_10_15_universal2.whl" + ] + } + }, + "ncore_pip_deps_311_pyzmq_cp311_cp311_macosx_10_9_x86_64_ed038a92": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyzmq-26.2.1-cp311-cp311-macosx_10_9_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyzmq==26.2.1", + "sha256": "ed038a921df836d2f538e509a59cb638df3e70ca0fcd70d0bf389dfcdf784d2a", + "urls": [ + "https://files.pythonhosted.org/packages/40/51/48fec8f990ee644f461ff14c8fe5caa341b0b9b3a0ad7544f8ef17d6f528/pyzmq-26.2.1-cp311-cp311-macosx_10_9_x86_64.whl" + ] + } + }, + "ncore_pip_deps_311_pyzmq_cp311_cp311_manylinux_2_17_aarch64_9027a7fc": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyzmq-26.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyzmq==26.2.1", + "sha256": "9027a7fcf690f1a3635dc9e55e38a0d6602dbbc0548935d08d46d2e7ec91f454", + "urls": [ + "https://files.pythonhosted.org/packages/c1/f4/f322b389727c687845e38470b48d7a43c18a83f26d4d5084603c6c3f79ca/pyzmq-26.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + ] + } + }, + "ncore_pip_deps_311_pyzmq_cp311_cp311_manylinux_2_17_x86_64_f0019cc8": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyzmq-26.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyzmq==26.2.1", + "sha256": "f0019cc804ac667fb8c8eaecdb66e6d4a68acf2e155d5c7d6381a5645bd93ae4", "urls": [ "https://files.pythonhosted.org/packages/b5/e2/45c0f6e122b562cb8c6c45c0dcac1160a4e2207385ef9b13463e74f93031/pyzmq-26.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" ] @@ -16919,7 +17261,7 @@ ] } }, - "ncore_pip_deps_311_zarr_py3_none_any_2795e20a": { + "ncore_pip_deps_311_zarr_py3_none_any_72c3dcee": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -16933,17 +17275,17 @@ "cp311_osx_x86_64", "cp311_windows_x86_64" ], - "filename": "zarr-2.18.4-py3-none-any.whl", + "filename": "zarr-3.1.4-py3-none-any.whl", "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "ncore_pip_deps_311", - "requirement": "zarr==2.18.4", - "sha256": "2795e20aff91093ce7e4da36ab1a138aededbd8ab66bf01fd01512e61d31e5d1", + "requirement": "zarr==3.1.4", + "sha256": "72c3dcee2d41527d21cd814cbaca1e6e23550701eb4fc30767ee53d925900eec", "urls": [ - "https://files.pythonhosted.org/packages/b4/d1/c84022a44afc7b7ccc442fba3daee56bdd03593d91ee4bc245a08e4fcc55/zarr-2.18.4-py3-none-any.whl" + "https://files.pythonhosted.org/packages/2d/8e/1b05f5b8a9fb1db01d1c09c3020125fc5c278d97843233dd6fa897a5f24f/zarr-3.1.4-py3-none-any.whl" ] } }, - "ncore_pip_deps_311_zarr_sdist_37790ede": { + "ncore_pip_deps_311_zarr_sdist_b0cf4eea": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -16963,13 +17305,13 @@ "--find-links", "https://download.pytorch.org/whl/cu128/torch" ], - "filename": "zarr-2.18.4.tar.gz", + "filename": "zarr-3.1.4.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "ncore_pip_deps_311", - "requirement": "zarr==2.18.4", - "sha256": "37790ededd0683ae1abe6ff90aa16c22543b3436810060f53d72c15e910c24bb", + "requirement": "zarr==3.1.4", + "sha256": "b0cf4eea8367ae444d7d0ea90b98d29ac0b7eb691fac2e0bcc2d2bd659522903", "urls": [ - "https://files.pythonhosted.org/packages/21/d1/764ca5b66d91b20dede66aedc6eb9ede3adbe5c61779e7378a7ecb010e87/zarr-2.18.4.tar.gz" + "https://files.pythonhosted.org/packages/c6/77/170915815b797861e0d74eb8246da2898f4069838d5f638de5d8bd1ab7c7/zarr-3.1.4.tar.gz" ] } }, @@ -20309,36 +20651,6 @@ ] } }, - "ncore_pip_deps_313_asciitree_sdist_4aa4b9b6": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cpu/torch" - ], - "filename": "asciitree-0.3.3.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "asciitree==0.3.3", - "sha256": "4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e", - "urls": [ - "https://files.pythonhosted.org/packages/2d/6a/885bc91484e1aa8f618f6f0228d76d0e67000b0fdd6090673b777e311913/asciitree-0.3.3.tar.gz" - ] - } - }, "ncore_pip_deps_313_cbor2_cp313_cp313_macosx_11_0_arm64_f850860e": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -20567,7 +20879,7 @@ ] } }, - "ncore_pip_deps_313_deprecated_py3_none_any_58204cf4": { + "ncore_pip_deps_313_donfig_py3_none_any_2a3175ce": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20581,17 +20893,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "deprecated-3.0.0-py3-none-any.whl", + "filename": "donfig-0.8.1.post1-py3-none-any.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "deprecated==3.0.0", - "sha256": "58204cf4a7f6270d547af5c278ee7a6bb56045a4b3d8441a1cd11660f41b7939", + "requirement": "donfig==0.8.1.post1", + "sha256": "2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", "urls": [ - "https://files.pythonhosted.org/packages/83/ae/676feae8e4644a6d7169951a97f61c56f416c73f67bf1761f2461d75cc81/deprecated-3.0.0-py3-none-any.whl" + "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl" ] } }, - "ncore_pip_deps_313_deprecated_sdist_16850204": { + "ncore_pip_deps_313_donfig_sdist_3bef3413": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20611,17 +20923,17 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "deprecated-3.0.0.tar.gz", + "filename": "donfig-0.8.1.post1.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "deprecated==3.0.0", - "sha256": "16850204d3a1e6bb0acd06bff48d96e8b0a0d25d1c52f71705405a0f4894192d", + "requirement": "donfig==0.8.1.post1", + "sha256": "3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", "urls": [ - "https://files.pythonhosted.org/packages/f7/9c/16649913bf14c73e0a9453782e148362ff2657067deff6aa9c7ebcddcc31/deprecated-3.0.0.tar.gz" + "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz" ] } }, - "ncore_pip_deps_313_fasteners_py3_none_any_9422c40d": { + "ncore_pip_deps_313_filelock_py3_none_any_2ce9818e": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20635,17 +20947,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "fasteners-0.20-py3-none-any.whl", + "filename": "filelock-4.1.0-py3-none-any.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "fasteners==0.20", - "sha256": "9422c40d1e350e4259f509fb2e608d6bc43c0136f79a00db1b49046029d0b3b7", + "requirement": "filelock==4.1.0", + "sha256": "2ce9818e3e2d8f284c1a964414447ef148d42a5fd5e2a477a7118e574b293ec1", "urls": [ - "https://files.pythonhosted.org/packages/51/ac/e5d886f892666d2d1e5cb8c1a41146e1d79ae8896477b1153a21711d3b44/fasteners-0.20-py3-none-any.whl" + "https://files.pythonhosted.org/packages/ee/86/032133892a5de43b5a98200b01aadcad68cc255e274a762f08b8a76d2912/filelock-4.1.0-py3-none-any.whl" ] } }, - "ncore_pip_deps_313_fasteners_sdist_55dce879": { + "ncore_pip_deps_313_filelock_sdist_ad7f724a": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20665,17 +20977,17 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "fasteners-0.20.tar.gz", + "filename": "filelock-4.1.0.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "fasteners==0.20", - "sha256": "55dce8792a41b56f727ba6e123fcaee77fd87e638a6863cec00007bfea84c8d8", + "requirement": "filelock==4.1.0", + "sha256": "ad7f724afef953e731b1cc39bcd3a09166d72ed7fcdf29e6e88b1c3235c6715d", "urls": [ - "https://files.pythonhosted.org/packages/2d/18/7881a99ba5244bfc82f06017316ffe93217dbbbcfa52b887caa1d4f2a6d3/fasteners-0.20.tar.gz" + "https://files.pythonhosted.org/packages/4c/58/6fd434bec86eff7c38a3168454cb132b762b2bea9b3ac094101a2f7bc32a/filelock-4.1.0.tar.gz" ] } }, - "ncore_pip_deps_313_filelock_py3_none_any_2ce9818e": { + "ncore_pip_deps_313_fsspec_py3_none_any_8dd6e646": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20689,17 +21001,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "filelock-4.1.0-py3-none-any.whl", + "filename": "fsspec-2026.9.0-py3-none-any.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "filelock==4.1.0", - "sha256": "2ce9818e3e2d8f284c1a964414447ef148d42a5fd5e2a477a7118e574b293ec1", + "requirement": "fsspec==2026.9.0", + "sha256": "8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f", "urls": [ - "https://files.pythonhosted.org/packages/ee/86/032133892a5de43b5a98200b01aadcad68cc255e274a762f08b8a76d2912/filelock-4.1.0-py3-none-any.whl" + "https://files.pythonhosted.org/packages/6c/c0/a98505f18594f1bce828bb159cec0fcf9860562f1a2c85913409fc8f3d9e/fsspec-2026.9.0-py3-none-any.whl" ] } }, - "ncore_pip_deps_313_filelock_sdist_ad7f724a": { + "ncore_pip_deps_313_fsspec_sdist_0f081479": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20719,17 +21031,17 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "filelock-4.1.0.tar.gz", + "filename": "fsspec-2026.9.0.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "filelock==4.1.0", - "sha256": "ad7f724afef953e731b1cc39bcd3a09166d72ed7fcdf29e6e88b1c3235c6715d", + "requirement": "fsspec==2026.9.0", + "sha256": "0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe", "urls": [ - "https://files.pythonhosted.org/packages/4c/58/6fd434bec86eff7c38a3168454cb132b762b2bea9b3ac094101a2f7bc32a/filelock-4.1.0.tar.gz" + "https://files.pythonhosted.org/packages/77/cd/9be253869fc42e764de7f3dedd6969af7d44ff9c3375214a3442a6f3fc08/fsspec-2026.9.0.tar.gz" ] } }, - "ncore_pip_deps_313_fsspec_py3_none_any_8dd6e646": { + "ncore_pip_deps_313_google_crc32c_cp313_cp313_macosx_12_0_arm64_f1dc17d9": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20743,17 +21055,113 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "fsspec-2026.9.0-py3-none-any.whl", + "filename": "google_crc32c-1.9.0-cp313-cp313-macosx_12_0_arm64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "fsspec==2026.9.0", - "sha256": "8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f", + "requirement": "google-crc32c==1.9.0", + "sha256": "f1dc17d987ddcc5eba12a7ce48f0eb93141dea236b170c1101151396edf2f0cf", "urls": [ - "https://files.pythonhosted.org/packages/6c/c0/a98505f18594f1bce828bb159cec0fcf9860562f1a2c85913409fc8f3d9e/fsspec-2026.9.0-py3-none-any.whl" + "https://files.pythonhosted.org/packages/3f/34/cb484e8b6174f130f8c6dc79c733a9dd8869b410ad6511fb6104c46b973a/google_crc32c-1.9.0-cp313-cp313-macosx_12_0_arm64.whl" ] } }, - "ncore_pip_deps_313_fsspec_sdist_0f081479": { + "ncore_pip_deps_313_google_crc32c_cp313_cp313_macosx_12_0_x86_64_f894a287": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp313-cp313-macosx_12_0_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "google-crc32c==1.9.0", + "sha256": "f894a2877650b56201d26a012a257b76d54a68834dc3913a93830ca8a047b075", + "urls": [ + "https://files.pythonhosted.org/packages/af/25/3e8e567bd48448e225ea27318ccf2b94e05124e7b8b97b13eaec9e127199/google_crc32c-1.9.0-cp313-cp313-macosx_12_0_x86_64.whl" + ] + } + }, + "ncore_pip_deps_313_google_crc32c_cp313_cp313_manylinux_2_17_aarch64_4488f155": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "google-crc32c==1.9.0", + "sha256": "4488f1553a9ab7e86cdedc833374a7e904031803b995dc0bd0be48c271fa6556", + "urls": [ + "https://files.pythonhosted.org/packages/f0/18/bee0dd59ae622482dc6463636c79e4bde7c954d061c859c9256362c9931a/google_crc32c-1.9.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl" + ] + } + }, + "ncore_pip_deps_313_google_crc32c_cp313_cp313_manylinux_2_17_x86_64_0568b17e": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "google-crc32c==1.9.0", + "sha256": "0568b17ed90ac596f29400d99e243fd0cc6276766183def888d1bf8d1dc13827", + "urls": [ + "https://files.pythonhosted.org/packages/fd/b6/e76e80fed5f2558273c7839e622f98095c9b36c719c7147e38e3c055cb70/google_crc32c-1.9.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" + ] + } + }, + "ncore_pip_deps_313_google_crc32c_cp313_cp313_win_amd64_8583ec21": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp313-cp313-win_amd64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "google-crc32c==1.9.0", + "sha256": "8583ec21d56b565d68ab2963cc7e21b3b271247c29b04286068255ef65f221bd", + "urls": [ + "https://files.pythonhosted.org/packages/87/34/165542bfa99dfef91a76471cc48cce74b8ff4e295722896087ab2b8e8611/google_crc32c-1.9.0-cp313-cp313-win_amd64.whl" + ] + } + }, + "ncore_pip_deps_313_google_crc32c_sdist_7b8c84c3": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20773,13 +21181,13 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "fsspec-2026.9.0.tar.gz", + "filename": "google_crc32c-1.9.0.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "fsspec==2026.9.0", - "sha256": "0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe", + "requirement": "google-crc32c==1.9.0", + "sha256": "7b8c84c3d159ab6817fe3f74e6e6cef099c3f95dcec3abc0d8afb1404642efbe", "urls": [ - "https://files.pythonhosted.org/packages/77/cd/9be253869fc42e764de7f3dedd6969af7d44ff9c3375214a3442a6f3fc08/fsspec-2026.9.0.tar.gz" + "https://files.pythonhosted.org/packages/fa/25/9cb0c1c31c45b893eb8f11ae70b3f4309432d59b5acaebca5dbe791729a4/google_crc32c-1.9.0.tar.gz" ] } }, @@ -21293,31 +21701,7 @@ ] } }, - "ncore_pip_deps_313_mypy_extensions_py3_none_any_1be4cccd": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "mypy_extensions-1.1.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "mypy-extensions==1.1.0", - "sha256": "1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", - "urls": [ - "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl" - ] - } - }, - "ncore_pip_deps_313_mypy_extensions_sdist_52e68efc": { + "ncore_pip_deps_313_msgspec_cp313_cp313_macosx_10_13_x86_64_f13c127a": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21331,23 +21715,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cpu/torch" - ], - "filename": "mypy_extensions-1.1.0.tar.gz", + "filename": "msgspec-0.22.0-cp313-cp313-macosx_10_13_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "mypy-extensions==1.1.0", - "sha256": "52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", + "requirement": "msgspec==0.22.0", + "sha256": "f13c127a945479bc9db057eb253b8851075c8e1ae07ffc967bfa1c5676203a86", "urls": [ - "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz" + "https://files.pythonhosted.org/packages/7f/62/5374fba2ede0408f4bd8b9b3a6c8464f8d0ea7ae9a2a064bd81ca492bd1e/msgspec-0.22.0-cp313-cp313-macosx_10_13_x86_64.whl" ] } }, - "ncore_pip_deps_313_networkx_py3_none_any_e3fd2c13": { + "ncore_pip_deps_313_msgspec_cp313_cp313_macosx_11_0_arm64_5aa24eb4": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21361,17 +21739,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "networkx-3.7-py3-none-any.whl", + "filename": "msgspec-0.22.0-cp313-cp313-macosx_11_0_arm64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "networkx==3.7", - "sha256": "e3fd2c13a7814cee3746340d8d7f8598a67f16a58bf47fb7f8793fab6efca1b0", + "requirement": "msgspec==0.22.0", + "sha256": "5aa24eb475d070ecbbe5b21080fc3ce4b0b76c60de25cfe0c9678d8fb44bb42f", "urls": [ - "https://files.pythonhosted.org/packages/7e/cd/fe58041e9011f307c490e3e17dd48cc516448f7c698a3f2d9d9d65d7e6a8/networkx-3.7-py3-none-any.whl" + "https://files.pythonhosted.org/packages/cc/e3/357baa8d2a9164a98dfd7ef9d3a58125df0ed981be909945bdd337be7194/msgspec-0.22.0-cp313-cp313-macosx_11_0_arm64.whl" ] } }, - "ncore_pip_deps_313_networkx_sdist_fd77a511": { + "ncore_pip_deps_313_msgspec_cp313_cp313_manylinux_2_17_aarch64_627bfdfe": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21385,23 +21763,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cpu/torch" - ], - "filename": "networkx-3.7.tar.gz", + "filename": "msgspec-0.22.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "networkx==3.7", - "sha256": "fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a", + "requirement": "msgspec==0.22.0", + "sha256": "627bfdfe5a4b3d916b3360b30f4cddeee3a084f56593e33527c6872fa8322ff9", "urls": [ - "https://files.pythonhosted.org/packages/dc/76/3af777226b63a5e64a6b36b1ec5855c14e2b94a37096d4760e595fc43511/networkx-3.7.tar.gz" + "https://files.pythonhosted.org/packages/fa/1b/9cc07718d1dee8ed5e89a265801d565bc0f15ead435ccb198f9c7bf92574/msgspec-0.22.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl" ] } }, - "ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_10_13_x86_64_e3d82b70": { + "ncore_pip_deps_313_msgspec_cp313_cp313_manylinux_2_17_x86_64_c6c310ef": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21415,17 +21787,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numcodecs-0.15.1-cp313-cp313-macosx_10_13_x86_64.whl", + "filename": "msgspec-0.22.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numcodecs==0.15.1", - "sha256": "e3d82b70500cf61e8d115faa0d0a76be6ecdc24a16477ee3279d711699ad85f3", + "requirement": "msgspec==0.22.0", + "sha256": "c6c310ef83e7e291b01a63298828f848348bb99e84a1098c4b3923c05674d032", "urls": [ - "https://files.pythonhosted.org/packages/78/57/acbc54b3419e5be65015e47177c76c0a73e037fd3ae2cde5808169194d4d/numcodecs-0.15.1-cp313-cp313-macosx_10_13_x86_64.whl" + "https://files.pythonhosted.org/packages/46/64/f33fdfe95aca76601194a7064d14816c7c22c4eccc1b03a5335785895fa3/msgspec-0.22.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" ] } }, - "ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_11_0_arm64_1d471a18": { + "ncore_pip_deps_313_msgspec_cp313_cp313_musllinux_1_2_aarch64_bc374ded": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21439,17 +21811,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numcodecs-0.15.1-cp313-cp313-macosx_11_0_arm64.whl", + "filename": "msgspec-0.22.0-cp313-cp313-musllinux_1_2_aarch64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numcodecs==0.15.1", - "sha256": "1d471a1829ce52d3f365053a2bd1379e32e369517557c4027ddf5ac0d99c591e", + "requirement": "msgspec==0.22.0", + "sha256": "bc374dedd5f85a5f4de2386dc5f737894ccb8c1ac18e9566ce66fd9839e6285d", "urls": [ - "https://files.pythonhosted.org/packages/b6/56/9863fa6dc679f40a31bea5e9713ee5507a31dcd3ee82ea4b1a9268ce52e8/numcodecs-0.15.1-cp313-cp313-macosx_11_0_arm64.whl" + "https://files.pythonhosted.org/packages/88/a6/7b5c4fb39e0bf2dabc8be923c33c39b07ba769a0ce6f0afbbdfaadb1f2f2/msgspec-0.22.0-cp313-cp313-musllinux_1_2_aarch64.whl" ] } }, - "ncore_pip_deps_313_numcodecs_cp313_cp313_manylinux_2_17_x86_64_1dfdea4a": { + "ncore_pip_deps_313_msgspec_cp313_cp313_musllinux_1_2_x86_64_6f48317f": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21463,17 +21835,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numcodecs-0.15.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "filename": "msgspec-0.22.0-cp313-cp313-musllinux_1_2_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numcodecs==0.15.1", - "sha256": "1dfdea4a67108205edfce99c1cb6cd621343bc7abb7e16a041c966776920e7de", + "requirement": "msgspec==0.22.0", + "sha256": "6f48317f05312bfdf78248f53933f830f07ab75cc1c813ac3ca4220cb3b5b019", "urls": [ - "https://files.pythonhosted.org/packages/fa/91/d96999b41e3146b6c0ce6bddc5ad85803cb4d743c95394562c2a4bb8cded/numcodecs-0.15.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "https://files.pythonhosted.org/packages/6c/e5/b4c5323b17ecfce45350695d40fc93e16856db957a53cbcf2f53007d6e12/msgspec-0.22.0-cp313-cp313-musllinux_1_2_x86_64.whl" ] } }, - "ncore_pip_deps_313_numcodecs_cp313_cp313_win_amd64_a4f7bdb2": { + "ncore_pip_deps_313_msgspec_cp313_cp313_win_amd64_0739b068": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21487,17 +21859,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numcodecs-0.15.1-cp313-cp313-win_amd64.whl", + "filename": "msgspec-0.22.0-cp313-cp313-win_amd64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numcodecs==0.15.1", - "sha256": "a4f7bdb26f1b34423cb56d48e75821223be38040907c9b5954eeb7463e7eb03c", + "requirement": "msgspec==0.22.0", + "sha256": "0739b068f31f2004a364f97679ba91f2f5ecd6ec2a5b4b890188ab5c57d20672", "urls": [ - "https://files.pythonhosted.org/packages/c3/32/233e5ede6568bdb044e6f99aaa9fa39827ff3109c6487fc137315f733586/numcodecs-0.15.1-cp313-cp313-win_amd64.whl" + "https://files.pythonhosted.org/packages/01/33/e591f9d3d8d6c9cfc02ae95f3e3c44920f2d18050f3f252c244e0f293a0e/msgspec-0.22.0-cp313-cp313-win_amd64.whl" ] } }, - "ncore_pip_deps_313_numcodecs_sdist_eeed77e4": { + "ncore_pip_deps_313_msgspec_sdist_0a13624a": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21517,17 +21889,17 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "numcodecs-0.15.1.tar.gz", + "filename": "msgspec-0.22.0.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numcodecs==0.15.1", - "sha256": "eeed77e4d6636641a2cc605fbc6078c7a8f2cc40f3dfa2b3f61e52e6091b04ff", + "requirement": "msgspec==0.22.0", + "sha256": "0a13624a4969159fe35d8c2a3d377b2b61bbd8585e327440d5e52725affcce38", "urls": [ - "https://files.pythonhosted.org/packages/63/fc/bb532969eb8236984ba65e4f0079a7da885b8ac0ce1f0835decbb3938a62/numcodecs-0.15.1.tar.gz" + "https://files.pythonhosted.org/packages/d0/e6/6dcf9306ff3c5e486578f3bf29ed11dfbdbbc2a8bf0caf7e07d392887fda/msgspec-0.22.0.tar.gz" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_macosx_10_13_x86_64_2377da2d": { + "ncore_pip_deps_313_mypy_extensions_py3_none_any_1be4cccd": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21541,17 +21913,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-macosx_10_13_x86_64.whl", + "filename": "mypy_extensions-1.1.0-py3-none-any.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "2377da2dd3ba2c1200956acbab2a358c83b8e1f8531191672d1cd6ad83250d53", + "requirement": "mypy-extensions==1.1.0", + "sha256": "1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", "urls": [ - "https://files.pythonhosted.org/packages/67/14/1c3ee0118a8fce08565a5d8482631608426a33af10a01077fada5dc7c119/numpy-2.5.4-cp313-cp313-macosx_10_13_x86_64.whl" + "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_macosx_11_0_arm64_7415db95": { + "ncore_pip_deps_313_mypy_extensions_sdist_52e68efc": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21565,17 +21937,23 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-macosx_11_0_arm64.whl", + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cpu/torch" + ], + "filename": "mypy_extensions-1.1.0.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "7415db95818b39ec475a5eea54d9e3b6bc83e3912158e46da3438cdce399804d", + "requirement": "mypy-extensions==1.1.0", + "sha256": "52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", "urls": [ - "https://files.pythonhosted.org/packages/83/8c/b0ea9477fb1f0d4484bbc5cba21678cc9969704d8d7f3f158d1db35f8e14/numpy-2.5.4-cp313-cp313-macosx_11_0_arm64.whl" + "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_macosx_14_0_arm64_6d6a71b9": { + "ncore_pip_deps_313_networkx_py3_none_any_e3fd2c13": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21589,17 +21967,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-macosx_14_0_arm64.whl", + "filename": "networkx-3.7-py3-none-any.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "6d6a71b9d9a97c03633aa12565ef2825ffa036cc1d99cfd50dacf0f128af4fe2", + "requirement": "networkx==3.7", + "sha256": "e3fd2c13a7814cee3746340d8d7f8598a67f16a58bf47fb7f8793fab6efca1b0", "urls": [ - "https://files.pythonhosted.org/packages/e2/84/6a3d75b3ba3dfe84ac0053450753d1e6d250a8bf80f66474cc46d1fb643f/numpy-2.5.4-cp313-cp313-macosx_14_0_arm64.whl" + "https://files.pythonhosted.org/packages/7e/cd/fe58041e9011f307c490e3e17dd48cc516448f7c698a3f2d9d9d65d7e6a8/networkx-3.7-py3-none-any.whl" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_macosx_14_0_x86_64_d8200f16": { + "ncore_pip_deps_313_networkx_sdist_fd77a511": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21613,17 +21991,23 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-macosx_14_0_x86_64.whl", + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cpu/torch" + ], + "filename": "networkx-3.7.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "d8200f16437b289a5bb927c6e184eccc3e8389bc0070fea4cd5b9e13c1757959", + "requirement": "networkx==3.7", + "sha256": "fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a", "urls": [ - "https://files.pythonhosted.org/packages/61/18/bb993f267ca20b376e07092a16793a5b31ed3138751e9ba480011a14d742/numpy-2.5.4-cp313-cp313-macosx_14_0_x86_64.whl" + "https://files.pythonhosted.org/packages/dc/76/3af777226b63a5e64a6b36b1ec5855c14e2b94a37096d4760e595fc43511/networkx-3.7.tar.gz" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_manylinux_2_27_aarch64_1c2e71b0": { + "ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_10_13_x86_64_de2c66db": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21637,17 +22021,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", + "filename": "numcodecs-0.17.0-cp313-cp313-macosx_10_13_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "1c2e71b04c6cad90026e544501bbe0ab9290fa8a4d845e7e8c0d124fb429c988", + "requirement": "numcodecs==0.17.0", + "sha256": "de2c66db238e74e66fe9be7e02b7e0129b75d3f812d38e4019eb0102cc2dcdf0", "urls": [ - "https://files.pythonhosted.org/packages/db/b6/135bb0953b61dc21c6cafa14b424ae666944e4899cf140e00c2b322a1a45/numpy-2.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl" + "https://files.pythonhosted.org/packages/11/f1/1d3d2bcb1240e5000f6647b5b0fd465b2b51ecef180bfa797a85df48cf2f/numcodecs-0.17.0-cp313-cp313-macosx_10_13_x86_64.whl" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_manylinux_2_27_x86_64_6ffa0766": { + "ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_11_0_arm64_69b9b468": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21661,17 +22045,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", + "filename": "numcodecs-0.17.0-cp313-cp313-macosx_11_0_arm64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "6ffa07666f8da0eef81d149934a626d0d95fbd6838432a33e66245423a9062c0", + "requirement": "numcodecs==0.17.0", + "sha256": "69b9b4685097c4d478a0c829debf4470555ec63e92cdd2c6b5f195460f1dc888", "urls": [ - "https://files.pythonhosted.org/packages/da/24/3bd070f3269dc609d8f26b2643f62ef91bb415841c0b294805aaf7fe06da/numpy-2.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl" + "https://files.pythonhosted.org/packages/64/81/64e2472a8b3a9fa26bccfc7d5fa876770a9027bd5cd77e5b4a7b807a0785/numcodecs-0.17.0-cp313-cp313-macosx_11_0_arm64.whl" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_musllinux_1_2_aarch64_2fa3328f": { + "ncore_pip_deps_313_numcodecs_cp313_cp313_manylinux_2_17_aarch64_7065b334": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21685,17 +22069,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-musllinux_1_2_aarch64.whl", + "filename": "numcodecs-0.17.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "2fa3328f784fc8277fc48026f6cad516f5c561c5d8e2e39b3c9e0c8f23223b34", + "requirement": "numcodecs==0.17.0", + "sha256": "7065b3349b73d54785aa89e00d0b97d80f664e9056757929d28151f9208dc04c", "urls": [ - "https://files.pythonhosted.org/packages/c7/8e/9d15bd356b0a019c965312b1a3c6a727cac4cae5bc40045fbc12ce4cff9c/numpy-2.5.4-cp313-cp313-musllinux_1_2_aarch64.whl" + "https://files.pythonhosted.org/packages/25/ea/2ab25f7e674cf1e78f123c5c2689d8a7dc85475554af0619bcce05cb32a9/numcodecs-0.17.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_musllinux_1_2_x86_64_b86966fb": { + "ncore_pip_deps_313_numcodecs_cp313_cp313_manylinux_2_17_x86_64_6c3342d9": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21709,17 +22093,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-musllinux_1_2_x86_64.whl", + "filename": "numcodecs-0.17.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "b86966fbe4ad7de710422175572bcdc75fdedadfb54bc6fab7deabccddd7780b", + "requirement": "numcodecs==0.17.0", + "sha256": "6c3342d91ed7cf59c1be84396edd364e936bb0ec9e366d24bb69689748d19625", "urls": [ - "https://files.pythonhosted.org/packages/dc/fe/9d5b560db964f15871885f2250795d15945f8699e17ef90c0c2ff4c875b2/numpy-2.5.4-cp313-cp313-musllinux_1_2_x86_64.whl" + "https://files.pythonhosted.org/packages/9d/96/b3bf9a31978d936654a73f2bb1036b92b6515164f170092d162419eb771c/numcodecs-0.17.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_win_amd64_8b4d2fd2": { + "ncore_pip_deps_313_numcodecs_cp313_cp313_win_amd64_a854e9c8": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21733,17 +22117,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-win_amd64.whl", + "filename": "numcodecs-0.17.0-cp313-cp313-win_amd64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "8b4d2fd2d34e5f8c9235ee787de5631a37a28402b15cb80814df973d2be54129", + "requirement": "numcodecs==0.17.0", + "sha256": "a854e9c89f58eeeb2453f3c1637d1916797edb6eaff26bc186a6cdb09d187092", "urls": [ - "https://files.pythonhosted.org/packages/90/8c/140a40398a66b4471211be1affdb6ed24c486d581bd28d07b7f2fcb69540/numpy-2.5.4-cp313-cp313-win_amd64.whl" + "https://files.pythonhosted.org/packages/5c/ec/47515bea31725376aa6f061c335326cc7437f863ab704b8e167e80734bfd/numcodecs-0.17.0-cp313-cp313-win_amd64.whl" ] } }, - "ncore_pip_deps_313_numpy_sdist_9a94cf75": { + "ncore_pip_deps_313_numcodecs_sdist_e8db2e33": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21763,17 +22147,17 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "numpy-2.5.4.tar.gz", + "filename": "numcodecs-0.17.0.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "9a94cf751c9ad8ebaa835bcd3d40dacf8534ad086b88c38029b65123c7999d2a", + "requirement": "numcodecs==0.17.0", + "sha256": "e8db2e337bdafd3bb5f891a2543b53b2b36a509ce9d587af2846db3715b6c8b9", "urls": [ - "https://files.pythonhosted.org/packages/95/b0/c7453d0b6e2073c3264468b106ee1563750cecc910965e67357e3698c83e/numpy-2.5.4.tar.gz" + "https://files.pythonhosted.org/packages/dd/ec/260cdb6304868de6db14eb31064bd2735c0200bcb3331d6b4c9e9be02a03/numcodecs-0.17.0.tar.gz" ] } }, - "ncore_pip_deps_313_opencv_python_headless_cp37_abi3_macosx_13_0_arm64_030ca5e0": { + "ncore_pip_deps_313_numpy_cp313_cp313_macosx_10_13_x86_64_2377da2d": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21787,17 +22171,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "opencv_python_headless-5.0.0.93-cp37-abi3-macosx_13_0_arm64.whl", + "filename": "numpy-2.5.4-cp313-cp313-macosx_10_13_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "opencv-python-headless==5.0.0.93", - "sha256": "030ca5e0837a2963ab36ef896baa9767eb8d2b83353fb28af5a521e40dd8756f", + "requirement": "numpy==2.5.4", + "sha256": "2377da2dd3ba2c1200956acbab2a358c83b8e1f8531191672d1cd6ad83250d53", "urls": [ - "https://files.pythonhosted.org/packages/53/7c/8c8097891c509d98cd128493835c95631c80be6a8f37ed9d25716c2e16f1/opencv_python_headless-5.0.0.93-cp37-abi3-macosx_13_0_arm64.whl" + "https://files.pythonhosted.org/packages/67/14/1c3ee0118a8fce08565a5d8482631608426a33af10a01077fada5dc7c119/numpy-2.5.4-cp313-cp313-macosx_10_13_x86_64.whl" ] } }, - "ncore_pip_deps_313_opencv_python_headless_cp37_abi3_macosx_14_0_x86_64_1e55af3a": { + "ncore_pip_deps_313_numpy_cp313_cp313_macosx_11_0_arm64_7415db95": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21811,7 +22195,253 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "opencv_python_headless-5.0.0.93-cp37-abi3-macosx_14_0_x86_64.whl", + "filename": "numpy-2.5.4-cp313-cp313-macosx_11_0_arm64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "7415db95818b39ec475a5eea54d9e3b6bc83e3912158e46da3438cdce399804d", + "urls": [ + "https://files.pythonhosted.org/packages/83/8c/b0ea9477fb1f0d4484bbc5cba21678cc9969704d8d7f3f158d1db35f8e14/numpy-2.5.4-cp313-cp313-macosx_11_0_arm64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_cp313_cp313_macosx_14_0_arm64_6d6a71b9": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "numpy-2.5.4-cp313-cp313-macosx_14_0_arm64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "6d6a71b9d9a97c03633aa12565ef2825ffa036cc1d99cfd50dacf0f128af4fe2", + "urls": [ + "https://files.pythonhosted.org/packages/e2/84/6a3d75b3ba3dfe84ac0053450753d1e6d250a8bf80f66474cc46d1fb643f/numpy-2.5.4-cp313-cp313-macosx_14_0_arm64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_cp313_cp313_macosx_14_0_x86_64_d8200f16": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "numpy-2.5.4-cp313-cp313-macosx_14_0_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "d8200f16437b289a5bb927c6e184eccc3e8389bc0070fea4cd5b9e13c1757959", + "urls": [ + "https://files.pythonhosted.org/packages/61/18/bb993f267ca20b376e07092a16793a5b31ed3138751e9ba480011a14d742/numpy-2.5.4-cp313-cp313-macosx_14_0_x86_64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_cp313_cp313_manylinux_2_27_aarch64_1c2e71b0": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "numpy-2.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "1c2e71b04c6cad90026e544501bbe0ab9290fa8a4d845e7e8c0d124fb429c988", + "urls": [ + "https://files.pythonhosted.org/packages/db/b6/135bb0953b61dc21c6cafa14b424ae666944e4899cf140e00c2b322a1a45/numpy-2.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_cp313_cp313_manylinux_2_27_x86_64_6ffa0766": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "numpy-2.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "6ffa07666f8da0eef81d149934a626d0d95fbd6838432a33e66245423a9062c0", + "urls": [ + "https://files.pythonhosted.org/packages/da/24/3bd070f3269dc609d8f26b2643f62ef91bb415841c0b294805aaf7fe06da/numpy-2.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_cp313_cp313_musllinux_1_2_aarch64_2fa3328f": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "numpy-2.5.4-cp313-cp313-musllinux_1_2_aarch64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "2fa3328f784fc8277fc48026f6cad516f5c561c5d8e2e39b3c9e0c8f23223b34", + "urls": [ + "https://files.pythonhosted.org/packages/c7/8e/9d15bd356b0a019c965312b1a3c6a727cac4cae5bc40045fbc12ce4cff9c/numpy-2.5.4-cp313-cp313-musllinux_1_2_aarch64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_cp313_cp313_musllinux_1_2_x86_64_b86966fb": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "numpy-2.5.4-cp313-cp313-musllinux_1_2_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "b86966fbe4ad7de710422175572bcdc75fdedadfb54bc6fab7deabccddd7780b", + "urls": [ + "https://files.pythonhosted.org/packages/dc/fe/9d5b560db964f15871885f2250795d15945f8699e17ef90c0c2ff4c875b2/numpy-2.5.4-cp313-cp313-musllinux_1_2_x86_64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_cp313_cp313_win_amd64_8b4d2fd2": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "numpy-2.5.4-cp313-cp313-win_amd64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "8b4d2fd2d34e5f8c9235ee787de5631a37a28402b15cb80814df973d2be54129", + "urls": [ + "https://files.pythonhosted.org/packages/90/8c/140a40398a66b4471211be1affdb6ed24c486d581bd28d07b7f2fcb69540/numpy-2.5.4-cp313-cp313-win_amd64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_sdist_9a94cf75": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cpu/torch" + ], + "filename": "numpy-2.5.4.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "9a94cf751c9ad8ebaa835bcd3d40dacf8534ad086b88c38029b65123c7999d2a", + "urls": [ + "https://files.pythonhosted.org/packages/95/b0/c7453d0b6e2073c3264468b106ee1563750cecc910965e67357e3698c83e/numpy-2.5.4.tar.gz" + ] + } + }, + "ncore_pip_deps_313_opencv_python_headless_cp37_abi3_macosx_13_0_arm64_030ca5e0": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "opencv_python_headless-5.0.0.93-cp37-abi3-macosx_13_0_arm64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "opencv-python-headless==5.0.0.93", + "sha256": "030ca5e0837a2963ab36ef896baa9767eb8d2b83353fb28af5a521e40dd8756f", + "urls": [ + "https://files.pythonhosted.org/packages/53/7c/8c8097891c509d98cd128493835c95631c80be6a8f37ed9d25716c2e16f1/opencv_python_headless-5.0.0.93-cp37-abi3-macosx_13_0_arm64.whl" + ] + } + }, + "ncore_pip_deps_313_opencv_python_headless_cp37_abi3_macosx_14_0_x86_64_1e55af3a": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "opencv_python_headless-5.0.0.93-cp37-abi3-macosx_14_0_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", "requirement": "opencv-python-headless==5.0.0.93", @@ -22294,14 +22924,254 @@ "filename": "pillow-12.3.0-cp313-cp313-win_amd64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pillow==12.3.0", - "sha256": "1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", + "requirement": "pillow==12.3.0", + "sha256": "1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", + "urls": [ + "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl" + ] + } + }, + "ncore_pip_deps_313_pillow_sdist_3b8182a7": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cpu/torch" + ], + "filename": "pillow-12.3.0.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pillow==12.3.0", + "sha256": "3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", + "urls": [ + "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz" + ] + } + }, + "ncore_pip_deps_313_pluggy_py3_none_any_e920276d": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "pluggy-1.6.0-py3-none-any.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pluggy==1.6.0", + "sha256": "e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", + "urls": [ + "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl" + ] + } + }, + "ncore_pip_deps_313_pluggy_sdist_7dcc130b": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cpu/torch" + ], + "filename": "pluggy-1.6.0.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pluggy==1.6.0", + "sha256": "7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", + "urls": [ + "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz" + ] + } + }, + "ncore_pip_deps_313_pygments_py3_none_any_2363c69b": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "pygments-2.21.0-py3-none-any.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pygments==2.21.0", + "sha256": "2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", + "urls": [ + "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl" + ] + } + }, + "ncore_pip_deps_313_pygments_sdist_610ca751": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cpu/torch" + ], + "filename": "pygments-2.21.0.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pygments==2.21.0", + "sha256": "610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", + "urls": [ + "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz" + ] + } + }, + "ncore_pip_deps_313_pytest_py3_none_any_37a86b45": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "pytest-9.1.1-py3-none-any.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pytest==9.1.1", + "sha256": "37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", + "urls": [ + "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl" + ] + } + }, + "ncore_pip_deps_313_pytest_sdist_1088fbde": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cpu/torch" + ], + "filename": "pytest-9.1.1.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pytest==9.1.1", + "sha256": "1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", + "urls": [ + "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz" + ] + } + }, + "ncore_pip_deps_313_pyyaml_cp313_cp313_macosx_10_13_x86_64_8da9669d": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pyyaml==6.0.3", + "sha256": "8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", + "urls": [ + "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl" + ] + } + }, + "ncore_pip_deps_313_pyyaml_cp313_cp313_macosx_11_0_arm64_2283a07e": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pyyaml==6.0.3", + "sha256": "2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", "urls": [ - "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl" + "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl" ] } }, - "ncore_pip_deps_313_pillow_sdist_3b8182a7": { + "ncore_pip_deps_313_pyyaml_cp313_cp313_manylinux_2_17_aarch64_ee292290": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -22315,23 +23185,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cpu/torch" - ], - "filename": "pillow-12.3.0.tar.gz", + "filename": "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pillow==12.3.0", - "sha256": "3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", + "requirement": "pyyaml==6.0.3", + "sha256": "ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", "urls": [ - "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz" + "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl" ] } }, - "ncore_pip_deps_313_pluggy_py3_none_any_e920276d": { + "ncore_pip_deps_313_pyyaml_cp313_cp313_manylinux_2_17_s390x_a33284e2": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -22345,17 +23209,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "pluggy-1.6.0-py3-none-any.whl", + "filename": "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pluggy==1.6.0", - "sha256": "e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", + "requirement": "pyyaml==6.0.3", + "sha256": "a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", "urls": [ - "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl" + "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl" ] } }, - "ncore_pip_deps_313_pluggy_sdist_7dcc130b": { + "ncore_pip_deps_313_pyyaml_cp313_cp313_manylinux_2_17_x86_64_0f29edc4": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -22369,23 +23233,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cpu/torch" - ], - "filename": "pluggy-1.6.0.tar.gz", + "filename": "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pluggy==1.6.0", - "sha256": "7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", + "requirement": "pyyaml==6.0.3", + "sha256": "0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", "urls": [ - "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz" + "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" ] } }, - "ncore_pip_deps_313_pygments_py3_none_any_2363c69b": { + "ncore_pip_deps_313_pyyaml_cp313_cp313_musllinux_1_2_aarch64_f7057c9a": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -22399,17 +23257,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "pygments-2.21.0-py3-none-any.whl", + "filename": "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pygments==2.21.0", - "sha256": "2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", + "requirement": "pyyaml==6.0.3", + "sha256": "f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", "urls": [ - "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl" + "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl" ] } }, - "ncore_pip_deps_313_pygments_sdist_610ca751": { + "ncore_pip_deps_313_pyyaml_cp313_cp313_musllinux_1_2_x86_64_eda16858": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -22423,23 +23281,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cpu/torch" - ], - "filename": "pygments-2.21.0.tar.gz", + "filename": "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pygments==2.21.0", - "sha256": "610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", + "requirement": "pyyaml==6.0.3", + "sha256": "eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", "urls": [ - "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz" + "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl" ] } }, - "ncore_pip_deps_313_pytest_py3_none_any_37a86b45": { + "ncore_pip_deps_313_pyyaml_cp313_cp313_win_amd64_79005a0d": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -22453,17 +23305,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "pytest-9.1.1-py3-none-any.whl", + "filename": "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pytest==9.1.1", - "sha256": "37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", + "requirement": "pyyaml==6.0.3", + "sha256": "79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", "urls": [ - "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl" + "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl" ] } }, - "ncore_pip_deps_313_pytest_sdist_1088fbde": { + "ncore_pip_deps_313_pyyaml_sdist_d7662337": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -22483,13 +23335,13 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "pytest-9.1.1.tar.gz", + "filename": "pyyaml-6.0.3.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pytest==9.1.1", - "sha256": "1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", + "requirement": "pyyaml==6.0.3", + "sha256": "d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", "urls": [ - "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz" + "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz" ] } }, @@ -23027,229 +23879,7 @@ ] } }, - "ncore_pip_deps_313_wrapt_cp313_cp313_macosx_10_13_x86_64_b312b3cc": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-cp313-cp313-macosx_10_13_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "b312b3cc87951faaed3cfef984d768ee8bee7f935d9cc929aaa9946b0b96a98c", - "urls": [ - "https://files.pythonhosted.org/packages/d6/4b/cc7bb5668f7ddc0e73e236e96a0c06cab8fddfca9c53538c9dffac62db6f/wrapt-2.5.0-cp313-cp313-macosx_10_13_x86_64.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_cp313_cp313_macosx_11_0_arm64_c57ddae2": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-cp313-cp313-macosx_11_0_arm64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "c57ddae24cf72eb6bd18112638a987cafe6109d90f2df111e6934362cc03ac1a", - "urls": [ - "https://files.pythonhosted.org/packages/4a/13/5d15ef0e2f42d5f084930dc4863e6e52c160c28301c8780aae170c58421c/wrapt-2.5.0-cp313-cp313-macosx_11_0_arm64.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_cp313_cp313_manylinux_2_17_aarch64_6058e12e": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "6058e12e9caa33468f9a36fb88c15a4bb30a479f997b37834b83abdbf062f264", - "urls": [ - "https://files.pythonhosted.org/packages/a4/f9/47ae1d7ef325c3f6c81ae3c1fb4a3fef9d98c8025ed676c0bfc1550903ce/wrapt-2.5.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_cp313_cp313_manylinux_2_5_x86_64_b95a6eca": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "b95a6eca3b927853529eea958310563c83140ae8451dd5dc4399c7da385dc4f3", - "urls": [ - "https://files.pythonhosted.org/packages/1e/02/c7174e78b0c38bb279b2d3c25a6bd7fb9d3b0200c3e5a8fad084e7cc3e85/wrapt-2.5.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_cp313_cp313_musllinux_1_2_aarch64_f4ef4935": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "f4ef4935962f7029b2058a99f1a47ccbffc3be919dddb3becb6c2c48eac3d9f0", - "urls": [ - "https://files.pythonhosted.org/packages/41/45/fc252bda5aa1ca01bc838d3b108778e786a2a13d0c52fd17c5f6179aa246/wrapt-2.5.0-cp313-cp313-musllinux_1_2_aarch64.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_cp313_cp313_musllinux_1_2_x86_64_a346408f": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "a346408f19b6d589bf029f25f65c0b4cdeed6302ef8f40da4e5d1552d22dc037", - "urls": [ - "https://files.pythonhosted.org/packages/81/8e/2b823fded8c3b815408c58633929812eacd29d824fe57548b7868c4ee422/wrapt-2.5.0-cp313-cp313-musllinux_1_2_x86_64.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_cp313_cp313_win_amd64_77f0a74f": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-cp313-cp313-win_amd64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "77f0a74ff6f6cf89f5b673a732d5afe1911a6e6b1c017260836fdfdf85518dc1", - "urls": [ - "https://files.pythonhosted.org/packages/ce/9a/51d95640e01d0ebdd04a7223755f076e4936b0c124ce99bb01a12b53e66c/wrapt-2.5.0-cp313-cp313-win_amd64.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_py3_none_any_107eea1a": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "107eea1a511e98a3a5033b0c2cb403fbb37f05dee6ac1fb85c0460d311ec278c", - "urls": [ - "https://files.pythonhosted.org/packages/87/7d/5ed859fad4b5eddd598a846150aaab2703730ed4886c5c5e03b0df0cfdd5/wrapt-2.5.0-py3-none-any.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_sdist_c48cdb6c": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cpu/torch" - ], - "filename": "wrapt-2.5.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "c48cdb6c904dca76d9915a579e4a5fab6b0c25f650c1019ce78a78effaf7a345", - "urls": [ - "https://files.pythonhosted.org/packages/3e/d2/a254a26d8ceaea87e0eee2e89fcfe53ddc1858418647493bb2937549ab6f/wrapt-2.5.0.tar.gz" - ] - } - }, - "ncore_pip_deps_313_zarr_py3_none_any_ac3dc403": { + "ncore_pip_deps_313_zarr_py3_none_any_38b54057": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -23263,17 +23893,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "zarr-2.18.7-py3-none-any.whl", + "filename": "zarr-3.4.1-py3-none-any.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "zarr==2.18.7", - "sha256": "ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223", + "requirement": "zarr==3.4.1", + "sha256": "38b540578a119352bdce02a720d7bfd99846e77f0728c58b1bab3d1f547526a0", "urls": [ - "https://files.pythonhosted.org/packages/5e/d8/9ffd8c237b3559945bb52103cf0eed64ea098f7b7f573f8d2962ef27b4b2/zarr-2.18.7-py3-none-any.whl" + "https://files.pythonhosted.org/packages/a6/82/0dbc9bc77b49cfb9268dc2b2b1dcd04346c8c7ed52e272b76a628a941656/zarr-3.4.1-py3-none-any.whl" ] } }, - "ncore_pip_deps_313_zarr_sdist_b2b8f66f": { + "ncore_pip_deps_313_zarr_sdist_b34bda11": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -23293,13 +23923,13 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "zarr-2.18.7.tar.gz", + "filename": "zarr-3.4.1.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "zarr==2.18.7", - "sha256": "b2b8f66f14dac4af66b180d2338819981b981f70e196c9a66e6bfaa9e59572f5", + "requirement": "zarr==3.4.1", + "sha256": "b34bda11ceb199c81ee78ecd42cd02f46c7f67a6d8a1e9bc501cafd5a1795356", "urls": [ - "https://files.pythonhosted.org/packages/da/1d/01cf9e3ab2d85190278efc3fca9f68563de35ae30ee59e7640e3af98abe3/zarr-2.18.7.tar.gz" + "https://files.pythonhosted.org/packages/3e/62/e8a36a4b65f01499c7aefcf103aabba0c37c018bf135fbf179f6ed2f01a0/zarr-3.4.1.tar.gz" ] } }, @@ -27715,7 +28345,7 @@ "accessible_pygments": "{\"ncore_pip_deps_311_accessible_pygments_py3_none_any_88ae3211\":[{\"filename\":\"accessible_pygments-0.0.5-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_accessible_pygments_sdist_40918d3e\":[{\"filename\":\"accessible_pygments-0.0.5.tar.gz\",\"version\":\"3.11\"}]}", "alabaster": "{\"ncore_pip_deps_311_alabaster_py3_none_any_b46733c0\":[{\"filename\":\"alabaster-0.7.16-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_alabaster_sdist_75a8b99c\":[{\"filename\":\"alabaster-0.7.16.tar.gz\",\"version\":\"3.11\"}]}", "anyio": "{\"ncore_pip_deps_311_anyio_py3_none_any_d4058288\":[{\"filename\":\"anyio-4.12.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_anyio_sdist_41cfcc3a\":[{\"filename\":\"anyio-4.12.1.tar.gz\",\"version\":\"3.11\"}]}", - "asciitree": "{\"ncore_pip_deps_311_asciitree_sdist_4aa4b9b6\":[{\"filename\":\"asciitree-0.3.3.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_asciitree_sdist_4aa4b9b6\":[{\"filename\":\"asciitree-0.3.3.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_asciitree_sdist_4aa4b9b6\":[{\"filename\":\"asciitree-0.3.3.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_asciitree_sdist_4aa4b9b6\":[{\"filename\":\"asciitree-0.3.3.tar.gz\",\"version\":\"3.8\"}]}", + "asciitree": "{\"ncore_pip_deps_312_asciitree_sdist_4aa4b9b6\":[{\"filename\":\"asciitree-0.3.3.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_38_asciitree_sdist_4aa4b9b6\":[{\"filename\":\"asciitree-0.3.3.tar.gz\",\"version\":\"3.8\"}]}", "asttokens": "{\"ncore_pip_deps_311_asttokens_py3_none_any_15a3ebc0\":[{\"filename\":\"asttokens-3.0.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_asttokens_sdist_71a4ee5d\":[{\"filename\":\"asttokens-3.0.1.tar.gz\",\"version\":\"3.11\"}]}", "astunparse": "{\"ncore_pip_deps_311_astunparse_py2_none_any_c2652417\":[{\"filename\":\"astunparse-1.6.3-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_astunparse_sdist_5ad93a84\":[{\"filename\":\"astunparse-1.6.3.tar.gz\",\"version\":\"3.11\"}]}", "attrs": "{\"ncore_pip_deps_311_attrs_py3_none_any_c75a69e2\":[{\"filename\":\"attrs-25.1.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_attrs_sdist_1c97078a\":[{\"filename\":\"attrs-25.1.0.tar.gz\",\"version\":\"3.11\"}]}", @@ -27734,15 +28364,16 @@ "debugpy": "{\"ncore_pip_deps_311_debugpy_cp311_cp311_macosx_14_0_universal2_36f48298\":[{\"filename\":\"debugpy-1.8.12-cp311-cp311-macosx_14_0_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_debugpy_cp311_cp311_manylinux_2_5_x86_64_a28ed481\":[{\"filename\":\"debugpy-1.8.12-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_debugpy_cp311_cp311_win_amd64_4703575b\":[{\"filename\":\"debugpy-1.8.12-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_debugpy_py2_none_any_274b6a20\":[{\"filename\":\"debugpy-1.8.12-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_debugpy_sdist_646530b0\":[{\"filename\":\"debugpy-1.8.12.tar.gz\",\"version\":\"3.11\"}]}", "decorator": "{\"ncore_pip_deps_311_decorator_py3_none_any_d316bb41\":[{\"filename\":\"decorator-5.2.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_decorator_sdist_65f26614\":[{\"filename\":\"decorator-5.2.1.tar.gz\",\"version\":\"3.11\"}]}", "defusedxml": "{\"ncore_pip_deps_311_defusedxml_py2_none_any_a352e7e4\":[{\"filename\":\"defusedxml-0.7.1-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_defusedxml_sdist_1bb3032d\":[{\"filename\":\"defusedxml-0.7.1.tar.gz\",\"version\":\"3.11\"}]}", - "deprecated": "{\"ncore_pip_deps_311_deprecated_py2_none_any_bd501178\":[{\"filename\":\"Deprecated-1.2.18-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_deprecated_sdist_422b6f6d\":[{\"filename\":\"deprecated-1.2.18.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_deprecated_py3_none_any_58204cf4\":[{\"filename\":\"deprecated-3.0.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_deprecated_sdist_16850204\":[{\"filename\":\"deprecated-3.0.0.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_deprecated_py3_none_any_58204cf4\":[{\"filename\":\"deprecated-3.0.0-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_deprecated_sdist_16850204\":[{\"filename\":\"deprecated-3.0.0.tar.gz\",\"version\":\"3.13\"}]}", + "deprecated": "{\"ncore_pip_deps_311_deprecated_py2_none_any_bd501178\":[{\"filename\":\"Deprecated-1.2.18-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_deprecated_sdist_422b6f6d\":[{\"filename\":\"deprecated-1.2.18.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_deprecated_py3_none_any_58204cf4\":[{\"filename\":\"deprecated-3.0.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_deprecated_sdist_16850204\":[{\"filename\":\"deprecated-3.0.0.tar.gz\",\"version\":\"3.12\"}]}", "descartes": "{\"ncore_pip_deps_311_descartes_py2_none_any_b7e412e7\":[{\"filename\":\"descartes-1.1.0-py2-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_descartes_sdist_135a5021\":[{\"filename\":\"descartes-1.1.0.tar.gz\",\"version\":\"3.11\"}]}", "docutils": "{\"ncore_pip_deps_311_docutils_py3_none_any_d0013f54\":[{\"filename\":\"docutils-0.22.4-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_docutils_sdist_4db53b1f\":[{\"filename\":\"docutils-0.22.4.tar.gz\",\"version\":\"3.11\"}]}", + "donfig": "{\"ncore_pip_deps_311_donfig_py3_none_any_2a3175ce\":[{\"filename\":\"donfig-0.8.1.post1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_donfig_sdist_3bef3413\":[{\"filename\":\"donfig-0.8.1.post1.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_313_donfig_py3_none_any_2a3175ce\":[{\"filename\":\"donfig-0.8.1.post1-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_donfig_sdist_3bef3413\":[{\"filename\":\"donfig-0.8.1.post1.tar.gz\",\"version\":\"3.13\"}]}", "dracopy": "{\"ncore_pip_deps_311_dracopy_cp311_cp311_macosx_10_9_x86_64_c4131151\":[{\"filename\":\"dracopy-2.0.0-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_dracopy_cp311_cp311_macosx_11_0_arm64_e16575ab\":[{\"filename\":\"dracopy-2.0.0-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_dracopy_cp311_cp311_manylinux_2_24_aarch64_418964da\":[{\"filename\":\"dracopy-2.0.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_dracopy_cp311_cp311_manylinux_2_24_x86_64_fa9a3215\":[{\"filename\":\"dracopy-2.0.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_dracopy_cp311_cp311_win_amd64_106c968f\":[{\"filename\":\"dracopy-2.0.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}]}", "embreex": "{\"ncore_pip_deps_311_embreex_cp311_cp311_macosx_10_9_x86_64_8154ab34\":[{\"filename\":\"embreex-2.17.7.post7-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_embreex_cp311_cp311_manylinux_2_12_x86_64_c4775a4d\":[{\"filename\":\"embreex-2.17.7.post7-cp311-cp311-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_embreex_cp311_cp311_win_amd64_a1157976\":[{\"filename\":\"embreex-2.17.7.post7-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}]}", "entrypoints": "{\"ncore_pip_deps_38_entrypoints_py3_none_any_f174b5ff\":[{\"filename\":\"entrypoints-0.4-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_entrypoints_sdist_b706edda\":[{\"filename\":\"entrypoints-0.4.tar.gz\",\"version\":\"3.8\"}]}", "exceptiongroup": "{\"ncore_pip_deps_38_exceptiongroup_py3_none_any_34328066\":[{\"filename\":\"exceptiongroup-1.1.3-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_exceptiongroup_sdist_097acd85\":[{\"filename\":\"exceptiongroup-1.1.3.tar.gz\",\"version\":\"3.8\"}]}", "executing": "{\"ncore_pip_deps_311_executing_py2_none_any_11387150\":[{\"filename\":\"executing-2.2.0-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_executing_sdist_5d108c02\":[{\"filename\":\"executing-2.2.0.tar.gz\",\"version\":\"3.11\"}]}", - "fasteners": "{\"ncore_pip_deps_311_fasteners_py3_none_any_758819cb\":[{\"filename\":\"fasteners-0.19-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fasteners_sdist_b4f37c3a\":[{\"filename\":\"fasteners-0.19.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_fasteners_py3_none_any_9422c40d\":[{\"filename\":\"fasteners-0.20-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_fasteners_sdist_55dce879\":[{\"filename\":\"fasteners-0.20.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_fasteners_py3_none_any_9422c40d\":[{\"filename\":\"fasteners-0.20-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_fasteners_sdist_55dce879\":[{\"filename\":\"fasteners-0.20.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_fasteners_py3_none_any_1d4caf5f\":[{\"filename\":\"fasteners-0.18-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_fasteners_sdist_cb7c13ef\":[{\"filename\":\"fasteners-0.18.tar.gz\",\"version\":\"3.8\"}]}", + "fasteners": "{\"ncore_pip_deps_312_fasteners_py3_none_any_9422c40d\":[{\"filename\":\"fasteners-0.20-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_fasteners_sdist_55dce879\":[{\"filename\":\"fasteners-0.20.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_38_fasteners_py3_none_any_1d4caf5f\":[{\"filename\":\"fasteners-0.18-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_fasteners_sdist_cb7c13ef\":[{\"filename\":\"fasteners-0.18.tar.gz\",\"version\":\"3.8\"}]}", "fastjsonschema": "{\"ncore_pip_deps_311_fastjsonschema_py3_none_any_c9e5b7e9\":[{\"filename\":\"fastjsonschema-2.21.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fastjsonschema_sdist_794d4f0a\":[{\"filename\":\"fastjsonschema-2.21.1.tar.gz\",\"version\":\"3.11\"}]}", "filelock": "{\"ncore_pip_deps_311_filelock_py3_none_any_96f5f634\":[{\"filename\":\"filelock-3.29.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_filelock_sdist_69974355\":[{\"filename\":\"filelock-3.29.0.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_filelock_py3_none_any_2ce9818e\":[{\"filename\":\"filelock-4.1.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_filelock_sdist_ad7f724a\":[{\"filename\":\"filelock-4.1.0.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_filelock_py3_none_any_2ce9818e\":[{\"filename\":\"filelock-4.1.0-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_filelock_sdist_ad7f724a\":[{\"filename\":\"filelock-4.1.0.tar.gz\",\"version\":\"3.13\"}]}", "fire": "{\"ncore_pip_deps_311_fire_py3_none_any_e43fd8a5\":[{\"filename\":\"fire-0.7.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fire_sdist_3b208f05\":[{\"filename\":\"fire-0.7.1.tar.gz\",\"version\":\"3.11\"}]}", @@ -27750,6 +28381,7 @@ "fonttools": "{\"ncore_pip_deps_311_fonttools_cp311_cp311_macosx_10_9_universal2_7ef04bc7\":[{\"filename\":\"fonttools-4.56.0-cp311-cp311-macosx_10_9_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_cp311_cp311_macosx_10_9_x86_64_ffda9b8c\":[{\"filename\":\"fonttools-4.56.0-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_cp311_cp311_manylinux_2_17_aarch64_e2e993e8\":[{\"filename\":\"fonttools-4.56.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_cp311_cp311_manylinux_2_17_x86_64_003548ea\":[{\"filename\":\"fonttools-4.56.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_cp311_cp311_musllinux_1_2_aarch64_bd982582\":[{\"filename\":\"fonttools-4.56.0-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_cp311_cp311_musllinux_1_2_x86_64_b23d30a2\":[{\"filename\":\"fonttools-4.56.0-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_cp311_cp311_win_amd64_14a3e3e6\":[{\"filename\":\"fonttools-4.56.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_py3_none_any_1088182f\":[{\"filename\":\"fonttools-4.56.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_sdist_a114d156\":[{\"filename\":\"fonttools-4.56.0.tar.gz\",\"version\":\"3.11\"}]}", "fsspec": "{\"ncore_pip_deps_311_fsspec_py3_none_any_7c771235\":[{\"filename\":\"fsspec-2025.10.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fsspec_sdist_b6789427\":[{\"filename\":\"fsspec-2025.10.0.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_fsspec_py3_none_any_8dd6e646\":[{\"filename\":\"fsspec-2026.9.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_fsspec_sdist_0f081479\":[{\"filename\":\"fsspec-2026.9.0.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_fsspec_py3_none_any_8dd6e646\":[{\"filename\":\"fsspec-2026.9.0-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_fsspec_sdist_0f081479\":[{\"filename\":\"fsspec-2026.9.0.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_fsspec_py3_none_any_efb87af3\":[{\"filename\":\"fsspec-2025.3.0-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_fsspec_sdist_a935fd1e\":[{\"filename\":\"fsspec-2025.3.0.tar.gz\",\"version\":\"3.8\"}]}", "gast": "{\"ncore_pip_deps_311_gast_py3_none_any_52b18231\":[{\"filename\":\"gast-0.6.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_gast_sdist_88fc5300\":[{\"filename\":\"gast-0.6.0.tar.gz\",\"version\":\"3.11\"}]}", + "google_crc32c": "{\"ncore_pip_deps_311_google_crc32c_cp311_cp311_macosx_12_0_arm64_457d0d9a\":[{\"filename\":\"google_crc32c-1.9.0-cp311-cp311-macosx_12_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_google_crc32c_cp311_cp311_macosx_12_0_x86_64_ccfe4002\":[{\"filename\":\"google_crc32c-1.9.0-cp311-cp311-macosx_12_0_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_google_crc32c_cp311_cp311_manylinux_2_17_aarch64_fbef61a3\":[{\"filename\":\"google_crc32c-1.9.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_google_crc32c_cp311_cp311_manylinux_2_17_x86_64_86764b99\":[{\"filename\":\"google_crc32c-1.9.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_google_crc32c_cp311_cp311_win_amd64_43a2dc26\":[{\"filename\":\"google_crc32c-1.9.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_google_crc32c_sdist_7b8c84c3\":[{\"filename\":\"google_crc32c-1.9.0.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_313_google_crc32c_cp313_cp313_macosx_12_0_arm64_f1dc17d9\":[{\"filename\":\"google_crc32c-1.9.0-cp313-cp313-macosx_12_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_google_crc32c_cp313_cp313_macosx_12_0_x86_64_f894a287\":[{\"filename\":\"google_crc32c-1.9.0-cp313-cp313-macosx_12_0_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_google_crc32c_cp313_cp313_manylinux_2_17_aarch64_4488f155\":[{\"filename\":\"google_crc32c-1.9.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_google_crc32c_cp313_cp313_manylinux_2_17_x86_64_0568b17e\":[{\"filename\":\"google_crc32c-1.9.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_google_crc32c_cp313_cp313_win_amd64_8583ec21\":[{\"filename\":\"google_crc32c-1.9.0-cp313-cp313-win_amd64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_google_crc32c_sdist_7b8c84c3\":[{\"filename\":\"google_crc32c-1.9.0.tar.gz\",\"version\":\"3.13\"}]}", "google_pasta": "{\"ncore_pip_deps_311_google_pasta_py3_none_any_b3248279\":[{\"filename\":\"google_pasta-0.2.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_google_pasta_sdist_c9f2c8df\":[{\"filename\":\"google-pasta-0.2.0.tar.gz\",\"version\":\"3.11\"}]}", "grpcio": "{\"ncore_pip_deps_311_grpcio_cp311_cp311_linux_armv7l_17325b0b\":[{\"filename\":\"grpcio-1.70.0-cp311-cp311-linux_armv7l.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_grpcio_cp311_cp311_macosx_10_14_universal2_dbe41ad1\":[{\"filename\":\"grpcio-1.70.0-cp311-cp311-macosx_10_14_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_grpcio_cp311_cp311_manylinux_2_17_aarch64_5ea67c72\":[{\"filename\":\"grpcio-1.70.0-cp311-cp311-manylinux_2_17_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_grpcio_cp311_cp311_manylinux_2_17_x86_64_e7831a0f\":[{\"filename\":\"grpcio-1.70.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_grpcio_cp311_cp311_musllinux_1_1_x86_64_d6376496\":[{\"filename\":\"grpcio-1.70.0-cp311-cp311-musllinux_1_1_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_grpcio_cp311_cp311_win_amd64_d24035d4\":[{\"filename\":\"grpcio-1.70.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_grpcio_sdist_8d1584a6\":[{\"filename\":\"grpcio-1.70.0.tar.gz\",\"version\":\"3.11\"}]}", "h11": "{\"ncore_pip_deps_311_h11_py3_none_any_63cf8bbe\":[{\"filename\":\"h11-0.16.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_h11_sdist_4e35b956\":[{\"filename\":\"h11-0.16.0.tar.gz\",\"version\":\"3.11\"}]}", @@ -27786,7 +28418,7 @@ "mistune": "{\"ncore_pip_deps_311_mistune_py2_none_any_bad7f5d4\":[{\"filename\":\"mistune-2.0.5-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_mistune_sdist_0246113c\":[{\"filename\":\"mistune-2.0.5.tar.gz\",\"version\":\"3.11\"}]}", "ml_dtypes": "{\"ncore_pip_deps_311_ml_dtypes_cp311_cp311_macosx_10_9_universal2_6c7ecb74\":[{\"filename\":\"ml_dtypes-0.5.4-cp311-cp311-macosx_10_9_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_ml_dtypes_cp311_cp311_manylinux_2_27_aarch64_bc11d7e8\":[{\"filename\":\"ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_ml_dtypes_cp311_cp311_manylinux_2_27_x86_64_19b9a535\":[{\"filename\":\"ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_ml_dtypes_cp311_cp311_win_amd64_7c23c54a\":[{\"filename\":\"ml_dtypes-0.5.4-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_ml_dtypes_sdist_8ab06a50\":[{\"filename\":\"ml_dtypes-0.5.4.tar.gz\",\"version\":\"3.11\"}]}", "mpmath": "{\"ncore_pip_deps_311_mpmath_py3_none_any_a0b2b9fe\":[{\"filename\":\"mpmath-1.3.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_mpmath_sdist_7a28eb2a\":[{\"filename\":\"mpmath-1.3.0.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_mpmath_py3_none_any_a0b2b9fe\":[{\"filename\":\"mpmath-1.3.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_mpmath_sdist_7a28eb2a\":[{\"filename\":\"mpmath-1.3.0.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_mpmath_py3_none_any_a0b2b9fe\":[{\"filename\":\"mpmath-1.3.0-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_mpmath_sdist_7a28eb2a\":[{\"filename\":\"mpmath-1.3.0.tar.gz\",\"version\":\"3.13\"}]}", - "msgspec": "{\"ncore_pip_deps_311_msgspec_cp311_cp311_macosx_10_9_x86_64_09e0efbf\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_macosx_11_0_arm64_23ee3787\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_manylinux_2_17_aarch64_81f4ac6f\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_manylinux_2_17_x86_64_bb4d873f\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_musllinux_1_2_aarch64_b92b8334\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_musllinux_1_2_x86_64_562c44b0\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_win_amd64_d1dcc93a\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_sdist_692349e5\":[{\"filename\":\"msgspec-0.20.0.tar.gz\",\"version\":\"3.11\"}]}", + "msgspec": "{\"ncore_pip_deps_311_msgspec_cp311_cp311_macosx_10_9_x86_64_09e0efbf\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_macosx_11_0_arm64_23ee3787\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_manylinux_2_17_aarch64_81f4ac6f\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_manylinux_2_17_x86_64_bb4d873f\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_musllinux_1_2_aarch64_b92b8334\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_musllinux_1_2_x86_64_562c44b0\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_win_amd64_d1dcc93a\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_sdist_692349e5\":[{\"filename\":\"msgspec-0.20.0.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_313_msgspec_cp313_cp313_macosx_10_13_x86_64_f13c127a\":[{\"filename\":\"msgspec-0.22.0-cp313-cp313-macosx_10_13_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_msgspec_cp313_cp313_macosx_11_0_arm64_5aa24eb4\":[{\"filename\":\"msgspec-0.22.0-cp313-cp313-macosx_11_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_msgspec_cp313_cp313_manylinux_2_17_aarch64_627bfdfe\":[{\"filename\":\"msgspec-0.22.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_msgspec_cp313_cp313_manylinux_2_17_x86_64_c6c310ef\":[{\"filename\":\"msgspec-0.22.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_msgspec_cp313_cp313_musllinux_1_2_aarch64_bc374ded\":[{\"filename\":\"msgspec-0.22.0-cp313-cp313-musllinux_1_2_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_msgspec_cp313_cp313_musllinux_1_2_x86_64_6f48317f\":[{\"filename\":\"msgspec-0.22.0-cp313-cp313-musllinux_1_2_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_msgspec_cp313_cp313_win_amd64_0739b068\":[{\"filename\":\"msgspec-0.22.0-cp313-cp313-win_amd64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_msgspec_sdist_0a13624a\":[{\"filename\":\"msgspec-0.22.0.tar.gz\",\"version\":\"3.13\"}]}", "mypy_extensions": "{\"ncore_pip_deps_311_mypy_extensions_py3_none_any_4392f6c0\":[{\"filename\":\"mypy_extensions-1.0.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_mypy_extensions_sdist_75dbf895\":[{\"filename\":\"mypy_extensions-1.0.0.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_mypy_extensions_py3_none_any_1be4cccd\":[{\"filename\":\"mypy_extensions-1.1.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_mypy_extensions_sdist_52e68efc\":[{\"filename\":\"mypy_extensions-1.1.0.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_mypy_extensions_py3_none_any_1be4cccd\":[{\"filename\":\"mypy_extensions-1.1.0-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_mypy_extensions_sdist_52e68efc\":[{\"filename\":\"mypy_extensions-1.1.0.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_mypy_extensions_py3_none_any_4392f6c0\":[{\"filename\":\"mypy_extensions-1.0.0-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_mypy_extensions_sdist_75dbf895\":[{\"filename\":\"mypy_extensions-1.0.0.tar.gz\",\"version\":\"3.8\"}]}", "namex": "{\"ncore_pip_deps_311_namex_py3_none_any_7ddb6c2b\":[{\"filename\":\"namex-0.0.8-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_namex_sdist_32a50f6c\":[{\"filename\":\"namex-0.0.8.tar.gz\",\"version\":\"3.11\"}]}", "nbclient": "{\"ncore_pip_deps_311_nbclient_py3_none_any_4ffee11e\":[{\"filename\":\"nbclient-0.10.2-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_nbclient_sdist_90b7fc6b\":[{\"filename\":\"nbclient-0.10.2.tar.gz\",\"version\":\"3.11\"}]}", @@ -27794,7 +28426,7 @@ "nbformat": "{\"ncore_pip_deps_311_nbformat_py3_none_any_3b48d6c8\":[{\"filename\":\"nbformat-5.10.4-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_nbformat_sdist_322168b1\":[{\"filename\":\"nbformat-5.10.4.tar.gz\",\"version\":\"3.11\"}]}", "nbsphinx": "{\"ncore_pip_deps_311_nbsphinx_py3_none_any_6e805e96\":[{\"filename\":\"nbsphinx-0.9.3-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_nbsphinx_sdist_ec339c86\":[{\"filename\":\"nbsphinx-0.9.3.tar.gz\",\"version\":\"3.11\"}]}", "networkx": "{\"ncore_pip_deps_311_networkx_py3_none_any_d47fbf30\":[{\"filename\":\"networkx-3.6.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_networkx_sdist_26b7c357\":[{\"filename\":\"networkx-3.6.1.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_networkx_py3_none_any_e3fd2c13\":[{\"filename\":\"networkx-3.7-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_networkx_sdist_fd77a511\":[{\"filename\":\"networkx-3.7.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_networkx_py3_none_any_e3fd2c13\":[{\"filename\":\"networkx-3.7-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_networkx_sdist_fd77a511\":[{\"filename\":\"networkx-3.7.tar.gz\",\"version\":\"3.13\"}]}", - "numcodecs": "{\"ncore_pip_deps_311_numcodecs_cp311_cp311_macosx_10_9_x86_64_698f1d59\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_cp311_cp311_macosx_11_0_arm64_bef8c8e6\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_cp311_cp311_manylinux_2_17_x86_64_cdfaef9f\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_cp311_cp311_win_amd64_e2547fa3\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_sdist_eeed77e4\":[{\"filename\":\"numcodecs-0.15.1.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_macosx_10_13_x86_64_b0a9d9cd\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-macosx_10_13_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_macosx_11_0_arm64_a34f0fe5\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-macosx_11_0_arm64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_manylinux_2_17_x86_64_c3a09e22\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_win_amd64_daed6066\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-win_amd64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_sdist_eeed77e4\":[{\"filename\":\"numcodecs-0.15.1.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_10_13_x86_64_e3d82b70\":[{\"filename\":\"numcodecs-0.15.1-cp313-cp313-macosx_10_13_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_11_0_arm64_1d471a18\":[{\"filename\":\"numcodecs-0.15.1-cp313-cp313-macosx_11_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_manylinux_2_17_x86_64_1dfdea4a\":[{\"filename\":\"numcodecs-0.15.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_win_amd64_a4f7bdb2\":[{\"filename\":\"numcodecs-0.15.1-cp313-cp313-win_amd64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_sdist_eeed77e4\":[{\"filename\":\"numcodecs-0.15.1.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_numcodecs_cp38_cp38_macosx_10_9_x86_64_0c240858\":[{\"filename\":\"numcodecs-0.11.0-cp38-cp38-macosx_10_9_x86_64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numcodecs_cp38_cp38_manylinux_2_17_x86_64_ee5bda16\":[{\"filename\":\"numcodecs-0.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numcodecs_cp38_cp38_win_amd64_bd05cdb8\":[{\"filename\":\"numcodecs-0.11.0-cp38-cp38-win_amd64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numcodecs_sdist_6c058b32\":[{\"filename\":\"numcodecs-0.11.0.tar.gz\",\"version\":\"3.8\"}]}", + "numcodecs": "{\"ncore_pip_deps_311_numcodecs_cp311_cp311_macosx_10_9_x86_64_698f1d59\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_cp311_cp311_macosx_11_0_arm64_bef8c8e6\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_cp311_cp311_manylinux_2_17_x86_64_cdfaef9f\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_cp311_cp311_win_amd64_e2547fa3\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_sdist_eeed77e4\":[{\"filename\":\"numcodecs-0.15.1.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_macosx_10_13_x86_64_b0a9d9cd\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-macosx_10_13_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_macosx_11_0_arm64_a34f0fe5\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-macosx_11_0_arm64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_manylinux_2_17_x86_64_c3a09e22\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_win_amd64_daed6066\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-win_amd64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_sdist_eeed77e4\":[{\"filename\":\"numcodecs-0.15.1.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_10_13_x86_64_de2c66db\":[{\"filename\":\"numcodecs-0.17.0-cp313-cp313-macosx_10_13_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_11_0_arm64_69b9b468\":[{\"filename\":\"numcodecs-0.17.0-cp313-cp313-macosx_11_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_manylinux_2_17_aarch64_7065b334\":[{\"filename\":\"numcodecs-0.17.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_manylinux_2_17_x86_64_6c3342d9\":[{\"filename\":\"numcodecs-0.17.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_win_amd64_a854e9c8\":[{\"filename\":\"numcodecs-0.17.0-cp313-cp313-win_amd64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_sdist_e8db2e33\":[{\"filename\":\"numcodecs-0.17.0.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_numcodecs_cp38_cp38_macosx_10_9_x86_64_0c240858\":[{\"filename\":\"numcodecs-0.11.0-cp38-cp38-macosx_10_9_x86_64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numcodecs_cp38_cp38_manylinux_2_17_x86_64_ee5bda16\":[{\"filename\":\"numcodecs-0.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numcodecs_cp38_cp38_win_amd64_bd05cdb8\":[{\"filename\":\"numcodecs-0.11.0-cp38-cp38-win_amd64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numcodecs_sdist_6c058b32\":[{\"filename\":\"numcodecs-0.11.0.tar.gz\",\"version\":\"3.8\"}]}", "numpy": "{\"ncore_pip_deps_311_numpy_cp311_cp311_macosx_10_9_x86_64_4c66707f\":[{\"filename\":\"numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numpy_cp311_cp311_macosx_11_0_arm64_edd8b5fe\":[{\"filename\":\"numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numpy_cp311_cp311_manylinux_2_17_aarch64_7ab55401\":[{\"filename\":\"numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numpy_cp311_cp311_manylinux_2_17_x86_64_666dbfb6\":[{\"filename\":\"numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numpy_cp311_cp311_musllinux_1_1_aarch64_96ff0b2a\":[{\"filename\":\"numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numpy_cp311_cp311_musllinux_1_1_x86_64_60dedbb9\":[{\"filename\":\"numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numpy_cp311_cp311_win_amd64_cd25bcec\":[{\"filename\":\"numpy-1.26.4-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numpy_sdist_2a02aba9\":[{\"filename\":\"numpy-1.26.4.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_macosx_10_13_x86_64_c6342f54\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-macosx_10_13_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_macosx_11_0_arm64_b11e8fda\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-macosx_11_0_arm64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_macosx_14_0_arm64_9cb18a32\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-macosx_14_0_arm64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_macosx_14_0_x86_64_aec3fc4b\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-macosx_14_0_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_manylinux_2_27_aarch64_fe4d21ab\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_manylinux_2_27_x86_64_fbde6962\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_musllinux_1_2_aarch64_381a7a3d\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-musllinux_1_2_aarch64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_musllinux_1_2_x86_64_b89d0aaa\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-musllinux_1_2_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_win_amd64_a7b1b635\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-win_amd64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_sdist_9a94cf75\":[{\"filename\":\"numpy-2.5.4.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_macosx_10_13_x86_64_2377da2d\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-macosx_10_13_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_macosx_11_0_arm64_7415db95\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-macosx_11_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_macosx_14_0_arm64_6d6a71b9\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-macosx_14_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_macosx_14_0_x86_64_d8200f16\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-macosx_14_0_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_manylinux_2_27_aarch64_1c2e71b0\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_manylinux_2_27_x86_64_6ffa0766\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_musllinux_1_2_aarch64_2fa3328f\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-musllinux_1_2_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_musllinux_1_2_x86_64_b86966fb\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-musllinux_1_2_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_win_amd64_8b4d2fd2\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-win_amd64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_sdist_9a94cf75\":[{\"filename\":\"numpy-2.5.4.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_numpy_cp38_cp38_macosx_10_9_x86_64_cf240200\":[{\"filename\":\"numpy-1.19.5-cp38-cp38-macosx_10_9_x86_64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numpy_cp38_cp38_manylinux_2_12_x86_64_a9d17f2b\":[{\"filename\":\"numpy-1.19.5-cp38-cp38-manylinux2010_x86_64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numpy_cp38_cp38_manylinux_2_17_aarch64_99abf4f3\":[{\"filename\":\"numpy-1.19.5-cp38-cp38-manylinux2014_aarch64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numpy_cp38_cp38_manylinux_2_5_x86_64_012426a4\":[{\"filename\":\"numpy-1.19.5-cp38-cp38-manylinux1_x86_64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numpy_cp38_cp38_win_amd64_811daee3\":[{\"filename\":\"numpy-1.19.5-cp38-cp38-win_amd64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numpy_sdist_a76f5024\":[{\"filename\":\"numpy-1.19.5.zip\",\"version\":\"3.8\"}]}", "nuscenes_devkit": "{\"ncore_pip_deps_311_nuscenes_devkit_py3_none_any_76cee0e7\":[{\"filename\":\"nuscenes_devkit-1.2.0-py3-none-any.whl\",\"version\":\"3.11\"}]}", "nvidia_cublas_cu12": "{\"ncore_pip_deps_311_nvidia_cublas_cu12_py3_none_manylinux_2_27_aarch64_93a4e0e3\":[{\"filename\":\"nvidia_cublas_cu12-12.8.3.14-py3-none-manylinux_2_27_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_nvidia_cublas_cu12_py3_none_manylinux_2_27_x86_64_3f0e05e7\":[{\"filename\":\"nvidia_cublas_cu12-12.8.3.14-py3-none-manylinux_2_27_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_nvidia_cublas_cu12_py3_none_win_amd64_9ae5eae5\":[{\"filename\":\"nvidia_cublas_cu12-12.8.3.14-py3-none-win_amd64.whl\",\"version\":\"3.11\"}]}", @@ -27841,6 +28473,7 @@ "pytest": "{\"ncore_pip_deps_311_pytest_py3_none_any_50e16d95\":[{\"filename\":\"pytest-8.3.4-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pytest_sdist_965370d0\":[{\"filename\":\"pytest-8.3.4.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_pytest_py3_none_any_37a86b45\":[{\"filename\":\"pytest-9.1.1-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_pytest_sdist_1088fbde\":[{\"filename\":\"pytest-9.1.1.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_pytest_py3_none_any_37a86b45\":[{\"filename\":\"pytest-9.1.1-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pytest_sdist_1088fbde\":[{\"filename\":\"pytest-9.1.1.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_pytest_py3_none_any_78bf1645\":[{\"filename\":\"pytest-7.4.0-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_pytest_sdist_b4bf8c45\":[{\"filename\":\"pytest-7.4.0.tar.gz\",\"version\":\"3.8\"}]}", "python_dateutil": "{\"ncore_pip_deps_311_python_dateutil_py2_none_any_a8b2bc7b\":[{\"filename\":\"python_dateutil-2.9.0.post0-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_python_dateutil_sdist_37dd5420\":[{\"filename\":\"python-dateutil-2.9.0.post0.tar.gz\",\"version\":\"3.11\"}]}", "pytz": "{\"ncore_pip_deps_311_pytz_py2_none_any_89dd22dc\":[{\"filename\":\"pytz-2025.1-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pytz_sdist_c2db42be\":[{\"filename\":\"pytz-2025.1.tar.gz\",\"version\":\"3.11\"}]}", + "pyyaml": "{\"ncore_pip_deps_311_pyyaml_cp311_cp311_macosx_10_13_x86_64_44edc647\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_cp311_cp311_macosx_11_0_arm64_652cb6ed\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_cp311_cp311_manylinux_2_17_aarch64_10892704\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_cp311_cp311_manylinux_2_17_s390x_850774a7\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_cp311_cp311_manylinux_2_17_x86_64_b8bb0864\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_cp311_cp311_musllinux_1_2_aarch64_1d37d57a\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_cp311_cp311_musllinux_1_2_x86_64_37503bfb\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_cp311_cp311_win_amd64_9f3bfb49\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_sdist_d7662337\":[{\"filename\":\"pyyaml-6.0.3.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_macosx_10_13_x86_64_8da9669d\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_macosx_11_0_arm64_2283a07e\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_manylinux_2_17_aarch64_ee292290\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_manylinux_2_17_s390x_a33284e2\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_manylinux_2_17_x86_64_0f29edc4\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_musllinux_1_2_aarch64_f7057c9a\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_musllinux_1_2_x86_64_eda16858\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_win_amd64_79005a0d\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-win_amd64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_sdist_d7662337\":[{\"filename\":\"pyyaml-6.0.3.tar.gz\",\"version\":\"3.13\"}]}", "pyzmq": "{\"ncore_pip_deps_311_pyzmq_cp311_cp311_macosx_10_15_universal2_c0598838\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-macosx_10_15_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_cp311_cp311_macosx_10_9_x86_64_ed038a92\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_cp311_cp311_manylinux_2_17_aarch64_9027a7fc\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_cp311_cp311_manylinux_2_17_x86_64_f0019cc8\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_cp311_cp311_manylinux_2_28_x86_64_f19dae58\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_cp311_cp311_musllinux_1_1_aarch64_f5eeeb82\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-musllinux_1_1_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_cp311_cp311_musllinux_1_1_x86_64_817fcd33\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-musllinux_1_1_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_cp311_cp311_win_amd64_ef29630f\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_sdist_17d72a74\":[{\"filename\":\"pyzmq-26.2.1.tar.gz\",\"version\":\"3.11\"}]}", "referencing": "{\"ncore_pip_deps_311_referencing_py3_none_any_d53ae300\":[{\"filename\":\"referencing-0.34.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_referencing_sdist_5773bd84\":[{\"filename\":\"referencing-0.34.0.tar.gz\",\"version\":\"3.11\"}]}", "remotezip": "{\"ncore_pip_deps_311_remotezip_py3_none_any_f70a4026\":[{\"filename\":\"remotezip-0.12.3-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_remotezip_sdist_bf1ebe2b\":[{\"filename\":\"remotezip-0.12.3.tar.gz\",\"version\":\"3.11\"}]}", @@ -27892,10 +28525,10 @@ "websockets": "{\"ncore_pip_deps_311_websockets_cp311_cp311_macosx_10_9_universal2_823c248b\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-macosx_10_9_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_cp311_cp311_macosx_10_9_x86_64_67899970\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_cp311_cp311_macosx_11_0_arm64_d50fd1ee\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_cp311_cp311_manylinux_2_17_aarch64_d99e5546\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_cp311_cp311_manylinux_2_5_x86_64_8dd8327c\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_cp311_cp311_musllinux_1_2_aarch64_8fdc5105\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_cp311_cp311_musllinux_1_2_x86_64_54479983\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_cp311_cp311_win_amd64_27ccee00\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_py3_none_any_f7a866fb\":[{\"filename\":\"websockets-15.0.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_sdist_82544de0\":[{\"filename\":\"websockets-15.0.1.tar.gz\",\"version\":\"3.11\"}]}", "werkzeug": "{\"ncore_pip_deps_311_werkzeug_py3_none_any_63a77fb8\":[{\"filename\":\"werkzeug-3.1.8-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_werkzeug_sdist_9bad61a4\":[{\"filename\":\"werkzeug-3.1.8.tar.gz\",\"version\":\"3.11\"}]}", "wheel": "{\"ncore_pip_deps_311_wheel_py3_none_any_708e7481\":[{\"filename\":\"wheel-0.45.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wheel_sdist_661e1abd\":[{\"filename\":\"wheel-0.45.1.tar.gz\",\"version\":\"3.11\"}]}", - "wrapt": "{\"ncore_pip_deps_311_wrapt_cp311_cp311_macosx_10_9_universal2_ff04ef6e\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-macosx_10_9_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_macosx_10_9_x86_64_4db983e7\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_macosx_11_0_arm64_9abc77a4\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_manylinux_2_17_aarch64_0b929ac1\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_manylinux_2_5_x86_64_1a7ed2d9\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_musllinux_1_2_aarch64_129a150f\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_musllinux_1_2_x86_64_9a2bce78\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_win_amd64_acc130bc\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_py3_none_any_b18f2d15\":[{\"filename\":\"wrapt-1.17.2-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_sdist_41388e9d\":[{\"filename\":\"wrapt-1.17.2.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_macosx_10_13_x86_64_b898caea\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-macosx_10_13_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_macosx_11_0_arm64_8837fbe7\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-macosx_11_0_arm64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_manylinux_2_17_aarch64_6761765c\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_manylinux_2_5_x86_64_0cabb9c1\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_musllinux_1_2_aarch64_6a9ee62a\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-musllinux_1_2_aarch64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_musllinux_1_2_x86_64_e716f47c\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-musllinux_1_2_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_win_amd64_ab45839c\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-win_amd64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_py3_none_any_107eea1a\":[{\"filename\":\"wrapt-2.5.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_sdist_c48cdb6c\":[{\"filename\":\"wrapt-2.5.0.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_wrapt_cp313_cp313_macosx_10_13_x86_64_b312b3cc\":[{\"filename\":\"wrapt-2.5.0-cp313-cp313-macosx_10_13_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_cp313_cp313_macosx_11_0_arm64_c57ddae2\":[{\"filename\":\"wrapt-2.5.0-cp313-cp313-macosx_11_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_cp313_cp313_manylinux_2_17_aarch64_6058e12e\":[{\"filename\":\"wrapt-2.5.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_cp313_cp313_manylinux_2_5_x86_64_b95a6eca\":[{\"filename\":\"wrapt-2.5.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_cp313_cp313_musllinux_1_2_aarch64_f4ef4935\":[{\"filename\":\"wrapt-2.5.0-cp313-cp313-musllinux_1_2_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_cp313_cp313_musllinux_1_2_x86_64_a346408f\":[{\"filename\":\"wrapt-2.5.0-cp313-cp313-musllinux_1_2_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_cp313_cp313_win_amd64_77f0a74f\":[{\"filename\":\"wrapt-2.5.0-cp313-cp313-win_amd64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_py3_none_any_107eea1a\":[{\"filename\":\"wrapt-2.5.0-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_sdist_c48cdb6c\":[{\"filename\":\"wrapt-2.5.0.tar.gz\",\"version\":\"3.13\"}]}", + "wrapt": "{\"ncore_pip_deps_311_wrapt_cp311_cp311_macosx_10_9_universal2_ff04ef6e\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-macosx_10_9_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_macosx_10_9_x86_64_4db983e7\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_macosx_11_0_arm64_9abc77a4\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_manylinux_2_17_aarch64_0b929ac1\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_manylinux_2_5_x86_64_1a7ed2d9\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_musllinux_1_2_aarch64_129a150f\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_musllinux_1_2_x86_64_9a2bce78\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_win_amd64_acc130bc\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_py3_none_any_b18f2d15\":[{\"filename\":\"wrapt-1.17.2-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_sdist_41388e9d\":[{\"filename\":\"wrapt-1.17.2.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_macosx_10_13_x86_64_b898caea\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-macosx_10_13_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_macosx_11_0_arm64_8837fbe7\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-macosx_11_0_arm64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_manylinux_2_17_aarch64_6761765c\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_manylinux_2_5_x86_64_0cabb9c1\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_musllinux_1_2_aarch64_6a9ee62a\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-musllinux_1_2_aarch64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_musllinux_1_2_x86_64_e716f47c\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-musllinux_1_2_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_win_amd64_ab45839c\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-win_amd64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_py3_none_any_107eea1a\":[{\"filename\":\"wrapt-2.5.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_sdist_c48cdb6c\":[{\"filename\":\"wrapt-2.5.0.tar.gz\",\"version\":\"3.12\"}]}", "xxhash": "{\"ncore_pip_deps_311_xxhash_cp311_cp311_macosx_10_9_x86_64_b47bbd8c\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_macosx_11_0_arm64_2b6821e9\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_manylinux_2_17_aarch64_8b29ee68\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_manylinux_2_17_ppc64le_6812c25f\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_manylinux_2_17_s390x_4ccbff01\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_manylinux_2_17_x86_64_297b7fbf\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_musllinux_1_2_aarch64_dea26ae1\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_musllinux_1_2_ppc64le_08d45aef\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-musllinux_1_2_ppc64le.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_musllinux_1_2_s390x_92914236\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-musllinux_1_2_s390x.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_musllinux_1_2_x86_64_51312c76\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_win_amd64_26734cdc\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_sdist_f0162a78\":[{\"filename\":\"xxhash-3.6.0.tar.gz\",\"version\":\"3.11\"}]}", "yourdfpy": "{\"ncore_pip_deps_311_yourdfpy_py3_none_any_8a187a8b\":[{\"filename\":\"yourdfpy-0.0.60-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_yourdfpy_sdist_2af2d8bd\":[{\"filename\":\"yourdfpy-0.0.60.tar.gz\",\"version\":\"3.11\"}]}", - "zarr": "{\"ncore_pip_deps_311_zarr_py3_none_any_2795e20a\":[{\"filename\":\"zarr-2.18.4-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zarr_sdist_37790ede\":[{\"filename\":\"zarr-2.18.4.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_zarr_py3_none_any_ac3dc403\":[{\"filename\":\"zarr-2.18.7-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_zarr_sdist_b2b8f66f\":[{\"filename\":\"zarr-2.18.7.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_zarr_py3_none_any_ac3dc403\":[{\"filename\":\"zarr-2.18.7-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_zarr_sdist_b2b8f66f\":[{\"filename\":\"zarr-2.18.7.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_zarr_py3_none_any_883305e8\":[{\"filename\":\"zarr-2.13.3-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_zarr_sdist_db24b090\":[{\"filename\":\"zarr-2.13.3.tar.gz\",\"version\":\"3.8\"}]}", + "zarr": "{\"ncore_pip_deps_311_zarr_py3_none_any_72c3dcee\":[{\"filename\":\"zarr-3.1.4-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zarr_sdist_b0cf4eea\":[{\"filename\":\"zarr-3.1.4.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_zarr_py3_none_any_ac3dc403\":[{\"filename\":\"zarr-2.18.7-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_zarr_sdist_b2b8f66f\":[{\"filename\":\"zarr-2.18.7.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_zarr_py3_none_any_38b54057\":[{\"filename\":\"zarr-3.4.1-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_zarr_sdist_b34bda11\":[{\"filename\":\"zarr-3.4.1.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_zarr_py3_none_any_883305e8\":[{\"filename\":\"zarr-2.13.3-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_zarr_sdist_db24b090\":[{\"filename\":\"zarr-2.13.3.tar.gz\",\"version\":\"3.8\"}]}", "zstandard": "{\"ncore_pip_deps_311_zstandard_cp311_cp311_macosx_10_9_x86_64_933b65d7\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_macosx_11_0_arm64_a3f79487\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_manylinux_2_17_aarch64_01582723\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_manylinux_2_17_ppc64le_5f1ad7bf\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_manylinux_2_17_s390x_06acb75e\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_manylinux_2_17_x86_64_9300d02e\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_musllinux_1_1_aarch64_bfd06b1c\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-musllinux_1_1_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_musllinux_1_1_x86_64_f373da2c\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-musllinux_1_1_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_musllinux_1_2_aarch64_6c0e5a65\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_musllinux_1_2_ppc64le_98750a30\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-musllinux_1_2_ppc64le.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_musllinux_1_2_s390x_22a086cf\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-musllinux_1_2_s390x.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_musllinux_1_2_x86_64_72d35d7a\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_win_amd64_daab68fa\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_sdist_7713e117\":[{\"filename\":\"zstandard-0.25.0.tar.gz\",\"version\":\"3.11\"}]}" }, "packages": [ @@ -27925,6 +28558,7 @@ "deprecated", "descartes", "docutils", + "donfig", "dracopy", "embreex", "entrypoints", @@ -27938,6 +28572,7 @@ "fonttools", "fsspec", "gast", + "google_crc32c", "google_pasta", "grpcio", "h11", @@ -28029,6 +28664,7 @@ "pytest", "python_dateutil", "pytz", + "pyyaml", "pyzmq", "referencing", "remotezip", diff --git a/deps/pip/requirements_3_11.in b/deps/pip/requirements_3_11.in index b31bf737..ba8ade94 100644 --- a/deps/pip/requirements_3_11.in +++ b/deps/pip/requirements_3_11.in @@ -26,6 +26,7 @@ # Public API restrictions for 3.11 # torch 2.7+ supports numpy 2.x natively, no constraint needed +zarr==3.1.4 # oldest supported zarr-python 3 (most recent zarr-python 3 is covered by python 3.13) --find-links https://download.pytorch.org/whl/cu128/torch torch==2.7.0+cu128 diff --git a/deps/pip/requirements_3_11.txt b/deps/pip/requirements_3_11.txt index d620e757..19e1ac35 100644 --- a/deps/pip/requirements_3_11.txt +++ b/deps/pip/requirements_3_11.txt @@ -22,9 +22,6 @@ anyio==4.12.1 \ --hash=sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703 \ --hash=sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c # via httpx -asciitree==0.3.3 \ - --hash=sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e - # via zarr asttokens==3.0.1 \ --hash=sha256:15a3ebc0f43c2d0a50eeafea25e19046c68398e487b9f1f5b517f7c0f40f976a \ --hash=sha256:71a4ee5de0bde6a31d64f6b13f2293ac190344478f081c3d1bccfcf5eacb0cb7 @@ -337,6 +334,10 @@ docutils==0.22.4 \ # nbsphinx # pydata-sphinx-theme # sphinx +donfig==0.8.1.post1 \ + --hash=sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d \ + --hash=sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52 + # via zarr dracopy==2.0.0 \ --hash=sha256:00ed24e138bcfbf2605d495c3a4e6c17b71cbd0b63b78e945d9d3c89d4fb9d69 \ --hash=sha256:05b5d4054402877f50221bc917e2943214a956d8d746e01b21f57ecc2d0adf21 \ @@ -416,10 +417,6 @@ executing==2.2.0 \ --hash=sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa \ --hash=sha256:5d108c028108fe2551d1a7b2e8b713341e2cb4fc0aa7dcf966fa4327a5226755 # via stack-data -fasteners==0.19 \ - --hash=sha256:758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237 \ - --hash=sha256:b4f37c3ac52d8a445af3a66bce57b33b5e90b97c696b7b984f530cf8f0ded09c - # via zarr fastjsonschema==2.21.1 \ --hash=sha256:794d4f0a58f848961ba16af7b9c85a3e88cd360df008c59aac6fc5ae9323b5d4 \ --hash=sha256:c9e5b7e908310918cf494a434eeb31384dd84a98b57a30bcb1f535015b554667 @@ -497,6 +494,39 @@ gast==0.6.0 \ --hash=sha256:52b182313f7330389f72b069ba00f174cfe2a06411099547288839c6cbafbd54 \ --hash=sha256:88fc5300d32c7ac6ca7b515310862f71e6fdf2c029bbec7c66c0f5dd47b6b1fb # via tensorflow +google-crc32c==1.9.0 \ + --hash=sha256:0568b17ed90ac596f29400d99e243fd0cc6276766183def888d1bf8d1dc13827 \ + --hash=sha256:0abe7e202c25909869c35672ab0f2fe748a7acf276eb78577332a7c38999740f \ + --hash=sha256:13179f7e3282617923e957b8e54b8f9c3968030f48640a9f47fd7c5c38c4a215 \ + --hash=sha256:16865b477d7941712cb0e0aad8ad4815e984fb5fc16d3fdaef7d986e26e53c95 \ + --hash=sha256:265233aff33d835f5b909584fe36ab29647b598c271b661a300001099109e53e \ + --hash=sha256:280f3a3e47af0eeba3a3e5aa7d311af77001812b8df80fb8beafcd0b40eaf7f1 \ + --hash=sha256:3abb18297d9ef0ab120531838be0e6d68c9fa876570e11c229c48f2edac23ce7 \ + --hash=sha256:43a2dc26f9be213fbe0b4fc4a1088c5d45cbfcb3247420ccc820f0fc3edeea86 \ + --hash=sha256:4488f1553a9ab7e86cdedc833374a7e904031803b995dc0bd0be48c271fa6556 \ + --hash=sha256:457d0d9a4718fd52b1494eac5c200ad25beeadbdc91843d550a003910838589f \ + --hash=sha256:51cb4e23a38ad4f495f35f87c233ca3ea6b9c4559e7ac383cdef786fab0f7977 \ + --hash=sha256:53fdafef58e230d0c946ab5f8446d123d9f548230a73b29c8b41c9546f268bc1 \ + --hash=sha256:56610f548f1b35c9568b9d1de30423480f505dae4991556072d5802820ff35c4 \ + --hash=sha256:5695c8b9327e040b2aba12c6659b0acb5995314ef0af0192da66e662e011103b \ + --hash=sha256:6a3b2c8a343c570ed8100a7627c20badfd92c6caa2067093a86be45af27f5b1b \ + --hash=sha256:7b8c84c3d159ab6817fe3f74e6e6cef099c3f95dcec3abc0d8afb1404642efbe \ + --hash=sha256:8535e75dfead304f30e9122b9ea2c0a570dbaa52c176a0a591540c7914c1e46d \ + --hash=sha256:8583ec21d56b565d68ab2963cc7e21b3b271247c29b04286068255ef65f221bd \ + --hash=sha256:86764b99e7a607830d93cb5b75e0ec3ff6cb06d3c274624418473cee701900d4 \ + --hash=sha256:8b91f41645b15a720357183fa5716682ada441873e3c462c15f9714be36f146b \ + --hash=sha256:af73200fa9791ccd380f3598235dba8d82b8af0905df045b3dc60b59836e8ddd \ + --hash=sha256:ccfe40021fd6afe23361175cf7551e3cef5fd34dc1ebe319f14993a83579e0eb \ + --hash=sha256:dee799544cae42a42b17a88e38b59cf2c271051dc001da2117a8ff240ffa0548 \ + --hash=sha256:e6b529a6a287104ec79d281c411685231200ce954a29c28ab8e5093cb6e130fb \ + --hash=sha256:e6e8be8a94436079cb5340f6d495d9d7ba30124d8b952703994c739c7c06e236 \ + --hash=sha256:f1dc17d987ddcc5eba12a7ce48f0eb93141dea236b170c1101151396edf2f0cf \ + --hash=sha256:f2b64641bca27497b986b9d87883014035aa904cb4fa333407c6752b3afee9ba \ + --hash=sha256:f894a2877650b56201d26a012a257b76d54a68834dc3913a93830ca8a047b075 \ + --hash=sha256:f97c3806dcea41c29c04965347b0e12481561b75e0045dc7a4f69d75dec5d9b1 \ + --hash=sha256:fb63a8d7fa2e95dcff1ca16af2f4d88b526fa5ff72d1696285884ac2d49b6963 \ + --hash=sha256:fbef61a3794e011c65fb4396a196cf123a7f474fe5a443db8e5dd7d751b9e6d4 + # via zarr google-pasta==0.2.0 \ --hash=sha256:4612951da876b1a10fe3960d7226f0c7682cf901e16ac06e473b267a5afa8954 \ --hash=sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed \ @@ -1563,6 +1593,7 @@ packaging==24.2 \ # sphinx # tensorboard # tensorflow + # zarr pandas==2.2.3 \ --hash=sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a \ --hash=sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d \ @@ -1960,6 +1991,81 @@ pytz==2025.1 \ --hash=sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57 \ --hash=sha256:c2db42be2a2518b28e65f9207c4d05e6ff547d1efa4086469ef855e4ab70178e # via pandas +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via donfig pyzmq==26.2.1 \ --hash=sha256:000760e374d6f9d1a3478a42ed0c98604de68c9e94507e5452951e598ebecfba \ --hash=sha256:004837cb958988c75d8042f5dac19a881f3d9b3b75b2f574055e22573745f841 \ @@ -2545,6 +2651,7 @@ typing-extensions==4.12.2 \ # torch # typing-inspect # viser + # zarr typing-inspect==0.9.0 \ --hash=sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f \ --hash=sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78 @@ -2946,10 +3053,12 @@ yourdfpy==0.0.60 \ --hash=sha256:2af2d8bdeea1b85b642590a3b4236fdb35746d7b3e38ce460a169c18d9c4f868 \ --hash=sha256:8a187a8b18c98db87c76e9a950581b3c875b761e00df83942526c17ea693166c # via viser -zarr==2.18.4 \ - --hash=sha256:2795e20aff91093ce7e4da36ab1a138aededbd8ab66bf01fd01512e61d31e5d1 \ - --hash=sha256:37790ededd0683ae1abe6ff90aa16c22543b3436810060f53d72c15e910c24bb - # via -r deps/pip/requirements_ncore.in +zarr==3.1.4 \ + --hash=sha256:72c3dcee2d41527d21cd814cbaca1e6e23550701eb4fc30767ee53d925900eec \ + --hash=sha256:b0cf4eea8367ae444d7d0ea90b98d29ac0b7eb691fac2e0bcc2d2bd659522903 + # via + # -r deps/pip/requirements_3_11.in + # -r deps/pip/requirements_ncore.in zstandard==0.25.0 \ --hash=sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64 \ --hash=sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a \ diff --git a/deps/pip/requirements_3_12.in b/deps/pip/requirements_3_12.in index 1bb6e8b9..a39491fc 100644 --- a/deps/pip/requirements_3_12.in +++ b/deps/pip/requirements_3_12.in @@ -17,7 +17,7 @@ -r requirements_tests.in -r requirements_ncore.in -# Public API restrictions for 3.12 +# Public API restrictions for 3.12: most recent zarr-python 2 zarr>=2.18.7,<3.0.0 # zarr-python 2.18.7 is the first version with the correct numcodecs<0.16 restriction # torch: same version as the python 3.11 environment diff --git a/deps/pip/requirements_3_13.in b/deps/pip/requirements_3_13.in index 5c2627ea..14d3debc 100644 --- a/deps/pip/requirements_3_13.in +++ b/deps/pip/requirements_3_13.in @@ -17,8 +17,8 @@ -r requirements_tests.in -r requirements_ncore.in -# Public API restrictions for 3.13 -zarr>=2.18.7,<3.0.0 # zarr-python 2.18.7 is the first version with the correct numcodecs<0.16 restriction +# Public API restrictions for 3.13: most recent zarr-python 3 +zarr>=3.4.0 # most recent zarr-python 3 (python>=3.12) # torch: most recent version # CPU-only on purpose: python 3.12 / 3.13 cover the CPU code paths, CUDA is tested with python 3.11 diff --git a/deps/pip/requirements_3_13.txt b/deps/pip/requirements_3_13.txt index 131dc3cd..29ff157c 100644 --- a/deps/pip/requirements_3_13.txt +++ b/deps/pip/requirements_3_13.txt @@ -3,9 +3,6 @@ --index-url https://pypi.org/simple --find-links https://download.pytorch.org/whl/cpu/torch -asciitree==0.3.3 \ - --hash=sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e - # via zarr cbor2==6.1.5 \ --hash=sha256:015ed73f10e1f7b67306d41e36e0d7dc40e4a2100bc5c29b7a7f039ad3dc9061 \ --hash=sha256:040cf628af473fe18cb6f56bdac556d2398102e56852aab5206fbeb3dbde6b52 \ @@ -77,13 +74,9 @@ dataclasses-json==0.6.7 \ --hash=sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a \ --hash=sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0 # via -r deps/pip/requirements_ncore.in -deprecated==3.0.0 \ - --hash=sha256:16850204d3a1e6bb0acd06bff48d96e8b0a0d25d1c52f71705405a0f4894192d \ - --hash=sha256:58204cf4a7f6270d547af5c278ee7a6bb56045a4b3d8441a1cd11660f41b7939 - # via numcodecs -fasteners==0.20 \ - --hash=sha256:55dce8792a41b56f727ba6e123fcaee77fd87e638a6863cec00007bfea84c8d8 \ - --hash=sha256:9422c40d1e350e4259f509fb2e608d6bc43c0136f79a00db1b49046029d0b3b7 +donfig==0.8.1.post1 \ + --hash=sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d \ + --hash=sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52 # via zarr filelock==4.1.0 \ --hash=sha256:2ce9818e3e2d8f284c1a964414447ef148d42a5fd5e2a477a7118e574b293ec1 \ @@ -95,6 +88,39 @@ fsspec==2026.9.0 \ # via # torch # universal-pathlib +google-crc32c==1.9.0 \ + --hash=sha256:0568b17ed90ac596f29400d99e243fd0cc6276766183def888d1bf8d1dc13827 \ + --hash=sha256:0abe7e202c25909869c35672ab0f2fe748a7acf276eb78577332a7c38999740f \ + --hash=sha256:13179f7e3282617923e957b8e54b8f9c3968030f48640a9f47fd7c5c38c4a215 \ + --hash=sha256:16865b477d7941712cb0e0aad8ad4815e984fb5fc16d3fdaef7d986e26e53c95 \ + --hash=sha256:265233aff33d835f5b909584fe36ab29647b598c271b661a300001099109e53e \ + --hash=sha256:280f3a3e47af0eeba3a3e5aa7d311af77001812b8df80fb8beafcd0b40eaf7f1 \ + --hash=sha256:3abb18297d9ef0ab120531838be0e6d68c9fa876570e11c229c48f2edac23ce7 \ + --hash=sha256:43a2dc26f9be213fbe0b4fc4a1088c5d45cbfcb3247420ccc820f0fc3edeea86 \ + --hash=sha256:4488f1553a9ab7e86cdedc833374a7e904031803b995dc0bd0be48c271fa6556 \ + --hash=sha256:457d0d9a4718fd52b1494eac5c200ad25beeadbdc91843d550a003910838589f \ + --hash=sha256:51cb4e23a38ad4f495f35f87c233ca3ea6b9c4559e7ac383cdef786fab0f7977 \ + --hash=sha256:53fdafef58e230d0c946ab5f8446d123d9f548230a73b29c8b41c9546f268bc1 \ + --hash=sha256:56610f548f1b35c9568b9d1de30423480f505dae4991556072d5802820ff35c4 \ + --hash=sha256:5695c8b9327e040b2aba12c6659b0acb5995314ef0af0192da66e662e011103b \ + --hash=sha256:6a3b2c8a343c570ed8100a7627c20badfd92c6caa2067093a86be45af27f5b1b \ + --hash=sha256:7b8c84c3d159ab6817fe3f74e6e6cef099c3f95dcec3abc0d8afb1404642efbe \ + --hash=sha256:8535e75dfead304f30e9122b9ea2c0a570dbaa52c176a0a591540c7914c1e46d \ + --hash=sha256:8583ec21d56b565d68ab2963cc7e21b3b271247c29b04286068255ef65f221bd \ + --hash=sha256:86764b99e7a607830d93cb5b75e0ec3ff6cb06d3c274624418473cee701900d4 \ + --hash=sha256:8b91f41645b15a720357183fa5716682ada441873e3c462c15f9714be36f146b \ + --hash=sha256:af73200fa9791ccd380f3598235dba8d82b8af0905df045b3dc60b59836e8ddd \ + --hash=sha256:ccfe40021fd6afe23361175cf7551e3cef5fd34dc1ebe319f14993a83579e0eb \ + --hash=sha256:dee799544cae42a42b17a88e38b59cf2c271051dc001da2117a8ff240ffa0548 \ + --hash=sha256:e6b529a6a287104ec79d281c411685231200ce954a29c28ab8e5093cb6e130fb \ + --hash=sha256:e6e8be8a94436079cb5340f6d495d9d7ba30124d8b952703994c739c7c06e236 \ + --hash=sha256:f1dc17d987ddcc5eba12a7ce48f0eb93141dea236b170c1101151396edf2f0cf \ + --hash=sha256:f2b64641bca27497b986b9d87883014035aa904cb4fa333407c6752b3afee9ba \ + --hash=sha256:f894a2877650b56201d26a012a257b76d54a68834dc3913a93830ca8a047b075 \ + --hash=sha256:f97c3806dcea41c29c04965347b0e12481561b75e0045dc7a4f69d75dec5d9b1 \ + --hash=sha256:fb63a8d7fa2e95dcff1ca16af2f4d88b526fa5ff72d1696285884ac2d49b6963 \ + --hash=sha256:fbef61a3794e011c65fb4396a196cf123a7f474fe5a443db8e5dd7d751b9e6d4 + # via zarr iniconfig==2.3.1 \ --hash=sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960 \ --hash=sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7 @@ -264,6 +290,90 @@ mpmath==1.3.0 \ --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c # via sympy +msgspec==0.22.0 \ + --hash=sha256:0067057df265795f742658b15dbe53f3b6f21d19dcfa53676db11088cfa41e0a \ + --hash=sha256:024138c51afd335d0b4dce401be33902caafac2b64f8c9f2509a378986175d98 \ + --hash=sha256:05dbc8268e50c9232ec72b9af1c7b13049aade4d1197764e38c427048706e046 \ + --hash=sha256:0666a1520cab86796612e794e71107e0fbf5e8ff3ddcdfcfff8f1d94b860d2f1 \ + --hash=sha256:0739b068f31f2004a364f97679ba91f2f5ecd6ec2a5b4b890188ab5c57d20672 \ + --hash=sha256:08826f5e5b0fa2f7a88592c396a243cfcc63d37e19f9d4fbe3b3f1be2fbdc404 \ + --hash=sha256:0922714feff5300aacd8ecd65fa828317ce4bf5212b3139258c0bfc0253cd80e \ + --hash=sha256:0a13624a4969159fe35d8c2a3d377b2b61bbd8585e327440d5e52725affcce38 \ + --hash=sha256:0b25dcbc108783cb72503ed705b9fbb8c3cb02ee5801923f44b5f038c91cc365 \ + --hash=sha256:0b31746da07cba0e330c6433a94a4699ad77d3aeb9638d1a320a7686b69f6249 \ + --hash=sha256:0dfadea8bdcfafc614bd031de55a8ede22b43445cfff6d8b77cc0c07d3edc8a8 \ + --hash=sha256:10d0d1d464960d99a949f7ca01ef8928e51c472433a5f5ab74b2d695fb830652 \ + --hash=sha256:12a887c4c06e4a771a2db32c9a80c7bb21866b12458025f636dcdc2253331c28 \ + --hash=sha256:1e547966017265c0d23342bcf2e027305dde40ea042d16694a9b96b4f696a052 \ + --hash=sha256:21460f54cee9208239b1a8421fdf25bffc77293e1daba88f585711ad839b9758 \ + --hash=sha256:21c887d4de397355f6635c2a037b1c067882dac5d132a1793d63bbf7cf5ca78e \ + --hash=sha256:221cbcbfa4478152b91d37dcfd4830e2be92773e8139e883f43773450ebacef8 \ + --hash=sha256:263e110955ed76fe0af2d79f819903b50a70dc0e7a752eb7aabe79d2e0a084fb \ + --hash=sha256:268594d0bae5510572599a6ab0364dd9de43c867d24a30856cd9f5edb63d8dc6 \ + --hash=sha256:27d9ef46c80884f9c4f323e0b18bec464287e872121e70f2cbe47335780bf597 \ + --hash=sha256:28f53f3604dd3e70225f7563c831628dbb03299b428f8e62aadb4b628e386874 \ + --hash=sha256:38c5b9bd347bc9abbcee40752be3c5117854e891ea7a1881a56d4b3dec58c5e7 \ + --hash=sha256:38f7022fbe91954b31afe3888a0af1b652e0f370fafdeb1d425f4a814d789c9f \ + --hash=sha256:3c789b5ccd07c0a3c09767108ee06e089b2875f2309a4569c2648f30a8d31dfa \ + --hash=sha256:3ca7d4cd69fbb66bd2da6211d3e79d40542d196c16c6d99bf838f76767ad35be \ + --hash=sha256:4600dbec738ed74e4c9bd35503e84701200ea7db344cfdeda80677b3ee53eb64 \ + --hash=sha256:4a663a8d7f6ad56ac1dbcba91e046ba8ebab7773ae72ef3dd3c47f8226919184 \ + --hash=sha256:508278300dd4efbd21cd3a4b2b016160a5feac98bc880d3673f6c06697baaf62 \ + --hash=sha256:57c282f474e17acf6bcf84f393c73afd45d6eba47cccff8b76b79c4fbb8a3b54 \ + --hash=sha256:5aa24eb475d070ecbbe5b21080fc3ce4b0b76c60de25cfe0c9678d8fb44bb42f \ + --hash=sha256:5e4f7e09cceac7dbf4c0761b8ae7df51c55b5df5e9af7aff2c895aac1ebea015 \ + --hash=sha256:614e2c827e0a3f934f3cf0cf4ba65210df8132b75a69a8a1f51bb3b2caf0ac5a \ + --hash=sha256:627bfdfe5a4b3d916b3360b30f4cddeee3a084f56593e33527c6872fa8322ff9 \ + --hash=sha256:65eea14bc65ccfeb8f3af62cb204841871e2961f002d7fa87dbe0f79dacf1c1c \ + --hash=sha256:6ad64f5c260866b0d543f89f50cee43628989c1433c5de7ce820281fa28a2611 \ + --hash=sha256:6ae370f92f3517f0e6f209ba7cc649c957b444868439197e046be07154667551 \ + --hash=sha256:6f48317f05312bfdf78248f53933f830f07ab75cc1c813ac3ca4220cb3b5b019 \ + --hash=sha256:71cbbdb39631064e2f2f9e9ac2b1b69931d72276eb5f9da4ed025726296bdbb6 \ + --hash=sha256:7293dee54de040cfa225c22151cc3d72f17cd674b5ebcb52f38fb9f5701592e6 \ + --hash=sha256:749899563d26b211379f142b8ffd7e2d7da149a51717798f0ce994dce50324f0 \ + --hash=sha256:7c1e76c6bd523141b9c05c2f8a70979cd0efedbd68855a66f292f8892c0b8fc7 \ + --hash=sha256:884c28c80b0a511595b29a9b04a3a230c3797369e4a033e6d5c6d9b5427f8e09 \ + --hash=sha256:885c6e0c89d6103648525fe62aa78d600054dedf7b3713d23b15d7ddb6d66a13 \ + --hash=sha256:8c8e84789918fbc15a503b92a829115ddd7567ecd3e4778bd418c56abbb86c11 \ + --hash=sha256:8d67582478b0eaabb899f2fb255c878ee7de57dff80eb73ab24f1865524ec441 \ + --hash=sha256:8f0a5c25516e2034b2db7767081759ff8996e214def9c43b3055f61e1be1caad \ + --hash=sha256:99c401861c5bb3a57f7d6423ea7ed4352cd57aa3f04f4fbe9f3e3e4564a10f08 \ + --hash=sha256:9a696f23f7c1ffb31fae308502e01a3965c3891d5c400f01d0d1096dbe77519e \ + --hash=sha256:a1dab6a99c759d1391ab2993388c1892746a697254f4b5dc6c059ca6e3bfbc8b \ + --hash=sha256:a52eba5c9528fd181fcec39d22b67aaa1dccc6cfe8e24d3f5d41130e6d04289d \ + --hash=sha256:a66b1766311e42371e509c996c3933b161c7ae0eabdf361af5316dec197e1022 \ + --hash=sha256:a6c8a3f210421e29d8f7e9815f106cf59d758665b7fe5428e61152ce24fe65d7 \ + --hash=sha256:a6db3806b3b76ca78064255eac6fa101a8a64fe6f698d80fbaf81fdfa21217d4 \ + --hash=sha256:a88d939d3fe4b8c7314645ebcd6e86c8c8a512ea7820d6550355973e803bc0f1 \ + --hash=sha256:a8b98ae215a102cbf6635f7df45f5c4af12f77fad1f7b71b9808fcf868a5735d \ + --hash=sha256:ab1e9e7531e353653b906cdd12a0220cc288a1e8e3436aabc65f4508d91b14d9 \ + --hash=sha256:b3113ebcceeb7693a915183c73d92c10bf5c62851dd187cab43bd025fb587419 \ + --hash=sha256:b5a169b5b03f0f2c7a296c002647db1dab75d2cd501bca34e32b71cab0261b56 \ + --hash=sha256:b60b43425a47eb9cfe987f6874e354ca7c760e58e295b4e2273ff03574df28a1 \ + --hash=sha256:b6d3ca19a8ff28d0a67a1824e2bff7ec649ec795c80a265f20ade4caa63080de \ + --hash=sha256:b962000e11dd34fb210a5a2c57a8a62b2d92b381c8cb3b05c075a83e38f8d645 \ + --hash=sha256:bc374dedd5f85a5f4de2386dc5f737894ccb8c1ac18e9566ce66fd9839e6285d \ + --hash=sha256:c3c510aba9015c085e514b75a9b3f1ed7c4591ae5e379655821b8bba51f30cc7 \ + --hash=sha256:c6c310ef83e7e291b01a63298828f848348bb99e84a1098c4b3923c05674d032 \ + --hash=sha256:c6f06576eced70462179a4b4638e84cf69fdbba37f44d13a64a21739c131a830 \ + --hash=sha256:cfc3d9557de9c806318725b702f3e664db33167bb42892079b693c69893fd33b \ + --hash=sha256:d2f950239ff1fc7322c6f9634807310265149cb168270d3ddcdda5b6ada13a28 \ + --hash=sha256:d7a738826936c72348c613061d260446f13c82b6fd7d5d7705b6911ab8dca2f3 \ + --hash=sha256:dce29a04966e31abf9b83b697c6d672486526dc5d03fcd6970cb56d5dc1fbeea \ + --hash=sha256:dd9568695911055440d2bb7099ed9098fc181d335daa772d0eb3fe8f31ba4efb \ + --hash=sha256:e0aa0cc3f18c35bab79bd7b87fde95d6274a9deddeebd1ea541f8066a5073165 \ + --hash=sha256:e79725246291516a7359caad5fb743ddc0ec66ed40d2381fb846325b5031504e \ + --hash=sha256:ebd211d7af79ed8710c64e9e8d4c0d02749bc20170e7ab4e1c5801ca7c99d25b \ + --hash=sha256:ec108e96fdaa8fdbe5bb993ec97a9d1faa69b3a521eecd71a6e5acbe0e29ae69 \ + --hash=sha256:f039ef5207b847f075a0a43020ee6140cd47505f890e47e157f2deb485c2dc96 \ + --hash=sha256:f13c127a945479bc9db057eb253b8851075c8e1ae07ffc967bfa1c5676203a86 \ + --hash=sha256:f2ddea9d78d09460f06c26a7a508adcd049761c3208776162b8eb79b8a032cff \ + --hash=sha256:f3413e3647275f787b21b4dfb4836a59a1a5acf1018ab1d45843b1d7edf15c22 \ + --hash=sha256:f7a923bcde480065c8e25967464cfb2a687ee67000bb43157e2d57e40eca7305 \ + --hash=sha256:fa3689b9dfcc663358ef23ba4299d7460f01108515b041a7d30d05908ac9c32f \ + --hash=sha256:fb1e129b81ac8fcf9ec649b081c6c8da1c7ea6f87cab336d46386abc2cd855c1 \ + --hash=sha256:feafe612034d49e9144340c0b5168ee4e22c2af4aaa2c1db11ae84e1aac9543b + # via zarr mypy-extensions==1.1.0 \ --hash=sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505 \ --hash=sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558 @@ -272,20 +382,28 @@ networkx==3.7 \ --hash=sha256:e3fd2c13a7814cee3746340d8d7f8598a67f16a58bf47fb7f8793fab6efca1b0 \ --hash=sha256:fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a # via torch -numcodecs==0.15.1 \ - --hash=sha256:1d471a1829ce52d3f365053a2bd1379e32e369517557c4027ddf5ac0d99c591e \ - --hash=sha256:1dfdea4a67108205edfce99c1cb6cd621343bc7abb7e16a041c966776920e7de \ - --hash=sha256:698f1d59511488b8fe215fadc1e679a4c70d894de2cca6d8bf2ab770eed34dfd \ - --hash=sha256:a34f0fe5e5f3b837bbedbeb98794a6d4a12eeeef8d4697b523905837900b5e1c \ - --hash=sha256:a4f7bdb26f1b34423cb56d48e75821223be38040907c9b5954eeb7463e7eb03c \ - --hash=sha256:b0a9d9cd29a0088220682dda4a9898321f7813ff7802be2bbb545f6e3d2f10ff \ - --hash=sha256:bef8c8e64fab76677324a07672b10c31861775d03fc63ed5012ca384144e4bb9 \ - --hash=sha256:c3a09e22140f2c691f7df26303ff8fa2dadcf26d7d0828398c0bc09b69e5efa3 \ - --hash=sha256:cdfaef9f5f2ed8f65858db801f1953f1007c9613ee490a1c56233cd78b505ed5 \ - --hash=sha256:daed6066ffcf40082da847d318b5ab6123d69ceb433ba603cb87c323a541a8bc \ - --hash=sha256:e2547fa3a7ffc9399cfd2936aecb620a3db285f2630c86c8a678e477741a4b3c \ - --hash=sha256:e3d82b70500cf61e8d115faa0d0a76be6ecdc24a16477ee3279d711699ad85f3 \ - --hash=sha256:eeed77e4d6636641a2cc605fbc6078c7a8f2cc40f3dfa2b3f61e52e6091b04ff +numcodecs==0.17.0 \ + --hash=sha256:0fc125d1c726c1937cde346e109e3662a2b4ff6be073289da7d124d172aceda5 \ + --hash=sha256:1837d4d1d646cecd3ab2d1ba22956295d709edea0bddc952737c647bec1d03c4 \ + --hash=sha256:1cce4bf2278ed74841c2088acfd38e67c3e5aa77e3bc1962ef0fa2931becbb12 \ + --hash=sha256:1ebd63cdb8985c66257bc037fcdff5f38637aff72d7ef62612ec46f2299e8749 \ + --hash=sha256:2e29732c5e3a83663e51b40007819d8fd0aae16a2322f7044ce13a2460a99e23 \ + --hash=sha256:3e4e351566b3ab2f6255a9d91c6c48e1d0f9ec6e2ae409a148e091a8fc0a80b0 \ + --hash=sha256:4a62e5a821ccfbe425bbdd9a079f8b6c41b7e796ff3c99324530561193a53047 \ + --hash=sha256:4c36f6fd14dc22939172145c24d3b3eab2410c34ed807906a5ece5f4541c7c43 \ + --hash=sha256:4d83befe67a51ba6a988c562209bf13836438c1b6dce23049d84ff42854af32d \ + --hash=sha256:4f43ba0d834ce012ed482996a7424df9077a47d5899ede2d1d54fe85e6eb12fa \ + --hash=sha256:657b1f9aa4b1025aa0fa7d4bd8d7492900950a11f636dff622bd208c0b99e35e \ + --hash=sha256:69b9b4685097c4d478a0c829debf4470555ec63e92cdd2c6b5f195460f1dc888 \ + --hash=sha256:6c3342d91ed7cf59c1be84396edd364e936bb0ec9e366d24bb69689748d19625 \ + --hash=sha256:6f1293581326e92293b142bd05b389f6682ed1ce333f36f116344bca340cfd10 \ + --hash=sha256:7065b3349b73d54785aa89e00d0b97d80f664e9056757929d28151f9208dc04c \ + --hash=sha256:8697a4631fedded77a75d333e4926b1eb3a11bc7d3e30213e7e565d6910526d0 \ + --hash=sha256:a854e9c89f58eeeb2453f3c1637d1916797edb6eaff26bc186a6cdb09d187092 \ + --hash=sha256:d30c69b4bdb1755af1022fa913e184eaadc4fc0cd38f736e483e8ad205e130d1 \ + --hash=sha256:de2c66db238e74e66fe9be7e02b7e0129b75d3f812d38e4019eb0102cc2dcdf0 \ + --hash=sha256:e8db2e337bdafd3bb5f891a2543b53b2b36a509ce9d587af2846db3715b6c8b9 \ + --hash=sha256:ecd0f6a10e3f8afbbb16ecc999d2b06aa2a31a2946f1c1a85d15d91a1ebcfef3 # via zarr numpy==2.5.4 \ --hash=sha256:067374eb538c34c745436365cf7b0112595c1d326f21ce4ff340f61230239fbb \ @@ -377,6 +495,7 @@ packaging==26.3 \ # via # marshmallow # pytest + # zarr parameterized==0.9.0 \ --hash=sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b \ --hash=sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1 @@ -486,6 +605,81 @@ pytest==9.1.1 \ --hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 \ --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c # via -r deps/pip/requirements_tests.in +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via donfig scipy==1.18.1 \ --hash=sha256:011413b7426b75012840e35649e00fe0a2c3bae89fed433876e3a99251572efc \ --hash=sha256:0ac49ea97594532dd44b7136094d35f5440fa06e6d9c6384a74c01764df388c5 \ @@ -566,8 +760,10 @@ typing-extensions==4.16.0 \ --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 # via # -r deps/pip/requirements_ncore.in + # numcodecs # torch # typing-inspect + # zarr typing-inspect==0.9.0 \ --hash=sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f \ --hash=sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78 @@ -576,112 +772,9 @@ universal-pathlib==0.3.10 \ --hash=sha256:4487cbc90730a48cfb64f811d99e14b6faed6d738420cd5f93f59f48e6930bfb \ --hash=sha256:dfaf2fb35683d2eb1287a3ed7b215e4d6016aa6eaf339c607023d22f90821c66 # via -r deps/pip/requirements_ncore.in -wrapt==2.5.0 \ - --hash=sha256:05246a100da68259af521b88788f131ba005465f1c95d358cc3c03ec5e351b52 \ - --hash=sha256:05b7a271e21598694dcb7a6d6224f04e80ecc24b00a31ec57c658196377b0b29 \ - --hash=sha256:0810e060e58f7960405172ad21080df8e7335841c9fe97417bd7d3f05af24f90 \ - --hash=sha256:0a7a369e7fca9fc8c2c50df634382009b09af416853da3d4515e4bb048a5b9ee \ - --hash=sha256:0cabb9c17ab79b2549d1f23b36f436473ad9253ef53995a817feba26fae69d5b \ - --hash=sha256:0d245ac03f5ae77f1eea6eb19edd9e778c2f772490c20496c2f1cd3a102ee1b6 \ - --hash=sha256:0dfc38cb672af51fc29696ba9c6f05d2315f5e62c4af2564e50f07f81198a163 \ - --hash=sha256:0ea62bc142f4fa8b2e0ab058f50699ccd679ef6199c8fa3cc1c2396c7a659000 \ - --hash=sha256:107eea1a511e98a3a5033b0c2cb403fbb37f05dee6ac1fb85c0460d311ec278c \ - --hash=sha256:1122f4f9e363da804ccba05a9f0f39baa3716eb82452c929258bc3f1420c899b \ - --hash=sha256:148052fc55013930217f531c6978e918ab210a12bd73cc9bd6de661a7adaf620 \ - --hash=sha256:181a45000506a6382eb337354ca7e8525690702f1ccf2eae4f23a210ff339543 \ - --hash=sha256:18baaf966bdc22dbdbe6e4323da8c17c0b7e8ea941dccf5426e741ae398953d9 \ - --hash=sha256:1a3c4035d2026b87ef23dd8d165f1f8d3853ee2bd02791cfd22bd8c6226c41ce \ - --hash=sha256:1b35ef7379323a149a6398f6261248bf48b61666210bd69e2ab24a9a9afdedc0 \ - --hash=sha256:1ccd22ef8690ca302425f26d4c3e8b24f1284d4351d20e49ce21f9c0dd58d64d \ - --hash=sha256:1d4da5f0e9a719471502b0db80d5c97503aeca796b7aeb9ab8f47403b2be76e6 \ - --hash=sha256:1ebc0d09906057ada57a32158a657364ca40b8f86e604da3e7d979069b601502 \ - --hash=sha256:1f17c5a3836397bf59fd57b0e5b0dc42969b1daa70d31aa361c6e13cbf138b5a \ - --hash=sha256:1ffb2823c95dbeb8a47fedfba9636b2afeb0a8ef94b66df97bd081bdfe5a263f \ - --hash=sha256:21cfe343ef9c2deb865ad0d5c57822266447c88dcf6d8805dd8c363fe367f30c \ - --hash=sha256:23a9d6cb6413359b76f030d6bbb75340b7669c69da245dde2919a4c93708993b \ - --hash=sha256:2ac82ef59ee05e259902bc7cf73dee5e6397845e8ccdc376d9d25536b59a877c \ - --hash=sha256:2e3d110d7f99644946f249927c346d9dba507a78815d1bcebdbf0d94c14c5649 \ - --hash=sha256:2fd8a61c31220840c7f52621cf51c961af5058bd009a55bcdf4a6732bdb13b35 \ - --hash=sha256:30cf86d1e35a57e772e4709a137c1731751255439368a41b1a2519561e385f4e \ - --hash=sha256:36703cafc2ec059e118c2175e6cb7ad7299c2924aecdcb1b7a7ebbf7a3e20c19 \ - --hash=sha256:3f5dfb867d3f58fde0d850a302f1164bb83f4a922b037882bcc7e0474aa8312f \ - --hash=sha256:425349a99b8c9540399d36620c376dc26e6aca93071cd6fafa239c2f1b5d53a4 \ - --hash=sha256:4343880acd72e74233baf092285aaaf4306244e31d7601828bd2600316027df0 \ - --hash=sha256:439523506fd0d9af4f76d75f2e46aa4979e164fecba6890859326ff06edbaf9c \ - --hash=sha256:47abb2bb7f15b416e72fbe5e68e49a6f09331dae6af1ca6055f5aa2251d2bd2f \ - --hash=sha256:513dd1f4a1f91030d5656d9f4b3af8aa490d4db1eb53481e6846b8a5ee7acfcd \ - --hash=sha256:5375ff1d2159e2ef847449e3dd2329441423d0a3d895d123273a67095fe7ca0c \ - --hash=sha256:5421acb5c363a9bc959122a8645e3f1f42010c932dc53885b11a5ff5b5a6d730 \ - --hash=sha256:557ebf4ce5568588368675014a2540405db687c2e4c7ad1eb83aa7e857be1864 \ - --hash=sha256:57fa1a3fd1279b3ca7655b943ad61d298f2a2464a4cdca7ff298058e408322f9 \ - --hash=sha256:58b2a87c65cbfb20917ec48ace47f71b1962c1f81dbf18a4052e3037abf72028 \ - --hash=sha256:5df50c6133a63071fb77a0a091ceae36e78467b728d65e711ded02510edd9855 \ - --hash=sha256:6058e12e9caa33468f9a36fb88c15a4bb30a479f997b37834b83abdbf062f264 \ - --hash=sha256:6119e5d5bda268171af1f68ab5e7544c618f02a9bdc7c3929a458b719b88cd03 \ - --hash=sha256:61253431e5a6f0eaeae71e2a101753fbaf4360b3e4dd8e118177d565ae119d40 \ - --hash=sha256:6269637d9a54990430b4a769df15833935a46c4d004d9fe8a153bbadf0b9a097 \ - --hash=sha256:63e58f96849f622ce769dcd705f83c7445cd9829bce1dae00e78bb031aec8096 \ - --hash=sha256:65f2ee406dc592a5b22a7dc6abac13e8a3e8de4b2ecf5dc3c22937865496e4b6 \ - --hash=sha256:672dd1bab4256311db1520b1b50e7a10cbaeaae2b0ac6bc5d858cc387ee605a2 \ - --hash=sha256:6761765cc520ff9616fb035c02a85d1d744f7f70edd4649718fd0d09c589eacf \ - --hash=sha256:691671ea05684f921ffc2e935fd3f9311c1795a10fbbfa006b46269733668f66 \ - --hash=sha256:6a31cc61ad1be4091f5b094c91d53d206609928af13b7b24573cb857a3bc07fc \ - --hash=sha256:6a9ee62a970075738909909bdbef3a7da9f7ae03dfca584db283547a29503b56 \ - --hash=sha256:7138b0e7990e5555a905c519e8dad17c1da1f20e08b283e202c414229065740f \ - --hash=sha256:734ea79e4707751fb7cc4f716123b2115cd16ed0c4926e803540aece22e3ba67 \ - --hash=sha256:76fb341d5a707a4f211631b8c77259b2df149147e9d9c245ae6ba3dd936bfdfb \ - --hash=sha256:77f0a74ff6f6cf89f5b673a732d5afe1911a6e6b1c017260836fdfdf85518dc1 \ - --hash=sha256:78b7bdaa8b27b7f7607c66bdb6ab15c1dcbd9e9a1556a253a347dad511f615d1 \ - --hash=sha256:79e68f0fd7d381b9bbd71776f602a2d5440d4d2077459128e02fd6607465422c \ - --hash=sha256:7e25e9697f60af41fb86b08697e470b1e7eb6cd6ac0eb25e4b1f519839adc271 \ - --hash=sha256:7e2af7ff9553c492f684a41d903ec41e37bf6267ee3206c17518a349201cae46 \ - --hash=sha256:8837fbe708cb9d8a2d32a37dee836d24a531f02560db26418e2b181986fa21cb \ - --hash=sha256:88bb24b9fdccb1d805258d5648533206eb58c58b6554985c47db53a89c11be85 \ - --hash=sha256:94ebe745d1b0ebd3af91e22c38961616d32a228bdf9c326e7d4a6007a623d48e \ - --hash=sha256:99f8ea48f14a71c5e2df8763e9a490e8af63096dcd67755b7bab0a4b74fc7cd7 \ - --hash=sha256:a039b009693b58f7e0cf6121851218c2227223e7601cc24ed4957ba0990fd25b \ - --hash=sha256:a10e9af5d4c5977d2d93e75d1662d076a8e8bf8c499dac7be4e78ebd8e0b59af \ - --hash=sha256:a145a7826eddea3eb5814903f98f93756042b919bb5305544cb1331daa2705b1 \ - --hash=sha256:a346408f19b6d589bf029f25f65c0b4cdeed6302ef8f40da4e5d1552d22dc037 \ - --hash=sha256:a45a5249a6965d91aac9f991fda7c17e6b8b41fe91592a6099f182bf53c82724 \ - --hash=sha256:a5bb346a34499e091e4fa23df58251ad192173c088e5413d893ca1c730c133c7 \ - --hash=sha256:aae2f5f4c77335a39ebe5a1c77d5519f6cefefc7dbff50bd551e3771d927e3fc \ - --hash=sha256:ab45839c912777e2738fed369636589c8b2a6d9c44ca56de0fd0814581d467c2 \ - --hash=sha256:b312b3cc87951faaed3cfef984d768ee8bee7f935d9cc929aaa9946b0b96a98c \ - --hash=sha256:b58138d19f34e32833e62de5e910bc2a8baae43310b921d783bd39b15227c2dd \ - --hash=sha256:b620d7559b6b2197c5730332fab0867ecf1c8cb74d45533ebbcbcad1eacf4616 \ - --hash=sha256:b640460f0ffb346b192686bd6fac5589e35a6c6640501c59a9fb6e82b0dd6bd8 \ - --hash=sha256:b898caea081303006decc562c7fca5126f7c96507e78dd8f1ae3285dfa50ddc7 \ - --hash=sha256:b95a6eca3b927853529eea958310563c83140ae8451dd5dc4399c7da385dc4f3 \ - --hash=sha256:bad63bb4dea3c58e8078a3a173259ac2df5442a437e49c632b4099c0250e803b \ - --hash=sha256:bc5607c1911c92530cb402ea90d818933cffb28bd8de9b453a2542279816d8c7 \ - --hash=sha256:bc6491d3008ecabf685b0746f03ad8241a0950336939b14addb03af39b51a316 \ - --hash=sha256:bcd42e7b69c8c1e33a29b79b28de03bdc08876a49745830f9162a3af860e06d0 \ - --hash=sha256:c3a78a3161b3a9bf07725822fd24d379c1f3f161b6db49166c4131096ea73b4c \ - --hash=sha256:c3dfb16e047c912e1a06bfbc45da752219f474897be85f55b8069f7936b08cc0 \ - --hash=sha256:c48cdb6c904dca76d9915a579e4a5fab6b0c25f650c1019ce78a78effaf7a345 \ - --hash=sha256:c57ddae24cf72eb6bd18112638a987cafe6109d90f2df111e6934362cc03ac1a \ - --hash=sha256:c932273bc43b068538f3874fa5e6c2a60f33fa0b11c1ebc7768652f6a0608943 \ - --hash=sha256:cbd45dfba6b5c1bfbabe1feb3c0f117fbd62416e98268d9a7cd9ad8802875356 \ - --hash=sha256:ce4cab32c37ef71e69cf88f909b7febd0dd79543e5ae3650e2b874e0f3d3b975 \ - --hash=sha256:d75d6366203c8d025c1a74bae0b565952187ddae79bd5c7bf10687652a56f020 \ - --hash=sha256:def66258d97ebf1e4e97def12c5daa542d1cc728a3da83ed3a43933f56df6dab \ - --hash=sha256:dfedf844892bf88f387cdeb9fa9bff1186904417c9f05087c99241bec24a65cc \ - --hash=sha256:e0345d4c1f7aa5a27075d28a7f0e9ed386729198045f1f08b47f8320d6bbda23 \ - --hash=sha256:e3c6fb1c1a516881353186bed9cfcb8899f968c03b3509720c79db0d967acf3b \ - --hash=sha256:e5d4885c5625c9d2dcb49458700851574e9d0ea046c7a265526e990282f8ae8e \ - --hash=sha256:e716f47c7f61e11709d3c0904213c94fc22999abf0c41461276cef886c1e8b4d \ - --hash=sha256:ea4fdc79c0045d6bb1603c109127145245cafe888588888444e1e37fbeadbac3 \ - --hash=sha256:f12e80c3089ebc03727d368f8205b811b5af2cd4a72b5e4cac75e901dd316e39 \ - --hash=sha256:f42239c89430eee2d8a6dec39e34677abdbb67fff63caf2467dd6124ea4d4d58 \ - --hash=sha256:f4ef4935962f7029b2058a99f1a47ccbffc3be919dddb3becb6c2c48eac3d9f0 \ - --hash=sha256:fd91203e156d610ecb28b9ccd7b764af7a7b38662d7c163090babab0d10def0c \ - --hash=sha256:fe09aac4837ec720af606a493e814dcc3f65631984e1b7231b589efcf9917024 \ - --hash=sha256:ff909b934b1958e31784d412abab5cbb0709fdbc01c86f22965e1d15331371ba - # via deprecated -zarr==2.18.7 \ - --hash=sha256:ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223 \ - --hash=sha256:b2b8f66f14dac4af66b180d2338819981b981f70e196c9a66e6bfaa9e59572f5 +zarr==3.4.1 \ + --hash=sha256:38b540578a119352bdce02a720d7bfd99846e77f0728c58b1bab3d1f547526a0 \ + --hash=sha256:b34bda11ceb199c81ee78ecd42cd02f46c7f67a6d8a1e9bc501cafd5a1795356 # via # -r deps/pip/requirements_3_13.in # -r deps/pip/requirements_ncore.in diff --git a/deps/pip/requirements_ncore.in b/deps/pip/requirements_ncore.in index 54bf97e8..d691be4c 100644 --- a/deps/pip/requirements_ncore.in +++ b/deps/pip/requirements_ncore.in @@ -24,10 +24,11 @@ pillow>=12.2.0; python_version>="3.9" pillow; python_version<"3.9" # zarr version rationales: -# - 2.17.0 contains crucial perf improvements ["Cache result of FSStore._fsspec_installed()" https://github.com/zarr-developers/zarr-python/pull/1581] +# - both zarr-python 2 and zarr-python 3 are supported # - 2.12.0 is the last version supporting python3.8 with old numpy versions, which we still want to support for now -# - we can't switch to zarr 3 yet unconditionally as it's not available for python3.8 anymore -zarr>=2.12.0,<3.0.0 +# - 2.17.0 contains crucial perf improvements ["Cache result of FSStore._fsspec_installed()" https://github.com/zarr-developers/zarr-python/pull/1581] +# - 3.1.4 is the oldest supported zarr-python 3 version (Array.async_array, Store ABC without set_partial_values) +zarr>=2.12.0,!=3.0.*,!=3.1.0,!=3.1.1,!=3.1.2,!=3.1.3 # cbor2 version rationale: # - 5.9.0 fixes CWE-674 / CVE-2026-26209 (uncontrolled recursion DoS) diff --git a/docs/data/formats.rst b/docs/data/formats.rst index 125612f7..86d32841 100644 --- a/docs/data/formats.rst +++ b/docs/data/formats.rst @@ -107,7 +107,7 @@ might attach raw GPS/IMU measurements to the poses component: ├── {component_specific_data}... │ └── generic_data/ (optional, only if generic data was set) - ├── {dataset_name} [shape] dtype (lz4-compressed zarr dataset) + ├── {array_name} [shape] dtype (lz4-compressed array) └── ... Writers use :meth:`~ncore.impl.data.v4.components.ComponentWriter.set_generic_data` @@ -414,12 +414,12 @@ Supported categories: Encoding ^^^^^^^^ -* ``RAW`` -- Numpy array stored as a zarr dataset regular compression. Shape +* ``RAW`` -- Numpy array stored as an array with regular compression. Shape is ``[H, W] + shape_suffix`` (e.g., ``[H, W, 2]`` for optical flow). Transparent quantization of raw labels is supported optionally (e.g., float32 depth quantized to uint16 with scale/offset). * ``IMAGE_ENCODED`` -- Pre-encoded image bytes (PNG, JPEG) stored as a 1-D - zarr uint8 dataset with no compression. Consumers can call ``get_encoded_data()`` for raw + uint8 array with no compression. Consumers can call ``get_encoded_data()`` for raw bytes (GPU-based decoding) or ``get_data()`` for Pillow-decoded numpy arrays. Instance naming convention @@ -486,7 +486,7 @@ A minimal custom component looks like: ... # collect data def finalize(self): - ... # write zarr datasets to self._group + ... # write arrays to self._group, e.g. self._group.create_array("velocities", ...) class Reader(ComponentReader): @staticmethod @@ -498,7 +498,16 @@ A minimal custom component looks like: return version == "v1" def get_velocities(self): - return self._group["velocities"][:], self._group["timestamps_us"][:] + return ( + self._group.array("velocities").read(), + self._group.array("timestamps_us").read(), + ) + +The groups of component writers / readers (``self._group``) are +``ncore.impl.data.nodes.Group`` instances with a typed API to create and read groups, arrays +(``create_array`` / ``array(path).read()``), binary blobs (``create_bytes_array`` / +``read_bytes``), and attributes. The wrapped zarr-python group is accessible via +``self._group.zarr``. Writers must ensure that all stored timestamps fall within the sequence's ``sequence_timestamp_interval_us`` time range. Existing datasets can be extended diff --git a/docs/data/storage_and_access.rst b/docs/data/storage_and_access.rst index 91df6110..1dee9f62 100644 --- a/docs/data/storage_and_access.rst +++ b/docs/data/storage_and_access.rst @@ -34,8 +34,8 @@ capability. appends a compressed index enabling O(1) key lookups and direct seeks to any chunk. -The ``.itar`` store implements the abstract zarr ``Store`` interface, so it can -be used as a drop-in replacement for directory stores in all NCore APIs. Via +The ``.itar`` store can be used as a drop-in replacement for directory stores in +all NCore APIs. Via `UPath `_, ``.itar`` containers can also be accessed transparently from cloud storage backends (e.g., S3, GCS) without requiring a local copy. @@ -137,6 +137,87 @@ random access without any index. .. |times| unicode:: U+00D7 .. |rarr| unicode:: U+2192 +.. _zarr_versions: + +zarr Versions and Formats +------------------------- + +NCore supports both major versions of the `zarr-python `_ +library, as well as both versions of the zarr on-disk format specification: + +* **zarr-python** (the installed library): ``2.x`` (Python |ge| 3.8) or ``3.x`` + (Python |ge| 3.11, version |ge| 3.1.4). The matching implementation is selected + automatically at import time. +* **zarr format** (the on-disk layout of component stores): format v2 (``.zgroup`` / + ``.zarray`` / ``.zattrs`` node metadata) or format v3 (``zarr.json`` node metadata). + +.. list-table:: + :header-rows: 1 + :widths: 30 35 35 + + * - + - zarr-python 2 + - zarr-python 3 + * - read zarr format v2 + - |check| + - |check| + * - read zarr format v3 + - |mdash| + - |check| + * - write zarr format v2 (default) + - |check| + - |check| + * - write zarr format v3 + - |mdash| + - |check| (opt-in) + +zarr format v2 data written with either zarr-python version is equivalent (chunk records +are byte-identical, node metadata documents are equivalent JSON) and can be read by all +NCore V4 versions (NCore |le| 19.8 supports zarr-python 2 and zarr format v2 only). Data is +therefore written in zarr format v2 by default. zarr format v3 can be selected explicitly +when writing, e.g. + +.. code-block:: python + + writer = SequenceComponentGroupsWriter(..., zarr_format=3) + +or via ``--zarr-format 3`` in the data converter CLIs. In zarr format v3, binary blobs +(e.g., encoded images) are stored as ``uint8`` arrays (zarr format v2: fixed-length bytes). + +Both formats store the consolidated metadata of each component group as a single +compressed record (``.zmetadata.cbor.xz`` for zarr format v2, CBOR + LZMA encoded; +``zarr.json.cbor.zlib`` for zarr format v3, the CBOR + zlib encoded root ``zarr.json`` +including the consolidated metadata of the hierarchy, which is only stored in this +compressed form). Hierarchies opened with consolidated metadata (the default of NCore +readers) resolve all zarr nodes from it, without reading node metadata records. With +zarr-python 3, NCore executes zarr's asynchronous I/O on the calling thread, avoiding +per-access thread hand-offs to zarr-python's global I/O thread. + +With zarr-python 3, numeric arrays of ``.itar`` archives (blosc compressed or +uncompressed, as written by NCore) are read by decompressing their chunks directly into +the output array (zarr-python's indexing determines the chunks), and large chunks of an +array are decoded concurrently. All other arrays and selections are read via +zarr-python's codec pipeline. Setting the environment variable +``NCORE_ZARR3_DIRECT_READS=0`` reads all arrays via zarr-python's codec pipeline. + +Records of ``.itar`` archives are write-once (archives are append-only): rewriting a +record fails. zarr-python 3 rewrites node metadata on every attribute update, so node +metadata records are kept in memory until the archive is closed, and only their final +versions are appended (parents before children). + +.. note:: + + The groups of component readers / writers (``self._group``) are + ``ncore.impl.data.nodes.Group`` instances, which support both zarr-python + versions and zarr formats with a typed API (e.g. ``create_array``, + ``array(path).read()``, ``create_bytes_array`` / ``read_bytes``, ``attrs``). The + wrapped zarr-python objects are accessible via ``.zarr`` for components implemented + with the zarr-python API of the installed version directly. + +.. |check| unicode:: U+2713 +.. |ge| unicode:: U+2265 +.. |le| unicode:: U+2264 + Loading V4 Data --------------- @@ -157,6 +238,50 @@ V4 sequences are loaded by specifying one or more component store paths: poses_readers = reader.open_component_readers(PosesComponent.Reader) camera_readers = reader.open_component_readers(CameraSensorComponent.Reader) +Asynchronous Reads +~~~~~~~~~~~~~~~~~~ + +All component reader methods reading array data have asynchronous variants +(``*_async``, e.g. ``get_frame_data_async``, ``get_frame_ray_bundle_data_async``, +``get_generic_data_async``), returning the same data. Awaiting multiple reads +concurrently overlaps their I/O and decompression (e.g., range requests to remote +storage), and the running event loop isn't blocked (blocking work, such as +decompression, runs on worker threads): + +.. code-block:: python + + import asyncio + + lidar = reader.open_component_readers(LidarSensorComponent.Reader)["lidar"] + + async def load_frames(timestamps_us): + return await asyncio.gather( + *(lidar.get_frame_ray_bundle_data_async(ts, "direction") for ts in timestamps_us) + ) + + directions = asyncio.run(load_frames(lidar.frames_timestamps_us[:, 1].tolist())) + +Metadata (attributes, names, timestamps) is held in memory and only provided +synchronously. Custom components can implement asynchronous accessors via +``self._group.array(path).read_async()``. + +.. note:: + + Each in-flight read holds its decompressed data in memory. When reading many large + frames, bound the number of concurrent reads (e.g., to the number of CPU cores, or + higher for high-latency remote storage) instead of gathering all of them at once: + + .. code-block:: python + + async def load_frames(timestamps_us, max_concurrent_reads=16): + semaphore = asyncio.Semaphore(max_concurrent_reads) + + async def load(ts): + async with semaphore: + return await lidar.get_frame_ray_bundle_data_async(ts, "direction") + + return await asyncio.gather(*(load(ts) for ts in timestamps_us)) + .. _cloud_storage_access: Cloud and Remote Storage Access @@ -361,6 +486,14 @@ Performance Recommendations This is especially important for remote stores where each metadata lookup would otherwise be a separate round-trip. +* **Keep the node cache enabled** (the default). Component readers cache the + groups / arrays they open (including their decoded metadata, but not their + data), so repeated accesses (e.g., of frames) skip re-opening them. The + ``node_cache_size`` parameter on + :class:`~ncore.data.v4.SequenceComponentGroupsReader` bounds the number of + cached nodes per component reader for very long sequences (``None`` caches + all accessed nodes, ``0`` disables caching). + * **Tune the block size and cache type** for your workload. The ``default_block_size`` and ``default_cache_type`` parameters on ``UPath`` control how much data is fetched per HTTP request and how it is cached diff --git a/ncore/BUILD.bazel b/ncore/BUILD.bazel index e71fe1f7..e1a5b44f 100644 --- a/ncore/BUILD.bazel +++ b/ncore/BUILD.bazel @@ -56,10 +56,11 @@ py_wheel( "dataclasses_json>=0.2.12", "pillow", # zarr version restrictions / suggestions rationale: + # - both zarr-python 2 and zarr-python 3 are supported # - 2.17.0 contains crucial perf improvements ["Cache result of FSStore._fsspec_installed()" https://github.com/zarr-developers/zarr-python/pull/1581] # - 2.12.0 is the last version supporting python3.8 with old numpy versions, which we still want to support for now - # - we can't switch to zarr 3 yet unconditionally as it's not available for python3.8 anymore - "zarr>=2.12.0,<3.0.0", + # - 3.1.4 is the oldest supported zarr-python 3 version (Array.async_array, Store ABC without set_partial_values) + "zarr>=2.12.0,!=3.0.*,!=3.1.0,!=3.1.1,!=3.1.2,!=3.1.3", "cbor2", "scipy", "torch", diff --git a/ncore/impl/data/BUILD.bazel b/ncore/impl/data/BUILD.bazel index 5d7237a7..c7206756 100644 --- a/ncore/impl/data/BUILD.bazel +++ b/ncore/impl/data/BUILD.bazel @@ -25,6 +25,7 @@ py_library( "types.py", ], deps = [ + ":pylib_json", ":pylib_util", "//ncore/impl/common:pylib_transformations", requirement("dataclasses_json"), @@ -33,13 +34,30 @@ py_library( ], ) +py_library( + name = "pylib_json", + srcs = [ + "_json.py", + ], +) + py_library( name = "pylib_stores", srcs = [ + "_backend.py", + "_zarr2.py", + "_zarr3.py", + "encoding.py", + "itar.py", + "nodes.py", "stores.py", ], deps = [ + ":pylib_json", requirement("cbor2"), + requirement("numcodecs"), + requirement("numpy"), + requirement("typing_extensions"), requirement("universal_pathlib"), requirement("zarr"), ], diff --git a/ncore/impl/data/_backend.py b/ncore/impl/data/_backend.py new file mode 100644 index 00000000..88cc4658 --- /dev/null +++ b/ncore/impl/data/_backend.py @@ -0,0 +1,244 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Interface of the zarr-python version specific backends of :mod:`ncore.impl.data.nodes` / :mod:`.stores`, and +types shared by the backends""" + +from __future__ import annotations + +import dataclasses +import sys +import typing + +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional, Tuple, Union, cast + +import numcodecs +import zarr + +from ncore.impl.data.itar import IndexedTarStore + + +if sys.version_info >= (3, 10): + from types import EllipsisType +else: # pragma: no cover + EllipsisType = type(Ellipsis) + +if TYPE_CHECKING: + import numpy as np + import numpy.typing as npt # type: ignore[import-not-found] + + from typing_extensions import Protocol +else: + Protocol = object + + +#: Key of the compressed consolidated metadata of zarr format v2 hierarchies (CBOR + LZMA encoded ``.zmetadata``) +V2_CONSOLIDATED_METADATA_KEY = ".zmetadata.cbor.xz" + +#: Key of the compressed consolidated metadata of zarr format v3 hierarchies (CBOR + zlib encoded root ``zarr.json`` +#: including the consolidated metadata of the hierarchy, which decodes several times faster than CBOR + LZMA). The root +#: ``zarr.json`` of such hierarchies is only stored in this compressed form. +V3_CONSOLIDATED_METADATA_KEY = "zarr.json.cbor.zlib" + +#: Basic selections of array data (integers, slices and Ellipsis) +Selection = Union[int, slice, EllipsisType, Tuple[Union[int, slice, EllipsisType], ...]] + +#: Attributes of nodes (JSON-serializable values, also nested mappings / sequences: values are JSON-encoded by zarr-python, +#: see :data:`ncore.impl.data.types.JsonLike` for the attributes read back) +Attributes = Mapping[str, object] + + +def attributes_dict(attributes: Attributes) -> Dict[str, Any]: + """A (shallow) copy of attributes as dictionary, as accepted by zarr-python""" + return dict(attributes) + + +class ZarrStore: + """A zarr-python store of the installed zarr-python version (e.g., of a directory)""" + + +#: Stores of zarr hierarchies: indexed tar archives, or zarr-python stores +StoreLike = Union[IndexedTarStore, ZarrStore] + + +def is_v2_metadata_key(key: str) -> bool: + """True for keys of zarr format v2 node metadata""" + return key.endswith((".zarray", ".zgroup", ".zattrs")) + + +#: Blosc compressors supported by zarr-python 2 (numcodecs ``Blosc``) and zarr-python 3 (``BloscCodec``) +BloscCname = Literal["blosclz", "lz4", "lz4hc", "zlib", "zstd"] + +#: Blosc shuffle modes (numcodecs ``Blosc`` encodes them as 0 / 1 / 2) +BloscShuffle = Literal["noshuffle", "shuffle", "bitshuffle"] + +_BLOSC_SHUFFLES: List[BloscShuffle] = ["noshuffle", "shuffle", "bitshuffle"] + + +@dataclasses.dataclass(frozen=True) +class BloscCompression: + """Blosc compression of array chunks""" + + #: Compressor used by blosc + cname: BloscCname = "lz4" + #: Compression level (0: no compression, 9: maximum compression) + clevel: int = 5 + #: Shuffle filter applied before compression + shuffle: BloscShuffle = "bitshuffle" + #: Block size in bytes (0: automatic) + blocksize: int = 0 + + def __post_init__(self) -> None: + if self.cname not in typing.get_args(BloscCname): + raise ValueError(f"Unsupported blosc compressor {self.cname!r}, supported: {typing.get_args(BloscCname)}") + if not 0 <= self.clevel <= 9: + raise ValueError(f"Blosc compression level needs to be in [0, 9], got {self.clevel}") + if self.shuffle not in _BLOSC_SHUFFLES: + raise ValueError(f"Unsupported blosc shuffle {self.shuffle!r}, supported: {_BLOSC_SHUFFLES}") + if self.blocksize < 0: + raise ValueError(f"Blosc block size needs to be non-negative, got {self.blocksize}") + + def to_config(self) -> Dict[str, object]: + """The configuration of the zarr format v3 ``blosc`` codec""" + return {"cname": self.cname, "clevel": self.clevel, "shuffle": self.shuffle, "blocksize": self.blocksize} + + @staticmethod + def from_config(config: Mapping[str, object]) -> BloscCompression: + """From the configuration of a zarr format v3 ``blosc`` codec""" + return BloscCompression( + cname=cast(BloscCname, config["cname"]), + clevel=cast(int, config["clevel"]), + shuffle=cast(BloscShuffle, config["shuffle"]), + blocksize=cast(int, config.get("blocksize", 0)), + ) + + def to_numcodecs(self) -> numcodecs.Blosc: + return numcodecs.Blosc( + cname=self.cname, + clevel=self.clevel, + shuffle=_BLOSC_SHUFFLES.index(self.shuffle), + blocksize=self.blocksize, + ) + + @staticmethod + def from_numcodecs(compressor: numcodecs.Blosc) -> BloscCompression: + config = compressor.get_config() + return BloscCompression( + cname=config["cname"], + clevel=config["clevel"], + shuffle=_BLOSC_SHUFFLES[config["shuffle"]], + blocksize=config["blocksize"], + ) + + +ZarrNode = Union[zarr.Group, zarr.Array] + + +class Backend(Protocol): + """zarr-python version specific operations of nodes and stores""" + + #: zarr formats supported by the zarr-python version + supported_zarr_formats: Tuple[int, ...] + + # -- nodes ----------------------------------------------------------------- + + def zarr_format(self, node: ZarrNode) -> int: + """The zarr format of a node""" + ... + + def open_member(self, group: zarr.Group, path: str) -> ZarrNode: + """Opens a (nested) member of a group (raises KeyError if it doesn't exist)""" + ... + + def member_names(self, group: zarr.Group) -> List[str]: + """Names of the direct members of a group""" + ... + + def has_member(self, group: zarr.Group, name: str) -> bool: + """True if a group has a direct member with the given name""" + ... + + def member_groups(self, group: zarr.Group) -> List[Tuple[str, zarr.Group]]: + """Direct member groups of a group""" + ... + + def create_group(self, group: zarr.Group, name: str, attributes: Optional[Attributes]) -> zarr.Group: + """Creates a member group (raises if a member with the name exists already)""" + ... + + def require_group(self, group: zarr.Group, name: str) -> zarr.Group: + """Returns a member group, creating it if it doesn't exist yet""" + ... + + def create_array( + self, + group: zarr.Group, + name: str, + data: npt.NDArray[np.generic], + chunks: Union[Tuple[int, ...], Literal["auto"]], + compression: Optional[BloscCompression], + attributes: Optional[Attributes], + ) -> zarr.Array: + """Creates a member array initialized with data""" + ... + + def read(self, array: zarr.Array, selection: Selection) -> object: + """Reads (a selection of) the data of an array (0-dimensional selections may result in numpy scalars)""" + ... + + async def read_async(self, array: zarr.Array, selection: Selection) -> object: + """Reads (a selection of) the data of an array asynchronously, without blocking the running event loop""" + ... + + def compression(self, array: zarr.Array) -> Optional[BloscCompression]: + """The blosc compression of an array (None if uncompressed, raises ValueError for other compressors)""" + ... + + # -- stores ---------------------------------------------------------------- + + def open_directory_store(self, path: str) -> ZarrStore: + """Opens a directory store (a directory per group / array, a file per record)""" + ... + + def create_root_group(self, store: StoreLike, zarr_format: Literal[2, 3], attributes: Attributes) -> zarr.Group: + """Creates the root group of a hierarchy in a store""" + ... + + def open_store(self, store: StoreLike, consolidated: bool) -> zarr.Group: + """Opens the root group of a hierarchy for reading, via its compressed consolidated metadata if consolidated + (raises KeyError if there is no compressed consolidated metadata)""" + ... + + def consolidate_store(self, store: StoreLike) -> None: + """Stores the compressed consolidated metadata of the hierarchy of a store""" + ... + + def group_store(self, group: zarr.Group) -> StoreLike: + """The store of a group (the indexed tar store for groups of indexed tar archives)""" + ... + + +__all__ = [ + "Attributes", + "attributes_dict", + "Backend", + "BloscCompression", + "Selection", + "StoreLike", + "V2_CONSOLIDATED_METADATA_KEY", + "V3_CONSOLIDATED_METADATA_KEY", + "ZarrStore", + "is_v2_metadata_key", +] diff --git a/ncore/impl/data/_json.py b/ncore/impl/data/_json.py new file mode 100644 index 00000000..d19174bf --- /dev/null +++ b/ncore/impl/data/_json.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""JSON-like values (independent of the other data modules)""" + +from __future__ import annotations + +from typing import Dict, List, Union + + +#: A JSON-serializable value +JsonLike = Union[ + Dict[str, "JsonLike"], + List["JsonLike"], + str, + int, + float, + bool, + None, +] + +#: A JSON-serializable document (a dictionary at the top level) +JsonDocument = Dict[str, JsonLike] diff --git a/ncore/impl/data/_zarr2.py b/ncore/impl/data/_zarr2.py new file mode 100644 index 00000000..181a99a9 --- /dev/null +++ b/ncore/impl/data/_zarr2.py @@ -0,0 +1,210 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""zarr-python 2 backend of :mod:`ncore.impl.data.nodes` and :mod:`ncore.impl.data.stores`""" + +from __future__ import annotations + +import asyncio +import json + +from typing import TYPE_CHECKING, Any, Iterator, List, Literal, Optional, Tuple, Union + +import numcodecs +import numpy as np +import zarr + +from numcodecs.compat import ensure_bytes + +from ncore.impl.data._backend import ( + V2_CONSOLIDATED_METADATA_KEY, + Attributes, + BloscCompression, + Selection, + StoreLike, + ZarrStore, + attributes_dict, + is_v2_metadata_key, +) +from ncore.impl.data.encoding import decode_cbor_xz, encode_cbor_xz +from ncore.impl.data.itar import IndexedTarStore + + +if TYPE_CHECKING: + import numpy.typing as npt # type: ignore[import-not-found] + + _StoreBase = object + _ConsolidatedMetadataStoreBase = object +else: + from zarr._storage.store import Store as _StoreBase + from zarr.storage import ConsolidatedMetadataStore as _ConsolidatedMetadataStoreBase + + +#: zarr-python 2 API (untyped, type checking may run against zarr-python 3) +_zarr: Any = zarr + + +def _z(node: object) -> Any: + """A zarr-python 2 group / array / store (untyped, see :data:`_zarr`)""" + return node + + +class _TarStore(_StoreBase): + """zarr-python 2 store of the records of an :class:`IndexedTarStore`""" + + _erasable = False + + def __init__(self, store: IndexedTarStore) -> None: + self.store = store + + def __getitem__(self, key: str) -> bytes: + return self.store[key] + + def __setitem__(self, key: str, value: object) -> None: + if self.store.mode != "w": + raise _zarr.errors.ReadOnlyError() + self.store[key] = ensure_bytes(value) + + def __delitem__(self, key: str) -> None: + raise NotImplementedError("IndexedTarStore: records can't be deleted") + + def __contains__(self, key: object) -> bool: + return key in self.store + + def __iter__(self) -> Iterator[str]: + return iter(self.store.keys()) + + def __len__(self) -> int: + return len(self.store) + + def __eq__(self, value: object) -> bool: + return isinstance(value, _TarStore) and self.store == value.store + + def __hash__(self) -> int: + return hash(self.store) + + def close(self) -> None: + self.store.close() + + +class _CompressedConsolidatedStore(_ConsolidatedMetadataStoreBase): + """A store serving all node metadata from the compressed consolidated metadata of the wrapped store""" + + def __init__(self, store: Any) -> None: + self.store = store + consolidated = decode_cbor_xz(store[V2_CONSOLIDATED_METADATA_KEY]) + if (consolidated_format := consolidated.get("zarr_consolidated_format")) != 1: + raise _zarr.errors.MetadataError(f"unsupported zarr consolidated metadata format: {consolidated_format}") + self.meta_store = _zarr.storage.KVStore(consolidated["metadata"]) + + +def _zarr_store(store: StoreLike) -> Any: + """The zarr-python 2 store of a store (wraps indexed tar stores)""" + return _TarStore(store) if isinstance(store, IndexedTarStore) else store + + +class Backend: + """zarr-python 2 implementation of the version specific operations of nodes and stores""" + + supported_zarr_formats: Tuple[int, ...] = (2,) + + # -- nodes ----------------------------------------------------------------- + + def zarr_format(self, node: Union[zarr.Group, zarr.Array]) -> int: + return 2 + + def open_member(self, group: zarr.Group, path: str) -> Union[zarr.Group, zarr.Array]: + return _z(group)[path] + + def member_names(self, group: zarr.Group) -> List[str]: + return list(_z(group).keys()) + + def has_member(self, group: zarr.Group, name: str) -> bool: + return name in _z(group) + + def member_groups(self, group: zarr.Group) -> List[Tuple[str, zarr.Group]]: + return list(_z(group).groups()) + + def create_group(self, group: zarr.Group, name: str, attributes: Optional[Attributes]) -> zarr.Group: + child = _z(group).create_group(name) + if attributes is not None: + child.attrs.update(attributes_dict(attributes)) # (writes .zattrs once) + return child + + def require_group(self, group: zarr.Group, name: str) -> zarr.Group: + return _z(group).require_group(name) + + def create_array( + self, + group: zarr.Group, + name: str, + data: npt.NDArray[np.generic], + chunks: Union[Tuple[int, ...], Literal["auto"]], + compression: Optional[BloscCompression], + attributes: Optional[Attributes], + ) -> zarr.Array: + array = _z(group).create_dataset( + name, + data=data, + chunks=True if chunks == "auto" else chunks, + compressor=None if compression is None else compression.to_numcodecs(), + ) + if attributes is not None: + array.attrs.update(attributes_dict(attributes)) + return array + + def read(self, array: zarr.Array, selection: Selection) -> object: + return _z(array)[selection] + + async def read_async(self, array: zarr.Array, selection: Selection) -> object: + # (zarr-python 2 is synchronous: reads run on worker threads, to not block the running event loop) + return await asyncio.get_running_loop().run_in_executor(None, self.read, array, selection) + + def compression(self, array: zarr.Array) -> Optional[BloscCompression]: + if (compressor := _z(array).compressor) is None: + return None + if not isinstance(compressor, numcodecs.Blosc): + raise ValueError(f"Unsupported compressor {compressor}") + return BloscCompression.from_numcodecs(compressor) + + # -- stores ---------------------------------------------------------------- + + def open_directory_store(self, path: str) -> ZarrStore: + return _zarr.storage.DirectoryStore(path) + + def create_root_group(self, store: StoreLike, zarr_format: Literal[2, 3], attributes: Attributes) -> zarr.Group: + root = _zarr.group(store=_zarr_store(store)) + root.attrs.put(attributes_dict(attributes)) + return root + + def open_store(self, store: StoreLike, consolidated: bool) -> zarr.Group: + zarr_store = _zarr_store(store) + if consolidated: + return _zarr.open_group(store=_CompressedConsolidatedStore(zarr_store), chunk_store=zarr_store, mode="r") + return _zarr.open_group(store=zarr_store, mode="r") + + def consolidate_store(self, store: StoreLike) -> None: + zarr_store = _zarr_store(store) + metadata = {key: json.loads(zarr_store[key]) for key in zarr_store if is_v2_metadata_key(key)} + zarr_store[V2_CONSOLIDATED_METADATA_KEY] = encode_cbor_xz({"zarr_consolidated_format": 1, "metadata": metadata}) + + def group_store(self, group: zarr.Group) -> StoreLike: + store = _z(group).store + if isinstance(store, _CompressedConsolidatedStore): + store = store.store + return store.store if isinstance(store, _TarStore) else store + + +backend = Backend() diff --git a/ncore/impl/data/_zarr3.py b/ncore/impl/data/_zarr3.py new file mode 100644 index 00000000..1edcb108 --- /dev/null +++ b/ncore/impl/data/_zarr3.py @@ -0,0 +1,756 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""zarr-python 3 backend of :mod:`ncore.impl.data.nodes` and :mod:`ncore.impl.data.stores` (only imported if +zarr-python 3 is installed). + +zarr-python 3 executes all I/O as coroutines. Its synchronous API submits each call to a global event loop running on +a dedicated I/O thread, and codecs additionally hand off (de)compression to worker threads. For many small accesses +(as typical for reading individual frames) these thread hand-offs dominate. The backend instead runs zarr-python's +coroutines on an event loop owned by the *calling* thread (see :func:`run`), which executes the same zarr-python code +paths without thread hand-offs, and lets multiple reader threads decode data concurrently. + +Arrays of indexed tar archives are read by decompressing their chunks directly into the output array (see +:func:`_direct_read`). +""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import contextlib +import json +import os +import sys +import threading +import warnings + +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterator, + Callable, + Coroutine, + Dict, + Iterable, + Iterator, + List, + Literal, + NamedTuple, + Optional, + Tuple, + TypeVar, + Union, + cast, +) + +import numcodecs +import numpy as np +import zarr +import zarr.api.asynchronous as zarr_async + +from zarr.abc.codec import Codec +from zarr.abc.store import ByteRequest, OffsetByteRequest, RangeByteRequest, Store, SuffixByteRequest +from zarr.codecs import BloscCodec +from zarr.core.buffer import Buffer, BufferPrototype, default_buffer_prototype +from zarr.core.group import AsyncGroup +from zarr.core.indexing import BasicIndexer +from zarr.core.sync import sync +from zarr.storage import LocalStore, StorePath + +from ncore.impl.data._backend import ( + V2_CONSOLIDATED_METADATA_KEY, + V3_CONSOLIDATED_METADATA_KEY, + Attributes, + BloscCompression, + Selection, + StoreLike, + ZarrStore, + attributes_dict, + is_v2_metadata_key, +) +from ncore.impl.data.encoding import decode_cbor_xz, decode_cbor_zlib, encode_cbor_xz, encode_cbor_zlib +from ncore.impl.data.itar import IndexedTarStore + + +if TYPE_CHECKING: + import numpy.typing as npt # type: ignore[import-not-found] + + +T = TypeVar("T") + + +# ----------------------------------------------------------------------------- +# Execution of coroutines on event loops of the calling threads +# ----------------------------------------------------------------------------- + + +class _InlineExecutor(concurrent.futures.ThreadPoolExecutor): + """Executor running submitted work immediately in the submitting thread (asyncio requires default executors to be + thread pools, which start no threads until work is submitted to them)""" + + def submit(self, fn, /, *args, **kwargs): # type: ignore[override] + future: concurrent.futures.Future = concurrent.futures.Future() + try: + future.set_result(fn(*args, **kwargs)) + except BaseException as e: # forwarded to the awaiting coroutine + future.set_exception(e) + return future + + +class _Loop: + """An event loop of a thread (closed when the thread ends)""" + + def __init__(self) -> None: + self.loop = asyncio.new_event_loop() + self.loop.set_default_executor(_InlineExecutor()) # (codecs offload (de)compression to the default executor) + self.pid = os.getpid() + + def __del__(self) -> None: + # Loops of parent processes aren't closed in forked processes: their selectors (e.g., epoll instances) are + # shared with the parent, closing them would unregister the parent's file descriptors (e.g., the self-pipe of + # the parent's loop, which then misses wake-ups by other threads) + if self.pid == os.getpid() and not self.loop.is_closed() and not self.loop.is_running(): + self.loop.close() + + +class _ThreadLoop(threading.local): + state: Optional[_Loop] = None + + +_thread_loop = _ThreadLoop() + + +def _on_own_loop() -> bool: + """True if running on the event loop of :func:`run` of the calling thread (blocking work runs inline), False on + other event loops (blocking work is offloaded to threads, to not block them)""" + state = _thread_loop.state + return state is not None and state.pid == os.getpid() and asyncio.get_running_loop() is state.loop + + +def run(coroutine: Coroutine[Any, Any, T]) -> T: + """Runs a zarr-python coroutine to completion on an event loop owned by the calling thread. + + The loop runs offloaded work (e.g., of codecs) inline. Processes forked by a thread with a loop create a new loop. + Within running event loops (e.g., of asynchronous applications), coroutines run on zarr-python's I/O loop. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + if (state := _thread_loop.state) is None or state.pid != os.getpid(): + state = _thread_loop.state = _Loop() # (see _Loop.__del__ for loops of parent processes) + return state.loop.run_until_complete(coroutine) + return sync(coroutine) + + +# ----------------------------------------------------------------------------- +# zarr-python 3 store of indexed tar archives +# ----------------------------------------------------------------------------- + + +def _byte_range(byte_range: Optional[ByteRequest], size: int) -> Tuple[int, int]: + """Maps a zarr-python byte request to the (start, end) range of a record of the given size""" + if byte_range is None: + return 0, size + if isinstance(byte_range, RangeByteRequest): + return min(byte_range.start, size), min(byte_range.end, size) + if isinstance(byte_range, OffsetByteRequest): + return min(byte_range.offset, size), size + if isinstance(byte_range, SuffixByteRequest): + return max(0, size - byte_range.suffix), size + raise TypeError(f"Unexpected byte range {byte_range}") + + +class _TarStore(Store): + """zarr-python 3 store of the records of an :class:`IndexedTarStore` (synchronous I/O)""" + + supports_writes: bool = True + supports_deletes: bool = False + supports_partial_writes: Literal[False] = False + supports_listing: bool = True + + def __init__(self, store: IndexedTarStore) -> None: + super().__init__(read_only=store.mode == "r") + self.store = store + self._is_open = True + + def __eq__(self, value: object) -> bool: + return isinstance(value, _TarStore) and self.store == value.store + + def __hash__(self) -> int: + return hash(self.store) + + def __repr__(self) -> str: + return f"_TarStore({self.store!r})" + + def with_read_only(self, read_only: bool = False) -> _TarStore: + if read_only != self.read_only: + raise NotImplementedError("IndexedTarStore: the access mode of archives can't be changed") + return self + + def close(self) -> None: + self.store.close() + self._is_open = False + + def _get( + self, key: str, prototype: Optional[BufferPrototype], byte_range: Optional[ByteRequest] + ) -> Optional[Buffer]: + try: + value = self.store[key] + except KeyError: + return None + start, end = _byte_range(byte_range, len(value)) + return (prototype or default_buffer_prototype()).buffer.from_bytes(value[start:end]) + + async def get( + self, key: str, prototype: BufferPrototype, byte_range: Optional[ByteRequest] = None + ) -> Optional[Buffer]: + return self._get(key, prototype, byte_range) + + async def get_partial_values( + self, prototype: BufferPrototype, key_ranges: Iterable[Tuple[str, Optional[ByteRequest]]] + ) -> List[Optional[Buffer]]: + return [self._get(key, prototype, byte_range) for key, byte_range in key_ranges] + + async def exists(self, key: str) -> bool: + return key in self.store + + async def set(self, key: str, value: Buffer) -> None: + self._check_writable() + self.store[key] = value.to_bytes() + + async def set_if_not_exists(self, key: str, value: Buffer) -> None: + if key not in self.store: + await self.set(key, value) + + async def delete(self, key: str) -> None: + """No-op for records that don't exist (zarr-python deletes them when creating nodes)""" + self._check_writable() + if key in self.store: + raise NotImplementedError("IndexedTarStore: records can't be deleted") + + async def delete_dir(self, prefix: str) -> None: + """No-op for prefixes without records (zarr-python deletes them when creating nodes)""" + self._check_writable() + prefix = prefix if prefix == "" or prefix.endswith("/") else prefix + "/" + if any(key.startswith(prefix) for key in self.store.keys()): + raise NotImplementedError("IndexedTarStore: records can't be deleted") + + async def list(self) -> AsyncIterator[str]: + for key in self.store.keys(): + yield key + + async def list_prefix(self, prefix: str) -> AsyncIterator[str]: + for key in self.store.keys(): + if key.startswith(prefix): + yield key + + async def list_dir(self, prefix: str) -> AsyncIterator[str]: + prefix = prefix.rstrip("/") + prefix = prefix + "/" if prefix else "" + children = {key[len(prefix) :].split("/", 1)[0]: None for key in self.store.keys() if key.startswith(prefix)} + for child in children: + yield child + + +def _zarr_store(store: StoreLike) -> Store: + """The zarr-python 3 store of a store (wraps indexed tar stores)""" + return _TarStore(store) if isinstance(store, IndexedTarStore) else cast(Store, store) + + +def _keys(store: Store) -> List[str]: + async def keys() -> List[str]: + return [key async for key in store.list()] + + return run(keys()) + + +def _get(store: Store, key: str) -> Optional[bytes]: + value = run(store.get(key, prototype=default_buffer_prototype())) + return None if value is None else value.to_bytes() + + +def _parent_grouped(metadata: dict, node_path: Callable[[str], str]) -> dict: + """Orders consolidated node metadata by node depth, with the members of each group adjacent. + + zarr-python < 3.4 nests flat consolidated metadata assuming this order (zarr-python#4226: the members of a group + are attached to the first run of keys of the group only), which isn't the order of hierarchies created in arbitrary + order (e.g., by multiple component writers). Keys are node metadata keys (zarr format v2) or node paths (zarr format + v3), ``node_path`` maps them to node paths. + """ + + def order(key: str) -> Tuple[int, str, str]: + parent, _, name = (path := node_path(key)).rpartition("/") + return path.count("/"), parent, name + + return {key: metadata[key] for key in sorted(metadata, key=order)} + + +def _v2_node_path(key: str) -> str: + return key.rpartition("/")[0] + + +def _v3_node_documents(store: Store) -> Dict[str, object]: + """The node metadata documents of all members of a zarr format v3 hierarchy, by node path""" + documents: Dict[str, object] = {} + for key in _keys(store): + if key.endswith("/zarr.json") and (value := _get(store, key)) is not None: + documents[key[: -len("/zarr.json")]] = json.loads(value) + return documents + + +@contextlib.contextmanager +def _ignored_warnings() -> Iterator[None]: + with warnings.catch_warnings(): + # compressed consolidated metadata records are not zarr nodes + warnings.filterwarnings("ignore", message="Object at .* is not recognized as a component") + yield + + +# ----------------------------------------------------------------------------- +# Direct chunk reads of indexed tar archives +# ----------------------------------------------------------------------------- + +#: Environment variable disabling direct reads (all reads via zarr-python's codec pipeline), e.g., to compare against +#: or to benefit from zarr-python improvements +DIRECT_READS_ENV = "NCORE_ZARR3_DIRECT_READS" + +#: Minimum number of decompressed bytes per chunk to decode the chunks of a read concurrently (smaller chunks are +#: cheaper to decode than to hand off to a thread pool) +_PARALLEL_CHUNK_BYTES = 2 << 20 + +#: Maximum number of concurrent reads (e.g., of multiple reader threads) decoding their chunks concurrently, further +#: concurrent reads decode their chunks sequentially (the reads themselves already utilize the cores) +_PARALLEL_READS = 2 + + +class _ParallelReads: + """Bounds the number of reads decoding their chunks concurrently (per process)""" + + def __init__(self) -> None: + self.semaphore = threading.BoundedSemaphore(_PARALLEL_READS) + self.executor: Optional[concurrent.futures.ThreadPoolExecutor] = None + self.lock = threading.Lock() + self.pid = os.getpid() + + def pool(self) -> concurrent.futures.ThreadPoolExecutor: + with self.lock: + if self.executor is None: + self.executor = concurrent.futures.ThreadPoolExecutor( + max_workers=min(8, os.cpu_count() or 1), thread_name_prefix="ncore_zarr3_chunks" + ) + return self.executor + + +_parallel_reads = _ParallelReads() + + +def _parallel() -> _ParallelReads: + """The parallel reads state of this process (re-created in forked processes, whose locks / threads are unusable)""" + global _parallel_reads + if _parallel_reads.pid != os.getpid(): + _parallel_reads = _ParallelReads() + return _parallel_reads + + +class _DirectRead(NamedTuple): + """How to read the chunks of an array directly (see :func:`_direct_read`)""" + + store: IndexedTarStore + path: str + dtype: np.dtype + chunks: Tuple[int, ...] + chunk_key: Callable[[Tuple[int, ...]], str] + compressor: Optional[numcodecs.Blosc] + fill_value: object + + +def _direct_reads_enabled() -> bool: + return os.environ.get(DIRECT_READS_ENV, "1").strip().lower() not in ("0", "false", "no", "off") + + +def _direct_read(array: zarr.Array) -> Optional[_DirectRead]: + """How to read the chunks of an array directly (None if the array is read via zarr-python). + + Supported are arrays of indexed tar archives opened for reading with a regular chunk grid, C order, numeric / + boolean data of native byte order or fixed-length bytes, no filters, and no compression or a single blosc compressor + (all arrays written by ncore). zarr-python determines the chunks and their selections (its indexers), but the + chunks are read and decompressed directly into the output array, instead of via zarr-python's codec pipeline, which + decodes each chunk into a temporary buffer and copies it (see zarr-python#2904). + + Planned per read (sub-microsecond), so nothing outlives the array. + """ + store_path = array.async_array.store_path + if not isinstance(store := store_path.store, _TarStore) or store.store.mode != "r": + return None + + metadata: Any = array.metadata + dtype = np.dtype(metadata.dtype.to_native_dtype()) + if dtype.hasobject or not dtype.isnative or dtype.kind not in "biufS": + return None + if metadata.zarr_format == 2: + if metadata.order != "C" or metadata.filters: + return None + compressor = metadata.compressor + chunks = tuple(metadata.chunks) + else: + codecs = list(metadata.codecs) + if not codecs or type(codecs[0]).__name__ != "BytesCodec": + return None + if dtype.itemsize > 1 and str(getattr(codecs[0].endian, "value", codecs[0].endian)) != sys.byteorder: + return None + if len(codecs) == 1: + compressor = None + elif len(codecs) == 2 and isinstance(codecs[1], BloscCodec): + compressor = codecs[1]._blosc_codec + else: + return None + if not hasattr(metadata.chunk_grid, "chunk_shape"): # (regular chunk grids only) + return None + chunks = tuple(metadata.chunk_grid.chunk_shape) + if compressor is not None and not isinstance(compressor, numcodecs.Blosc): + return None + return _DirectRead( + store.store, store_path.path, dtype, chunks, metadata.encode_chunk_key, compressor, metadata.fill_value + ) + + +def _is_basic_selection(selection: Selection) -> bool: + """True for selections of (tuples of) integers, slices with unit step, and Ellipsis""" + for item in selection if isinstance(selection, tuple) else (selection,): + if item is Ellipsis or (isinstance(item, (int, np.integer)) and not isinstance(item, (bool, np.bool_))): + continue + if isinstance(item, slice) and item.step in (None, 1): + continue + return False + return True + + +def _chunk_grid(array: zarr.Array) -> Any: + """zarr-python's chunk grid of an array (an array attribute since zarr-python 3.2, metadata before)""" + if (grid := getattr(array.async_array, "_chunk_grid", None)) is not None: + return grid + return cast(Any, array.metadata).chunk_grid + + +class _DirectReadOp: + """A direct read of a basic selection of an array: the chunks to read (see :meth:`keys`), and their decoding into + the output array (see :meth:`decode`)""" + + def __init__(self, plan: _DirectRead, array: zarr.Array, selection: Selection) -> None: + self.plan = plan + self.shape = tuple(array.metadata.shape) + indexer = BasicIndexer(selection, self.shape, _chunk_grid(array)) + self.out_shape = tuple(indexer.shape) + self.projections: List[Any] = list(indexer) if all(indexer.shape) else [] + self.keys = [self._key(projection) for projection in self.projections] + + def _key(self, projection: Any) -> str: + key = self.plan.chunk_key(projection.chunk_coords) + return f"{self.plan.path}/{key}" if self.plan.path else key + + @property + def is_single_raw_chunk(self) -> bool: + """True for selections of complete uncompressed single-chunk arrays (e.g., encoded images), whose chunk bytes + are the result""" + plan = self.plan + return ( + plan.compressor is None + and len(self.projections) == 1 + and self.projections[0].is_complete_chunk + and plan.chunks == self.shape == self.out_shape + ) + + @property + def decodes_concurrently(self) -> bool: + """True for reads of multiple large chunks (see :data:`_PARALLEL_CHUNK_BYTES`)""" + chunk_bytes = self.plan.dtype.itemsize * int(np.prod(self.plan.chunks)) + return len(self.projections) > 1 and chunk_bytes >= _PARALLEL_CHUNK_BYTES + + def decode(self, raws: List[Optional[bytes]], fetch: Optional[Callable[[str], Optional[bytes]]] = None) -> Any: + """Decodes the chunks (None: not stored, read as the fill value) into the output array. With ``fetch``, chunks + are read by the decoding threads (raws are ignored)""" + plan = self.plan + if self.is_single_raw_chunk and (raw := raws[0] if fetch is None else fetch(self.keys[0])) is not None: + # (bytearray: results are writeable, as results of reads via zarr-python) + return np.frombuffer(bytearray(raw), dtype=plan.dtype).reshape(self.shape) + + out = np.empty(self.out_shape, dtype=plan.dtype) + + def decode_chunk(index: int) -> None: + projection = self.projections[index] + raw = raws[index] if fetch is None else fetch(self.keys[index]) + target = out[projection.out_selection] if projection.out_selection != () else out + if raw is None: # chunk not stored (fill value) + target[...] = np.asarray(plan.fill_value, dtype=plan.dtype) if plan.fill_value is not None else 0 + return + if ( + plan.compressor is not None + and projection.is_complete_chunk + and target.shape == plan.chunks + and target.flags.c_contiguous + ): + plan.compressor.decode(raw, out=target) + return + decoded = plan.compressor.decode(raw) if plan.compressor is not None else raw + selected = np.frombuffer(decoded, dtype=plan.dtype).reshape(plan.chunks)[projection.chunk_selection] + target[...] = selected.reshape(target.shape) # (integer selections drop axes) + + parallel = _parallel() + if self.decodes_concurrently and parallel.semaphore.acquire(blocking=False): + try: + for future in [parallel.pool().submit(decode_chunk, i) for i in range(len(self.projections))]: + future.result() + finally: + parallel.semaphore.release() + else: + for index in range(len(self.projections)): + decode_chunk(index) + return out # (0-d arrays for 0-d results, see nodes.Array.read) + + +def _get_record(store: IndexedTarStore, key: str) -> Optional[bytes]: + try: + return store[key] + except KeyError: + return None + + +async def _get_record_async(store: IndexedTarStore, key: str) -> Optional[bytes]: + try: + return await store.read_async(key) + except KeyError: + return None + + +def _read_direct(plan: _DirectRead, array: zarr.Array, selection: Selection) -> Any: + """Reads a basic selection of an array, decompressing its chunks directly into the output array. + + Reads of multiple large chunks (see :data:`_PARALLEL_CHUNK_BYTES`) read and decode their chunks concurrently on a + thread pool (positional reads and blosc decompression release the GIL), unless :data:`_PARALLEL_READS` reads do so + already. All other reads run on the calling thread. + """ + op = _DirectReadOp(plan, array, selection) + return op.decode([], fetch=lambda key: _get_record(plan.store, key)) + + +async def _read_direct_async(plan: _DirectRead, array: zarr.Array, selection: Selection) -> Any: + """Asynchronous :func:`_read_direct`: chunks of remote archives are read via concurrent asynchronous range + requests. Blocking work (local reads, decompression) runs on the calling thread on the event loop of :func:`run`, + on worker threads on other event loops (e.g., of asynchronous applications), so they aren't blocked.""" + op = _DirectReadOp(plan, array, selection) + if plan.store.is_remote: + raws = list(await asyncio.gather(*(_get_record_async(plan.store, key) for key in op.keys))) + if _on_own_loop(): + return op.decode(raws) + return await asyncio.get_running_loop().run_in_executor(_read_executor(), op.decode, raws) + if _on_own_loop(): + return op.decode([], fetch=lambda key: _get_record(plan.store, key)) + return await asyncio.get_running_loop().run_in_executor( + _read_executor(), lambda: op.decode([], fetch=lambda key: _get_record(plan.store, key)) + ) + + +def _read_executor() -> concurrent.futures.ThreadPoolExecutor: + """Executor of blocking parts of reads awaited on event loops other than the one of :func:`run` (per process)""" + global _read_executor_state + if _read_executor_state is None or _read_executor_state[0] != os.getpid(): + _read_executor_state = ( + os.getpid(), + concurrent.futures.ThreadPoolExecutor( + max_workers=min(32, (os.cpu_count() or 1) + 4), thread_name_prefix="ncore_zarr3_reads" + ), + ) + return _read_executor_state[1] + + +_read_executor_state: Optional[Tuple[int, concurrent.futures.ThreadPoolExecutor]] = None + + +# ----------------------------------------------------------------------------- +# Backend +# ----------------------------------------------------------------------------- + + +class Backend: + """zarr-python 3 implementation of the version specific operations of nodes and stores""" + + supported_zarr_formats: Tuple[int, ...] = (2, 3) + + # -- nodes ----------------------------------------------------------------- + + def zarr_format(self, node: Union[zarr.Group, zarr.Array]) -> int: + return node.metadata.zarr_format + + def open_member(self, group: zarr.Group, path: str) -> Union[zarr.Group, zarr.Array]: + node = run(group._async_group.getitem(path)) + return zarr.Group(node) if isinstance(node, AsyncGroup) else zarr.Array(node) + + def member_names(self, group: zarr.Group) -> List[str]: + with _ignored_warnings(): # (members of groups with consolidated metadata are listed without I/O) + return list(group.keys()) + + def has_member(self, group: zarr.Group, name: str) -> bool: + return name in self.member_names(group) + + def member_groups(self, group: zarr.Group) -> List[Tuple[str, zarr.Group]]: + with _ignored_warnings(): + return list(group.groups()) + + def create_group(self, group: zarr.Group, name: str, attributes: Optional[Attributes]) -> zarr.Group: + return group.create_group(name, attributes=None if attributes is None else attributes_dict(attributes)) + + def require_group(self, group: zarr.Group, name: str) -> zarr.Group: + return group.require_group(name) + + def create_array( + self, + group: zarr.Group, + name: str, + data: npt.NDArray[np.generic], + chunks: Union[Tuple[int, ...], Literal["auto"]], + compression: Optional[BloscCompression], + attributes: Optional[Attributes], + ) -> zarr.Array: + compressor: Union[numcodecs.Blosc, Codec, None] = None + if compression is not None: + if group.metadata.zarr_format == 2: + compressor = compression.to_numcodecs() + else: + compressor = BloscCodec(**cast(Dict[str, Any], compression.to_config())) + return group.create_array( + name, + data=data, + chunks=chunks, + compressors=compressor, + attributes=None if attributes is None else attributes_dict(attributes), + # (store chunks equal to the fill value too, as zarr-python 2: identical records of both versions) + config={"write_empty_chunks": True}, + ) + + def read(self, array: zarr.Array, selection: Selection) -> object: + if _direct_reads_enabled() and _is_basic_selection(selection) and (plan := _direct_read(array)) is not None: + return _read_direct(plan, array, selection) + return run(array.async_array.getitem(selection)) + + async def read_async(self, array: zarr.Array, selection: Selection) -> object: + if _direct_reads_enabled() and _is_basic_selection(selection) and (plan := _direct_read(array)) is not None: + return await _read_direct_async(plan, array, selection) + if _on_own_loop(): + return await array.async_array.getitem(selection) + # (zarr-python's codec pipeline decodes on the calling event loop: run it on a worker thread's own loop) + return await asyncio.get_running_loop().run_in_executor(_read_executor(), self.read, array, selection) + + def compression(self, array: zarr.Array) -> Optional[BloscCompression]: + if not (compressors := array.compressors): + return None + if len(compressors) == 1: + if isinstance(compressor := compressors[0], numcodecs.Blosc): # (zarr format v2) + return BloscCompression.from_numcodecs(compressor) + if isinstance(compressor, BloscCodec): # (zarr format v3) + return BloscCompression.from_config(cast(Dict[str, object], compressor.to_dict()["configuration"])) + raise ValueError(f"Unsupported compressors {compressors}") + + # -- stores ---------------------------------------------------------------- + + def open_directory_store(self, path: str) -> ZarrStore: + return cast(ZarrStore, LocalStore(path)) + + def create_root_group(self, store: StoreLike, zarr_format: Literal[2, 3], attributes: Attributes) -> zarr.Group: + node = run( + zarr_async.create_group( + store=_zarr_store(store), zarr_format=zarr_format, attributes=attributes_dict(attributes) + ) + ) + return zarr.Group(node) + + def open_store(self, store: StoreLike, consolidated: bool) -> zarr.Group: + zarr_store = _zarr_store(store) + + if consolidated: + # the compressed consolidated metadata of zarr format v2 / v3 hierarchies (root zarr.json documents of zarr + # format v3 hierarchies with consolidated metadata are only stored compressed) + if (compressed := _get(zarr_store, V3_CONSOLIDATED_METADATA_KEY)) is not None: + return self._open_v3(zarr_store, decode_cbor_zlib(compressed), consolidated=True) + if (compressed := _get(zarr_store, V2_CONSOLIDATED_METADATA_KEY)) is None: + raise KeyError(V2_CONSOLIDATED_METADATA_KEY) + else: + if (compressed := _get(zarr_store, V3_CONSOLIDATED_METADATA_KEY)) is not None: + return self._open_v3(zarr_store, decode_cbor_zlib(compressed), consolidated=False) + if (root := _get(zarr_store, "zarr.json")) is not None: # (zarr format v3 without consolidated metadata) + return self._open_v3(zarr_store, json.loads(root), consolidated=False) + return zarr.Group( + run(zarr_async.open_group(store=zarr_store, zarr_format=2, mode="r", use_consolidated=False)) + ) + + document = decode_cbor_xz(compressed) + if (consolidated_format := document.get("zarr_consolidated_format")) != 1: + raise ValueError(f"Unsupported zarr consolidated metadata format: {consolidated_format}") + metadata = document["metadata"] + + # the root group of the hierarchy, with the node metadata of all members (no node metadata reads) + prototype = default_buffer_prototype() + root = AsyncGroup._from_bytes_v2( + StorePath(zarr_store, ""), + prototype.buffer.from_bytes(json.dumps(metadata[".zgroup"]).encode()), + prototype.buffer.from_bytes(json.dumps(metadata.get(".zattrs", {})).encode()), + prototype.buffer.from_bytes( + json.dumps({**document, "metadata": _parent_grouped(metadata, _v2_node_path)}).encode() + ), + ) + return zarr.Group(root) + + def _open_v3(self, store: Store, document: Dict[str, Any], consolidated: bool) -> zarr.Group: + """Opens the root group of a zarr format v3 hierarchy from its root document""" + if consolidated: + nodes = document["consolidated_metadata"] + nodes = {**nodes, "metadata": _parent_grouped(nodes["metadata"], str)} + document = {**document, "consolidated_metadata": nodes} + else: + # (members are opened from their node metadata records) + document = {key: value for key, value in document.items() if key != "consolidated_metadata"} + with _ignored_warnings(): + return zarr.Group(AsyncGroup.from_dict(StorePath(store, ""), document)) + + def consolidate_store(self, store: StoreLike) -> None: + zarr_store = _zarr_store(store) + if (root := _get(zarr_store, "zarr.json")) is not None: + # zarr format v3: the root document with the consolidated metadata of the hierarchy (as written by + # zarr-python's consolidate_metadata) + document = { + **json.loads(root), + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": _v3_node_documents(zarr_store), + }, + } + value = encode_cbor_zlib(document) + run(zarr_store.set(V3_CONSOLIDATED_METADATA_KEY, default_buffer_prototype().buffer.from_bytes(value))) + return + + metadata = {} + for key in _keys(zarr_store): + if is_v2_metadata_key(key) and (value := _get(zarr_store, key)) is not None: + metadata[key] = json.loads(value) + value = encode_cbor_xz({"zarr_consolidated_format": 1, "metadata": metadata}) + run(zarr_store.set(V2_CONSOLIDATED_METADATA_KEY, default_buffer_prototype().buffer.from_bytes(value))) + + def group_store(self, group: zarr.Group) -> StoreLike: + store = group.store + return store.store if isinstance(store, _TarStore) else cast(ZarrStore, store) + + +backend = Backend() diff --git a/ncore/impl/data/encoding.py b/ncore/impl/data/encoding.py new file mode 100644 index 00000000..bc6b4ce8 --- /dev/null +++ b/ncore/impl/data/encoding.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compressed binary encodings of JSON-like documents (record indices of indexed tar archives, compressed consolidated +zarr metadata)""" + +from __future__ import annotations + +import io +import lzma +import zlib + +from typing import Any + +import cbor2 + + +def encode_cbor_xz(document: Any) -> bytes: + """Encodes a document as CBOR + LZMA (xz container format)""" + with io.BytesIO() as buffer: + with lzma.open(buffer, "wb", format=lzma.FORMAT_XZ) as lzma_file: + cbor2.dump(document, lzma_file) + return buffer.getvalue() + + +def decode_cbor_xz(data: bytes) -> Any: + """Decodes a CBOR + LZMA (xz container format) encoded document""" + return cbor2.loads(lzma.LZMADecompressor().decompress(data)) + + +def encode_cbor_zlib(document: Any) -> bytes: + """Encodes a document as CBOR + zlib (decodes several times faster than CBOR + LZMA)""" + return zlib.compress(cbor2.dumps(document), 9) + + +def decode_cbor_zlib(data: bytes) -> Any: + """Decodes a CBOR + zlib encoded document""" + return cbor2.loads(zlib.decompress(data)) + + +__all__ = [ + "decode_cbor_xz", + "decode_cbor_zlib", + "encode_cbor_xz", + "encode_cbor_zlib", +] diff --git a/ncore/impl/data/itar.py b/ncore/impl/data/itar.py new file mode 100644 index 00000000..855492da --- /dev/null +++ b/ncore/impl/data/itar.py @@ -0,0 +1,404 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Indexed tar archives (``.itar``), independent of zarr-python. + +An indexed tar archive is a regular tar archive with a compressed record index appended after the tar end-of-archive +blocks, enabling O(1) lookups of record payloads. :class:`IndexedTarStore` provides raw access to the records of such +archives; zarr-python version specific store adapters (see :mod:`ncore.impl.data.stores`) store zarr hierarchies in +them. + +Records are *write-once*: archives are append-only, and writing a record twice raises a ``ValueError``. The only +exception are zarr node metadata records (see :func:`is_node_metadata_key`), which zarr-python 3 rewrites on every +update of a node: they are kept in memory while writing and are appended (top-down) when closing the archive. +""" + +from __future__ import annotations + +import asyncio +import io +import logging +import os +import struct +import tarfile +import threading +import weakref + +from enum import IntEnum, unique +from pathlib import Path +from typing import IO, Any, Callable, ClassVar, Dict, Iterator, List, Literal, NamedTuple, Optional, Tuple, Union, cast + +from upath import UPath + +from ncore.impl.data.encoding import decode_cbor_xz, encode_cbor_xz + + +_logger = logging.getLogger(__name__) + + +#: Names of the records of zarr node metadata (zarr format v2 and v3) +NODE_METADATA_NAMES = (".zarray", ".zgroup", ".zattrs", "zarr.json") + + +def is_node_metadata_key(key: str) -> bool: + """True for keys of zarr node metadata records (zarr format v2 and v3)""" + return key.rsplit("/", 1)[-1] in NODE_METADATA_NAMES + + +class IndexedTarStore: + """The records of an indexed tar archive, opened for reading or (truncating) writing. + + Parameters + ---------- + itar_path : string, Path or UPath + Location of the archive (local or remote). + mode : string, optional, default 'r' + 'r' to read an existing archive, or 'w' to create (truncate) and write an archive. + index_tail_read_size : int or None, optional, default 1 << 20 (1 MiB) + Maximum bytes read from the end of the archive in a single I/O call to load the record index. Covers both the + small index header and (ideally) the compressed index. Values smaller than the tar block size (512 bytes), or + None, result in separate reads of the index header and index, which is more robust for large indices. + + Records are accessed via a mapping interface (``store[key]``, ``store[key] = value``, ``key in store``, + ``keys()``, ``len(store)``). Records of the index tail read are served from memory, so records stored near the end + of the archive don't repeat its high-latency I/O. Local archives are read via positional reads (``os.pread``), + which don't serialize reads of multiple threads. + + Archives opened for writing need to be closed (``close()``, or by using the store as context manager), which + appends the node metadata records and the record index. Until then, node metadata records can be rewritten (see + the module documentation), and are served from memory. + + All methods are thread-safe. + """ + + class Record(NamedTuple): + """The payload of a record within the archive file""" + + offset: int + size: int + + class TailBuffer(NamedTuple): + """Bytes read from the end of the archive file when loading the index""" + + start: int #: File offset of the first byte + data: bytes + + @unique + class IndexType(IntEnum): + """Serialization formats of the record index""" + + CBOR_LZMA_XZ_V1 = 1 + + #: Index header (little-endian): magic (4 bytes), index type (uint32), index offset (uint64), index size (uint32) + INDEX_HEADER_FORMAT: ClassVar[str] = "<4sIQI" + INDEX_HEADER_MAGIC: ClassVar[bytes] = b"itar" + + #: Index type of written archives + INDEX_TYPE: ClassVar[IndexType] = IndexType.CBOR_LZMA_XZ_V1 + + #: Decoders / encoders of the supported index types (the index is a structure of arrays: record keys, payload + #: offsets and payload sizes, ordered by payload offsets) + INDEX_CODECS: ClassVar[Dict[IndexType, Tuple[Callable[[bytes], Any], Callable[[Any], bytes]]]] = { + IndexType.CBOR_LZMA_XZ_V1: (decode_cbor_xz, encode_cbor_xz), + } + + def __init__( + self, + itar_path: Union[str, Path, UPath], + mode: Literal["r", "w"] = "r", + index_tail_read_size: Optional[int] = 1 << 20, # 1 MiB by default + ) -> None: + if mode not in ("r", "w"): + raise ValueError(f"IndexedTarStore: unsupported mode {mode!r}, supported are 'r' and 'w'") + self._mode: Literal["r", "w"] = mode + + # convert str / Path to an absolute UPath (with the UPath-internal `file://` protocol for local files) + itar_upath = UPath(itar_path) + if itar_upath.protocol == "": + itar_upath = UPath("file://" + str(itar_upath)) + self._itar_upath = itar_upath.absolute() + + # the tarfile module isn't thread-safe: writing and file-object based reading are serialized + self._lock = threading.RLock() + + self._records: Dict[str, IndexedTarStore.Record] = {} + self._node_metadata: Dict[str, bytes] = {} # (write mode, written when closing) + self._tail: Optional[IndexedTarStore.TailBuffer] = None + self._tar_file: Optional[tarfile.TarFile] = None + self._file: IO[bytes] = self._open_file(truncate=True) + if mode == "r": + self._records, self._tail = self._load_index(self._file, index_tail_read_size) + else: + self._tar_file = tarfile.TarFile(fileobj=self._file, mode="w") + + self._pread_fd: Optional[int] = None + self._pread_finalizer: Optional[weakref.finalize] = None + self._open_positional_reads() + + @property + def mode(self) -> Literal["r", "w"]: + """Access mode of the archive ('r' for reading, 'w' for writing)""" + return self._mode + + @property + def itar_upath(self) -> UPath: + """Absolute location of the archive""" + return self._itar_upath + + def __eq__(self, value: object) -> bool: + return isinstance(value, IndexedTarStore) and (self._itar_upath, self._mode) == (value._itar_upath, value._mode) + + def __hash__(self) -> int: + return hash((self._itar_upath, self._mode)) + + def __repr__(self) -> str: + return f"IndexedTarStore('{self._itar_upath}', mode='{self._mode}')" + + # ------------------------------------------------------------------------- + # Records + # ------------------------------------------------------------------------- + + def keys(self) -> List[str]: + """Returns the keys of all records (in archive order, followed by node metadata records that are written when + closing the archive)""" + with self._lock: + return list(self._records) + list(self._node_metadata) + + def __iter__(self) -> Iterator[str]: + return iter(self.keys()) + + def __len__(self) -> int: + with self._lock: + return len(self._records) + len(self._node_metadata) + + def __contains__(self, key: object) -> bool: + with self._lock: + return key in self._records or key in self._node_metadata + + def __getitem__(self, key: str) -> bytes: + """Reads the payload of a record (raises KeyError if it doesn't exist)""" + if self._node_metadata: + with self._lock: + if (value := self._node_metadata.get(key)) is not None: + return value + record = self._records[key] + + # (records end before the index, which ends the tail read: records starting within the tail read are part of it) + if (tail := self._tail) is not None and record.offset >= tail.start: + start = record.offset - tail.start + return tail.data[start : start + record.size] + + if self._pread_fd is not None: + return self._pread(self._pread_fd, record.offset, record.size) + + with self._lock: + # restore the position afterwards (the position to append records to when writing) + position = self._file.tell() + self._file.seek(record.offset) + value = self._file.read(record.size) + self._file.seek(position) + return value + + @property + def is_remote(self) -> bool: + """True for archives opened for reading on asynchronous fsspec filesystems (e.g., s3fs, gcsfs, http), whose + records are read via asynchronous range requests by :meth:`read_async`""" + return self._async_fs() is not None + + async def read_async(self, key: str) -> bytes: + """Reads the payload of a record asynchronously (raises KeyError if it doesn't exist). + + Records of remote archives (see :attr:`is_remote`) are read via asynchronous range requests (concurrent reads + overlap), all other records synchronously (see :meth:`__getitem__`). + """ + if (fs := self._async_fs()) is None or key in self._node_metadata: + return self[key] + record = self._records[key] + if ((tail := self._tail) is not None and record.offset >= tail.start) or record.size == 0: + return self[key] + + request = fs._cat_file(self._itar_upath.path, start=record.offset, end=record.offset + record.size) + if (fs_loop := getattr(fs, "loop", None)) is None or fs_loop is asyncio.get_running_loop(): + return await request + # (filesystems are bound to fsspec's I/O event loop, their sessions can't be used on other event loops) + return await asyncio.wrap_future(asyncio.run_coroutine_threadsafe(request, fs_loop)) + + def _async_fs(self) -> Any: + """The asynchronous fsspec filesystem of remote archives opened for reading (None otherwise)""" + if self._mode != "r" or self._pread_fd is not None: + return None + fs = self._itar_upath.fs + return fs if getattr(fs, "async_impl", False) and hasattr(fs, "_cat_file") else None + + def __setitem__(self, key: str, value: Union[bytes, bytearray, memoryview]) -> None: + """Appends a record (requires write mode, raises ValueError if the record exists already). Node metadata + records are appended when closing the archive (and can be rewritten until then).""" + if self._mode != "w": + raise ValueError("IndexedTarStore: archive is opened for reading and doesn't support writing") + payload = bytes(value) + + with self._lock: + if self._file.closed: + raise ValueError("IndexedTarStore: archive is closed") + if is_node_metadata_key(key): + self._node_metadata[key] = payload + return + if key in self._records: + raise ValueError(f"IndexedTarStore: record {key} already exists, records are write-once") + self._append(key, payload) + + def _append(self, key: str, payload: bytes) -> None: + with self._lock: + # the header of a record can span multiple tar blocks (long keys), the payload is padded to full blocks + info = tarfile.TarInfo(key) + info.size = len(payload) + cast(tarfile.TarFile, self._tar_file).addfile(info, fileobj=io.BytesIO(payload)) + padded_size = -(-len(payload) // tarfile.BLOCKSIZE) * tarfile.BLOCKSIZE + self._records[key] = self.Record(self._file.tell() - padded_size, len(payload)) + + def __delitem__(self, key: str) -> None: + raise NotImplementedError("IndexedTarStore: records can't be deleted") + + # ------------------------------------------------------------------------- + # Lifecycle + # ------------------------------------------------------------------------- + + def __enter__(self) -> IndexedTarStore: + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + self.close() + + def close(self) -> None: + """Closes the archive (when writing: appends the record index, which is required to read the archive)""" + with self._lock: + if self._file.closed: + return + if self._mode == "w": + # (deterministic order independent of the order of node updates: parents before children) + node_metadata, self._node_metadata = self._node_metadata, {} + for key in sorted(node_metadata, key=lambda key: (key.count("/"), key)): + self._append(key, node_metadata[key]) + # closing the tar file appends the end-of-archive blocks, but keeps the file object open + cast(tarfile.TarFile, self._tar_file).close() + self._save_index(self._file, self._records) + self._file.close() + self._close_positional_reads() + + def reload_resources(self) -> None: + """Re-opens the file objects (required in processes forked after opening the archive)""" + with self._lock: + position = self._file.tell() + self._file.close() + self._file = self._open_file(truncate=False) + self._file.seek(position) + if self._tar_file is not None: + self._tar_file.fileobj = self._file + self._close_positional_reads() + self._open_positional_reads() + + def _open_file(self, truncate: bool) -> IO[bytes]: + if self._mode == "r": + return self._itar_upath.open("rb") + # writing requires a readable file (universal_pathlib <= 0.2.6, the last version supporting python 3.8, + # supports "wb+" / "rb+" without declaring them in its type annotations) + return self._itar_upath.open("wb+" if truncate else "rb+") # type: ignore[call-overload] + + def _open_positional_reads(self) -> None: + """Opens a file descriptor for positional reads of local archives opened for reading, which is closed with the + archive (or when the store is garbage-collected)""" + if self._mode == "r" and self._itar_upath.protocol in ("file", "local") and hasattr(os, "pread"): + self._pread_fd = os.open(self._itar_upath.path, os.O_RDONLY) + self._pread_finalizer = weakref.finalize(self, os.close, self._pread_fd) + + def _close_positional_reads(self) -> None: + if self._pread_finalizer is not None: + self._pread_finalizer() + self._pread_finalizer = None + self._pread_fd = None + + # ------------------------------------------------------------------------- + # Positional reads / index serialization + # ------------------------------------------------------------------------- + + @staticmethod + def _pread(fd: int, offset: int, size: int) -> bytes: + value = os.pread(fd, size, offset) + while len(value) < size: # positional reads may return fewer bytes than requested + if not (chunk := os.pread(fd, size - len(value), offset + len(value))): + raise EOFError(f"IndexedTarStore: archive ends within record at offset {offset}") + value += chunk + return value + + @classmethod + def _load_index( + cls, file: IO[bytes], tail_read_size: Optional[int] + ) -> Tuple[Dict[str, IndexedTarStore.Record], IndexedTarStore.TailBuffer]: + """Loads the record index from the end of an archive file (with a single read if the index lies within the tail + read), and returns it together with the tail read""" + file.seek(0, os.SEEK_END) + file_size = file.tell() + + tail_size = min(file_size, max(tarfile.BLOCKSIZE, tail_read_size or 0)) + file.seek(file_size - tail_size) + tail = cls.TailBuffer(file_size - tail_size, file.read(tail_size)) + + # the index header is stored in the last tar block + header_offset = len(tail.data) - tarfile.BLOCKSIZE + magic, index_type, index_offset, index_size = struct.unpack_from( + cls.INDEX_HEADER_FORMAT, tail.data, header_offset + ) + if magic != cls.INDEX_HEADER_MAGIC: + raise ValueError("IndexedTarStore: invalid index header, can't load indexed tar archive") + if index_type not in cls.INDEX_CODECS: + raise TypeError(f"IndexedTarStore: unsupported index type {index_type}") + decode_index = cls.INDEX_CODECS[cls.IndexType(index_type)][0] + + if index_offset >= tail.start: + index = tail.data[index_offset - tail.start : index_offset - tail.start + index_size] + else: + file.seek(index_offset) + index = file.read(index_size) + + table = decode_index(index) + _logger.debug(f"IndexedTarStore: loaded index of {len(table['items'])} records ({index_size} bytes)") + records = { + key: cls.Record(offset, size) + for key, offset, size in zip(table["items"], table["offset_datas"], table["sizes"]) + } + return records, tail + + @classmethod + def _save_index(cls, file: IO[bytes], records: Dict[str, IndexedTarStore.Record]) -> None: + """Appends the record index and its header to a finalized archive file (after the end-of-archive blocks)""" + + def pad_to_block() -> None: + if remainder := file.tell() % tarfile.BLOCKSIZE: + file.write(tarfile.NUL * (tarfile.BLOCKSIZE - remainder)) + + index_offset = file.tell() + assert index_offset % tarfile.BLOCKSIZE == 0, "IndexedTarStore: archive end not at a tar block boundary" + + table = sorted(((key, record.offset, record.size) for key, record in records.items()), key=lambda item: item[1]) + keys, offsets, sizes = (list(column) for column in zip(*table)) if table else ([], [], []) + index = cls.INDEX_CODECS[cls.INDEX_TYPE][1]({"items": keys, "offset_datas": offsets, "sizes": sizes}) + file.write(index) + pad_to_block() + + file.write( + struct.pack(cls.INDEX_HEADER_FORMAT, cls.INDEX_HEADER_MAGIC, cls.INDEX_TYPE, index_offset, len(index)) + ) + pad_to_block() + _logger.debug(f"IndexedTarStore: saved index of {len(records)} records ({len(index)} bytes)") diff --git a/ncore/impl/data/nodes.py b/ncore/impl/data/nodes.py new file mode 100644 index 00000000..af63fde9 --- /dev/null +++ b/ncore/impl/data/nodes.py @@ -0,0 +1,356 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Groups and arrays of NCore data stores. + +:class:`Group` and :class:`Array` wrap zarr-python groups and arrays with a restricted, typed API, implemented by a +backend of the installed zarr-python version (see :mod:`ncore.impl.data._backend`). The wrapped zarr-python objects +are accessible via the ``zarr`` properties (for APIs of the installed zarr-python version not exposed by NCore). +""" + +from __future__ import annotations + +import collections +import threading + +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Literal, Optional, Tuple, Type, TypeVar, Union, cast + +import numpy as np +import zarr + +from zarr import Array as ZarrArray +from zarr import Group as ZarrGroup + +from ncore.impl.data._backend import Attributes, Backend, BloscCompression, Selection, attributes_dict +from ncore.impl.data._json import JsonLike + + +if TYPE_CHECKING: + import numpy.typing as npt # type: ignore[import-not-found] + + +def _installed_backend() -> Backend: + """The backend of the installed zarr-python version""" + if int(zarr.__version__.split(".", 1)[0]) >= 3: + from ncore.impl.data._zarr3 import backend as zarr3_backend + + return zarr3_backend + + from ncore.impl.data._zarr2 import backend as zarr2_backend + + return zarr2_backend + + +_backend: Backend = _installed_backend() + +#: zarr formats supported by the installed zarr-python version +SUPPORTED_ZARR_FORMATS: Tuple[int, ...] = _backend.supported_zarr_formats + + +def check_zarr_format(zarr_format: int) -> Literal[2, 3]: + """Validates that a zarr format is supported by the installed zarr-python version (raises ValueError otherwise)""" + if zarr_format not in SUPPORTED_ZARR_FORMATS: + raise ValueError( + f"zarr format v{zarr_format} is not supported by zarr-python {zarr.__version__} " + f"(supported: {', '.join(f'v{f}' for f in SUPPORTED_ZARR_FORMATS)})" + ) + return 2 if zarr_format == 2 else 3 + + +T = TypeVar("T") + + +class NodeCache: + """Cache of the opened nodes of a hierarchy opened for reading, by node path. + + Opening a node constructs its zarr-python object (decoding its metadata), which dominates repeated reads of small + arrays (e.g., of frames). Cached nodes reference their metadata only, never array data. ``max_entries=None`` caches + all opened nodes, ``max_entries=N`` the N most recently used nodes (evicted nodes are re-opened on access). + + All methods are thread-safe. + """ + + def __init__(self, max_entries: Optional[int] = None) -> None: + if max_entries is not None and max_entries < 1: + raise ValueError(f"max_entries needs to be positive or None (unbounded), got {max_entries}") + self._max_entries = max_entries + self._nodes: collections.OrderedDict[str, Union[ZarrGroup, ZarrArray]] = collections.OrderedDict() + self._lock = threading.Lock() + + @property + def max_entries(self) -> Optional[int]: + return self._max_entries + + def __len__(self) -> int: + return len(self._nodes) + + def keys(self) -> List[str]: + """The paths of the cached nodes, from least to most recently used""" + with self._lock: + return list(self._nodes) + + def _open(self, group: ZarrGroup, path: str) -> Union[ZarrGroup, ZarrArray]: + key = _join(group.path, path) + with self._lock: + if (node := self._nodes.get(key)) is not None: + self._nodes.move_to_end(key) + return node + + # (concurrent first accesses open nodes redundantly, which is cheaper than serializing all accesses) + node = _backend.open_member(group, path) + + with self._lock: + self._nodes[key] = node + if self._max_entries is not None: + while len(self._nodes) > self._max_entries: + self._nodes.popitem(last=False) + return node + + +class _Node: + """Common functionality of groups and arrays""" + + def __init__(self, node: Union[ZarrGroup, ZarrArray]) -> None: + self._node = node + + @property + def path(self) -> str: + """The path of the node within its store ("" for root groups)""" + return self._node.path + + @property + def zarr_format(self) -> int: + """The zarr format of the node""" + return _backend.zarr_format(self._node) + + @property + def attrs(self) -> Dict[str, JsonLike]: + """A copy of the attributes of the node""" + return cast(Dict[str, JsonLike], dict(self._node.attrs.asdict())) + + def attr_str(self, name: str) -> str: + """Returns a string attribute (raises KeyError if missing, TypeError for other types)""" + return _typed_attr(self, name, str) + + def attr_int(self, name: str) -> int: + """Returns an integer attribute (raises KeyError if missing, TypeError for other types)""" + return _typed_attr(self, name, int) + + def attr_dict(self, name: str) -> Dict[str, JsonLike]: + """Returns a dictionary attribute (raises KeyError if missing, TypeError for other types)""" + return _typed_attr(self, name, dict) + + def attr_list(self, name: str) -> List[JsonLike]: + """Returns a list attribute (raises KeyError if missing, TypeError for other types)""" + return _typed_attr(self, name, list) + + def update_attrs(self, attributes: Attributes) -> None: + """Updates (merges) attributes of the node""" + self._node.attrs.update(attributes_dict(attributes)) + + def __repr__(self) -> str: + return f"{type(self).__name__}({self.path!r}, zarr_format={self.zarr_format})" + + +class Array(_Node): + """An array of an NCore data store""" + + def __init__(self, array: ZarrArray) -> None: + super().__init__(array) + self._array = array + + @property + def zarr(self) -> ZarrArray: + """The wrapped zarr-python array""" + return self._array + + @property + def shape(self) -> Tuple[int, ...]: + return tuple(self._array.shape) + + @property + def dtype(self) -> np.dtype: + return np.dtype(self._array.dtype) + + @property + def compression(self) -> Optional[BloscCompression]: + """The blosc compression of the array (None if uncompressed, raises ValueError for other compressors)""" + return _backend.compression(self._array) + + def read(self, selection: Selection = Ellipsis) -> npt.NDArray[Any]: + """Reads (a selection of) the data of the array (0-dimensional selections result in 0-d arrays)""" + return np.asarray(_backend.read(self._array, selection)) + + async def read_async(self, selection: Selection = Ellipsis) -> npt.NDArray[Any]: + """Reads (a selection of) the data of the array asynchronously, see :meth:`read`. + + Concurrent reads (e.g., via ``asyncio.gather``) overlap their I/O (e.g., range requests of remote archives) and + decompression, the running event loop isn't blocked (blocking work runs on worker threads). + """ + return np.asarray(await _backend.read_async(self._array, selection)) + + def read_bytes(self) -> bytes: + """Reads a binary blob stored by :meth:`Group.create_bytes_array`""" + return self._blob(_backend.read(self._array, ())) + + async def read_bytes_async(self) -> bytes: + """Reads a binary blob asynchronously, see :meth:`read_bytes` and :meth:`read_async`""" + return self._blob(await _backend.read_async(self._array, ())) + + def _blob(self, value: object) -> bytes: + if isinstance(value, bytes): # fixed-length bytes scalar (trailing NUL bytes are stripped) + return bytes(value) + if isinstance(value, np.ndarray) and value.dtype.kind == "S" and value.ndim == 0: + # (0-d fixed-length bytes arrays of some zarr-python 3 versions, zarr-python#4466) + return value.tobytes().rstrip(b"\x00") + if isinstance(value, np.ndarray) and value.dtype == np.uint8: + return value.tobytes() + raise TypeError(f"Array {self.path!r} doesn't store a binary blob ({self.dtype})") + + +class Group(_Node): + """A group of an NCore data store. + + Members of groups using a :class:`NodeCache` are opened via the cache (also members of member groups). + """ + + def __init__(self, group: ZarrGroup, cache: Optional[NodeCache] = None) -> None: + super().__init__(group) + self._group = group + self._cache = cache + + @property + def zarr(self) -> ZarrGroup: + """The wrapped zarr-python group""" + return self._group + + @property + def cache(self) -> Optional[NodeCache]: + """The node cache used to open members (None if not cached)""" + return self._cache + + def with_cache(self, cache: Optional[NodeCache]) -> Group: + """Returns the group, using the given node cache to open its members""" + return Group(self._group, cache) + + # -- reading --------------------------------------------------------------- + + def _open(self, path: str) -> Union[ZarrGroup, ZarrArray]: + if self._cache is not None: + return self._cache._open(self._group, path) + return _backend.open_member(self._group, path) + + def node(self, path: str) -> Union[Group, Array]: + """Returns a (nested) member group or array (raises KeyError if it doesn't exist)""" + node = self._open(path) + return Group(node, self._cache) if isinstance(node, ZarrGroup) else Array(node) + + def group(self, path: str) -> Group: + """Returns a (nested) member group (raises KeyError if it doesn't exist or isn't a group)""" + if not isinstance(node := self._open(path), ZarrGroup): + raise KeyError(f"{_join(self.path, path)} is not a group") + return Group(node, self._cache) + + def array(self, path: str) -> Array: + """Returns a (nested) member array (raises KeyError if it doesn't exist or isn't an array)""" + if not isinstance(node := self._open(path), ZarrArray): + raise KeyError(f"{_join(self.path, path)} is not an array") + return Array(node) + + def members(self) -> List[str]: + """Names of all direct members""" + return _backend.member_names(self._group) + + def __contains__(self, name: str) -> bool: + """True if the group has a direct member with the given name""" + return _backend.has_member(self._group, name) + + def groups(self) -> Iterator[Tuple[str, Group]]: + """Iterates over the direct member groups (ordered by name)""" + for name, group in sorted(_backend.member_groups(self._group), key=lambda member: member[0]): + yield name, Group(group, self._cache) + + # -- writing --------------------------------------------------------------- + + def create_group(self, name: str, *, attributes: Optional[Attributes] = None) -> Group: + """Creates a member group (raises if a member with the name exists already)""" + return Group(_backend.create_group(self._group, name, attributes)) + + def require_group(self, name: str) -> Group: + """Returns a member group, creating it if it doesn't exist yet""" + return Group(_backend.require_group(self._group, name)) + + def create_array( + self, + name: str, + data: npt.ArrayLike, + *, + dtype: Optional[npt.DTypeLike] = None, + chunks: Union[Tuple[int, ...], Literal["auto"], None] = None, + compression: Optional[BloscCompression] = BloscCompression(), + attributes: Optional[Attributes] = None, + ) -> Array: + """Creates a member array initialized with data. + + Parameters + ---------- + name + Name of the array within the group. + data + Data of the array (defines the shape of the array, and its dtype if ``dtype`` isn't given). + dtype + Optional dtype of the array (the data is converted to it). + chunks + Chunk shape, a single chunk by default (zero-sized dimensions are clamped to 1). "auto" uses the chunk + shape heuristic of zarr-python. + compression + Blosc compression of the chunks (default: lz4, level 5, bit-shuffle), or None for uncompressed chunks. + attributes + Optional attributes of the array. + """ + values = np.asarray(data, dtype=dtype) + if chunks != "auto": + chunks = tuple(max(1, int(size)) for size in (values.shape if chunks is None else chunks)) + return Array(_backend.create_array(self._group, name, values, chunks, compression, attributes)) + + def create_bytes_array(self, name: str, data: bytes, *, attributes: Optional[Attributes] = None) -> Array: + """Stores a binary blob (e.g., an encoded image) as uncompressed member array. + + zarr format v2: 0-dimensional fixed-length bytes array (``|S``, the layout of ncore <= 19.8). zarr format v3: + 1-dimensional ``uint8`` array (fixed-length bytes are not part of the zarr format v3 specification). + """ + value = np.asarray(data) if self.zarr_format == 2 else np.frombuffer(data, dtype=np.uint8) + return self.create_array(name, value, compression=None, attributes=attributes) + + +def _typed_attr(node: _Node, name: str, expected: Type[T]) -> T: + value = node._node.attrs[name] + if not isinstance(value, expected) or (isinstance(value, bool) and expected is int): + raise TypeError(f"Attribute {name!r} of {node.path!r} is not of type {expected.__name__}: {value!r}") + return cast(T, value) + + +def _join(parent: str, path: str) -> str: + return f"{parent}/{path}" if parent else path + + +__all__ = [ + "Array", + "BloscCompression", + "Group", + "NodeCache", + "SUPPORTED_ZARR_FORMATS", + "check_zarr_format", +] diff --git a/ncore/impl/data/stores.py b/ncore/impl/data/stores.py index a7a588f1..a48ffd4e 100644 --- a/ncore/impl/data/stores.py +++ b/ncore/impl/data/stores.py @@ -13,494 +13,70 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import annotations +"""Stores of NCore data: zarr hierarchies in indexed tar archives (:class:`IndexedTarStore`) or directories, with +compressed consolidated metadata""" -import io -import logging -import lzma -import os -import struct -import tarfile +from __future__ import annotations -from dataclasses import dataclass, field -from enum import IntEnum, auto, unique from pathlib import Path -from threading import RLock -from typing import IO, Any, Dict, Iterator, Literal, NamedTuple, Optional, Union, cast +from typing import Union -import cbor2 -import zarr - -from numcodecs import compat from upath import UPath -from zarr._storage.store import Store -from zarr.util import json_loads - - -_logger = logging.getLogger(__name__) - - -class IndexedTarStore(Store): - """A zarr store over *indexed* tar files - - Parameters - ---------- - itar_path : string - Location of the tar file (needs to end with '.itar'). - mode : string, optional, default 'r' - One of 'r' to read an existing file, or 'w' to truncate and write a new - file. - index_tail_read_size : int or None, optional, default 1 << 20 (1 MiB) - Maximum bytes read from the tail of the file in a single I/O call to load the tar index. - Covers both the small index header and (ideally) the compressed index payload. - If not specified, perform separate reads for header and payload, - which is more robust for large indices, but requires two I/O calls. - Will be increased to at least one tar block size if specified smaller, - since the header is located in the last block. - Default 1 MiB size fits typical use cases, but may be increased if the compressed - index size is expected to be larger or decreased if remote access chunk sizes are smaller. - - Read mode retains the byte range loaded for the index (the tail read) as an in-memory - cache. Any store key whose payload lies entirely in that range is served from the cache - without an additional read, so overlapping zarr chunks near the end of the archive do - not repeat the same high-latency tail I/O. - - After modifying a IndexedTarStore, the ``close()`` method must be called, otherwise - essential data will not be written to the underlying files. The IndexedTarStore - class also supports the context manager protocol, which ensures the ``close()`` - method is called on leaving the context, e.g.:: - - >>> with IndexedTarStore('data/array.itar', mode='w') as store: - ... z = zarr.zeros((10, 10), chunks=(5, 5), store=store) - ... z[...] = 42 - ... # no need to call store.close() - - """ - - _erasable = False - - @dataclass - class TarRecord: - """A file record within a tar file""" - - offset_data: int - size: int - - @dataclass - class TarRecordIndex: - """All file records within a tar file""" - - records: Dict[str, IndexedTarStore.TarRecord] = field(default_factory=dict) - - class TailBuffer(NamedTuple): - """Cached byte range from the tail of the archive file. - - Retains the raw bytes of the single tail I/O read performed when the - index is loaded so that payloads falling entirely within that range can - be served from memory without an additional file read. - """ - - start: int #: File byte offset where the tail read began - data: bytes #: Raw bytes read from *start* to the end of the file - - def __init__( - self, - itar_path: Union[str, Path, UPath], - mode: Literal["r", "w"] = "r", - index_tail_read_size: Optional[int] = 1 << 20, # 1 MiB by default - ) -> None: - if mode not in ["r", "w"]: - raise ValueError("TarRecordIndex: only r/w modes supported") - - # store properties - self.mode = mode - - # Current understanding is that tarfile module in stdlib is not thread-safe, - # and so locking is required for both read and write. However, this has not - # been investigated in detail, perhaps no lock is needed if mode='r'. - self.mutex = RLock() - - # convert str / Path to absolute UPath uncondtionally - itar_upath = UPath(itar_path) - if itar_upath.protocol == "": - # use UPath-internal `file://` protocol for local files - itar_upath = UPath("file://" + str(itar_upath)) - - self.itar_upath = itar_upath.absolute() - - self._tail_buffer: Optional[IndexedTarStore.TailBuffer] = None - - # open file object (require file to be both writeable and readable when writing) and - # tar file (only required for write mode, read mode directly uses file object) - self.tar_file_object: IO[Any] - self.tar_file: Optional[tarfile.TarFile] = None - if self.mode == "r": - self.tar_file_object = self.itar_upath.open("rb") - else: - # universal_path for Python 3.8 (<=0.2.6) doesn't expose a - # write/read mode in it's static type-hints, although "wb+" is still accepted - # if the FS supports it, so ignore type-checker here - self.tar_file_object = self.itar_upath.open("wb+") # type: ignore[call-overload] - - self.tar_file = tarfile.TarFile(fileobj=self.tar_file_object, mode="w") - - # init / load index table - if self.mode == "r": - self.index, self._tail_buffer = self._load_tar_index(self.tar_file_object, index_tail_read_size) - else: - self.index = self.TarRecordIndex() - - def __delitem__(self, _: str) -> None: - raise NotImplementedError("Deleting items is not supported") - - def __iter__(self) -> Iterator[str]: - with self.mutex: - return iter(self.index.records.keys()) - - def __len__(self) -> int: - with self.mutex: - return len(self.index.records) - - def __contains__(self, item: object) -> bool: - with self.mutex: - return item in self.index.records - - def __getitem__(self, item: str) -> bytes: - with self.mutex: - # Query index for file record - record = self.index.records[item] # raises KeyError if not in archive - - # Check if payload is contained in tail buffer - if (tail := self._tail_buffer) is not None: - tail_start = tail.start - tail_end = tail_start + len(tail.data) - offset_data = record.offset_data - if offset_data >= tail_start and offset_data + record.size <= tail_end: - return tail.data[offset_data - tail_start : offset_data - tail_start + record.size] - - # Remember current tar file position - current_position = self.tar_file_object.tell() - - # Read the value - self.tar_file_object.seek(record.offset_data) - value = self.tar_file_object.read(record.size) - - # Return tar file to previous location - self.tar_file_object.seek(current_position) - - return value - - def __setitem__(self, item: str, value) -> None: - if self.mode != "w": - raise zarr.errors.ReadOnlyError - - with self.mutex: - if item in self.index.records: - raise ValueError(f"{item} already exists, update is not supported") - - value_bytes: bytes = compat.ensure_bytes(value) - value_size: int = len(value_bytes) - - # Remember current tar file position, which is the start of the header - header_start_position = self.tar_file_object.tell() - - # Store value in tar-file (will pre-pend a potentially *multi*-block header depending on item path-lengths) - tarinfo = tarfile.TarInfo(item) - tarinfo.size = value_size - - cast(tarfile.TarFile, self.tar_file).addfile(tarinfo, fileobj=io.BytesIO(value_bytes)) - - # End position after writing both header and payload - end_position = self.tar_file_object.tell() - - # Determine the effective value's payload size as a multiple of blocksize - payload_size = value_size - if remainder := payload_size % tarfile.BLOCKSIZE: - payload_size += tarfile.BLOCKSIZE - remainder - - # Determine the effective header-size (can be multiple blocks for long path names) - header_size = end_position - header_start_position - payload_size - - # Construct record from reconstructed size-information - record = self.TarRecord( - # Effective start of the data in the tar file (current tar file position + header-size) - header_start_position + header_size, - # Length of the data - value_size, - ) - - self.index.records[item] = record - - def __enter__(self) -> IndexedTarStore: - return self - - def __exit__(self, exc_type, exc_value, traceback) -> None: - self.close() - - def close(self) -> None: - """Needs to be called after finishing updating the store""" - with self.mutex: - if self.mode == "w": - # Closing the tar file appends two finishing blocks to the end of the file - # if in write mode, but doesn't close the internal file object yet - cast(tarfile.TarFile, self.tar_file).close() - - # Add index if writing - self._save_tar_index(self.tar_file_object, self.index) - - self.tar_file_object.close() - - def reload_resources(self) -> None: - """Reloads the tar file object *only* - useful to re-initialize the store in multi-process 'fork()' settings""" - with self.mutex: - # get current tar file path and seek positions, and close file object - current_position = self.tar_file_object.tell() - self.tar_file_object.close() - - # reload file object (require file to be both writeable and readable when writing) - if self.mode == "r": - self.tar_file_object = self.itar_upath.open("rb") - else: - # universal_path for Python 3.8 (<=0.2.6) doesn't expose a - # write/read mode in it's static type-hints, although "rb+" is still accepted - # if the FS supports it, so ignore type-checker here - self.tar_file_object = self.itar_upath.open("rb+") # type: ignore[call-overload] - - if self.mode == "w": - # assign the new file object to the tar file (only required for write mode) - cast(tarfile.TarFile, self.tar_file).fileobj = self.tar_file_object - - # seek to previous position - self.tar_file_object.seek(current_position) - - # Methods / constants for storing index header and payload - INDEX_HEADER_MAGIC = b"itar" - - # Index header binary format (20-bytes) - # - # tuple[TarRecordIndex, TailBuffer]: - """Loads a tar record index from the end of a tar file object. - - Returns the parsed index and a :class:`TailBuffer` representing the - single tail read used to locate the index. Callers may reuse the - buffer to satisfy reads for payloads that fall entirely within that range. - """ - # Determine tail read size - if index_tail_read_size is None or index_tail_read_size < tarfile.BLOCKSIZE: - index_tail_read_size = tarfile.BLOCKSIZE # explicit separate reads - - original_file_position = tar_file_object.tell() - - # Determine file size - tar_file_object.seek(0, os.SEEK_END) - file_size = tar_file_object.tell() - - # Read up to index_tail_read_size from the tail in one read call - tail_buffer_size = min(file_size, index_tail_read_size) - tar_file_object.seek(file_size - tail_buffer_size) - tail_buffer = tar_file_object.read(tail_buffer_size) - - # Extract the header from the last 512-byte block (it's guaranteed that the header fits into a single block) - header_binary_size = struct.calcsize(cls.INDEX_HEADER_FORMAT) - header_offset_in_tail_buffer = tail_buffer_size - tarfile.BLOCKSIZE - header = cls.IndexHeader._make( - struct.unpack( - cls.INDEX_HEADER_FORMAT, - tail_buffer[header_offset_in_tail_buffer : header_offset_in_tail_buffer + header_binary_size], - ) - ) - - if header.magic != cls.INDEX_HEADER_MAGIC: - raise ValueError("IndexedTarStore: invalid index header, can't load indexed tar file") - - # Try to extract the compressed index payload from the tail buffer - index_start_in_file = file_size - tail_buffer_size - if header.offset >= index_start_in_file: - # Payload is fully within the buffer we already read - index_start_in_tail_buffer = header.offset - index_start_in_file - index_binary = tail_buffer[index_start_in_tail_buffer : index_start_in_tail_buffer + header.size] - else: - # Rare: compressed index > index_tail_read_size — fall back to a second read - tar_file_object.seek(header.offset) - index_binary = tar_file_object.read(header.size) - - tar_file_object.seek(original_file_position) - - if header.type == cls.IndexType.CBOR_LZMA_XZ_V1.value: - _logger.debug(f"IndexedTarStore: lzma-compressed (xz archive format) index load size={len(index_binary)}") - - # load table (SOA) - table = cbor2.loads(lzma.LZMADecompressor().decompress(index_binary)) - items = table["items"] - offset_datas = table["offset_datas"] - sizes = table["sizes"] - else: - raise TypeError(f"IndexedTarStore: unsupported header type {header.type}") - - # Construct record index from loaded table - index = cls.TarRecordIndex({item: cls.TarRecord(offset_datas[i], sizes[i]) for i, item in enumerate(items)}) - return index, cls.TailBuffer(start=index_start_in_file, data=tail_buffer) - - @classmethod - def _save_tar_index(cls, tar_file_object: IO[Any], index: TarRecordIndex) -> None: - """Saves a tar record index at the end of a tar file object (needs to be finalized / have two empty blocks appended already)""" - - def fill_block() -> None: - # Fill up block with zeros - _, remainder = divmod(tar_file_object.tell(), tarfile.BLOCKSIZE) - if remainder > 0: - tar_file_object.write(tarfile.NUL * (tarfile.BLOCKSIZE - remainder)) - - assert tar_file_object.tell() % tarfile.BLOCKSIZE == 0, "Tar file not at block boundary" - - # Remember where we are storing the index - index_offset = tar_file_object.tell() - - assert index_offset % tarfile.BLOCKSIZE == 0, "Tar file not at block boundary" - - # Reformat index table as SOA (sorted by offset) - table = [(item, record.offset_data, record.size) for (item, record) in index.records.items()] - items, offset_datas, sizes = list(zip(*sorted(table, key=lambda data: data[1]))) if len(table) else ([], [], []) - - # Append compressed table to tar file - with io.BytesIO() as index_buffer: - # Compress table to in-memory buffer - with lzma.open(index_buffer, "wb", format=lzma.FORMAT_XZ) as lzma_file: - cbor2.dump({"items": items, "offset_datas": offset_datas, "sizes": sizes}, lzma_file) - - index_binary = index_buffer.getvalue() - index_size = len(index_binary) - - _logger.debug(f"IndexedTarStore lzma-compressed index store size={index_size}") - - # Append buffer to tar file - tar_file_object.write(index_binary) - - fill_block() - - # Create index header block - assert struct.calcsize(cls.INDEX_HEADER_FORMAT) <= tarfile.BLOCKSIZE, ( - "Index header larger than single block size" - ) - header_binary = struct.pack( - cls.INDEX_HEADER_FORMAT, - cls.INDEX_HEADER_MAGIC, - cls.IndexType.CBOR_LZMA_XZ_V1.value, - index_offset, - index_size, - ) - _logger.debug(f"IndexedTarStore: header store size={len(header_binary)}") - - # Append index header to tar file - tar_file_object.write(header_binary) - fill_block() - - -def consolidate_compressed_metadata(store: zarr.storage.BaseStore, metadata_key=".zmetadata.cbor.xz") -> None: - """Consolidate all metadata for groups and arrays within the given store - into a single compressed cbor resource and put it under the given key. - - See Also - -------- - zarr.consolidate_metadata - """ - store = zarr.storage.normalize_store_arg(store, mode="w") - - version = store._store_version - - if version == 2: - - def is_zarr_key(key) -> bool: - return key.endswith(".zarray") or key.endswith(".zgroup") or key.endswith(".zattrs") - - else: - raise NotImplementedError("Only supporting V2 stores") - - # Collect all meta-data - out = { - "zarr_consolidated_format": 1, - "metadata": {key: json_loads(store[key]) for key in store if is_zarr_key(key)}, - } - with io.BytesIO() as metadata_buffer: - # Compress meta-data to in-memory buffer - with lzma.open(metadata_buffer, "wb") as lzma_file: - cbor2.dump(out, lzma_file) +from ncore.impl.data._backend import ( + V2_CONSOLIDATED_METADATA_KEY, + V3_CONSOLIDATED_METADATA_KEY, + Attributes, + StoreLike, + ZarrStore, +) +from ncore.impl.data.itar import IndexedTarStore +from ncore.impl.data.nodes import Group, _backend, check_zarr_format - store[metadata_key] = metadata_buffer.getvalue() +def open_directory_store(path: Union[str, Path, UPath]) -> ZarrStore: + """Opens a directory store (a directory per group / array, a file per record) for reading or writing""" + return _backend.open_directory_store(str(path)) -class ConsolidatedCompressedMetadataStore(zarr.storage.ConsolidatedMetadataStore): - """A layer over other storage, where the metadata has been consolidated into a single compressed key.""" - # Overwrite constructor to perform decompression of metadata - def __init__(self, store: zarr.storage.StoreLike, metadata_key=".zmetadata.cbor.xz") -> None: - self.store = Store._ensure_store(store) +def create_root_group(store: StoreLike, attributes: Attributes, *, zarr_format: int = 2) -> Group: + """Creates the root group of a hierarchy in a store, in zarr format v2 (default, readable with all zarr-python + versions) or v3 (requires zarr-python 3 for writing and reading)""" + return Group(_backend.create_root_group(store, check_zarr_format(zarr_format), attributes)) - # retrieve consolidated metadata - meta = cbor2.loads(lzma.LZMADecompressor().decompress(self.store[metadata_key])) - # check format of consolidated metadata - consolidated_format = meta.get("zarr_consolidated_format", None) - if consolidated_format != 1: - raise zarr.MetadataError("unsupported zarr consolidated metadata format: %s" % consolidated_format) +def open_store(store: StoreLike, consolidated: bool = True) -> Group: + """Opens the root group of the hierarchy of a store for reading, via its compressed consolidated metadata if + consolidated (raises KeyError if the store has no compressed consolidated metadata)""" + return Group(_backend.open_store(store, consolidated)) - # decode metadata - self.meta_store: zarr.storage.Store = zarr.KVStore(meta["metadata"]) +def consolidate_store(store: StoreLike) -> None: + """Stores the compressed consolidated metadata of the hierarchy of a store (after writing all nodes)""" + _backend.consolidate_store(store) -def open_compressed_consolidated( - store: zarr.storage.StoreLike, metadata_key=".zmetadata.cbor.xz", mode="r+", **kwargs -) -> zarr.hierarchy.Group: - """Open group using metadata previously consolidated and compressed into a single key. - See Also - -------- - consolidate_compressed_metadata - zarr.open_consolidated - """ +def close_store(store: StoreLike) -> None: + """Closes a store (required for indexed tar stores opened for writing)""" + if isinstance(store, IndexedTarStore): + store.close() - # normalize parameters - zarr_version = kwargs.get("zarr_version") - store = zarr.storage.normalize_store_arg( - store, storage_options=kwargs.get("storage_options"), mode=mode, zarr_version=zarr_version - ) - if mode not in {"r", "r+"}: - raise ValueError("invalid mode, expected either 'r' or 'r+'; found {!r}".format(mode)) - path = kwargs.pop("path", None) - if store._store_version == 2: - ConsolidatedStoreClass = ConsolidatedCompressedMetadataStore - else: - raise NotImplementedError("Only supporting V2 stores") +def group_store(group: Group) -> StoreLike: + """The store of a group (the :class:`IndexedTarStore` of groups of indexed tar archives)""" + return _backend.group_store(group.zarr) - # setup metadata store - meta_store = ConsolidatedStoreClass(store, metadata_key=metadata_key) - # pass through - chunk_store = kwargs.pop("chunk_store", None) or store - return zarr.convenience.open(store=meta_store, chunk_store=chunk_store, mode=mode, path=path, **kwargs) +__all__ = [ + "IndexedTarStore", + "StoreLike", + "V2_CONSOLIDATED_METADATA_KEY", + "V3_CONSOLIDATED_METADATA_KEY", + "ZarrStore", + "close_store", + "consolidate_store", + "create_root_group", + "group_store", + "open_directory_store", + "open_store", +] diff --git a/ncore/impl/data/stores_test.py b/ncore/impl/data/stores_test.py index 030cb0c6..f3029dc0 100644 --- a/ncore/impl/data/stores_test.py +++ b/ncore/impl/data/stores_test.py @@ -13,195 +13,1182 @@ # See the License for the specific language governing permissions and # limitations under the License. -import itertools +import asyncio +import concurrent.futures +import gc +import hashlib +import json +import os +import struct +import tarfile import tempfile +import threading import unittest +import zlib from pathlib import Path -from typing import IO, Any, Optional +from typing import Any, Dict, List, Optional, Tuple, cast +from unittest import mock +import cbor2 import numpy as np import parameterized import zarr -from ncore.impl.data.stores import IndexedTarStore, consolidate_compressed_metadata, open_compressed_consolidated +from ncore.impl.data import nodes +from ncore.impl.data.encoding import decode_cbor_xz +from ncore.impl.data.itar import is_node_metadata_key +from ncore.impl.data.stores import ( + V2_CONSOLIDATED_METADATA_KEY, + V3_CONSOLIDATED_METADATA_KEY, + IndexedTarStore, + StoreLike, + close_store, + consolidate_store, + create_root_group, + group_store, + open_directory_store, + open_store, +) -COMPRESSED_CONSOLIDATED_VALUES = [False, True] -INDEX_TAIL_READ_SIZES = [ - None, - 1 << 10, - 1 << 20, - 1, - 512, - 513, -] # None (explicit separate reads), 0.5 MiB, 1 MiB (default), 1 byte / 512 / 513 bytes (edge cases resulting in separate reads) +#: None / values < 512 (separate reads of index header and index), 1 KiB, 1 MiB (default), edge cases around 512 +INDEX_TAIL_READ_SIZES = [None, 1, 512, 513, 1 << 10, 1 << 20] + +_ROOT_ATTRIBUTES = {"some": "thing"} + +ZARR_PYTHON_3 = int(zarr.__version__.split(".", 1)[0]) >= 3 + +#: All zarr formats (tests of zarr formats unsupported by the installed zarr-python version are skipped) +ZARR_FORMATS = (2, 3) + + +def _require(test: unittest.TestCase, zarr_format: int) -> None: + if zarr_format not in nodes.SUPPORTED_ZARR_FORMATS: + test.skipTest(f"zarr format v{zarr_format} requires zarr-python 3") + + +def _fill_reference(root: nodes.Group) -> None: + rng = np.random.default_rng(0) + root.create_array("foo", rng.random((3, 3, 3)), chunks=(2, 3, 3)) + sub = root.create_group("subgroup", attributes={"other": [1, 2, 3]}) + sub.create_array("foo", rng.random((5, 5, 5)), attributes={"unit": "m"}) + sub.create_array("empty", np.zeros((0, 3), dtype=np.float32)) + sub.create_bytes_array("blob", b"\x89PNG\x00some binary data", attributes={"format": "png"}) + + +def _check_reference(test: unittest.TestCase, root: nodes.Group) -> None: + rng = np.random.default_rng(0) + np.testing.assert_array_equal(root.array("foo").read(), rng.random((3, 3, 3))) + np.testing.assert_array_equal(root.array("subgroup/foo").read(), rng.random((5, 5, 5))) + test.assertEqual(root.array("subgroup/empty").shape, (0, 3)) + test.assertEqual(root.array("subgroup/blob").read_bytes(), b"\x89PNG\x00some binary data") + test.assertEqual(root.attrs, _ROOT_ATTRIBUTES) + test.assertEqual(root.group("subgroup").attrs, {"other": [1, 2, 3]}) + test.assertEqual(root.array("subgroup/foo").attrs, {"unit": "m"}) + test.assertEqual(sorted(root.members()), ["foo", "subgroup"]) + test.assertEqual(sorted(root.group("subgroup").members()), ["blob", "empty", "foo"]) + test.assertEqual([name for name, _ in root.groups()], ["subgroup"]) + test.assertIn("blob", root.group("subgroup")) + test.assertNotIn("missing", root.group("subgroup")) + with test.assertRaises(KeyError): + root.group("foo") + with test.assertRaises(KeyError): + root.array("subgroup") + with test.assertRaises(KeyError): + root.array("missing") + + +def _decode_v3_root(value: bytes) -> Any: + return cbor2.loads(zlib.decompress(value)) + + +def _write(store: StoreLike, consolidate: bool = True, zarr_format: int = 2) -> None: + _fill_reference(create_root_group(store, _ROOT_ATTRIBUTES, zarr_format=zarr_format)) + if consolidate: + consolidate_store(store) + close_store(store) class TestIndexedTarStore(unittest.TestCase): - """Test to verify functionality of IndexedTarStore""" + """Records of indexed tar archives""" def setUp(self): - # Fill a reference group with an in-memory store - self.g_ref = zarr.open(store=zarr.MemoryStore()) - self.g_ref.create_dataset("foo", data=np.random.default_rng().random((3, 3, 3))) - self.g_ref.attrs.update({"some": "thing"}) - self.g_ref.require_group("subgroup").create_dataset("foo", data=np.random.default_rng().random((5, 5, 5))) - - def check_with_reference(self, group): - """Verifies all values of a group against the reference""" - self.assertIsNone(np.testing.assert_array_equal(self.g_ref["foo"][()], group["foo"][()])) - self.assertIsNone( - np.testing.assert_array_equal(self.g_ref["subgroup"]["foo"][()], group["subgroup"]["foo"][()]) - ) - self.assertDictEqual(self.g_ref.attrs.asdict(), group.attrs.asdict()) + self._tmp = tempfile.TemporaryDirectory() + self.path = os.path.join(self._tmp.name, "store.itar") + + def tearDown(self): + self._tmp.cleanup() + + def test_records(self): + records = {"a": b"0123456789", "dir/b": b"x" * 513, "empty": b"", "l" * 150 + "/long-key": bytes(range(256))} + with IndexedTarStore(self.path, mode="w") as store: + for key, value in records.items(): + store[key] = value + self.assertEqual(store["dir/b"], records["dir/b"]) # readable while writing + with self.assertRaises(ValueError): # write-once + store["a"] = b"again" + + store = IndexedTarStore(self.path) + self.assertEqual(store.keys(), list(records)) + self.assertEqual(len(store), len(records)) + self.assertEqual({key: store[key] for key in store}, records) + self.assertIn("a", store) + self.assertNotIn("missing", store) + with self.assertRaises(KeyError): + store["missing"] + with self.assertRaises(ValueError): # read-only + store["c"] = b"c" + with self.assertRaises(NotImplementedError): + del store["a"] + + # archives are regular tar archives + with tarfile.open(self.path) as tar: + self.assertEqual({m.name: _read_member(tar, m) for m in tar.getmembers()}, records) + + @parameterized.parameterized.expand([(size,) for size in INDEX_TAIL_READ_SIZES]) + def test_index_tail_read_sizes(self, index_tail_read_size: Optional[int]): + records = {f"r{i}": bytes([i]) * (i * 97) for i in range(40)} + with IndexedTarStore(self.path, mode="w") as store: + for key, value in records.items(): + store[key] = value + + store = IndexedTarStore(self.path, index_tail_read_size=index_tail_read_size) + self.assertEqual({key: store[key] for key in store}, records) + + def test_records_within_the_index_tail_read_require_no_reads(self): + with IndexedTarStore(self.path, mode="w") as store: + store["first"] = b"1" * 1000 + store["last"] = b"2" * 1000 + + store = IndexedTarStore(self.path, index_tail_read_size=Path(self.path).stat().st_size) + with mock.patch("os.pread", side_effect=AssertionError("unexpected read")): + self.assertEqual(store["first"], b"1" * 1000) + self.assertEqual(store["last"], b"2" * 1000) + + def test_records_partially_within_the_index_tail_read(self): + """Records starting before the index tail read are read from the archive file""" + with IndexedTarStore(self.path, mode="w") as store: + store["large"] = bytes(range(256)) * 64 # 16 KiB + store["small"] = b"s" + + file_size = Path(self.path).stat().st_size + store = IndexedTarStore(self.path, index_tail_read_size=file_size - 1024) # ends within "large" + self.assertEqual(store["large"], bytes(range(256)) * 64) + self.assertEqual(store["small"], b"s") + + def test_positional_reads(self): + """Local archives are read via positional reads""" + with IndexedTarStore(self.path, mode="w") as store: + store["a"] = b"a" * 4096 + + store = IndexedTarStore(self.path, index_tail_read_size=None) + with mock.patch("os.pread", side_effect=os.pread) as pread: + self.assertEqual(store["a"], b"a" * 4096) + self.assertGreater(pread.call_count, 0) + + def test_dropped_stores_release_file_descriptors(self): + with IndexedTarStore(self.path, mode="w") as store: + store["a"] = b"a" + + def open_fds() -> int: + return len(os.listdir(f"/proc/{os.getpid()}/fd")) + + IndexedTarStore(self.path)["a"] + gc.collect() + before = open_fds() + for _ in range(20): + IndexedTarStore(self.path)["a"] + gc.collect() + self.assertEqual(open_fds(), before) + + def test_reload_resources(self): + """Reloading resources neither truncates archives opened for writing nor affects reads""" + with IndexedTarStore(self.path, mode="w") as store: + store["first"] = b"first payload" + store.reload_resources() + self.assertEqual(store["first"], b"first payload") + store["second"] = b"second payload" + + store = IndexedTarStore(self.path, index_tail_read_size=None) + store.reload_resources() + self.assertEqual(store["first"], b"first payload") + self.assertEqual(store["second"], b"second payload") + + def test_invalid_archives(self): + invalid_mode: Any = "a" + with self.assertRaises(ValueError): + IndexedTarStore(self.path, mode=invalid_mode) + Path(self.path).write_bytes(b"\0" * 2048) + with self.assertRaises(ValueError): + IndexedTarStore(self.path) + + def test_unsupported_index_types(self): + with IndexedTarStore(self.path, mode="w") as store: + store["a"] = b"a" + data = bytearray(Path(self.path).read_bytes()) + header = len(data) - tarfile.BLOCKSIZE # (the index header is stored in the last tar block) + magic, _, offset, size = struct.unpack_from(IndexedTarStore.INDEX_HEADER_FORMAT, data, header) + struct.pack_into(IndexedTarStore.INDEX_HEADER_FORMAT, data, header, magic, 99, offset, size) + Path(self.path).write_bytes(bytes(data)) + with self.assertRaisesRegex(TypeError, "unsupported index type 99"): + IndexedTarStore(self.path) + + def test_empty(self): + with IndexedTarStore(self.path, mode="w"): + pass + self.assertEqual(IndexedTarStore(self.path).keys(), []) + + def test_node_metadata_records(self): + """Node metadata records can be rewritten while writing, they are appended when closing (parents first)""" + with IndexedTarStore(self.path, mode="w") as store: + store["g/a/zarr.json"] = b"a v1" + store["g/a/c/0"] = b"chunk" + store["zarr.json"] = b"root v1" + store["g/a/zarr.json"] = b"a v2" # rewritten + store[".zattrs"] = b"attrs" + self.assertEqual(store["g/a/zarr.json"], b"a v2") # readable while writing + self.assertEqual(len(store), 4) + self.assertIn("zarr.json", store) + with self.assertRaises(ValueError): # other records are write-once + store["g/a/c/0"] = b"again" + with self.assertRaises(ValueError): # closed + store["zarr.json"] = b"root v2" + + store = IndexedTarStore(self.path) + self.assertEqual(store.keys(), ["g/a/c/0", ".zattrs", "zarr.json", "g/a/zarr.json"]) + self.assertEqual(store["g/a/zarr.json"], b"a v2") + self.assertEqual(store["zarr.json"], b"root v1") + with tarfile.open(self.path) as tar: # (written once each) + self.assertEqual([m.name for m in tar.getmembers()], store.keys()) + + def test_node_metadata_keys(self): + for key in (".zarray", ".zgroup", ".zattrs", "zarr.json", "a/b/.zarray", "a/zarr.json"): + self.assertTrue(is_node_metadata_key(key), key) + for key in ("zarr.json.cbor.zlib", ".zmetadata.cbor.xz", "a/0.0", "a/c/0/0", "a/x.zarray", "a/zarr.json/0"): + self.assertFalse(is_node_metadata_key(key), key) + + +class TestStores(unittest.TestCase): + """zarr hierarchies in indexed tar archives and directories""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.itar_path = os.path.join(self._tmp.name, "store.itar") + self.directory_path = os.path.join(self._tmp.name, "store.zarr") + + def tearDown(self): + self._tmp.cleanup() + + def _store(self, store_type: str, mode: str) -> StoreLike: + if store_type == "itar": + return IndexedTarStore(self.itar_path, mode="w" if mode == "w" else "r") + return open_directory_store(self.directory_path) @parameterized.parameterized.expand( - itertools.product( - INDEX_TAIL_READ_SIZES, - ) + [ + (store_type, consolidated, zarr_format) + for store_type in ("itar", "directory") + for consolidated in (True, False) + for zarr_format in ZARR_FORMATS + ] ) - def test_reserialization(self, index_tail_read_size: Optional[int]): - """Make sure storing / loading of regular zarr data to .itar files works correctly""" + def test_reserialization(self, store_type: str, consolidated: bool, zarr_format: int): + _require(self, zarr_format) + _write(self._store(store_type, "w"), zarr_format=zarr_format) - # re-serialize to .itar archive - with tempfile.NamedTemporaryFile(suffix=".itar") as f: - with IndexedTarStore(f.name, mode="w") as s_itar_out: # closes file on exit - zarr.copy_store(self.g_ref.store, s_itar_out) + store = self._store(store_type, "r") + root = open_store(store, consolidated) + _check_reference(self, root) + self.assertEqual(root.zarr_format, zarr_format) + self.assertEqual(root.array("subgroup/foo").zarr_format, zarr_format) - # reload store from file - store = IndexedTarStore(f.name, index_tail_read_size=index_tail_read_size) - g_reload = zarr.open(store=store, mode="r") + if isinstance(store, IndexedTarStore): + self.assertIs(group_store(root), store) + self.assertIs(group_store(root.group("subgroup")), store) + store.reload_resources() + _check_reference(self, root) - # check all data was correctly serialized / deserialized - self.check_with_reference(g_reload) + @parameterized.parameterized.expand( + [(size, zarr_format) for size in INDEX_TAIL_READ_SIZES for zarr_format in ZARR_FORMATS] + ) + def test_consolidated_open_reads_no_node_metadata(self, index_tail_read_size: Optional[int], zarr_format: int): + _require(self, zarr_format) + _write(self._store("itar", "w"), zarr_format=zarr_format) - # check reloading resources is functional - store.reload_resources() - self.check_with_reference(g_reload) + store = IndexedTarStore(self.itar_path, index_tail_read_size=index_tail_read_size) + with mock.patch.object( + IndexedTarStore, "__getitem__", autospec=True, side_effect=IndexedTarStore.__getitem__ + ) as getitem: + _check_reference(self, open_store(store, consolidated=True)) + read_keys = [call.args[1] for call in getitem.call_args_list] + self.assertIn(V2_CONSOLIDATED_METADATA_KEY if zarr_format == 2 else V3_CONSOLIDATED_METADATA_KEY, read_keys) + self.assertEqual([key for key in read_keys if is_node_metadata_key(key)], []) @parameterized.parameterized.expand( - itertools.product( - INDEX_TAIL_READ_SIZES, - ) + [(store_type, zarr_format) for store_type in ("itar", "directory") for zarr_format in ZARR_FORMATS] ) - def test_compressed_consolidated(self, index_tail_read_size: Optional[int]): - """Make sure compressed consolidated meta data is stored/loaded correctly""" + def test_missing_consolidated_metadata(self, store_type: str, zarr_format: int): + _require(self, zarr_format) + _write(self._store(store_type, "w"), consolidate=False, zarr_format=zarr_format) + with self.assertRaises(KeyError): + open_store(self._store(store_type, "r"), consolidated=True) + _check_reference(self, open_store(self._store(store_type, "r"), consolidated=False)) - # serialize to .itar archive (will also serialize compressed-consolidated meta-data) - with tempfile.NamedTemporaryFile(suffix=".itar") as f: - with IndexedTarStore(f.name, mode="w") as s_itar_out: # closes file on exit - zarr.copy_store(self.g_ref.store, s_itar_out) + @parameterized.parameterized.expand([(zarr_format,) for zarr_format in ZARR_FORMATS]) + def test_unconsolidated_open_reads_node_metadata(self, zarr_format: int): + """Hierarchies opened without consolidated metadata are resolved from their node metadata records (e.g., nodes + added after consolidating, as component writers extending existing directory stores)""" + _require(self, zarr_format) + root = create_root_group(store := self._store("directory", "w"), _ROOT_ATTRIBUTES, zarr_format=zarr_format) + root.create_array("before", np.zeros(2)) + consolidate_store(store) + root.create_array("after", np.ones(2)) - # consolidate compress meta-data - consolidate_compressed_metadata(s_itar_out) + self.assertEqual( + sorted(open_store(self._store("directory", "r"), consolidated=False).members()), ["after", "before"] + ) + self.assertEqual(sorted(open_store(self._store("directory", "r"), consolidated=True).members()), ["before"]) - # reload store from file with compressed consolidated meta-data - store = IndexedTarStore(f.name, index_tail_read_size=index_tail_read_size) - g_reload = open_compressed_consolidated(store=store, mode="r") + @parameterized.parameterized.expand([(zarr_format,) for zarr_format in ZARR_FORMATS]) + def test_stores_are_identical(self, zarr_format: int): + """Indexed tar archives and directories store the same records""" + _require(self, zarr_format) + _write(self._store("itar", "w"), zarr_format=zarr_format) + _write(self._store("directory", "w"), zarr_format=zarr_format) - # check all data was correctly serialized / deserialized - self.check_with_reference(g_reload) + archive = IndexedTarStore(self.itar_path) + files = { + str(path.relative_to(self.directory_path)): path.read_bytes() + for path in Path(self.directory_path).rglob("*") + if path.is_file() + } + self.assertEqual(sorted(archive.keys()), sorted(files)) + for key in archive.keys(): + if key == V2_CONSOLIDATED_METADATA_KEY: # (ordered by the record order of the stores) + self.assertEqual(decode_cbor_xz(archive[key]), decode_cbor_xz(files[key])) + elif key == V3_CONSOLIDATED_METADATA_KEY: + self.assertEqual(_decode_v3_root(archive[key]), _decode_v3_root(files[key])) + else: + self.assertEqual(archive[key], files[key], key) - # check reloading resources is functional - store.reload_resources() - self.check_with_reference(g_reload) + def test_directory_stores_are_not_truncated(self): + Path(self.directory_path).mkdir() + (Path(self.directory_path) / "other").write_bytes(b"other") + _write(self._store("directory", "w")) + self.assertEqual((Path(self.directory_path) / "other").read_bytes(), b"other") + + #: sha256 of the records of the hierarchy of :meth:`_write_golden` written by ncore 19.8 (zarr-python 2) + GOLDEN_RECORDS = { + ".zattrs": "0e326aaa71db8ecc9051e106a4a70695515ad95a95ff348445699fd79f907623", + ".zgroup": "2383746e67b4bcc2762b3f100f06c3fa2d5f149ab5a8e5da5d33521464a01959", + ".zmetadata.cbor.xz": "245a85421d308c67ea2e452746f0628fe0b3bb3a337af3fec3b46e75b044a4cc", + "foo/.zarray": "28db182434d2702c2ccc384e0c1d386359f308bd275821655377b09a40c610fa", + "foo/0.0.0": "e9361a616760d41ac367d54b2349f18184e8a226563ca34047e691cf393a9610", + "foo/1.0.0": "8611647425ac52bdfd586f7342bdb768d363ae139aadafa7130c7a4368015614", + "subgroup/.zattrs": "b638e55387a56bef81862ccd44889e0068a67b04cf00a4c4e90783cf2940e160", + "subgroup/.zgroup": "2383746e67b4bcc2762b3f100f06c3fa2d5f149ab5a8e5da5d33521464a01959", + "subgroup/bar/.zarray": "6c83cd89670edfe4f9f12cf48f7de5f55ed9cefbfa38b168f1ab14dfb082c449", + "subgroup/bar/0": "3c7acfa845b57df9e3a46779d4f17c7eb9d697d63dd8b2c30c176c6fec90051b", + "subgroup/blob/.zarray": "dcb9ebabd3bb2d803fd59453b16ba414f6d55d4451ba597170b4a90fae4e749b", + "subgroup/blob/0": "5399a4c8461dd3fd590854ff73ec4e88db9462165e9160375113dfec0c741e0c", + } + + def _write_golden(self) -> IndexedTarStore: + store = IndexedTarStore(self.itar_path, mode="w") + root = create_root_group(store, {"some": "thing", "nested": {"a": [1, 2.5, None]}}) + root.create_array( + "foo", + np.arange(27, dtype=np.float64).reshape(3, 3, 3), + chunks=(2, 3, 3), + compression=nodes.BloscCompression(shuffle="shuffle"), + ) + sub = root.create_group("subgroup", attributes={"other": [1, 2, 3]}) + sub.create_array("bar", np.arange(10, dtype=np.uint16), compression=None) + sub.create_bytes_array("blob", b"\x89PNG\x00data") + consolidate_store(store) + close_store(store) + return IndexedTarStore(self.itar_path) + + @unittest.skipIf(ZARR_PYTHON_3, "zarr-python 2 metadata serialization") + def test_archives_of_ncore_19_8(self): + """Records are byte-identical to records written by ncore 19.8, archives are deterministic (node metadata + records are appended when closing archives)""" + store = self._write_golden() + self.assertEqual({key: hashlib.sha256(store[key]).hexdigest() for key in store}, self.GOLDEN_RECORDS) + # data records in creation order, followed by node metadata records (parents first) + self.assertEqual( + store.keys(), + [ + "foo/0.0.0", + "foo/1.0.0", + "subgroup/bar/0", + "subgroup/blob/0", + ".zmetadata.cbor.xz", + ".zattrs", + ".zgroup", + "foo/.zarray", + "subgroup/.zattrs", + "subgroup/.zgroup", + "subgroup/bar/.zarray", + "subgroup/blob/.zarray", + ], + ) + digest = hashlib.sha256(Path(self.itar_path).read_bytes()).hexdigest() + self.assertIn( + digest, + { + # (python's tarfile module encodes the device numbers of tar headers differently before python 3.9) + "f2f98df32a018a2f178f3561e21bb14eb3bba9da51e00956bb09ed53177000ea", # python 3.8 + "10c02f6fdbaad3c048af23c47eb50fcb539d660d1487df6c42c6e14c14b98a13", # python >= 3.9 + }, + ) + + def test_archives_equivalent_to_ncore_19_8(self): + """Chunk records are byte-identical to chunk records written by ncore 19.8, node metadata documents are + equivalent (zarr-python 3 serializes them differently, and stores empty attributes of arrays)""" + store = self._write_golden() + # (zarr-python 3 writes the chunks of an array concurrently, in any order) + chunks = sorted(key for key in store if not is_node_metadata_key(key) and key != ".zmetadata.cbor.xz") + self.assertEqual(chunks, ["foo/0.0.0", "foo/1.0.0", "subgroup/bar/0", "subgroup/blob/0"]) + self.assertEqual( + {key: hashlib.sha256(store[key]).hexdigest() for key in chunks}, + {key: self.GOLDEN_RECORDS[key] for key in chunks}, + ) + + metadata = decode_cbor_xz(store[".zmetadata.cbor.xz"])["metadata"] + self.assertEqual(metadata, {key: json.loads(store[key]) for key in store if is_node_metadata_key(key)}) + self.assertEqual(metadata[".zattrs"], {"some": "thing", "nested": {"a": [1, 2.5, None]}}) + self.assertEqual(metadata["subgroup/.zattrs"], {"other": [1, 2, 3]}) + self.assertEqual(metadata.get("foo/.zattrs", {}), {}) + self.assertEqual( + {key: value for key, value in metadata["foo/.zarray"].items() if key != "dimension_separator"}, + { + "shape": [3, 3, 3], + "chunks": [2, 3, 3], + "dtype": " int: + return threading.get_ident() + + def run_in_thread() -> int: + return _zarr3.run(thread()) + + self.assertEqual(_zarr3.run(thread()), threading.get_ident()) + with concurrent.futures.ThreadPoolExecutor(2) as pool: + for future in [pool.submit(run_in_thread) for _ in range(4)]: + self.assertNotEqual(future.result(), threading.get_ident()) + + # offloaded work (e.g., of codecs) runs inline + async def offloaded() -> int: + return await asyncio.get_running_loop().run_in_executor(None, threading.get_ident) + + self.assertEqual(_zarr3.run(offloaded()), threading.get_ident()) + + def test_coroutines_within_event_loops(self): + from ncore.impl.data import _zarr3 + + async def value() -> int: + return 42 + + async def main() -> int: + return _zarr3.run(value()) # (runs on zarr-python's I/O loop) + + self.assertEqual(asyncio.run(main()), 42) + + @unittest.skipUnless(hasattr(os, "fork"), "requires fork") + def test_forked_processes(self): + """Forked processes don't reuse the event loop of the forking thread (its selector is shared with the parent)""" + from ncore.impl.data import _zarr3 + + async def loop_and_pid() -> Tuple[int, int]: + return id(asyncio.get_running_loop()), os.getpid() + + parent_loop, _ = _zarr3.run(loop_and_pid()) # (creates the loop of this thread) + read_fd, write_fd = os.pipe() + if (pid := os.fork()) == 0: # child (continues on this thread) + try: + child_loop, child_pid = _zarr3.run(loop_and_pid()) + os.write(write_fd, b"ok" if child_loop != parent_loop and child_pid == os.getpid() else b"reused") + finally: + os._exit(0) + os.close(write_fd) + result = os.read(read_fd, 16) + os.close(read_fd) + os.waitpid(pid, 0) + self.assertEqual(result, b"ok") + self.assertEqual(_zarr3.run(loop_and_pid())[0], parent_loop) # (the parent keeps its loop) + + # the parent's loop is still woken by other threads (children don't close the selector shared with the parent) + async def woken_by_thread() -> int: + loop, future = asyncio.get_running_loop(), asyncio.get_running_loop().create_future() + timer = threading.Timer(0.05, lambda: loop.call_soon_threadsafe(future.set_result, 42)) + timer.start() + return await asyncio.wait_for(future, timeout=5) + + self.assertEqual(_zarr3.run(woken_by_thread()), 42) + + def test_byte_ranges(self): + from ncore.impl.data import _zarr3 + from ncore.impl.data._zarr3 import ( + OffsetByteRequest, + RangeByteRequest, + SuffixByteRequest, + default_buffer_prototype, + ) + + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "store.itar") + with IndexedTarStore(path, mode="w") as store: + store["r"] = b"0123456789" + tar_store = _zarr3._TarStore(IndexedTarStore(path)) + + def get(byte_range: Any) -> Optional[bytes]: + value = _zarr3.run(tar_store.get("r", default_buffer_prototype(), byte_range)) + return None if value is None else value.to_bytes() + + self.assertEqual(get(None), b"0123456789") + self.assertEqual(get(RangeByteRequest(2, 5)), b"234") + self.assertEqual(get(RangeByteRequest(8, 20)), b"89") + self.assertEqual(get(OffsetByteRequest(7)), b"789") + self.assertEqual(get(SuffixByteRequest(3)), b"789") + self.assertEqual(get(SuffixByteRequest(30)), b"0123456789") + self.assertIsNone(_zarr3.run(tar_store.get("missing", default_buffer_prototype()))) + + +@unittest.skipUnless(ZARR_PYTHON_3, "zarr-python 3 backend") +class TestDirectReads(unittest.TestCase): + """Direct chunk reads of arrays of indexed tar archives (zarr-python 3) are identical to zarr-python's reads""" + + #: Arrays (dtype, shape, chunks, compression, missing chunks) of all supported kinds + ARRAYS = { + "f4_multi": (np.float32, (7, 5, 3), (3, 2, 3), nodes.BloscCompression(), ()), + "u2_single_uncompressed": (np.uint16, (11,), (11,), None, ()), + "i8_multi_uncompressed": (np.int64, (9, 4), (4, 3), None, ()), + "bool": (np.bool_, (6, 6), (4, 4), nodes.BloscCompression(cname="zstd", shuffle="noshuffle"), ()), + "u1_partial_last": (np.uint8, (13,), (5,), nodes.BloscCompression(shuffle="shuffle"), ()), + "f8_missing": (np.float64, (6, 2), (2, 2), nodes.BloscCompression(), ("1.0", "c/1/0")), + "f4_empty": (np.float32, (0, 3), (1, 3), nodes.BloscCompression(), ()), + } + + SELECTIONS = [ + Ellipsis, + (), + 0, + -1, + (slice(1, 4),), + (slice(None), 1), + (2, slice(0, 2)), + (slice(2, 2),), + (Ellipsis, 0), + (slice(-3, None),), + (0, slice(None), slice(1, 2)), # (dropped axes next to selected single elements) + (slice(0, 3), slice(0, 2), 1), # (complete chunks, not complete arrays) + ] + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + + def tearDown(self): + self._tmp.cleanup() + + def _write(self, zarr_format: int, skip_records: Tuple[str, ...] = ()) -> str: + """Writes all arrays (chunks of `missing chunks` dropped from the archive) and returns the archive path""" + rng = np.random.default_rng(1) + path = os.path.join(self._tmp.name, f"v{zarr_format}.itar") + full = os.path.join(self._tmp.name, f"v{zarr_format}.full.itar") + root = create_root_group(store := IndexedTarStore(full, mode="w"), {}, zarr_format=zarr_format) + for name, (dtype, shape, chunks, compression, _) in self.ARRAYS.items(): + data = rng.random(shape) > 0.5 if dtype == np.bool_ else (rng.random(shape) * 1000).astype(dtype) + root.create_array(name, data, chunks=chunks, compression=compression) + root.create_bytes_array("blob", b"\x89PNG\x00binary") + root.create_bytes_array("blob_missing", b"some bytes") + consolidate_store(store) + close_store(store) + + # copy without the missing chunks (stores of hierarchies written partially, e.g. by other writers) + source = IndexedTarStore(full) + missing = {f"{name}/{key}" for name, spec in self.ARRAYS.items() for key in spec[4]} + missing |= {"blob_missing/0", "blob_missing/c/0"} + with IndexedTarStore(path, mode="w") as copy: + for key in source.keys(): + if key not in missing: + copy[key] = source[key] + return path + + def _reads(self, path: str) -> Dict[str, Any]: + root = open_store(IndexedTarStore(path)) + out: Dict[str, Any] = {} + for name, (_, shape, *_rest) in self.ARRAYS.items(): + array = root.array(name) + for selection in self.SELECTIONS: + try: + out[f"{name}[{selection!r}]"] = array.read(selection) + except IndexError as e: # (e.g., integer selections of empty arrays) + out[f"{name}[{selection!r}]"] = type(e) + out["blob"] = root.array("blob").read_bytes() + # (fill value of missing chunks; zarr-python returns empty fixed-length bytes of dtype |S0) + out["blob_missing"] = root.array("blob_missing").read_bytes() + return out + + @parameterized.parameterized.expand([(zarr_format,) for zarr_format in ZARR_FORMATS]) + def test_direct_reads_match_zarr(self, zarr_format: int): + from ncore.impl.data import _zarr3 + + path = self._write(zarr_format) + with mock.patch.object(_zarr3, "_read_direct", wraps=_zarr3._read_direct) as direct: + actual = self._reads(path) + self.assertGreater(direct.call_count, len(actual) - 10) # (direct reads were used) + with mock.patch.dict(os.environ, {_zarr3.DIRECT_READS_ENV: "0"}): + with mock.patch.object(_zarr3, "_read_direct", side_effect=AssertionError("direct read")): + expected = self._reads(path) + + self.assertEqual(sorted(actual), sorted(expected)) + for key, value in expected.items(): + if isinstance(value, type): + self.assertIs(actual[key], value, key) + continue + actual_value = actual[key] + if isinstance(value, bytes): + self.assertEqual(actual_value, value, key) + continue + value = np.asarray(value) + self.assertEqual((actual_value.dtype, actual_value.shape), (value.dtype, value.shape), key) + np.testing.assert_array_equal(actual_value, value, key) + self.assertTrue(actual_value.flags.writeable, key) + + def test_missing_chunks_of_fixed_length_bytes(self): + """Missing chunks of fixed-length bytes read as the fill value (empty bytes), as with zarr-python""" + root = open_store(IndexedTarStore(self._write(2))) + self.assertEqual(root.array("blob_missing").read_bytes(), b"") + + def test_unsupported_selections_use_zarr(self): + from ncore.impl.data import _zarr3 + + root = open_store(IndexedTarStore(self._write(3))) + array = root.array("f4_multi") + with mock.patch.object(_zarr3, "_read_direct", side_effect=AssertionError("direct read")): + expected = array.read((slice(0, 6, 2),)) # (non-unit step) + np.testing.assert_array_equal(expected, array.read()[0:6:2]) + + def test_partial_chunk_selections(self): + """Selections of parts of compressed chunks of the output array (no in-place decompression)""" + root = open_store(IndexedTarStore(self._write(2))) + data = root.array("f4_multi").read() + for selection in [(slice(1, 3), slice(0, 2)), (slice(0, 3), slice(0, 2), slice(0, 2)), (slice(3, 7),)]: + np.testing.assert_array_equal(root.array("f4_multi").read(selection), data[selection], str(selection)) + + def test_stores_opened_for_writing_use_zarr(self): + """(node metadata of archives opened for writing is not final)""" + from ncore.impl.data import _zarr3 + + root = create_root_group(IndexedTarStore(os.path.join(self._tmp.name, "w.itar"), mode="w"), {}) + self.assertIsNone(_zarr3._direct_read(root.create_array("a", np.arange(4)).zarr)) + + def test_non_native_byte_order_uses_zarr(self): + from ncore.impl.data import _zarr3 + + path = os.path.join(self._tmp.name, "be.itar") + data = np.arange(6, dtype=">u4") + root = create_root_group(store := IndexedTarStore(path, mode="w"), {}) + root.create_array("a", data) + consolidate_store(store) + close_store(store) + array = open_store(IndexedTarStore(path)).array("a") + self.assertIsNone(_zarr3._direct_read(array.zarr)) + np.testing.assert_array_equal(array.read(), data) + + def test_directory_stores_use_zarr(self): + from ncore.impl.data import _zarr3 + + root = create_root_group(store := open_directory_store(os.path.join(self._tmp.name, "dir")), {}) + root.create_array("a", np.arange(4)) + consolidate_store(store) + self.assertIsNone( + _zarr3._direct_read(open_store(open_directory_store(os.path.join(self._tmp.name, "dir"))).array("a").zarr) + ) + + def test_large_chunks_are_decoded_concurrently(self): + from ncore.impl.data import _zarr3 + + path = os.path.join(self._tmp.name, "large.itar") + data = np.arange(4 * (1 << 20), dtype=np.float32).reshape(4, -1) # (4 chunks of 4 MiB) + root = create_root_group(store := IndexedTarStore(path, mode="w"), {}) + root.create_array("a", data, chunks=(1, data.shape[1])) + consolidate_store(store) + close_store(store) + + array = open_store(IndexedTarStore(path)).array("a") + threads: List[str] = [] + decode = np.frombuffer + + def record(*args: Any, **kwargs: Any) -> Any: + threads.append(threading.current_thread().name) + return decode(*args, **kwargs) + + with mock.patch.object(_zarr3.np, "frombuffer", side_effect=record): + np.testing.assert_array_equal(array.read((slice(None), slice(0, 10))), data[:, :10]) + self.assertEqual(len(threads), 4) + self.assertTrue(all(name.startswith("ncore_zarr3_chunks") for name in threads), threads) + + # reads beyond the bound decode their chunks on the calling thread + threads.clear() + parallel = _zarr3._parallel() + for _ in range(_zarr3._PARALLEL_READS): + parallel.semaphore.acquire() + try: + with mock.patch.object(_zarr3.np, "frombuffer", side_effect=record): + array.read((slice(None), slice(0, 10))) + finally: + for _ in range(_zarr3._PARALLEL_READS): + parallel.semaphore.release() + self.assertEqual(threads, [threading.current_thread().name] * 4) + + @unittest.skipUnless(hasattr(os, "fork"), "requires fork") + def test_forked_processes(self): + """Forked processes don't inherit the parallel reads state (its locks / threads are unusable in children)""" + from ncore.impl.data import _zarr3 + + parent = _zarr3._parallel() + read_fd, write_fd = os.pipe() + if (pid := os.fork()) == 0: + try: + os.write(write_fd, b"ok" if _zarr3._parallel() is not parent else b"inherited") + finally: + os._exit(0) + os.close(write_fd) + result = os.read(read_fd, 16) + os.close(read_fd) + os.waitpid(pid, 0) + self.assertEqual(result, b"ok") + self.assertIs(_zarr3._parallel(), parent) + + +class TestAsyncReads(unittest.TestCase): + """Asynchronous reads return the same data as synchronous reads, without blocking the running event loop""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.zarr_format = max(nodes.SUPPORTED_ZARR_FORMATS) + self.path = os.path.join(self._tmp.name, "store.itar") + _write(IndexedTarStore(self.path, mode="w"), zarr_format=self.zarr_format) + + def tearDown(self): + self._tmp.cleanup() + + def _reads(self, root: nodes.Group) -> List[Any]: + return [ + root.array("foo").read(), + root.array("subgroup/foo").read((slice(1, 3),)), + root.array("subgroup/foo").read((2, slice(None), 0)), + root.array("subgroup/empty").read(), + root.array("subgroup/blob").read_bytes(), + ] + + async def _reads_async(self, root: nodes.Group) -> List[Any]: + return list( + await asyncio.gather( + root.array("foo").read_async(), + root.array("subgroup/foo").read_async((slice(1, 3),)), + root.array("subgroup/foo").read_async((2, slice(None), 0)), + root.array("subgroup/empty").read_async(), + root.array("subgroup/blob").read_bytes_async(), + ) + ) + + def _assert_equal(self, actual: List[Any], expected: List[Any]) -> None: + self.assertEqual(len(actual), len(expected)) + for a, e in zip(actual, expected): + if isinstance(e, bytes): + self.assertEqual(a, e) + else: + self.assertEqual((a.dtype, a.shape), (e.dtype, e.shape)) + np.testing.assert_array_equal(a, e) + + @parameterized.parameterized.expand([("direct",), ("zarr",)]) + def test_async_reads(self, mode: str): + root = open_store(IndexedTarStore(self.path)) + expected = self._reads(root) + with mock.patch.dict(os.environ, {"NCORE_ZARR3_DIRECT_READS": "1" if mode == "direct" else "0"}): + self._assert_equal(asyncio.run(self._reads_async(root)), expected) # (an application's event loop) + if ZARR_PYTHON_3: + from ncore.impl.data import _zarr3 + + self._assert_equal(_zarr3.run(self._reads_async(root)), expected) # (the loop of the calling thread) + + @parameterized.parameterized.expand([("direct",), ("zarr",)]) + def test_async_reads_dont_block_event_loops(self, mode: str): + """Blocking parts of reads awaited on event loops of applications (record reads, decompression) run on + worker threads""" + root = open_store(IndexedTarStore(self.path)) + expected = self._reads(root) + threads = set() + getitem = IndexedTarStore.__getitem__ + + def recording_getitem(store: IndexedTarStore, key: str) -> bytes: + threads.add(threading.current_thread().name) + return getitem(store, key) + + with mock.patch.dict(os.environ, {"NCORE_ZARR3_DIRECT_READS": "1" if mode == "direct" else "0"}): + with mock.patch.object(IndexedTarStore, "__getitem__", recording_getitem): + self._assert_equal(asyncio.run(self._reads_async(root)), expected) + self.assertTrue(threads, "no record reads") + self.assertNotIn(threading.current_thread().name, threads) + + @parameterized.parameterized.expand([("direct",), ("zarr",)]) + def test_async_reads_on_the_loop_of_the_calling_thread_run_inline(self, mode: str): + """Reads awaited on the event loop of the calling thread (see _zarr3.run) don't hand off to threads""" + if not ZARR_PYTHON_3: + self.skipTest("event loops of the calling threads (zarr-python 3)") + from ncore.impl.data import _zarr3 + + root = open_store(IndexedTarStore(self.path)) + expected = self._reads(root) + threads = set() + getitem = IndexedTarStore.__getitem__ + + def recording_getitem(store: IndexedTarStore, key: str) -> bytes: + threads.add(threading.current_thread().name) + return getitem(store, key) + + with mock.patch.dict(os.environ, {"NCORE_ZARR3_DIRECT_READS": "1" if mode == "direct" else "0"}): + with mock.patch.object(IndexedTarStore, "__getitem__", recording_getitem): + self._assert_equal(_zarr3.run(self._reads_async(root)), expected) + self.assertEqual(threads, {threading.current_thread().name}) + + def test_sync_reads_within_event_loops(self): + """Synchronous reads also work within running event loops""" + root = open_store(IndexedTarStore(self.path)) + expected = self._reads(root) + + async def main() -> List[Any]: + return self._reads(open_store(IndexedTarStore(self.path))) + + self._assert_equal(asyncio.run(main()), expected) + + @unittest.skipUnless(ZARR_PYTHON_3, "range requests of remote archives (zarr-python 3)") + def test_remote_range_requests(self): + """Chunks of remote archives are read via asynchronous range requests on the event loop of their filesystem, + also when awaited on other event loops (e.g., fsspec's http filesystem binds its sessions to fsspec's I/O + loop)""" + import fsspec.asyn + + class LoopBoundFileSystem(fsspec.asyn.AsyncFileSystem): + """A filesystem whose coroutines only work on its own event loop (as fsspec's http filesystem)""" + + async_impl = True + + async def _cat_file(self, path: str, start: Optional[int] = None, end: Optional[int] = None, **_: Any): + if asyncio.get_running_loop() is not self.loop: + raise RuntimeError("Timeout context manager should be used inside a task") + requests.append((start, end)) + with open(path, "rb") as f: + f.seek(start or 0) + return f.read(None if end is None else end - (start or 0)) + + requests: List[Tuple[Optional[int], Optional[int]]] = [] + fs = LoopBoundFileSystem(loop=fsspec.asyn.get_loop()) + store = IndexedTarStore(self.path, index_tail_read_size=None) # (chunks aren't read with the index) + root = open_store(store) + expected = self._reads(root) + with mock.patch.object(IndexedTarStore, "_async_fs", lambda self: fs): + self.assertTrue(store.is_remote) + self._assert_equal(asyncio.run(self._reads_async(root)), expected) + self.assertGreaterEqual(len(requests), 4) # (range requests of the chunks) + self.assertFalse(IndexedTarStore(self.path).is_remote) # (local archives use positional reads) + + +@parameterized.parameterized_class(("zarr_format",), [(zarr_format,) for zarr_format in ZARR_FORMATS]) +class TestNodes(unittest.TestCase): + """Typed node API""" + + zarr_format: int + + def setUp(self): + _require(self, self.zarr_format) + self._tmp = tempfile.TemporaryDirectory() + self.root = create_root_group( + open_directory_store(self._tmp.name), + {"s": "x", "i": 3, "b": True, "l": [1], "d": {"k": 1}}, + zarr_format=self.zarr_format, ) + + def tearDown(self): + self._tmp.cleanup() + + def test_typed_attributes(self): + self.assertEqual(self.root.attr_str("s"), "x") + self.assertEqual(self.root.attr_int("i"), 3) + self.assertEqual(self.root.attr_list("l"), [1]) + self.assertEqual(self.root.attr_dict("d"), {"k": 1}) + with self.assertRaises(TypeError): + self.root.attr_int("b") # (booleans aren't integers) + with self.assertRaises(TypeError): + self.root.attr_str("i") + with self.assertRaises(KeyError): + self.root.attr_str("missing") + + self.root.update_attrs({"i": 4, "new": None}) + self.assertEqual(self.root.attrs, {"s": "x", "i": 4, "b": True, "l": [1], "d": {"k": 1}, "new": None}) + + def test_arrays(self): + array = self.root.create_array("a", [1.5, 2.5], dtype=np.float32, attributes={"unit": "m"}) + self.assertEqual((array.shape, array.dtype, array.attrs), ((2,), np.dtype(np.float32), {"unit": "m"})) + self.assertEqual(array.compression, nodes.BloscCompression()) + np.testing.assert_array_equal(array.read((slice(1, 2),)), [2.5]) + self.assertEqual(array.read((0,)).shape, ()) # 0-d arrays for scalar selections + + self.assertIsNone(self.root.create_array("u", np.arange(3), compression=None).compression) + self.assertEqual(self.root.create_array("c", np.zeros((10, 0)), chunks=(4, 0)).zarr.chunks, (4, 1)) + with self.assertRaises(TypeError): + array.read_bytes() + + def test_bytes_arrays(self): + for data in (b"\x89PNG\x00binary", b"", b"\x00"): + array = self.root.create_bytes_array(f"blob{len(data)}", data, attributes={"format": "png"}) + self.assertEqual(array.zarr_format, self.zarr_format) + self.assertEqual(array.read_bytes(), data.rstrip(b"\x00") if self.zarr_format == 2 else data) + self.assertEqual(array.attrs, {"format": "png"}) + self.assertIsNone(array.compression) + # (zarr format v2: fixed-length bytes, as ncore <= 19.8, which strip trailing NUL bytes) + self.assertEqual(self.root.array("blob11").dtype, np.dtype("S11") if self.zarr_format == 2 else np.uint8) + + @parameterized.parameterized.expand( + [(None,), (nodes.BloscCompression(),), (nodes.BloscCompression("zstd", 3, "shuffle", 256),)] + ) + def test_compression(self, compression: Optional[nodes.BloscCompression]): + array = self.root.create_array("a", np.arange(1000, dtype=np.int32), compression=compression) + self.assertEqual(array.compression, compression) + np.testing.assert_array_equal(array.read(), np.arange(1000)) + + def test_groups(self): + self.root.create_group("b") + self.root.create_group("a").create_group("nested") + self.assertEqual([name for name, _ in self.root.groups()], ["a", "b"]) + self.assertEqual(self.root.require_group("a").group("nested").path, "a/nested") + self.assertEqual(self.root.group("a/nested").path, "a/nested") + with self.assertRaises(Exception): + self.root.create_group("a") + + def test_nodes(self): + """Members of unknown kind (groups or arrays)""" + self.root.create_group("g").create_array("a", np.arange(3)) + assert isinstance(group := self.root.node("g"), nodes.Group) + self.assertEqual(group.path, "g") + assert isinstance(array := self.root.node("g/a"), nodes.Array) + np.testing.assert_array_equal(array.read(), np.arange(3)) + with self.assertRaises(KeyError): + self.root.node("g/missing") + + +class TestNodeCache(unittest.TestCase): + """Node caches of groups (disabled / unbounded / least recently used)""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + path = os.path.join(self._tmp.name, "store.itar") + store = IndexedTarStore(path, mode="w") + sub = create_root_group(store, {}).create_group("g") + for i in range(8): + sub.create_array(f"a{i}", np.full(3, i)) + consolidate_store(store) + close_store(store) + self.root = open_store(IndexedTarStore(path)) + + def tearDown(self): + self._tmp.cleanup() + + def test_uncached(self): + self.assertIsNone(self.root.cache) + self.assertIsNot(self.root.array("g/a1").zarr, self.root.array("g/a1").zarr) + + def test_unbounded(self): + root = self.root.with_cache(cache := nodes.NodeCache()) + for i in range(8): + np.testing.assert_array_equal(root.array(f"g/a{i}").read(), [i] * 3) + self.assertEqual(len(cache), 8) + self.assertIs(root.array("g/a3").zarr, root.array("g/a3").zarr) + + def test_member_groups_share_the_cache(self): + root = self.root.with_cache(cache := nodes.NodeCache()) + group = root.group("g") + self.assertIs(group.cache, cache) + node = root.node("g") + assert isinstance(node, nodes.Group) + self.assertIs(node.cache, cache) + self.assertIs(cast(nodes.Array, root.node("g/a2")).zarr, root.array("g/a2").zarr) + self.assertIs(group.array("a3").zarr, root.array("g/a3").zarr) + self.assertIs(next(root.groups())[1].cache, cache) + self.assertEqual(cache.keys(), ["g", "g/a2", "g/a3"]) + + def test_least_recently_used(self): + root = self.root.with_cache(cache := nodes.NodeCache(max_entries=3)) + a0 = root.array("g/a0").zarr + root.array("g/a1") + root.array("g/a2") + self.assertIs(root.array("g/a0").zarr, a0) # (a1 is the least recently used node now) + root.array("g/a3") + self.assertEqual(cache.keys(), ["g/a2", "g/a0", "g/a3"]) + for i in range(8): # evicted nodes are re-opened + np.testing.assert_array_equal(root.array(f"g/a{i}").read(), [i] * 3) + self.assertEqual(len(cache), 3) + + def test_missing_nodes_are_not_cached(self): + root = self.root.with_cache(cache := nodes.NodeCache()) + with self.assertRaises(KeyError): + root.array("g/missing") + self.assertEqual(len(cache), 0) + + def test_invalid_sizes(self): + with self.assertRaises(ValueError): + nodes.NodeCache(max_entries=0) + + +class TestBloscCompression(unittest.TestCase): + @parameterized.parameterized.expand( + [({"cname": "snappy"},), ({"clevel": 10},), ({"clevel": -1},), ({"shuffle": "byte"},), ({"blocksize": -1},)] ) - def test_empty(self, compressed_consolidate: bool, index_tail_read_size: Optional[int]): - """Verify edge case of serialization of empty store is possible without errors""" - with tempfile.NamedTemporaryFile(suffix=".itar") as f: - with IndexedTarStore(f.name, mode="w") as s_itar_out: # closes file on exit - # Don't write any zarr data (still serializes empty tar / seek tables) - - if compressed_consolidate: - consolidate_compressed_metadata(s_itar_out) - - with IndexedTarStore(f.name, index_tail_read_size=index_tail_read_size) as s_itar_in: - # Loading store should work without errors - - # But loading a non-existing group should then fail - with self.assertRaises(zarr.errors.PathNotFoundError): - if compressed_consolidate: - open_compressed_consolidated(s_itar_in, mode="r") - else: - zarr.open(store=s_itar_in, mode="r") - - def test_reload_resources_preserves_write_mode_archive(self): - """Verify writer resource reload does not truncate already-written archive data.""" - with tempfile.NamedTemporaryFile(suffix=".itar") as f: - with IndexedTarStore(f.name, mode="w") as store: - store["first"] = b"first payload" - store.tar_file_object.flush() - - size_before_reload = Path(f.name).stat().st_size - self.assertGreater(size_before_reload, 0) - - store.reload_resources() - - self.assertEqual(Path(f.name).stat().st_size, size_before_reload) - self.assertEqual(store["first"], b"first payload") - - store["second"] = b"second payload" - self.assertEqual(store["first"], b"first payload") - self.assertEqual(store["second"], b"second payload") - - with IndexedTarStore(f.name, mode="r") as store: - self.assertEqual(store["first"], b"first payload") - self.assertEqual(store["second"], b"second payload") - - def test_index_tail_cache_avoids_extra_file_reads(self): - """Payloads fully covered by the index tail read must not trigger further ``read()`` calls.""" - - class _CountingReader(IO[Any]): - __slots__ = ("_f", "read_calls") - - def __init__(self, f: IO[Any]): - self._f = f - self.read_calls = 0 - - def read(self, n=-1): - self.read_calls += 1 - return self._f.read(n) - - def seek(self, *args, **kwargs): - return self._f.seek(*args, **kwargs) - - def tell(self): - return self._f.tell() - - def close(self): - return self._f.close() - - def __getattr__(self, name): - return getattr(self._f, name) - - with tempfile.NamedTemporaryFile(suffix=".itar") as f: - g_ref = zarr.open(store=zarr.MemoryStore()) - g_ref.create_dataset("x", data=np.array([1.0, 2.0])) - with IndexedTarStore(f.name, mode="w") as s_out: - zarr.copy_store(g_ref.store, s_out) - - file_size = Path(f.name).stat().st_size - self.assertGreater(file_size, 0) - store = IndexedTarStore(f.name, index_tail_read_size=file_size) - - # Wrap the file object to intercept read counts - inner = store.tar_file_object - counter = _CountingReader(inner) - store.tar_file_object = counter - self.assertEqual(counter.read_calls, 0) - - # Read every key via the tail buffer and verify no extra file reads occurred - values = {key: store[key] for key in store} - self.assertEqual(counter.read_calls, 0) - - # Verify actual data was served from the store - self.assertGreater(len(values), 0) - g_reload = zarr.open(store=store, mode="r") - self.assertIsNone(np.testing.assert_array_equal(g_ref["x"][()], g_reload["x"][()])) + def test_invalid_parameters(self, kwargs: Any): + with self.assertRaises(ValueError): + nodes.BloscCompression(**kwargs) + + @parameterized.parameterized.expand([(shuffle,) for shuffle in ("noshuffle", "shuffle", "bitshuffle")]) + def test_numcodecs(self, shuffle: Any): + compression = nodes.BloscCompression(cname="zstd", clevel=3, shuffle=shuffle, blocksize=1024) + self.assertEqual(nodes.BloscCompression.from_numcodecs(compression.to_numcodecs()), compression) + + +def _read_member(tar: tarfile.TarFile, member: tarfile.TarInfo) -> bytes: + file = tar.extractfile(member) + assert file is not None + return file.read() diff --git a/ncore/impl/data/types.py b/ncore/impl/data/types.py index e4e29cf4..3fc7a697 100644 --- a/ncore/impl/data/types.py +++ b/ncore/impl/data/types.py @@ -49,7 +49,7 @@ import numpy.typing as npt # type: ignore[import-not-found] from ncore.impl.common.transformations import PoseGraphInterpolator, transform_bbox, transform_point_cloud -from ncore.impl.data import util +from ncore.impl.data import _json, util if sys.version_info >= (3, 11): @@ -61,15 +61,7 @@ ## JSON-like structures -JsonLike = Union[ - Dict[str, "JsonLike"], - List["JsonLike"], - str, - int, - float, - bool, - None, -] +JsonLike = _json.JsonLike ## Data classes representing stored data types diff --git a/ncore/impl/data/v4/BUILD.bazel b/ncore/impl/data/v4/BUILD.bazel index b8a88a24..5b8e5ae1 100644 --- a/ncore/impl/data/v4/BUILD.bazel +++ b/ncore/impl/data/v4/BUILD.bazel @@ -99,3 +99,20 @@ pytest_test( "@rules_python//python/runfiles", ], ) + +pytest_test( + name = "pytest_formats", + srcs = ["formats_test.py"], + data = [ + "@test-data-v4-trimmed//:zarr_v2", + "@test-data-v4-trimmed//:zarr_v3", + ], + deps = [ + ":pylib_components", + "//ncore/impl/data:pylib_stores", + requirement("numpy"), + requirement("parameterized"), + requirement("universal_pathlib"), + "@rules_python//python/runfiles", + ], +) diff --git a/ncore/impl/data/v4/components.py b/ncore/impl/data/v4/components.py index bf95083d..fe160980 100644 --- a/ncore/impl/data/v4/components.py +++ b/ncore/impl/data/v4/components.py @@ -44,17 +44,13 @@ import dataclasses_json import numpy as np import PIL.Image as PILImage -import zarr -import zarr.storage -from numcodecs import Blosc from typing_extensions import Concatenate, ParamSpec from upath import UPath -from zarr._storage.store import Store from ncore.impl.common.transformations import HalfClosedInterval from ncore.impl.common.util import MD5Hasher, unpack_optional -from ncore.impl.data import stores, types, util +from ncore.impl.data import nodes, stores, types, util from ncore.impl.data.types import PointCloud @@ -72,32 +68,30 @@ VERSION = "v4" -def _normalize_chunks( - shape: "Tuple[int, ...]", - *, - require_nonzero_dims: "Optional[Tuple[int, ...]]" = None, -) -> "Tuple[int, ...]": - """Normalize a chunk shape by replacing zero dimensions with 1. +def _check_nonzero_dims(shape: Tuple[int, ...], dims: Tuple[int, ...], name: str) -> None: + """Asserts that the given dimensions of the shape of the named data are non-zero. - Zarr v2 rejects chunk tuples containing zeros (division by zero in internal - bookkeeping). This helper ensures that every dimension in the chunk tuple is - at least 1 while the *logical* data shape passed to ``create_dataset`` via - the ``data=`` argument remains unchanged. + Other dimensions are allowed to be zero (arrays with zero-sized dimensions are supported). Args: - shape: The data shape to derive a chunk shape from. - require_nonzero_dims: Optional tuple of dimension indices that MUST be - non-zero (raises AssertionError otherwise). When ``None`` (the - default) all dimensions are allowed to be zero and get clamped to 1 - in the returned chunk shape. - - Returns: - A chunk-safe tuple where any zero dimension is replaced with 1. + shape: The shape of the data. + dims: Indices of the dimensions that MUST be non-zero (raises AssertionError otherwise). + name: Name of the data (for error messages). """ - if require_nonzero_dims is not None: - for d in require_nonzero_dims: - assert shape[d] != 0, f"Dimension {d} must be non-zero, got shape {shape}" - return tuple(max(1, s) for s in shape) + for d in dims: + assert shape[d] != 0, f"Dimension {d} of {name} must be non-zero, got shape {shape}" + + +def _frame_id( + # end-of-frame timestamp, or start-of-frame / end-of-frame timestamps + timestamps_us: Union[int, npt.NDArray[np.uint64]], +) -> int: + """Returns the frame ID (the end-of-frame timestamp) of a frame""" + + if isinstance(timestamps_us, np.ndarray): + return cast(np.ndarray, timestamps_us)[1].item() # end-of-frame timestamp is frame ID + + return timestamps_us @dataclass @@ -142,6 +136,7 @@ def from_reader( store_base_name: str, sequence_reader: SequenceComponentGroupsReader, store_type: Literal["itar", "directory"] = "itar", # valid values: ['itar', 'directory'] + zarr_format: Literal[2, 3] = 2, ) -> SequenceComponentGroupsWriter: """Creates a SequenceComponentGroupsWriter from an existing SequenceComponentGroupsReader instance to share consistent per-sequence meta-data""" @@ -152,6 +147,7 @@ def from_reader( sequence_timestamp_interval_us=sequence_reader.sequence_timestamp_interval_us, generic_meta_data=sequence_reader.generic_meta_data, store_type=store_type, + zarr_format=zarr_format, ) def __init__( @@ -166,6 +162,9 @@ def __init__( generic_meta_data: Dict[str, types.JsonLike], # Zarr store type: either serialize as .itar archive store (default / production) or plain "directory" store (simpler for introspection / asynchronous / external setup) store_type: Literal["itar", "directory"] = "itar", # valid values: ['itar', 'directory'] + # On-disk zarr format of the component stores: format v2 (default) is readable with all zarr-python versions + # (and all ncore versions), format v3 requires zarr-python 3 for writing *and* reading + zarr_format: Literal[2, 3] = 2, ): """ Instantiate sequence component groups writer and initialize the default data groups and file stores for a given sequence and sensor IDs @@ -180,9 +179,10 @@ def __init__( # Individual stores for each group are initialized lazily on-demand (indexed tar file or zarr directories) self._stores_rootgroups: dict[ - str, Tuple[zarr.Group, UPath] + str, Tuple[nodes.Group, UPath] ] = {} # maps component group names to stores, store path, and base groups self._store_type = store_type + self._zarr_format = nodes.check_zarr_format(zarr_format) # registered component writers self._component_writers: dict[ @@ -201,7 +201,12 @@ def sequence_timestamp_interval_us(self) -> HalfClosedInterval: def generic_meta_data(self) -> Dict[str, types.JsonLike]: return self._generic_meta_data - def get_base_group(self, component_group_name: Optional[str]) -> zarr.Group: + @property + def zarr_format(self) -> int: + """The on-disk zarr format of the written component stores""" + return self._zarr_format + + def get_base_group(self, component_group_name: Optional[str]) -> nodes.Group: """Lazily initializes ncore base-groups and underlying stores on demand""" if component_group_name is None: @@ -209,7 +214,7 @@ def get_base_group(self, component_group_name: Optional[str]) -> zarr.Group: component_group_name = "" if (store_rootgroup := self._stores_rootgroups.get(component_group_name)) is not None: - # Store already initialized, return it's root group + # Store already initialized, return its root group return store_rootgroup[0] # Store doesn't exist yet, create it @@ -221,23 +226,21 @@ def get_base_group(self, component_group_name: Optional[str]) -> zarr.Group: # append group name as suffix to store name if given store_name += f"-{component_group_name}" - store: Store + store: stores.StoreLike if self._store_type == "itar": # container-based zarr stores ..zarr.itar store_path = self._output_dir_path / f"{self._store_base_name}.{store_name}.zarr.itar" store = stores.IndexedTarStore(store_path, mode="w") elif self._store_type == "directory": - # directory-based zarr stores ..zarr.zarr + # directory-based zarr stores ..zarr store_path = self._output_dir_path / f"{self._store_base_name}.{store_name}.zarr" - store = zarr.storage.DirectoryStore(store_path) + store = stores.open_directory_store(store_path) else: raise ValueError(f"Unknown store type {self._store_type}") - # Create root group in store - root_group = zarr.group(store=store) - - # Store dataset associated meta-data to root - root_group.attrs.put( + # Create root group in store, storing dataset associated meta-data to root + root_group = stores.create_root_group( + store, { "sequence_id": self._sequence_id, "sequence_timestamp_interval_us": { @@ -247,7 +250,8 @@ def get_base_group(self, component_group_name: Optional[str]) -> zarr.Group: "generic_meta_data": self._generic_meta_data, "version": VERSION, "component_group_name": component_group_name, - } + }, + zarr_format=self._zarr_format, ) # Create store / base-group mapping @@ -267,32 +271,27 @@ def finalize(self) -> List[UPath]: # Write deferred component metadata and generic data of all writers for component_writer in self._component_writers.values(): - # Merge generic_meta_data with regular component's meta data - (cw_group := component_writer._group).attrs.put( + # Merge generic_meta_data with regular component's meta data (component groups are created without + # attributes by register_component_writer(), so this sets the complete component attributes) + (cw_group := component_writer._group).update_attrs( {**component_writer._component_meta_data, "generic_meta_data": component_writer._generic_meta_data} ) # Write generic data arrays if any if component_writer._generic_data: - compressor = Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE) gd_group = cw_group.require_group("generic_data") for name, array in component_writer._generic_data.items(): - gd_group.create_dataset( - name, - data=array, - chunks=_normalize_chunks(array.shape), - compressor=compressor, - ) + gd_group.create_array(name, array) # Make sure the stores are consolidated and closed ret = [] for root_group, store_path in self._stores_rootgroups.values(): - store = root_group.store + store = stores.group_store(root_group) - stores.consolidate_compressed_metadata(store) + stores.consolidate_store(store) # Finish writing all files - store.close() + stores.close_store(store) ret.append(store_path) @@ -300,7 +299,7 @@ def finalize(self) -> List[UPath]: def register_component_writer( self, - component_writer_type: "Callable[Concatenate[zarr.Group, HalfClosedInterval, P], CW]", + component_writer_type: "Callable[Concatenate[nodes.Group, HalfClosedInterval, P], CW]", component_instance_name: str, group_name: Optional[str] = None, generic_meta_data: Optional[Dict[str, types.JsonLike]] = None, @@ -387,6 +386,7 @@ def __init__( open_consolidated: bool = True, itar_index_tail_read_size: int = 1 << 20, # 1 MiB default max_threads: int | None = None, + node_cache_size: int | None = None, ): """Initialize a SequenceComponentReader for a virtual sequence represented by a list of components. @@ -402,28 +402,39 @@ def __init__( single I/O call to initialize the tar file index to minimize reads. Default 1 MiB size fits typical use cases, but may be increased if the compressed index size is expected to be larger or decreased if remote access chunk sizes are smaller. - max_threads: The maximum number of threads used to load the different components (if None, - use interpreter-default number of threads for a ThreadPoolExecutor) + max_threads: The maximum number of threads used to open the component stores concurrently (if + None, use interpreter-default number of threads for a ThreadPoolExecutor) + node_cache_size: Number of opened groups / arrays (with their decoded meta-data, but not their data) + each component reader caches for repeated accesses (e.g., of frames). None caches + all accessed nodes for the lifetime of the reader, a positive number the most recently + used nodes only, and 0 disables caching. """ + if node_cache_size is not None and node_cache_size < 0: + raise ValueError(f"node_cache_size needs to be None or non-negative, got {node_cache_size}") + self._node_cache_size = node_cache_size + component_group_upaths: List[UPath] = self.expand_component_group_paths(component_group_paths) assert len(component_group_upaths), "No component inputs provided" # Load component stores concurrently (to hide latency) and check for sequence consistency - self._component_stores: Dict[str, Tuple[zarr.Group, UPath]] = {} # use str as the generic path / URL type + self._component_stores: Dict[str, Tuple[nodes.Group, UPath]] = {} # use str as the generic path / URL type + # Component stores are opened concurrently (hiding the latency of remote archive index reads), their + # hierarchies are opened from their consolidated metadata sequentially (pure-Python CPU work that threads + # only slow down) with concurrent.futures.ThreadPoolExecutor(max_workers=max_threads) as executor: - def thread_load_component_store(component_store_upath: UPath) -> Tuple[zarr.Group, UPath]: - """Thread-executed shard opening""" + def thread_open_component_store(component_store_upath: UPath) -> Tuple[stores.StoreLike, UPath]: + """Thread-executed component store opening""" # Make sure paths are absolute at this point component_store_upath = component_store_upath.absolute() _logger.info(f"SequenceStoreReader: Loading component store {component_store_upath}") - component_store: Store + component_store: stores.StoreLike if component_store_upath.is_file(): if not component_store_upath.name.endswith(".zarr.itar"): # not a supported file-based store format @@ -435,61 +446,50 @@ def thread_load_component_store(component_store_upath: UPath) -> Tuple[zarr.Grou component_store_upath, mode="r", index_tail_read_size=itar_index_tail_read_size ) else: - component_store = zarr.storage.DirectoryStore(component_store_upath) - - component_root = ( - stores.open_compressed_consolidated(store=component_store, mode="r") - if open_consolidated - else zarr.open(store=component_store, mode="r") - ) + component_store = stores.open_directory_store(component_store_upath) - return cast(zarr.Group, component_root), component_store_upath + return component_store, component_store_upath - for future in concurrent.futures.as_completed( - [ - executor.submit(thread_load_component_store, component_store_upath) - for component_store_upath in component_group_upaths - ] - ): - # Note: thread completion order is not relevant here - component_root, component_store_path = future.result() + opened_component_stores = list(executor.map(thread_open_component_store, component_group_upaths)) - component_root_attrs = dict(component_root.attrs.items()) + for component_store, component_store_path in opened_component_stores: + component_root = stores.open_store(component_store, open_consolidated) - # Check sequence compatibility - component_root_version = component_root_attrs["version"] - if component_root_version not in [VERSION]: - raise RuntimeError( - f"Can't load V4 component store {component_store_path} with incompatible data version {component_root_version}" - ) - - component_store_sequence_id = component_root_attrs["sequence_id"] - component_store_sequence_timestamp_interval_us = HalfClosedInterval( - component_root_attrs["sequence_timestamp_interval_us"]["start"], - component_root_attrs["sequence_timestamp_interval_us"]["stop"], + # Check sequence compatibility + component_root_version = component_root.attr_str("version") + if component_root_version not in [VERSION]: + raise RuntimeError( + f"Can't load V4 component store {component_store_path} with incompatible data version {component_root_version}" ) - component_store_generic_meta_data = component_root_attrs["generic_meta_data"] - if not self._component_stores: - self._sequence_id: str = component_store_sequence_id - self._sequence_timestamp_interval_us = component_store_sequence_timestamp_interval_us - self._generic_meta_data: Dict[str, types.JsonLike] = component_store_generic_meta_data - self._version: str = component_root_version + component_store_sequence_id = component_root.attr_str("sequence_id") + sequence_timestamp_interval_us = component_root.attr_dict("sequence_timestamp_interval_us") + component_store_sequence_timestamp_interval_us = HalfClosedInterval( + cast(int, sequence_timestamp_interval_us["start"]), + cast(int, sequence_timestamp_interval_us["stop"]), + ) + component_store_generic_meta_data = component_root.attr_dict("generic_meta_data") + + if not self._component_stores: + self._sequence_id: str = component_store_sequence_id + self._sequence_timestamp_interval_us = component_store_sequence_timestamp_interval_us + self._generic_meta_data: Dict[str, types.JsonLike] = component_store_generic_meta_data + self._version: str = component_root_version - if not self._sequence_id == component_store_sequence_id: - raise RuntimeError("Can't load component store from different sequences") - if not self._sequence_timestamp_interval_us == component_store_sequence_timestamp_interval_us: - raise RuntimeError("Can't load component store with different sequence timestamp intervals") - if not self._generic_meta_data == component_store_generic_meta_data: - raise RuntimeError("Can't load component store with different generic meta-data") - if not self._version == component_root_version: - raise RuntimeError("Can't load shards from different data versions") + if not self._sequence_id == component_store_sequence_id: + raise RuntimeError("Can't load component store from different sequences") + if not self._sequence_timestamp_interval_us == component_store_sequence_timestamp_interval_us: + raise RuntimeError("Can't load component store with different sequence timestamp intervals") + if not self._generic_meta_data == component_store_generic_meta_data: + raise RuntimeError("Can't load component store with different generic meta-data") + if not self._version == component_root_version: + raise RuntimeError("Can't load shards from different data versions") - component_group_name = component_root_attrs["component_group_name"] - if component_group_name in self._component_stores: - raise RuntimeError(f"Component group {component_group_name} loaded multiple times") + component_group_name = component_root.attr_str("component_group_name") + if component_group_name in self._component_stores: + raise RuntimeError(f"Component group {component_group_name} loaded multiple times") - self._component_stores[component_group_name] = (component_root, component_store_path) + self._component_stores[component_group_name] = (component_root, component_store_path) # Check version-compatibility if self._version != VERSION: @@ -497,15 +497,8 @@ def thread_load_component_store(component_store_upath: UPath) -> Tuple[zarr.Grou def reload_resources(self) -> None: """Trigger a reload of each itar store - useful to re-initialize file objects in multi-process settings""" - component_store: Union[zarr.Group, stores.ConsolidatedCompressedMetadataStore] - for component_store, _ in self._component_stores.values(): - # unwind one layer of possible consolidated metadata store - if isinstance( - compressed_consolidated_store := component_store.store, stores.ConsolidatedCompressedMetadataStore - ): - component_store = compressed_consolidated_store - - if isinstance(store := component_store.store, stores.IndexedTarStore): + for component_root, _ in self._component_stores.values(): + if isinstance(store := stores.group_store(component_root), stores.IndexedTarStore): store.reload_resources() @property @@ -533,20 +526,24 @@ def open_component_readers( ret = {} for component_root_group, _ in self._component_stores.values(): - if (component_group := component_root_group.get(component_reader_type.get_component_name())) is None: + if (component_name := component_reader_type.get_component_name()) not in component_root_group: continue # instantiate a reader for each of the components - for component_instance_name, component_group in component_group.items(): + for component_instance_name, component_group in component_root_group.group(component_name).groups(): assert component_instance_name not in ret, ( f"Component instance {component_instance_name} encountered multiple times" ) # check if the reader supports the component version - if not component_reader_type.supports_component_version(component_group.attrs["component_version"]): + if not component_reader_type.supports_component_version(component_group.attr_str("component_version")): continue - ret[component_instance_name] = component_reader_type(component_instance_name, component_group) + # each reader caches the nodes of its component + cache = None if self._node_cache_size == 0 else nodes.NodeCache(self._node_cache_size) + ret[component_instance_name] = component_reader_type( + component_instance_name, component_group.with_cache(cache) + ) return ret @@ -557,14 +554,13 @@ def get_sequence_meta(self) -> SequenceMeta: component_stores_info: List[SequenceMeta.ComponentStoreMeta] = [] for component_root_group, component_store_path in self._component_stores.values(): components: Dict[str, Dict[str, SequenceMeta.ComponentInstanceMeta]] = {} - for component_name, component in component_root_group.items(): + for component_name, component in component_root_group.groups(): # collect component names and instances component_instances: Dict[str, SequenceMeta.ComponentInstanceMeta] = {} - for component_instance_name, component_instance in component.items(): - component_instance_attrs = component_instance.attrs + for component_instance_name, component_instance in component.groups(): component_instances[component_instance_name] = SequenceMeta.ComponentInstanceMeta( - version=component_instance_attrs["component_version"], - generic_meta_data=component_instance_attrs["generic_meta_data"], + version=component_instance.attr_str("component_version"), + generic_meta_data=component_instance.attr_dict("generic_meta_data"), ) components[component_name] = component_instances @@ -612,7 +608,7 @@ def get_component_version() -> str: """Returns the version of the current component writer""" ... - def __init__(self, component_group: zarr.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: + def __init__(self, component_group: nodes.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: """Initializes a component writer targeting the given component group and sequence time interval""" self._group = component_group @@ -652,7 +648,7 @@ def finalize(self) -> None: Called by :meth:`SequenceComponentGroupsWriter.finalize` before component metadata is persisted. Override to flush buffered data - (e.g., create zarr datasets/groups). + (e.g., create arrays / groups). """ pass @@ -661,7 +657,7 @@ class ComponentReader(ABC): """Base class for V4 component readers. Subclasses must implement :meth:`get_component_name` and - :meth:`supports_component_version`. The underlying zarr group is + :meth:`supports_component_version`. The underlying component group is accessible as ``self._group``; component metadata is available via the :attr:`instance_name`, :attr:`component_version`, and :attr:`generic_meta_data` properties. @@ -677,14 +673,16 @@ def get_component_name() -> str: def supports_component_version(version: str) -> bool: """Returns true if the component version is supported by the reader""" - def __init__(self, component_instance_name: str, component_group: zarr.Group) -> None: + def __init__(self, component_instance_name: str, component_group: nodes.Group) -> None: """Initializes a component reader for a given component instance name and group""" self._instance_name = component_instance_name self._group = component_group # Preload component meta-data and generic data group (if existing) - self._component_meta_data: Dict = dict(self._group.attrs) - self._generic_data_group: Optional[zarr.Group] = self._group.get("generic_data") + self._component_meta_data: Dict[str, types.JsonLike] = self._group.attrs + self._generic_data_group: Optional[nodes.Group] = ( + self._group.group("generic_data") if "generic_data" in self._group else None + ) @property def instance_name(self) -> str: @@ -694,12 +692,12 @@ def instance_name(self) -> str: @property def component_version(self) -> str: """Returns the component version of the loaded component""" - return self._component_meta_data["component_version"] + return cast(str, self._component_meta_data["component_version"]) @property def generic_meta_data(self) -> Dict[str, types.JsonLike]: """Returns the generic meta data of the loaded component""" - return self._component_meta_data["generic_meta_data"] + return cast(Dict[str, types.JsonLike], self._component_meta_data["generic_meta_data"]) def has_generic_data(self, name: str) -> bool: """Returns True if a named generic data array exists on this component""" @@ -707,17 +705,24 @@ def has_generic_data(self, name: str) -> bool: def get_generic_data_names(self) -> List[str]: """Returns the list of all generic data array names on this component""" - return list(self._generic_data_group.keys()) if self._generic_data_group is not None else [] + return self._generic_data_group.members() if self._generic_data_group is not None else [] def get_generic_data(self, name: str) -> np.ndarray: """Returns a named generic data array from this component. Raises KeyError if the name does not exist. """ + return self._generic_data_array(name).read() + + async def get_generic_data_async(self, name: str) -> np.ndarray: + """Asynchronous :meth:`get_generic_data`""" + return await self._generic_data_array(name).read_async() + + def _generic_data_array(self, name: str) -> nodes.Array: if self._generic_data_group is None: raise KeyError("Component has no generic_data") - if (generic_data := self._generic_data_group.get(name)) is not None: - return np.array(generic_data) + if name in self._generic_data_group: + return self._generic_data_group.array(name) raise KeyError(f"Generic data '{name}' not found. Available: {self.get_generic_data_names()}") @@ -751,7 +756,7 @@ def get_component_version() -> str: """Returns the version of the current component writer""" return "v1" - def __init__(self, component_group: zarr.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: + def __init__(self, component_group: nodes.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: """Initializes the current component writer targeting the given component group and sequence time interval""" super().__init__(component_group, sequence_timestamp_interval_us) @@ -760,8 +765,8 @@ def __init__(self, component_group: zarr.Group, sequence_timestamp_interval_us: def finalize(self) -> None: """Actually store the json-encoded pose data""" - self._group.create_group("static_poses").attrs.put(self.data["static_poses"]) - self._group.create_group("dynamic_poses").attrs.put(self.data["dynamic_poses"]) + self._group.create_group("static_poses", attributes=self.data["static_poses"]) + self._group.create_group("dynamic_poses", attributes=self.data["dynamic_poses"]) def store_static_pose( self, @@ -859,10 +864,18 @@ def supports_component_version(version: str) -> bool: """Returns true if the component version is supported by the reader""" return version == "v1" + def _static_poses(self) -> Dict[str, Dict[str, Any]]: + """Returns the json-encoded static poses data""" + return cast(Dict[str, Dict[str, Any]], self._group.group("static_poses").attrs) + + def _dynamic_poses(self) -> Dict[str, Dict[str, Any]]: + """Returns the json-encoded dynamic poses data""" + return cast(Dict[str, Dict[str, Any]], self._group.group("dynamic_poses").attrs) + def get_static_poses(self) -> Generator[Tuple[Tuple[str, str], npt.NDArray[np.floating]]]: """Returns all static poses (rigid transformations) between named coordinate frames, if available""" - for key, static_pose in self._group["static_poses"].attrs.items(): + for key, static_pose in self._static_poses().items(): yield ast.literal_eval(key), np.array(static_pose["pose"], dtype=static_pose["dtype"]) def get_dynamic_poses( @@ -870,7 +883,7 @@ def get_dynamic_poses( ) -> Generator[Tuple[Tuple[str, str], Tuple[npt.NDArray[np.floating], npt.NDArray[np.uint64]]]]: """Returns all dynamic poses (time-dependent rigid transformations) between named coordinate frames, if available""" - for key, dynamic_poses in self._group["dynamic_poses"].attrs.items(): + for key, dynamic_poses in self._dynamic_poses().items(): yield ( ast.literal_eval(key), ( @@ -883,7 +896,7 @@ def get_static_pose(self, source_frame_id: str, target_frame_id: str) -> npt.NDA """Returns static pose (rigid transformation) between two named coordinate frames, if available""" if ( - static_pose := self._group["static_poses"].attrs.get( + static_pose := self._static_poses().get( key := str((validate_frame_name(source_frame_id), validate_frame_name(target_frame_id))) ) ) is None: @@ -897,7 +910,7 @@ def get_dynamic_pose( """Returns dynamic poses (time-dependent rigid transformations) between two named coordinate frames, if available""" if ( - dynamic_poses := self._group["dynamic_poses"].attrs.get( + dynamic_poses := self._dynamic_poses().get( key := str((validate_frame_name(source_frame_id), validate_frame_name(target_frame_id))) ) ) is None: @@ -926,7 +939,7 @@ def get_component_version() -> str: """Returns the version of the current intrinsic calibration component""" return "v1" - def __init__(self, component_group: zarr.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: + def __init__(self, component_group: nodes.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: """Initializes the current component writer targeting the given component group and sequence time interval""" super().__init__(component_group, sequence_timestamp_interval_us) @@ -945,7 +958,7 @@ def store_camera_intrinsics( meta_data = types.encode_camera_model_parameters(camera_model_parameters) - self._cameras_group.create_group(camera_id).attrs.put(meta_data) + self._cameras_group.create_group(camera_id, attributes=meta_data) return self @@ -960,7 +973,7 @@ def store_lidar_intrinsics( # Prepare meta-data containing the serialization of the mandatory lidar model meta_data = types.encode_lidar_model_parameters(lidar_model_parameters) - self._lidars_group.create_group(lidar_id).attrs.put(meta_data) + self._lidars_group.create_group(lidar_id, attributes=meta_data) return self @@ -979,16 +992,14 @@ def supports_component_version(version: str) -> bool: def get_camera_model_parameters(self, camera_id: str) -> types.CameraModelParameters: """Returns the camera model associated with the requested camera sensor""" - return types.decode_camera_model_parameters(cast(zarr.Group, self._group["cameras"][camera_id]).attrs) + return types.decode_camera_model_parameters(self._group.group(f"cameras/{camera_id}").attrs) def get_lidar_model_parameters(self, lidar_id: str) -> Optional[types.LidarModelParameters]: """Returns the lidar model associated with the requested lidar sensor""" - lidars_group = self._group["lidars"] - - if lidar_id not in lidars_group: + if lidar_id not in self._group.group("lidars"): return None - return types.decode_lidar_model_parameters(cast(zarr.Group, lidars_group[lidar_id]).attrs) + return types.decode_lidar_model_parameters(self._group.group(f"lidars/{lidar_id}").attrs) class MasksComponent: @@ -1009,7 +1020,7 @@ def get_component_version() -> str: """Returns the version of the current sensor masks component""" return "v1" - def __init__(self, component_group: zarr.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: + def __init__(self, component_group: nodes.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: """Initializes the current component writer targeting the given component group and sequence time interval""" super().__init__(component_group, sequence_timestamp_interval_us) @@ -1024,8 +1035,8 @@ def store_camera_masks( """Store camera-associated masks""" # Store mask names - (camera_grp := self._cameras_group.create_group(camera_id)).attrs.put( - {"mask_names": list(mask_images.keys())} + camera_grp = self._cameras_group.create_group( + camera_id, attributes={"mask_names": list(mask_images.keys())} ) # Store mask images @@ -1034,9 +1045,7 @@ def store_camera_masks( FORMAT = "png" mask_image.save(buffer, format=FORMAT, optimize=True) # encodes as png # store mask data (uncompressed, as already encoded) - camera_grp.create_dataset(mask_name, data=np.asarray(buffer.getvalue()), compressor=None).attrs[ - "format" - ] = FORMAT + camera_grp.create_bytes_array(mask_name, buffer.getvalue(), attributes={"format": FORMAT}) return self @@ -1056,14 +1065,23 @@ def supports_component_version(version: str) -> bool: def get_camera_mask_names(self, camera_id: str) -> List[str]: """Returns all constant camera mask names""" - return list(cast(zarr.Group, self._group["cameras"][camera_id]).attrs.get("mask_names", [])) + return list(cast(List[str], self._group.group(f"cameras/{camera_id}").attrs.get("mask_names", []))) def get_camera_mask_image(self, camera_id: str, mask_name: str) -> PILImage.Image: """Returns constant named camera mask image""" - mask_dataset = cast(zarr.Array, cast(zarr.Group, self._group["cameras"][camera_id])[mask_name]) + mask_array = self._group.array(f"cameras/{camera_id}/{mask_name}") - return PILImage.open(io.BytesIO(cast(np.bytes_, mask_dataset[()])), formats=[mask_dataset.attrs["format"]]) + return PILImage.open(io.BytesIO(mask_array.read_bytes()), formats=[mask_array.attr_str("format")]) + + async def get_camera_mask_image_async(self, camera_id: str, mask_name: str) -> PILImage.Image: + """Asynchronous :meth:`get_camera_mask_image`""" + + mask_array = self._group.array(f"cameras/{camera_id}/{mask_name}") + + return PILImage.open( + io.BytesIO(await mask_array.read_bytes_async()), formats=[mask_array.attr_str("format")] + ) def get_camera_mask_images(self, camera_id: str) -> Generator[Tuple[str, PILImage.Image]]: """Returns all constant named camera mask images""" @@ -1075,7 +1093,7 @@ def get_camera_mask_images(self, camera_id: str) -> Generator[Tuple[str, PILImag class BaseSensorComponentWriter(ComponentWriter): """Base class for all sensor component writers""" - def __init__(self, component_group: zarr.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: + def __init__(self, component_group: nodes.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: """Initializes the current component writer targeting the given component group and sequence time interval""" super().__init__(component_group, sequence_timestamp_interval_us) @@ -1103,21 +1121,16 @@ def finalize(self) -> None: ) # Store as meta-data of frames group - self._frames_group.attrs.put({"frames_timestamps_us": frames_timestamps_us.tolist()}) + self._frames_group.update_attrs({"frames_timestamps_us": frames_timestamps_us.tolist()}) def _get_frame_group( self, # end-of-frame timestamp, or start-of-frame / end-of-frame timestamps timestamps_us: Union[int, npt.NDArray[np.uint64]], - ) -> zarr.Group: + ) -> nodes.Group: """Returns the group of a frame, initializing it if required""" - if isinstance(timestamps_us, np.ndarray): - frame_id = cast(np.ndarray, timestamps_us)[1].item() # end-of-frame timestamp is frame ID - else: - frame_id = timestamps_us - - return self._frames_group.require_group(str(frame_id)) + return self._frames_group.require_group(str(_frame_id(timestamps_us))) def _store_base_frame( self, @@ -1126,7 +1139,7 @@ def _store_base_frame( # generic per-frame data (key-value pairs, *not* dimension / dtype validated) and meta-data generic_data: Dict[str, npt.NDArray[Any]], generic_meta_data: Dict[str, types.JsonLike], - ) -> zarr.Group: + ) -> nodes.Group: # Sanity / timestamp consistency checks assert np.shape(frame_timestamps_us) == (2,) assert frame_timestamps_us.dtype == np.dtype("uint64") @@ -1149,16 +1162,11 @@ def _store_base_frame( self._frames_timestamps_us[frame_timestamps_us[1].item()] = frame_timestamps_us[0].item() # Store additional generic frame data and meta-data (not dimension / dtype checked) - (frame_generic_data_group := frame_group.create_group("generic_data")).attrs.put(generic_meta_data) + frame_generic_data_group = frame_group.create_group("generic_data", attributes=generic_meta_data) for name, value in generic_data.items(): - frame_generic_data_group.create_dataset( - name, - data=value, - # we are not accessing sub-ranges, so disable chunking - chunks=_normalize_chunks(value.shape), - # use compression that is fast to decode on modern hardware - compressor=Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE), - ) + # we are not accessing sub-ranges, so store a single chunk (default) - use the default compression, + # which is fast to decode on modern hardware + frame_generic_data_group.create_array(name, value) return frame_group @@ -1166,7 +1174,7 @@ def _store_base_frame( class BaseSensorComponentReader(ComponentReader): """Base class for all sensor component readers""" - def __init__(self, component_instance_name: str, component_group: zarr.Group) -> None: + def __init__(self, component_instance_name: str, component_group: nodes.Group) -> None: """Initializes a component reader for a given component instance name and group""" super().__init__(component_instance_name, component_group) @@ -1174,7 +1182,9 @@ def __init__(self, component_instance_name: str, component_group: zarr.Group) -> raise RuntimeError("Sensor component doesn't contain any frames") # preload frame timestamps and create map - self._frames_timestamps_us = np.array(self._group["frames"].attrs["frames_timestamps_us"], dtype=np.uint64) + self._frames_timestamps_us = np.array( + self._group.group("frames").attr_list("frames_timestamps_us"), dtype=np.uint64 + ) self._frame_end_to_frame_timestamps_us = { end: np.array([self._frames_timestamps_us[i, 0], end], dtype=np.uint64) for i, end in enumerate(self._frames_timestamps_us[:, 1]) @@ -1184,19 +1194,32 @@ def _get_frame_group( self, # end-of-frame timestamp, or start-of-frame / end-of-frame timestamps timestamps_us: Union[int, npt.NDArray[np.uint64]], - ) -> zarr.Group: + ) -> nodes.Group: """Returns the group of a frame""" - if isinstance(timestamps_us, np.ndarray): - frame_id = cast(np.ndarray, timestamps_us)[1].item() # end-of-frame timestamp is frame ID - else: - frame_id = timestamps_us + return self._group.group(self._frame_path(timestamps_us)) + + @staticmethod + def _frame_path( + # end-of-frame timestamp, or start-of-frame / end-of-frame timestamps + timestamps_us: Union[int, npt.NDArray[np.uint64]], + # optional name of a frame member + name: Optional[str] = None, + ) -> str: + """Returns the path of a frame group (or of one of its members) within the component group""" + + frame_path = f"frames/{_frame_id(timestamps_us)}" - return cast(zarr.Group, self._group["frames"][str(frame_id)]) + return frame_path if name is None else f"{frame_path}/{name}" + + def _get_frame_generic_data_group(self, timestamp_us: int) -> nodes.Group: + """Returns the generic data group of a frame""" + + return self._group.group(self._frame_path(timestamp_us, "generic_data")) @property def frames_timestamps_us(self) -> npt.NDArray[np.uint64]: - return np.array(self._group["frames"].attrs["frames_timestamps_us"], dtype=np.uint64) + return np.array(self._group.group("frames").attr_list("frames_timestamps_us"), dtype=np.uint64) @property def frames_count(self) -> int: @@ -1209,7 +1232,7 @@ def get_frame_timestamps_us(self, timestamp_us: int) -> npt.NDArray[np.uint64]: def get_frame_generic_data_names(self, timestamp_us: int) -> List[str]: """List of all generic frame-data names""" - return list(cast(zarr.Group, self._get_frame_group(timestamp_us)["generic_data"]).keys()) + return self._get_frame_generic_data_group(timestamp_us).members() def has_frame_generic_data(self, timestamp_us: int, name: str) -> bool: """Signals if named generic frame-data exists""" @@ -1219,12 +1242,17 @@ def has_frame_generic_data(self, timestamp_us: int, name: str) -> bool: def get_frame_generic_data(self, timestamp_us: int, name: str) -> npt.NDArray[Any]: """Returns generic frame-data for a specific frame and name""" - return np.array(self._get_frame_group(timestamp_us)["generic_data"][name]) + return self._get_frame_generic_data_group(timestamp_us).array(name).read() + + async def get_frame_generic_data_async(self, timestamp_us: int, name: str) -> npt.NDArray[Any]: + """Asynchronous :meth:`get_frame_generic_data`""" + + return await self._get_frame_generic_data_group(timestamp_us).array(name).read_async() def get_frame_generic_meta_data(self, timestamp_us: int) -> Dict[str, types.JsonLike]: """Returns generic frame meta-data for a specific frame""" - return dict(self._get_frame_group(timestamp_us)["generic_data"].attrs) + return self._get_frame_generic_data_group(timestamp_us).attrs class BaseRayBundleSensorComponentWriter(BaseSensorComponentWriter): @@ -1244,21 +1272,16 @@ def _store_frame_ray_bundle( ) -> None: ## Initialize ray bundle group frame_group = self._get_frame_group(frame_timestamps_us) - (ray_bundle_group := frame_group.create_group("ray_bundle")).attrs.put({"n_rays": n_rays}) + ray_bundle_group = frame_group.create_group("ray_bundle", attributes={"n_rays": n_rays}) - # Store per-ray data + # Store per-ray data (zero-sized chunk dimensions are clamped to 1, default compression is fast to decode on + # modern hardware) for name, (ray_data_data, chunks) in ray_data.items(): assert len(ray_data_data) == n_rays, f"{name} doesn't have required ray count" - ray_bundle_group.create_dataset( - name, - data=ray_data_data, - chunks=_normalize_chunks(chunks), - # use compression that is fast to decode on modern hardware - compressor=Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE), - ) + ray_bundle_group.create_array(name, ray_data_data, chunks=chunks) ## Initialize ray bundle returns group - (ray_bundle_returns_group := frame_group.create_group("ray_bundle_returns")).attrs.put({"n_returns": n_returns}) + ray_bundle_returns_group = frame_group.create_group("ray_bundle_returns", attributes={"n_returns": n_returns}) # Store per-return data absent_mask = None @@ -1291,13 +1314,7 @@ def _store_frame_ray_bundle( # validate absent mask consistency assert np.array_equal(absent_mask, local_absent_mask), f"Inconsistent NaN masks in return data {name}" - ray_bundle_returns_group.create_dataset( - name, - data=return_data_data, - chunks=_normalize_chunks(chunks), - # use compression that is fast to decode on modern hardware - compressor=Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE), - ) + ray_bundle_returns_group.create_array(name, return_data_data, chunks=chunks) if absent_mask is None: # Initialize empty absent mask if no per-return data was provided @@ -1305,34 +1322,32 @@ def _store_frame_ray_bundle( valid_mask_packed = np.packbits(~absent_mask) - frame_group.create_dataset( + # we are not accessing sub-ranges, so store a single chunk (default) + frame_group.create_array( "ray_bundle_returns_valid_mask_packed", - data=valid_mask_packed, - # we are not accessing sub-ranges, so disable chunking - chunks=_normalize_chunks(valid_mask_packed.shape), - # use compression that is fast to decode on modern hardware - compressor=Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE), - ).attrs.put({"n_returns": n_returns, "n_rays": n_rays}) + valid_mask_packed, + attributes={"n_returns": n_returns, "n_rays": n_rays}, + ) class BaseRayBundleSensorComponentReader(BaseSensorComponentReader): """Base class for all ray bundle sensor component readers""" # per-ray data - def _get_ray_bundle_group(self, timestamp_us: int) -> zarr.Group: + def _get_ray_bundle_group(self, timestamp_us: int) -> nodes.Group: """Returns the ray bundle group of a frame""" - return cast(zarr.Group, self._get_frame_group(timestamp_us)["ray_bundle"]) + return self._group.group(self._frame_path(timestamp_us, "ray_bundle")) def get_frame_ray_bundle_count(self, timestamp_us: int) -> int: """Returns the number of rays and ray returns for a specific frame""" - return self._get_ray_bundle_group(timestamp_us).attrs["n_rays"] + return self._get_ray_bundle_group(timestamp_us).attr_int("n_rays") def get_frame_ray_bundle_data_names(self, timestamp_us: int) -> List[str]: """List of all ray bundle data names for a frame""" - return list(self._get_ray_bundle_group(timestamp_us).keys()) + return self._get_ray_bundle_group(timestamp_us).members() def has_frame_ray_bundle_data(self, timestamp_us: int, name: str) -> bool: """Signals if named ray bundle data exists for a frame""" @@ -1342,23 +1357,28 @@ def has_frame_ray_bundle_data(self, timestamp_us: int, name: str) -> bool: def get_frame_ray_bundle_data(self, timestamp_us: int, name: str) -> npt.NDArray[Any]: """Returns named ray bundle data for a frame""" - return np.array(self._get_ray_bundle_group(timestamp_us)[name]) + return self._get_ray_bundle_group(timestamp_us).array(name).read() + + async def get_frame_ray_bundle_data_async(self, timestamp_us: int, name: str) -> npt.NDArray[Any]: + """Asynchronous :meth:`get_frame_ray_bundle_data`""" + + return await self._get_ray_bundle_group(timestamp_us).array(name).read_async() # per-ray return data - def _get_ray_bundle_returns_group(self, timestamp_us: int) -> zarr.Group: + def _get_ray_bundle_returns_group(self, timestamp_us: int) -> nodes.Group: """Returns the ray bundle returns group of a frame""" - return cast(zarr.Group, self._get_frame_group(timestamp_us)["ray_bundle_returns"]) + return self._group.group(self._frame_path(timestamp_us, "ray_bundle_returns")) def get_frame_ray_bundle_return_count(self, timestamp_us: int) -> int: """Returns the number of ray returns for a specific frame""" - return self._get_ray_bundle_returns_group(timestamp_us).attrs["n_returns"] + return self._get_ray_bundle_returns_group(timestamp_us).attr_int("n_returns") def get_frame_ray_bundle_return_data_names(self, timestamp_us: int) -> List[str]: """List of all ray bundle return data names for a frame""" - return list(self._get_ray_bundle_returns_group(timestamp_us).keys()) + return self._get_ray_bundle_returns_group(timestamp_us).members() def has_frame_ray_bundle_return_data(self, timestamp_us: int, name: str) -> bool: """Signals if named ray bundle return data exists for a frame""" @@ -1368,30 +1388,45 @@ def has_frame_ray_bundle_return_data(self, timestamp_us: int, name: str) -> bool def get_frame_ray_bundle_return_valid_mask(self, timestamp_us: int) -> npt.NDArray[np.bool_]: """Returns the per-ray return valid mask for a frame""" - valid_mask_packed = self._get_frame_group(timestamp_us)["ray_bundle_returns_valid_mask_packed"] + valid_mask_packed = self._valid_mask_packed_array(timestamp_us) + return self._unpack_valid_mask(valid_mask_packed, valid_mask_packed.read()) - attrs = valid_mask_packed.attrs - n_returns, n_rays = attrs["n_returns"], attrs["n_rays"] + async def get_frame_ray_bundle_return_valid_mask_async(self, timestamp_us: int) -> npt.NDArray[np.bool_]: + """Asynchronous :meth:`get_frame_ray_bundle_return_valid_mask`""" - return ( - np.unpackbits(np.array(valid_mask_packed), count=n_returns * n_rays) - .astype(np.bool_) - .reshape((n_returns, n_rays)) - ) + valid_mask_packed = self._valid_mask_packed_array(timestamp_us) + return self._unpack_valid_mask(valid_mask_packed, await valid_mask_packed.read_async()) + + def _valid_mask_packed_array(self, timestamp_us: int) -> nodes.Array: + return self._group.array(self._frame_path(timestamp_us, "ray_bundle_returns_valid_mask_packed")) + + @staticmethod + def _unpack_valid_mask(valid_mask_packed: nodes.Array, packed: npt.NDArray[np.uint8]) -> npt.NDArray[np.bool_]: + n_returns, n_rays = valid_mask_packed.attr_int("n_returns"), valid_mask_packed.attr_int("n_rays") + return np.unpackbits(packed, count=n_returns * n_rays).astype(np.bool_).reshape((n_returns, n_rays)) def get_frame_ray_bundle_return_data( self, timestamp_us: int, name: str, return_index: Optional[int] ) -> npt.NDArray[np.float32]: """Returns named ray bundle return data for a frame, optionally indexed by return index to accelerate data-retrieval""" - return_array = self._get_ray_bundle_returns_group(timestamp_us)[ - name - ] # only references the underlying Array, don't load it's data yet + # only references the underlying array, don't load its data yet + return_array = self._get_ray_bundle_returns_group(timestamp_us).array(name) if return_index is None: - return np.array(return_array[slice(return_array.shape[0]), ...]) # load all returns + return return_array.read() # load all returns else: - return np.array(return_array[return_index, ...]) # load specific return only + return return_array.read((return_index, Ellipsis)) # load specific return only + + async def get_frame_ray_bundle_return_data_async( + self, timestamp_us: int, name: str, return_index: Optional[int] + ) -> npt.NDArray[np.float32]: + """Asynchronous :meth:`get_frame_ray_bundle_return_data`""" + + return_array = self._get_ray_bundle_returns_group(timestamp_us).array(name) + if return_index is None: + return await return_array.read_async() + return await return_array.read_async((return_index, Ellipsis)) class CameraSensorComponent: @@ -1427,9 +1462,7 @@ def store_frame( frame_group = self._store_base_frame(frame_timestamps_us, generic_data, generic_meta_data) # Store image data (uncompressed, as already encoded) - frame_group.create_dataset("image", data=np.asarray(image_binary_data), compressor=None).attrs["format"] = ( - image_format - ) + frame_group.create_bytes_array("image", image_binary_data, attributes={"format": image_format}) return self @@ -1449,23 +1482,31 @@ def supports_component_version(version: str) -> bool: class EncodedImageDataHandle: """References encoded image data without loading it""" - def __init__(self, image_dataset: zarr.Array) -> None: - self._image_dataset = image_dataset + def __init__(self, image_array: nodes.Array) -> None: + self._image_array = image_array def get_data(self) -> types.EncodedImageData: """Loads the referenced encoded image data to memory""" + return types.EncodedImageData(self._image_array.read_bytes(), self._image_array.attr_str("format")) + + async def get_data_async(self) -> types.EncodedImageData: + """Asynchronous :meth:`get_data`""" return types.EncodedImageData( - cast(np.bytes_, self._image_dataset[()]), self._image_dataset.attrs["format"] + await self._image_array.read_bytes_async(), self._image_array.attr_str("format") ) def get_frame_handle(self, timestamp_us: int) -> EncodedImageDataHandle: """Returns the frame's encoded image data""" - return self.EncodedImageDataHandle(cast(zarr.Array, self._get_frame_group(timestamp_us)["image"])) + return self.EncodedImageDataHandle(self._group.array(self._frame_path(timestamp_us, "image"))) def get_frame_data(self, timestamp_us: int) -> types.EncodedImageData: """Returns the frame's encoded image data""" return self.get_frame_handle(timestamp_us).get_data() + async def get_frame_data_async(self, timestamp_us: int) -> types.EncodedImageData: + """Asynchronous :meth:`get_frame_data`""" + return await self.get_frame_handle(timestamp_us).get_data_async() + def get_frame_image(self, timestamp_us: int) -> PILImage.Image: """Returns the frame's decoded image data""" return self.get_frame_data(timestamp_us).get_decoded_image() @@ -1706,7 +1747,7 @@ def store_observations( ) obs_dict_list.append(obs.to_dict()) - self._group.create_group("cuboids").attrs.put({"cuboid_track_observations": obs_dict_list}) + self._group.create_group("cuboids", attributes={"cuboid_track_observations": obs_dict_list}) return self @@ -1726,8 +1767,8 @@ def supports_component_version(version: str) -> bool: def get_observations(self) -> Generator[types.CuboidTrackObservation]: """Returns all stored cuboid track observations""" - for obs in self._group["cuboids"].attrs["cuboid_track_observations"]: - yield types.CuboidTrackObservation.from_dict(obs) + for obs in self._group.group("cuboids").attr_list("cuboid_track_observations"): + yield types.CuboidTrackObservation.from_dict(cast(Dict[str, Any], obs)) class PointCloudsComponent: @@ -1771,7 +1812,7 @@ def get_component_version() -> str: def __init__( self, - component_group: zarr.Group, + component_group: nodes.Group, sequence_timestamp_interval_us: HalfClosedInterval, coordinate_unit: PointCloud.CoordinateUnit, attribute_schemas: Optional[Dict[str, PointCloudsComponent.AttributeSchema]] = None, @@ -1785,7 +1826,7 @@ def __init__( # Pre-create the pcs group self._pcs_group = self._group.require_group("pcs") - self._pcs_group.attrs.put( + self._pcs_group.update_attrs( { "coordinate_unit": coordinate_unit.name, "attribute_schemas": {name: s.to_dict() for name, s in self._attribute_schemas.items()}, @@ -1812,7 +1853,6 @@ def store_pc( attributes = unpack_optional(attributes, default={}) generic_data = unpack_optional(generic_data, default={}) generic_meta_data = unpack_optional(generic_meta_data, default={}) - compressor = Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE) # -- Validate xyz -- assert xyz.dtype == np.dtype("float32") @@ -1843,7 +1883,7 @@ def store_pc( pc_group = self._pcs_group.require_group(str(len(self._pc_timestamps))) # Store per-pc metadata (timestamp lives in source-level pc_timestamps_us array) - pc_group.attrs.put( + pc_group.update_attrs( { "reference_frame_id": reference_frame_id, "generic_meta_data": generic_meta_data, @@ -1851,43 +1891,23 @@ def store_pc( ) # Store xyz - pc_group.create_dataset( - "xyz", - data=xyz, - chunks=_normalize_chunks(xyz.shape, require_nonzero_dims=(1,)), - compressor=compressor, - ) + _check_nonzero_dims(xyz.shape, (1,), "xyz") + pc_group.create_array("xyz", xyz) # Store schema-declared attributes for attr_name, attr_array in attributes.items(): - pc_group.create_dataset( - attr_name, - data=attr_array, - chunks=_normalize_chunks(attr_array.shape), - compressor=compressor, - ) + pc_group.create_array(attr_name, attr_array) # Store generic data gd_group = pc_group.require_group("generic_data") for gd_name, gd_array in generic_data.items(): - gd_group.create_dataset( - gd_name, - data=gd_array, - chunks=_normalize_chunks(gd_array.shape), - compressor=compressor, - ) + gd_group.create_array(gd_name, gd_array) self._pc_timestamps.append(reference_frame_timestamp_us) def finalize(self) -> None: """Write pc_timestamps_us array (derived from per-pc reference_frame_timestamp_us values).""" - ts_array = np.array(self._pc_timestamps, dtype=np.uint64) - self._group.create_dataset( - "pc_timestamps_us", - data=ts_array, - chunks=_normalize_chunks(ts_array.shape), - compressor=Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE), - ) + self._group.create_array("pc_timestamps_us", np.array(self._pc_timestamps, dtype=np.uint64)) # -------------------------------------------------------------------------- # Reader @@ -1904,16 +1924,16 @@ def get_component_name() -> str: def supports_component_version(version: str) -> bool: return version == "v1" - def __init__(self, component_instance_name: str, component_group: zarr.Group) -> None: + def __init__(self, component_instance_name: str, component_group: nodes.Group) -> None: super().__init__(component_instance_name, component_group) - self._pc_timestamps_us: npt.NDArray[np.uint64] = np.array(self._group["pc_timestamps_us"][:]) + self._pc_timestamps_us: npt.NDArray[np.uint64] = self._group.array("pc_timestamps_us").read() - pcs_attrs = self._group["pcs"].attrs - self._coordinate_unit = PointCloud.CoordinateUnit[pcs_attrs["coordinate_unit"]] + pcs_group = self._group.group("pcs") + self._coordinate_unit = PointCloud.CoordinateUnit[pcs_group.attr_str("coordinate_unit")] self._attribute_schemas: Dict[str, PointCloudsComponent.AttributeSchema] = { - name: PointCloudsComponent.AttributeSchema.from_dict(s) - for name, s in pcs_attrs["attribute_schemas"].items() + name: PointCloudsComponent.AttributeSchema.from_dict(cast(Dict[str, Any], s)) + for name, s in pcs_group.attr_dict("attribute_schemas").items() } # -- properties -------------------------------------------------------- @@ -1942,32 +1962,52 @@ def get_attribute_schema(self, name: str) -> PointCloudsComponent.AttributeSchem # -- per-pc data access ------------------------------------------------ - def _pc_group(self, pc_index: int) -> zarr.Group: - return cast(zarr.Group, self._group["pcs"][str(pc_index)]) + @staticmethod + def _pc_path(pc_index: int, name: Optional[str] = None) -> str: + """Returns the path of a point cloud group (or of one of its members) within the component group""" + return f"pcs/{pc_index}" if name is None else f"pcs/{pc_index}/{name}" + + def _pc_group(self, pc_index: int, name: Optional[str] = None) -> nodes.Group: + return self._group.group(self._pc_path(pc_index, name)) + + def _pc_array(self, pc_index: int, name: str) -> nodes.Array: + return self._group.array(self._pc_path(pc_index, name)) def get_pc_xyz(self, pc_index: int) -> npt.NDArray[np.float32]: - return np.array(self._pc_group(pc_index)["xyz"][:]) + return self._pc_array(pc_index, "xyz").read() + + async def get_pc_xyz_async(self, pc_index: int) -> npt.NDArray[np.float32]: + """Asynchronous :meth:`get_pc_xyz`""" + return await self._pc_array(pc_index, "xyz").read_async() def get_pc_attribute(self, pc_index: int, name: str) -> npt.NDArray[Any]: - return np.array(self._pc_group(pc_index)[name][:]) + return self._pc_array(pc_index, name).read() + + async def get_pc_attribute_async(self, pc_index: int, name: str) -> npt.NDArray[Any]: + """Asynchronous :meth:`get_pc_attribute`""" + return await self._pc_array(pc_index, name).read_async() def get_pc_reference_frame_id(self, pc_index: int) -> str: - return str(self._pc_group(pc_index).attrs["reference_frame_id"]) + return self._pc_group(pc_index).attr_str("reference_frame_id") def get_pc_reference_frame_timestamp_us(self, pc_index: int) -> int: - return int(self._group["pc_timestamps_us"][pc_index]) + return int(self._group.array("pc_timestamps_us").read(pc_index)) def get_pc_generic_data_names(self, pc_index: int) -> List[str]: - return list(cast(zarr.Group, self._pc_group(pc_index)["generic_data"]).keys()) + return self._pc_group(pc_index, "generic_data").members() def has_pc_generic_data(self, pc_index: int, name: str) -> bool: - return name in self._pc_group(pc_index)["generic_data"] + return name in self._pc_group(pc_index, "generic_data") def get_pc_generic_data(self, pc_index: int, name: str) -> npt.NDArray[Any]: - return np.array(self._pc_group(pc_index)["generic_data"][name][:]) + return self._pc_array(pc_index, f"generic_data/{name}").read() + + async def get_pc_generic_data_async(self, pc_index: int, name: str) -> npt.NDArray[Any]: + """Asynchronous :meth:`get_pc_generic_data`""" + return await self._pc_array(pc_index, f"generic_data/{name}").read_async() def get_pc_generic_meta_data(self, pc_index: int) -> Dict[str, types.JsonLike]: - return dict(self._pc_group(pc_index).attrs.get("generic_meta_data", {})) + return cast(Dict[str, types.JsonLike], self._pc_group(pc_index).attrs.get("generic_meta_data", {})) class CameraLabelsComponent: @@ -1992,7 +2032,7 @@ def get_component_version() -> str: def __init__( self, - component_group: zarr.Group, + component_group: nodes.Group, sequence_timestamp_interval_us: HalfClosedInterval, descriptor: types.CameraLabelDescriptor, ) -> None: @@ -2002,7 +2042,7 @@ def __init__( # Initialize labels group and timestamps list self._labels_group = self._group.require_group("labels") - self._labels_group.attrs.put( + self._labels_group.update_attrs( { "descriptor": descriptor.to_dict(), } @@ -2030,7 +2070,6 @@ def store_label( Optional per-label metadata. """ generic_meta_data = unpack_optional(generic_meta_data, default={}) - compressor = Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE) # Sanity checks assert timestamp_us in self._sequence_timestamp_interval_us, ( @@ -2067,38 +2106,32 @@ def store_label( q.quantized_dtype ) - label_group.create_dataset( - "data", - data=stored, - chunks=_normalize_chunks(stored.shape, require_nonzero_dims=(0, 1)), - compressor=compressor, - ) + _check_nonzero_dims(stored.shape, (0, 1), "label data") + label_group.create_array("data", stored) elif self._descriptor.label_schema.encoding == types.LabelEncoding.IMAGE_ENCODED: assert isinstance(data, bytes), "IMAGE_ENCODED encoding requires bytes" - label_group.create_dataset( + # store encoded image data as uncompressed 1-d uint8 array (as already encoded), with chunks + # determined by zarr-python's chunk shape heuristic + label_group.create_array( "data", - data=np.asarray(bytearray(data), dtype=np.uint8), - compressor=None, - ).attrs["format"] = self._descriptor.label_schema.encoded_format + np.frombuffer(data, dtype=np.uint8), + chunks="auto", + compression=None, + attributes={"format": self._descriptor.label_schema.encoded_format}, + ) else: raise ValueError(f"Unsupported label encoding: {self._descriptor.label_schema.encoding}") - label_group.attrs["generic_meta_data"] = generic_meta_data + label_group.update_attrs({"generic_meta_data": generic_meta_data}) self._timestamps.append(timestamp_us) def finalize(self) -> None: """Write sorted timestamps_us array.""" - ts_array = np.array(sorted(self._timestamps), dtype=np.uint64) - self._group.create_dataset( - "timestamps_us", - data=ts_array, - chunks=_normalize_chunks(ts_array.shape), - compressor=Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE), - ) + self._group.create_array("timestamps_us", np.array(sorted(self._timestamps), dtype=np.uint64)) # -------------------------------------------------------------------------- # Reader @@ -2115,12 +2148,14 @@ def get_component_name() -> str: def supports_component_version(version: str) -> bool: return version == "v1" - def __init__(self, component_instance_name: str, component_group: zarr.Group) -> None: + def __init__(self, component_instance_name: str, component_group: nodes.Group) -> None: super().__init__(component_instance_name, component_group) - self._timestamps_us: npt.NDArray[np.uint64] = np.array(self._group["timestamps_us"][:]) + self._timestamps_us: npt.NDArray[np.uint64] = self._group.array("timestamps_us").read() self._timestamp_to_index: Dict[int, int] = {int(ts): i for i, ts in enumerate(self._timestamps_us)} - self._descriptor = types.CameraLabelDescriptor.from_dict(self._group["labels"].attrs["descriptor"]) + self._descriptor = types.CameraLabelDescriptor.from_dict( + self._group.group("labels").attr_dict("descriptor") + ) # -- properties -------------------------------------------------------- @@ -2138,11 +2173,11 @@ def timestamps_us(self) -> npt.NDArray[np.uint64]: # -- per-label access -------------------------------------------------- - def _label_group(self, timestamp_us: int) -> zarr.Group: + def _label_group(self, timestamp_us: int) -> nodes.Group: assert timestamp_us in self._timestamp_to_index, ( f"Unknown timestamp: {timestamp_us}. Available: {list(self._timestamp_to_index.keys())[:5]}..." ) - return cast(zarr.Group, self._group["labels"][str(timestamp_us)]) + return self._group.group(f"labels/{timestamp_us}") class CameraLabelHandle: """References label data without eagerly loading it. @@ -2153,7 +2188,7 @@ class CameraLabelHandle: def __init__( self, - label_group: zarr.Group, + label_group: nodes.Group, descriptor: types.CameraLabelDescriptor, timestamp_us: int, generic_meta_data: Dict[str, types.JsonLike], @@ -2181,35 +2216,50 @@ def get_data(self) -> npt.NDArray[Any]: For RAW encoding, applies de-quantization if specified in the schema. For IMAGE_ENCODED encoding, decodes the image bytes via PIL. """ + data = self._label_group.array("data") if self._descriptor.label_schema.encoding == types.LabelEncoding.RAW: - arr = np.array(self._label_group["data"][:]) - - # De-quantize if configured - if (q := self._descriptor.label_schema.quantization) is not None: - arr = (arr.astype(q.intermediate_dtype) * q.scale + q.offset).astype( - self._descriptor.label_schema.dtype - ) - - return arr - + return self._decode_raw(data.read()) elif self._descriptor.label_schema.encoding == types.LabelEncoding.IMAGE_ENCODED: - raw_bytes = bytes(self._label_group["data"][:]) - image = PILImage.open(io.BytesIO(raw_bytes)) - - return np.asarray(image, dtype=self._descriptor.label_schema.dtype) + return self._decode_image(data.read_bytes()) + else: + raise ValueError(f"Unsupported label encoding: {self._descriptor.label_schema.encoding}") + async def get_data_async(self) -> npt.NDArray[Any]: + """Asynchronous :meth:`get_data`""" + data = self._label_group.array("data") + if self._descriptor.label_schema.encoding == types.LabelEncoding.RAW: + return self._decode_raw(await data.read_async()) + elif self._descriptor.label_schema.encoding == types.LabelEncoding.IMAGE_ENCODED: + return self._decode_image(await data.read_bytes_async()) else: raise ValueError(f"Unsupported label encoding: {self._descriptor.label_schema.encoding}") + def _decode_raw(self, arr: npt.NDArray[Any]) -> npt.NDArray[Any]: + # De-quantize if configured + if (q := self._descriptor.label_schema.quantization) is not None: + arr = (arr.astype(q.intermediate_dtype) * q.scale + q.offset).astype( + self._descriptor.label_schema.dtype + ) + return arr + + def _decode_image(self, raw_bytes: bytes) -> npt.NDArray[Any]: + return np.asarray(PILImage.open(io.BytesIO(raw_bytes)), dtype=self._descriptor.label_schema.dtype) + def get_encoded_data(self) -> Optional[bytes]: """Return the raw encoded bytes for IMAGE_ENCODED labels, or None for RAW.""" if self._descriptor.label_schema.encoding == types.LabelEncoding.IMAGE_ENCODED: - return bytes(self._label_group["data"][:]) + return self._label_group.array("data").read_bytes() + return None + + async def get_encoded_data_async(self) -> Optional[bytes]: + """Asynchronous :meth:`get_encoded_data`""" + if self._descriptor.label_schema.encoding == types.LabelEncoding.IMAGE_ENCODED: + return await self._label_group.array("data").read_bytes_async() return None def get_label(self, timestamp_us: int) -> CameraLabelHandle: """Return a lazy handle to the label data at the given timestamp.""" label_group = self._label_group(timestamp_us) return self.CameraLabelHandle( - label_group, self._descriptor, timestamp_us, label_group.attrs["generic_meta_data"] + label_group, self._descriptor, timestamp_us, label_group.attr_dict("generic_meta_data") ) diff --git a/ncore/impl/data/v4/components_test.py b/ncore/impl/data/v4/components_test.py index 5ae85ba0..9187886c 100644 --- a/ncore/impl/data/v4/components_test.py +++ b/ncore/impl/data/v4/components_test.py @@ -16,9 +16,11 @@ import dataclasses import io import tempfile +import threading import unittest -from typing import Dict, Literal, Tuple, cast +from typing import Any, Dict, Literal, Optional, Tuple, cast +from unittest import mock import numpy as np import PIL.Image as PILImage @@ -29,6 +31,8 @@ from ncore.impl.common.transformations import HalfClosedInterval from ncore.impl.common.util import unpack_optional +from ncore.impl.data import nodes, stores +from ncore.impl.data.nodes import BloscCompression from ncore.impl.data.types import ( BBox3, BivariateWindshieldModelParameters, @@ -65,29 +69,39 @@ ) -@parameterized_class( - ("store_type"), - [ - ("itar",), - ("directory",), - ], -) +#: Store types and zarr formats of the component tests (zarr formats unsupported by the installed zarr-python version are +#: skipped) +STORE_TYPES_AND_ZARR_FORMATS = [ + (store_type, zarr_format) for store_type in ("itar", "directory") for zarr_format in (2, 3) +] + + +def _require_zarr_format(test: unittest.TestCase, zarr_format: int) -> None: + if zarr_format not in nodes.SUPPORTED_ZARR_FORMATS: + test.skipTest(f"zarr format v{zarr_format} requires zarr-python 3") + + +@parameterized_class(("store_type", "zarr_format"), STORE_TYPES_AND_ZARR_FORMATS) class TestData4Reload(unittest.TestCase): """Test to verify functionality of V4 data writer + loader""" store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] def setUp(self): + _require_zarr_format(self, self.zarr_format) # Make printed errors more representable numerically np.set_printoptions(floatmode="unique", linewidth=200, suppress=True) @parameterized.expand( [ - (False,), - (True,), + (False, None), + (True, None), + (True, 0), # uncached nodes + (True, 1), # nodes evicted on most accesses ] ) - def test_reload(self, open_consolidated: bool): + def test_reload(self, open_consolidated: bool, node_cache_size: Optional[int]): """Test to make sure serialized data is faithfully reloaded""" tempdir = tempfile.TemporaryDirectory() @@ -101,6 +115,7 @@ def test_reload(self, open_consolidated: bool): ref_sequence_timestamp_interval_us := HalfClosedInterval(int(0 * 1e6), int(1 * 1e6) + 1) ), store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data=cast(Dict[str, JsonLike], ref_generic_sequence_meta_data := {"some": 1, "key": 1.2}), ) @@ -506,6 +521,7 @@ def normalize_points(vectors: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: store_base_name=store_writer._store_base_name, sequence_reader=SequenceComponentGroupsReader(store_paths, open_consolidated=open_consolidated), store_type=self.store_type, + zarr_format=self.zarr_format, ) # Store cuboids with the new writer @@ -573,7 +589,9 @@ def normalize_points(vectors: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: store_paths += store_writer.finalize() ## Reload sequence and verify consistency - store_reader = SequenceComponentGroupsReader(store_paths, open_consolidated=open_consolidated) + store_reader = SequenceComponentGroupsReader( + store_paths, open_consolidated=open_consolidated, node_cache_size=node_cache_size + ) # check sequence data self.assertEqual(store_reader.sequence_id, ref_sequence_id) @@ -1096,6 +1114,7 @@ def test_component_generic_data_roundtrip(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -1160,6 +1179,7 @@ def test_component_generic_data_backwards_compat(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -1205,6 +1225,7 @@ def test_component_generic_data_meta_overwrite(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -1254,6 +1275,7 @@ def test_component_generic_data_meta_only(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -1294,13 +1316,7 @@ def test_component_generic_data_meta_only(self) -> None: tempdir.cleanup() -@parameterized_class( - ("store_type"), - [ - ("itar",), - ("directory",), - ], -) +@parameterized_class(("store_type", "zarr_format"), STORE_TYPES_AND_ZARR_FORMATS) class TestDataNewComponent(unittest.TestCase): """ Test to demonstrate how to extend an existing dataset with a new custom component. @@ -1312,8 +1328,10 @@ class TestDataNewComponent(unittest.TestCase): """ store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] def setUp(self): + _require_zarr_format(self, self.zarr_format) np.set_printoptions(floatmode="unique", linewidth=200, suppress=True) def test_new_component_extension(self): @@ -1342,6 +1360,7 @@ def test_new_component_extension(self): sequence_id=sequence_id, sequence_timestamp_interval_us=(timestamp_interval := HalfClosedInterval(int(0 * 1e6), int(10 * 1e6) + 1)), store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={"dataset": "test", "version": 1.0}, ) @@ -1413,21 +1432,23 @@ def store_velocity( return self def finalize(self): - """Write all velocity data to zarr storage""" + """Write all velocity data to the component group""" if self.velocities: velocities_array = np.stack(self.velocities) timestamps_array = np.array(self.timestamps, dtype=np.uint64) - # Store as zarr arrays - self._group.create_dataset( + # Store as arrays + self._group.create_array( "velocities", - data=velocities_array, + velocities_array, dtype=velocities_array.dtype, + compression=BloscCompression(shuffle="shuffle"), ) - self._group.create_dataset( + self._group.create_array( "timestamps_us", - data=timestamps_array, + timestamps_array, dtype=np.uint64, + compression=BloscCompression(shuffle="shuffle"), ) class Reader(ComponentReader): @@ -1444,8 +1465,8 @@ def supports_component_version(version: str) -> bool: def get_velocities(self) -> Tuple[np.ndarray, np.ndarray]: """Returns (velocities, timestamps_us) arrays""" - velocities = np.array(self._group["velocities"][:]) - timestamps_us = np.array(self._group["timestamps_us"][:]) + velocities = self._group.array("velocities").read() + timestamps_us = self._group.array("timestamps_us").read() return velocities, timestamps_us # ============================================================================ @@ -1471,6 +1492,7 @@ def get_velocities(self) -> Tuple[np.ndarray, np.ndarray]: output_dir_path=UPath(tempdir.name), # Same directory as initial store_base_name=sequence_id + "-extension", store_type=self.store_type, + zarr_format=self.zarr_format, ) # Now add our custom velocity component to the extended dataset @@ -1605,9 +1627,10 @@ def finalize(self): accelerations_array = np.stack(self.accelerations) timestamps_array = np.array(self.timestamps, dtype=np.uint64) - self._group.create_dataset("velocities", data=velocities_array) - self._group.create_dataset("accelerations", data=accelerations_array) # NEW - self._group.create_dataset("timestamps_us", data=timestamps_array) + compression = BloscCompression(shuffle="shuffle") + self._group.create_array("velocities", velocities_array, compression=compression) + self._group.create_array("accelerations", accelerations_array, compression=compression) # NEW + self._group.create_array("timestamps_us", timestamps_array, compression=compression) # Create a backward-compatible reader that can read both v1 and v2 class VelocityComponentBackwardCompatibleReader(ComponentReader): @@ -1624,15 +1647,15 @@ def supports_component_version(version: str) -> bool: def get_velocities(self) -> Tuple[np.ndarray, np.ndarray]: """Returns velocities (works for both v1 and v2)""" - velocities = np.array(self._group["velocities"][:]) - timestamps_us = np.array(self._group["timestamps_us"][:]) + velocities = self._group.array("velocities").read() + timestamps_us = self._group.array("timestamps_us").read() return velocities, timestamps_us def get_accelerations(self) -> np.ndarray: """Returns accelerations (only available in v2)""" if self.component_version == "v1": raise ValueError("Acceleration data not available in v1") - return np.array(self._group["accelerations"][:]) + return self._group.array("accelerations").read() # Test that v1 reader cannot read v2 data v2_store_writer = SequenceComponentGroupsWriter( @@ -1641,6 +1664,7 @@ def get_accelerations(self) -> np.ndarray: sequence_id="test_v2", sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={"version": "v2_test"}, ) @@ -1704,19 +1728,15 @@ def get_accelerations(self) -> np.ndarray: # Version compatibility tests passed - all tests completed successfully -@parameterized_class( - ("store_type"), - [ - ("itar",), - ("directory",), - ], -) +@parameterized_class(("store_type", "zarr_format"), STORE_TYPES_AND_ZARR_FORMATS) class TestPointCloudsComponent(unittest.TestCase): """Round-trip tests for the PointCloudsComponent Writer/Reader.""" store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] def setUp(self): + _require_zarr_format(self, self.zarr_format) np.set_printoptions(floatmode="unique", linewidth=200, suppress=True) def _make_writer_reader( @@ -1735,6 +1755,7 @@ def _make_writer_reader( sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -2059,19 +2080,15 @@ def test_generic_data_and_metadata(self): tmpdir.cleanup() -@parameterized_class( - ("store_type"), - [ - ("itar",), - ("directory",), - ], -) +@parameterized_class(("store_type", "zarr_format"), STORE_TYPES_AND_ZARR_FORMATS) class TestCameraLabelsComponent(unittest.TestCase): """Round-trip tests for the CameraLabelsComponent Writer/Reader.""" store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] def setUp(self): + _require_zarr_format(self, self.zarr_format) np.set_printoptions(floatmode="unique", linewidth=200, suppress=True) def _make_writer( @@ -2095,6 +2112,7 @@ def _make_writer( sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -2420,6 +2438,7 @@ def test_multiple_label_types_per_camera(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type="directory", + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -2582,6 +2601,7 @@ def test_reject_empty_camera_id(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -2748,24 +2768,116 @@ def test_schema_json_roundtrip(self) -> None: self.assertIsNone(rt.quantization) +class TestReaderOpening(unittest.TestCase): + """Component stores are opened concurrently, their hierarchies sequentially on the calling thread""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + writer = SequenceComponentGroupsWriter( + output_dir_path=UPath(self._tmp.name), + store_base_name="seq", + sequence_id="seq", + sequence_timestamp_interval_us=HalfClosedInterval(0, 10_000_001), + generic_meta_data={}, + ) + for name in ("a", "b", "c"): + writer.register_component_writer(PosesComponent.Writer, name, f"poses_{name}").store_static_pose( + "x", "y", np.eye(4, dtype=np.float32) + ) + self.store_paths = writer.finalize() + + def tearDown(self): + self._tmp.cleanup() + + def test_hierarchies_are_opened_on_the_calling_thread(self): + threads = [] + open_store = stores.open_store + + def recording_open_store(*args: Any, **kwargs: Any) -> Any: + threads.append(threading.current_thread()) + return open_store(*args, **kwargs) + + with mock.patch.object(stores, "open_store", side_effect=recording_open_store): + reader = SequenceComponentGroupsReader(self.store_paths, max_threads=4) + self.assertEqual(len(threads), len(self.store_paths)) + self.assertEqual(set(threads), {threading.current_thread()}) + self.assertEqual(sorted(reader.open_component_readers(PosesComponent.Reader)), ["a", "b", "c"]) + + +class TestReaderNodeCaches(unittest.TestCase): + """Component readers cache the opened nodes of their component (sized by node_cache_size)""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + writer = SequenceComponentGroupsWriter( + output_dir_path=UPath(self._tmp.name), + store_base_name="seq", + sequence_id="seq", + sequence_timestamp_interval_us=HalfClosedInterval(0, 10_000_001), + generic_meta_data={}, + ) + lidar_writer = writer.register_component_writer(LidarSensorComponent.Writer, "lidar", "lidar") + for i in range(4): + direction = np.tile(np.array([1.0, 0.0, 0.0], dtype=np.float32), (5, 1)) + lidar_writer.store_frame( + direction=direction, + timestamp_us=np.full(5, 10 + i * 1_000_000, dtype=np.uint64), + model_element=None, + distance_m=np.full((1, 5), i, dtype=np.float32), + intensity=np.ones((1, 5), dtype=np.float32), + frame_timestamps_us=np.array([10 + i * 1_000_000, 20 + i * 1_000_000], dtype=np.uint64), + generic_data={}, + generic_meta_data={}, + ) + self.store_paths = writer.finalize() + + def tearDown(self): + self._tmp.cleanup() + + @parameterized.expand([(None,), (0,), (1,), (3,)]) + def test_node_cache_sizes(self, node_cache_size: Optional[int]): + reader = SequenceComponentGroupsReader(self.store_paths, node_cache_size=node_cache_size) + lidar = reader.open_component_readers(LidarSensorComponent.Reader)["lidar"] + frames = [int(ts) for ts in lidar.frames_timestamps_us[:, 1]] + for _ in range(2): + for i, ts in enumerate(frames): + np.testing.assert_array_equal(lidar.get_frame_ray_bundle_return_data(ts, "distance_m", 0), [i] * 5) + + cache = lidar._group.cache + if node_cache_size == 0: + self.assertIsNone(cache) + return + assert cache is not None + self.assertEqual(cache.max_entries, node_cache_size) + # every frame read opens the frame's returns group and its distance_m array + accessed = ["lidars/lidar/frames"] + [ + f"lidars/lidar/frames/{ts}/ray_bundle_returns{suffix}" for ts in frames for suffix in ("", "/distance_m") + ] + if node_cache_size is None: + self.assertEqual(sorted(cache.keys()), sorted(accessed)) + else: + # the most recently used nodes only + self.assertEqual(cache.keys(), accessed[-node_cache_size:]) + + def test_invalid_node_cache_size(self): + with self.assertRaises(ValueError): + SequenceComponentGroupsReader(self.store_paths, node_cache_size=-1) + + # ============================================================================== -# Tests for zero-dimension array support (_normalize_chunks) +# Tests for zero-dimension array support # ============================================================================== -@parameterized_class( - ("store_type"), - [ - ("itar",), - ("directory",), - ], -) +@parameterized_class(("store_type", "zarr_format"), STORE_TYPES_AND_ZARR_FORMATS) class TestZeroDimArraySupport(unittest.TestCase): """Tests that arrays with zero-length dimensions can be stored and read back correctly.""" store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] def setUp(self): + _require_zarr_format(self, self.zarr_format) np.set_printoptions(floatmode="unique", linewidth=200, suppress=True) # ------------------------------------------------------------------ @@ -2782,6 +2894,7 @@ def test_point_cloud_zero_points(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -2845,6 +2958,7 @@ def test_point_cloud_zero_points_multidim_generic_data(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -2887,6 +3001,7 @@ def test_lidar_zero_rays(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -2953,6 +3068,7 @@ def test_radar_zero_rays(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -3009,6 +3125,7 @@ def test_camera_label_zero_shape_suffix_dim(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -3062,6 +3179,7 @@ def test_component_generic_data_zero_dim(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -3136,6 +3254,7 @@ def test_per_frame_generic_data_zero_dim(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) diff --git a/ncore/impl/data/v4/formats_test.py b/ncore/impl/data/v4/formats_test.py new file mode 100644 index 00000000..a109ab26 --- /dev/null +++ b/ncore/impl/data/v4/formats_test.py @@ -0,0 +1,394 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cross-format / cross-runtime consistency of NCore V4 test data + +Compares the zarr format v2 and zarr format v3 serializations of the same (time-trimmed) test sequence, as well as +re-serializations performed by the zarr-python version under test. +""" + +from __future__ import annotations + +import asyncio +import tempfile +import unittest + +from typing import Any, Dict, List, Literal, Optional, cast + +import numpy as np +import parameterized + +from python.runfiles import Runfiles +from upath import UPath + +from ncore.impl.data import nodes +from ncore.impl.data.v4.components import ( + CameraSensorComponent, + CuboidsComponent, + IntrinsicsComponent, + LidarSensorComponent, + MasksComponent, + PosesComponent, + RadarSensorComponent, + SequenceComponentGroupsReader, + SequenceComponentGroupsWriter, +) + + +_RUNFILES = Runfiles.Create() +assert _RUNFILES is not None +_RUNFILES_: Runfiles = _RUNFILES + +_SEQUENCE = "c9b05cf4-afb9-11ec-b3c2-00044bf65fcb@1648597318700123-1648599151600035" + + +def _dataset(name: str) -> UPath: + path = UPath(_RUNFILES_.Rlocation(f"test-data-v4-trimmed/{name}/{_SEQUENCE}.json") or "") + assert path.exists(), f"test data {name} not found (missing data dependency?)" + return path + + +_TRIMMED_V2 = _dataset("zarr-v2") +_TRIMMED_V3 = _dataset("zarr-v3") + + +#: Frame subsampling of the reserialization test (every N-th frame, plus the last frame of each sensor) +_FRAME_STRIDE = 5 + + +def _subsampled(frames_timestamps_us: np.ndarray) -> np.ndarray: + """Subsampled frames (every _FRAME_STRIDE-th frame, plus the last frame)""" + indices = sorted(set(range(0, len(frames_timestamps_us), _FRAME_STRIDE)) | {len(frames_timestamps_us) - 1}) + return frames_timestamps_us[[i for i in indices if i >= 0]] + + +def _summary(reader: SequenceComponentGroupsReader, subsampled: bool = False, frames: bool = True) -> Dict[str, Any]: + """Collects a comparable summary of all data of a sequence (optionally of the subsampled frames only, or without + masks and sensor frames)""" + + def selected(timestamps_us: np.ndarray) -> np.ndarray: + return _subsampled(timestamps_us) if subsampled else timestamps_us + + out: Dict[str, Any] = { + "sequence_id": reader.sequence_id, + "interval": (reader.sequence_timestamp_interval_us.start, reader.sequence_timestamp_interval_us.stop), + "generic_meta_data": reader.generic_meta_data, + } + + for name, poses in sorted(reader.open_component_readers(PosesComponent.Reader).items()): + out[f"poses/{name}/static"] = {k: v.tolist() for k, v in poses.get_static_poses()} + out[f"poses/{name}/dynamic"] = {k: (p.tolist(), t.tolist()) for k, (p, t) in poses.get_dynamic_poses()} + + for name, intrinsics in sorted(reader.open_component_readers(IntrinsicsComponent.Reader).items()): + out[f"intrinsics/{name}"] = (intrinsics._group.group("cameras").attrs, intrinsics._group.group("lidars").attrs) + + for name, cuboids in sorted(reader.open_component_readers(CuboidsComponent.Reader).items()): + out[f"cuboids/{name}"] = [obs.to_dict() for obs in cuboids.get_observations()] + + if not frames: + return out + + for name, masks in sorted(reader.open_component_readers(MasksComponent.Reader).items()): + out[f"masks/{name}"] = { + camera_id: { + mask_name: np.asarray(image).tobytes() for mask_name, image in masks.get_camera_mask_images(camera_id) + } + for camera_id in masks._group.group("cameras").members() + } + + for name, cameras in sorted(reader.open_component_readers(CameraSensorComponent.Reader).items()): + frames: List[object] = [] + for ts in selected(cameras.frames_timestamps_us)[:, 1]: + ts = int(ts) + data = cameras.get_frame_data(ts) + frames.append( + ( + ts, + data.get_encoded_image_format(), + data.get_encoded_image_data(), + { + n: cameras.get_frame_generic_data(ts, n).tobytes() + for n in sorted(cameras.get_frame_generic_data_names(ts)) + }, + cameras.get_frame_generic_meta_data(ts), + ) + ) + out[f"cameras/{name}"] = frames + + for component in (LidarSensorComponent, RadarSensorComponent): + for name, sensor in sorted(reader.open_component_readers(component.Reader).items()): + sensor_frames = [] + for ts in selected(sensor.frames_timestamps_us)[:, 1]: + ts = int(ts) + sensor_frames.append( + ( + ts, + { + n: sensor.get_frame_ray_bundle_data(ts, n).tobytes() + for n in sorted(sensor.get_frame_ray_bundle_data_names(ts)) + }, + { + n: sensor.get_frame_ray_bundle_return_data(ts, n, None).tobytes() + for n in sorted(sensor.get_frame_ray_bundle_return_data_names(ts)) + }, + sensor.get_frame_ray_bundle_return_valid_mask(ts).tobytes(), + { + n: sensor.get_frame_generic_data(ts, n).tobytes() + for n in sorted(sensor.get_frame_generic_data_names(ts)) + }, + sensor.get_frame_generic_meta_data(ts), + ) + ) + out[f"{component.COMPONENT_NAME}/{name}"] = sensor_frames + + return out + + +async def _summary_async(reader: SequenceComponentGroupsReader) -> Dict[str, Any]: + """:func:`_summary` via the asynchronous reader APIs, all frames of all sensors read concurrently""" + out = _summary(reader, subsampled=False, frames=False) + + async def masks(masks: MasksComponent.Reader) -> Dict[str, Any]: + out: Dict[str, Any] = {} + for camera_id in masks._group.group("cameras").members(): + out[camera_id] = {} + for mask_name in masks.get_camera_mask_names(camera_id): + image = await masks.get_camera_mask_image_async(camera_id, mask_name) + out[camera_id][mask_name] = np.asarray(image).tobytes() + return out + + async def camera_frame(cameras: CameraSensorComponent.Reader, ts: int) -> object: + data, generic = await asyncio.gather( + cameras.get_frame_data_async(ts), + asyncio.gather( + *(cameras.get_frame_generic_data_async(ts, n) for n in sorted(cameras.get_frame_generic_data_names(ts))) + ), + ) + names = sorted(cameras.get_frame_generic_data_names(ts)) + return ( + ts, + data.get_encoded_image_format(), + data.get_encoded_image_data(), + {n: value.tobytes() for n, value in zip(names, generic)}, + cameras.get_frame_generic_meta_data(ts), + ) + + async def sensor_frame(sensor: Any, ts: int) -> object: + ray_names = sorted(sensor.get_frame_ray_bundle_data_names(ts)) + return_names = sorted(sensor.get_frame_ray_bundle_return_data_names(ts)) + generic_names = sorted(sensor.get_frame_generic_data_names(ts)) + rays, returns, mask, generic = await asyncio.gather( + asyncio.gather(*(sensor.get_frame_ray_bundle_data_async(ts, n) for n in ray_names)), + asyncio.gather(*(sensor.get_frame_ray_bundle_return_data_async(ts, n, None) for n in return_names)), + sensor.get_frame_ray_bundle_return_valid_mask_async(ts), + asyncio.gather(*(sensor.get_frame_generic_data_async(ts, n) for n in generic_names)), + ) + return ( + ts, + {n: v.tobytes() for n, v in zip(ray_names, rays)}, + {n: v.tobytes() for n, v in zip(return_names, returns)}, + mask.tobytes(), + {n: v.tobytes() for n, v in zip(generic_names, generic)}, + sensor.get_frame_generic_meta_data(ts), + ) + + for name, masks_reader in sorted(reader.open_component_readers(MasksComponent.Reader).items()): + out[f"masks/{name}"] = await masks(masks_reader) + for name, cameras in sorted(reader.open_component_readers(CameraSensorComponent.Reader).items()): + out[f"cameras/{name}"] = list( + await asyncio.gather(*(camera_frame(cameras, int(ts)) for ts in cameras.frames_timestamps_us[:, 1])) + ) + for component in (LidarSensorComponent, RadarSensorComponent): + for name, sensor in sorted(reader.open_component_readers(component.Reader).items()): + out[f"{component.COMPONENT_NAME}/{name}"] = list( + await asyncio.gather(*(sensor_frame(sensor, int(ts)) for ts in sensor.frames_timestamps_us[:, 1])) + ) + return out + + +class TestZarrFormats(unittest.TestCase): + """zarr format v2 and v3 serializations of a sequence need to be identical in content""" + + reference: Dict[str, Any] # summary of the zarr format v2 test data + reference_subsampled: Dict[str, Any] # summary of the subsampled frames of the zarr format v2 test data + + @classmethod + def setUpClass(cls) -> None: + cls.reference = _summary(SequenceComponentGroupsReader([_TRIMMED_V2])) + cls.reference_subsampled = _summary(SequenceComponentGroupsReader([_TRIMMED_V2]), subsampled=True) + + def test_v2_v3_consistency(self): + if 3 not in nodes.SUPPORTED_ZARR_FORMATS: + # zarr format v3 data can't be read with zarr-python 2 + with self.assertRaises(Exception): + SequenceComponentGroupsReader([_TRIMMED_V3]) + return + + _assert_summaries_equal(self, self.reference, _summary(SequenceComponentGroupsReader([_TRIMMED_V3]))) + + @parameterized.parameterized.expand([(None,), (0,), (1,), (16,)]) + def test_node_cache_sizes(self, node_cache_size: Optional[int]): + """All node cache configurations of the readers return the same data, and caches respect their bounds""" + reader = SequenceComponentGroupsReader([_TRIMMED_V2], node_cache_size=node_cache_size) + _assert_summaries_equal(self, self.reference, _summary(reader)) + + lidar = next(iter(reader.open_component_readers(LidarSensorComponent.Reader).values())) + frames = lidar.frames_timestamps_us[:, 1] + for _ in range(2): + for ts in frames: + lidar.get_frame_ray_bundle_data(int(ts), "direction") + if node_cache_size == 0: + self.assertIsNone(lidar._group.cache) + else: + assert (cache := lidar._group.cache) is not None + self.assertEqual(cache.max_entries, node_cache_size) + self.assertLessEqual(len(cache), node_cache_size or len(cache)) + frame_entries = [key for key in cache.keys() if key.endswith("/ray_bundle/direction")] + self.assertEqual(len(frame_entries), min(len(frames), node_cache_size or len(frames))) + + @parameterized.parameterized.expand([("v2",), ("v3",)]) + def test_async_reads(self, data: str): + """Asynchronous reader APIs (all frames read concurrently) return the same data as the synchronous APIs""" + if data == "v3" and 3 not in nodes.SUPPORTED_ZARR_FORMATS: + self.skipTest("zarr format v3 data requires zarr-python 3") + reader = SequenceComponentGroupsReader([_TRIMMED_V2 if data == "v2" else _TRIMMED_V3]) + _assert_summaries_equal(self, self.reference, asyncio.run(_summary_async(reader))) + + def test_invalid_node_cache_size(self): + with self.assertRaises(ValueError): + SequenceComponentGroupsReader([_TRIMMED_V2], node_cache_size=-1) + + @parameterized.parameterized.expand([(f,) for f in nodes.SUPPORTED_ZARR_FORMATS]) + def test_reserialization(self, zarr_format: int): + """Re-serializing (subsampled frames of) all components of the test data with the installed zarr-python version + via the public component APIs preserves all content""" + reader = SequenceComponentGroupsReader([_TRIMMED_V2]) + + with tempfile.TemporaryDirectory() as output_dir: + writer = SequenceComponentGroupsWriter.from_reader( + UPath(output_dir), "copy", reader, zarr_format=cast(Literal[2, 3], zarr_format) + ) + _copy_components(reader, writer) + copy = _summary(SequenceComponentGroupsReader(writer.finalize())) # (contains the subsampled frames only) + + self.assertEqual(self.reference_subsampled.keys(), copy.keys()) + for component in ("poses/", "intrinsics/", "masks/", "cameras/", "lidars/", "radars/", "cuboids/"): + self.assertTrue(any(key.startswith(component) for key in copy), f"no {component} data in the test data") + _assert_summaries_equal(self, self.reference_subsampled, copy) + + +def _assert_summaries_equal(test: unittest.TestCase, expected: Dict[str, Any], actual: Dict[str, Any]) -> None: + """Compares sequence summaries per component / frame / field (without expensive diffs of large payloads)""" + test.assertEqual(sorted(expected), sorted(actual)) + for key, expected_value in expected.items(): + actual_value = actual[key] + if not isinstance(expected_value, list): + test.assertTrue(expected_value == actual_value, key) + continue + test.assertEqual(len(expected_value), len(actual_value), key) + for index, (expected_frame, actual_frame) in enumerate(zip(expected_value, actual_value)): + if not isinstance(expected_frame, tuple): + test.assertTrue(expected_frame == actual_frame, f"{key}[{index}]") + continue + for field, (expected_field, actual_field) in enumerate(zip(expected_frame, actual_frame)): + if isinstance(expected_field, dict): + expected_items = cast(Dict[str, object], expected_field) + actual_items = cast(Dict[str, object], actual_field) + test.assertEqual(sorted(expected_items), sorted(actual_items), f"{key}[{index}][{field}]") + for name, expected_item in expected_items.items(): + test.assertTrue(expected_item == actual_items[name], f"{key}[{index}][{field}][{name}] differs") + else: + test.assertTrue(expected_field == actual_field, f"{key}[{index}][{field}] differs") + + +def _copy_components(reader: SequenceComponentGroupsReader, writer: SequenceComponentGroupsWriter) -> None: + """Copies (subsampled frames of) all components of a sequence via the public component APIs""" + + for name, poses in reader.open_component_readers(PosesComponent.Reader).items(): + poses_writer = writer.register_component_writer(PosesComponent.Writer, name, name, poses.generic_meta_data) + for (source, target), pose in poses.get_static_poses(): + poses_writer.store_static_pose(source, target, pose) + for (source, target), (trajectory, timestamps_us) in poses.get_dynamic_poses(): + poses_writer.store_dynamic_pose(source, target, trajectory, timestamps_us) + + for name, intrinsics in reader.open_component_readers(IntrinsicsComponent.Reader).items(): + intrinsics_writer = writer.register_component_writer( + IntrinsicsComponent.Writer, name, name, intrinsics.generic_meta_data + ) + for camera_id in intrinsics._group.group("cameras").members(): + intrinsics_writer.store_camera_intrinsics(camera_id, intrinsics.get_camera_model_parameters(camera_id)) + for lidar_id in intrinsics._group.group("lidars").members(): + if (lidar_model := intrinsics.get_lidar_model_parameters(lidar_id)) is not None: + intrinsics_writer.store_lidar_intrinsics(lidar_id, lidar_model) + + for name, masks in reader.open_component_readers(MasksComponent.Reader).items(): + masks_writer = writer.register_component_writer(MasksComponent.Writer, name, name, masks.generic_meta_data) + for camera_id in masks._group.group("cameras").members(): + masks_writer.store_camera_masks(camera_id, dict(masks.get_camera_mask_images(camera_id))) + + for name, cameras in reader.open_component_readers(CameraSensorComponent.Reader).items(): + camera_writer = writer.register_component_writer( + CameraSensorComponent.Writer, name, name, cameras.generic_meta_data + ) + for frame_timestamps_us in _subsampled(cameras.frames_timestamps_us): + ts = int(frame_timestamps_us[1]) + data = cameras.get_frame_data(ts) + camera_writer.store_frame( + data.get_encoded_image_data(), + data.get_encoded_image_format(), + frame_timestamps_us, + {n: cameras.get_frame_generic_data(ts, n) for n in cameras.get_frame_generic_data_names(ts)}, + cameras.get_frame_generic_meta_data(ts), + ) + + for name, lidar in reader.open_component_readers(LidarSensorComponent.Reader).items(): + lidar_writer = writer.register_component_writer( + LidarSensorComponent.Writer, name, name, lidar.generic_meta_data + ) + for frame_timestamps_us in _subsampled(lidar.frames_timestamps_us): + ts = int(frame_timestamps_us[1]) + ray_names = lidar.get_frame_ray_bundle_data_names(ts) + lidar_writer.store_frame( + direction=lidar.get_frame_ray_bundle_data(ts, "direction"), + timestamp_us=lidar.get_frame_ray_bundle_data(ts, "timestamp_us"), + model_element=( + lidar.get_frame_ray_bundle_data(ts, "model_element") if "model_element" in ray_names else None + ), + distance_m=lidar.get_frame_ray_bundle_return_data(ts, "distance_m", None), + intensity=lidar.get_frame_ray_bundle_return_data(ts, "intensity", None), + frame_timestamps_us=frame_timestamps_us, + generic_data={n: lidar.get_frame_generic_data(ts, n) for n in lidar.get_frame_generic_data_names(ts)}, + generic_meta_data=lidar.get_frame_generic_meta_data(ts), + ) + + for name, radar in reader.open_component_readers(RadarSensorComponent.Reader).items(): + radar_writer = writer.register_component_writer( + RadarSensorComponent.Writer, name, name, radar.generic_meta_data + ) + for frame_timestamps_us in _subsampled(radar.frames_timestamps_us): + ts = int(frame_timestamps_us[1]) + radar_writer.store_frame( + direction=radar.get_frame_ray_bundle_data(ts, "direction"), + timestamp_us=radar.get_frame_ray_bundle_data(ts, "timestamp_us"), + distance_m=radar.get_frame_ray_bundle_return_data(ts, "distance_m", None), + frame_timestamps_us=frame_timestamps_us, + generic_data={n: radar.get_frame_generic_data(ts, n) for n in radar.get_frame_generic_data_names(ts)}, + generic_meta_data=radar.get_frame_generic_meta_data(ts), + ) + + for name, cuboids in reader.open_component_readers(CuboidsComponent.Reader).items(): + writer.register_component_writer( + CuboidsComponent.Writer, name, name, cuboids.generic_meta_data + ).store_observations(list(cuboids.get_observations())) diff --git a/ncore/impl/data_converter/base.py b/ncore/impl/data_converter/base.py index 7a692bf6..347a813b 100644 --- a/ncore/impl/data_converter/base.py +++ b/ncore/impl/data_converter/base.py @@ -20,7 +20,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Optional, Tuple +from typing import Literal, Optional, Tuple from upath import UPath @@ -45,6 +45,9 @@ class BaseDataConverterConfig: debug: bool debug_port: int + ## Output + zarr_format: Literal[2, 3] # On-disk zarr format of written component stores (3 requires zarr-python 3) + class BaseDataConverter(ABC): """ @@ -62,6 +65,9 @@ def __init__(self, config: BaseDataConverterConfig) -> None: self.output_dir = UPath(config.output_dir) + # On-disk zarr format of written component stores + self.zarr_format: Literal[2, 3] = config.zarr_format + # External sensor selection overwrites # Store `None`` for `_active__ids` in case all sensors should be used, as the # actual full list of sensor ids will be passed via `get_active__ids()` diff --git a/tools/data_converter/argoverse2/BUILD.bazel b/tools/data_converter/argoverse2/BUILD.bazel index a645eb51..cf67b628 100644 --- a/tools/data_converter/argoverse2/BUILD.bazel +++ b/tools/data_converter/argoverse2/BUILD.bazel @@ -67,6 +67,7 @@ alias( # Integration test for the Argoverse 2 converter (requires AV2_DIR env var) pytest_test( name = "pytest_converter", + size = "large", # converts the test sequence for all store types / zarr formats srcs = ["converter_test.py"], python_versions = ["3.11"], tags = ["manual"], # Only run when explicitly requested (needs external data) diff --git a/tools/data_converter/argoverse2/converter.py b/tools/data_converter/argoverse2/converter.py index 7c999e1a..ceba5c5a 100644 --- a/tools/data_converter/argoverse2/converter.py +++ b/tools/data_converter/argoverse2/converter.py @@ -245,6 +245,7 @@ def convert_sequence(self, sequence_id: str) -> None: sequence_id=log_id, sequence_timestamp_interval_us=sequence_timestamp_interval_us, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={ "source_dataset": "argoverse2", "argoverse2_split": self._split, diff --git a/tools/data_converter/argoverse2/converter_test.py b/tools/data_converter/argoverse2/converter_test.py index c70bc25e..e76c7fca 100644 --- a/tools/data_converter/argoverse2/converter_test.py +++ b/tools/data_converter/argoverse2/converter_test.py @@ -33,6 +33,7 @@ from parameterized import parameterized_class from upath import UPath +from ncore.impl.data import nodes from ncore.impl.data.types import ( IdealPinholeCameraModelParameters, RowOffsetStructuredSpinningLidarModelParameters, @@ -52,11 +53,8 @@ @parameterized_class( - ("store_type",), - [ - ("itar",), - ("directory",), - ], + ("store_type", "zarr_format"), + [(store_type, zarr_format) for store_type in ("itar", "directory") for zarr_format in nodes.SUPPORTED_ZARR_FORMATS], ) class TestArgoverse2Converter(unittest.TestCase): """Integration tests for the Argoverse 2 data converter. @@ -66,6 +64,7 @@ class TestArgoverse2Converter(unittest.TestCase): """ store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] @classmethod def setUpClass(cls): @@ -97,6 +96,7 @@ def setUpClass(cls): split=cls.split, log_id=cls.log_id, store_type=cls.store_type, + zarr_format=cls.zarr_format, component_group_profile="separate-sensors", store_sequence_meta=True, ) @@ -284,7 +284,7 @@ def test_lidar_model_reconstructs_directions(self): ts = int(reader.frames_timestamps_us[0, 1]) direction = reader.get_frame_ray_bundle_data(ts, "direction") elem = reader.get_frame_ray_bundle_data(ts, "model_element") - distance = np.asarray(reader._get_ray_bundle_returns_group(ts)["distance_m"])[0] + distance = reader._get_ray_bundle_returns_group(ts).array("distance_m").read()[0] far = np.isfinite(distance) & (distance > 20.0) & (np.linalg.norm(direction, axis=1) > 0) self.assertGreater(int(far.sum()), 100, f"{unit}: too few far returns to validate") @@ -352,7 +352,7 @@ def test_cuboids_align_with_lidar(self): rig_poses, pose_ts = poses_reader.get_dynamic_pose("rig", "world") direction = lidar_reader.get_frame_ray_bundle_data(ts, "direction") - distance = np.asarray(lidar_reader._get_ray_bundle_returns_group(ts)["distance_m"])[0] + distance = lidar_reader._get_ray_bundle_returns_group(ts).array("distance_m").read()[0] point_ts = lidar_reader.get_frame_ray_bundle_data(ts, "timestamp_us").astype(np.int64) valid = np.isfinite(distance) & (distance > 0) pts_sensor = direction[valid] * distance[valid, None] diff --git a/tools/data_converter/cli.py b/tools/data_converter/cli.py index 2e394587..89eb38e3 100644 --- a/tools/data_converter/cli.py +++ b/tools/data_converter/cli.py @@ -44,6 +44,13 @@ @click.option( "--debug-port", type=int, default=5678, help="The port on which debugpy will wait for a client to connect" ) +@click.option( + "--zarr-format", + type=click.Choice(["2", "3"]), + default="2", + callback=lambda _ctx, _param, value: int(value), + help="On-disk zarr format of written component stores (format 3 requires zarr-python 3 for writing and reading)", +) @click.option("--no-cameras", is_flag=True, default=False, help="Disable exporting any camera sensor") @click.option( "--camera-id", diff --git a/tools/data_converter/colmap/converter.py b/tools/data_converter/colmap/converter.py index 2cb28c7c..1e3e4d26 100644 --- a/tools/data_converter/colmap/converter.py +++ b/tools/data_converter/colmap/converter.py @@ -265,6 +265,7 @@ def convert_sequence(self, sequence_id: str) -> None: sequence_id=self.sequence_name, sequence_timestamp_interval_us=sequence_timestamp_interval_us, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data=self.generic_meta_data, ) @@ -768,6 +769,7 @@ def scannetpp_v4(ctx, **kwargs): lidar_ids=base.lidar_ids, no_radars=base.no_radars, radar_ids=base.radar_ids, + zarr_format=base.zarr_format, colmap_dir="dslr/colmap", images_dir=images_dir, masks_dir=masks_dir, diff --git a/tools/data_converter/kitti/BUILD.bazel b/tools/data_converter/kitti/BUILD.bazel index 04946c07..e9e8a5e3 100644 --- a/tools/data_converter/kitti/BUILD.bazel +++ b/tools/data_converter/kitti/BUILD.bazel @@ -59,6 +59,7 @@ alias( # Integration test for the KITTI converter (requires KITTI_RAW_DIR env var) pytest_test( name = "pytest_converter", + size = "large", # converts the test sequence for all store types / zarr formats srcs = ["converter_test.py"], python_versions = ["3.11"], tags = ["manual"], # Only run when explicitly requested (needs external data) diff --git a/tools/data_converter/kitti/converter.py b/tools/data_converter/kitti/converter.py index 39b464a3..919039af 100644 --- a/tools/data_converter/kitti/converter.py +++ b/tools/data_converter/kitti/converter.py @@ -211,6 +211,7 @@ def convert_sequence(self, sequence_id: str) -> None: sequence_id=sequence_name, sequence_timestamp_interval_us=sequence_timestamp_interval_us, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) diff --git a/tools/data_converter/kitti/converter_test.py b/tools/data_converter/kitti/converter_test.py index 17121df6..5c59ae38 100644 --- a/tools/data_converter/kitti/converter_test.py +++ b/tools/data_converter/kitti/converter_test.py @@ -26,6 +26,7 @@ from parameterized import parameterized_class from upath import UPath +from ncore.impl.data import nodes from ncore.impl.data.types import IdealPinholeCameraModelParameters from ncore.impl.data.v4.components import ( CameraSensorComponent, @@ -39,11 +40,8 @@ @parameterized_class( - ("store_type"), - [ - ("itar",), - ("directory",), - ], + ("store_type", "zarr_format"), + [(store_type, zarr_format) for store_type in ("itar", "directory") for zarr_format in nodes.SUPPORTED_ZARR_FORMATS], ) class TestKittiConverter(unittest.TestCase): """Integration tests for KITTI raw data converter. @@ -54,6 +52,7 @@ class TestKittiConverter(unittest.TestCase): """ store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] @classmethod def setUpClass(cls): @@ -78,6 +77,7 @@ def setUpClass(cls): debug=False, debug_port=5678, store_type=cls.store_type, + zarr_format=cls.zarr_format, component_group_profile="separate-sensors", store_sequence_meta=True, ) diff --git a/tools/data_converter/nuscenes/BUILD.bazel b/tools/data_converter/nuscenes/BUILD.bazel index e16cb101..8eb13151 100644 --- a/tools/data_converter/nuscenes/BUILD.bazel +++ b/tools/data_converter/nuscenes/BUILD.bazel @@ -65,6 +65,7 @@ alias( # Integration test for the nuScenes converter (requires NUSCENES_DIR env var) pytest_test( name = "pytest_converter", + size = "large", # converts the test sequence for all store types / zarr formats srcs = ["converter_test.py"], python_versions = ["3.11"], tags = ["manual"], # Only run when explicitly requested (needs external data) diff --git a/tools/data_converter/nuscenes/converter.py b/tools/data_converter/nuscenes/converter.py index e8b31915..0332a22b 100644 --- a/tools/data_converter/nuscenes/converter.py +++ b/tools/data_converter/nuscenes/converter.py @@ -280,6 +280,7 @@ def convert_sequence(self, sequence_id: str) -> None: sequence_id=sequence_output_name, sequence_timestamp_interval_us=sequence_timestamp_interval_us, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={ "source_dataset": "nuscenes", "nuscenes_version": self._version, diff --git a/tools/data_converter/nuscenes/converter_test.py b/tools/data_converter/nuscenes/converter_test.py index 6e5edb83..649205e9 100644 --- a/tools/data_converter/nuscenes/converter_test.py +++ b/tools/data_converter/nuscenes/converter_test.py @@ -33,6 +33,7 @@ from parameterized import parameterized_class from upath import UPath +from ncore.impl.data import nodes from ncore.impl.data.types import IdealPinholeCameraModelParameters, RowOffsetStructuredSpinningLidarModelParameters from ncore.impl.data.v4.components import ( CameraSensorComponent, @@ -49,11 +50,8 @@ @parameterized_class( - ("store_type",), - [ - ("itar",), - ("directory",), - ], + ("store_type", "zarr_format"), + [(store_type, zarr_format) for store_type in ("itar", "directory") for zarr_format in nodes.SUPPORTED_ZARR_FORMATS], ) class TestNuScenesConverter(unittest.TestCase): """Integration tests for nuScenes data converter. @@ -63,6 +61,7 @@ class TestNuScenesConverter(unittest.TestCase): """ store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] @classmethod def setUpClass(cls): @@ -97,6 +96,7 @@ def setUpClass(cls): scene_token=cls.scene_token, scene_name=None, store_type=cls.store_type, + zarr_format=cls.zarr_format, component_group_profile="separate-sensors", store_sequence_meta=True, ) @@ -294,7 +294,7 @@ def test_lidar_model_reproduces_point_cloud(self): # Native: direction * distance direction = lidar_reader.get_frame_ray_bundle_data(ts, "direction") - distance_2d = np.array(lidar_reader._get_ray_bundle_returns_group(ts)["distance_m"]) # [R, N] + distance_2d = lidar_reader._get_ray_bundle_returns_group(ts).array("distance_m").read() # [R, N] distance = distance_2d[0] # first return # Model element diff --git a/tools/data_converter/pai/converter.py b/tools/data_converter/pai/converter.py index f6bd419c..ba79f23e 100644 --- a/tools/data_converter/pai/converter.py +++ b/tools/data_converter/pai/converter.py @@ -175,6 +175,7 @@ def _init_pai_fields(self, config) -> None: self.seek_sec: float | None = config.seek_sec self.duration_sec: float | None = config.duration_sec self.store_type: Literal["itar", "directory"] = config.store_type + self.zarr_format: Literal[2, 3] = config.zarr_format self.component_group_profile: Literal["default", "separate-sensors", "separate-all"] = ( config.component_group_profile ) @@ -286,6 +287,7 @@ def _convert_clip(self, clip_id: str) -> None: sequence_id=sequence_id, sequence_timestamp_interval_us=self.sequence_timestamp_interval_us, store_type=self.store_type, # can also be "directory" + zarr_format=self.zarr_format, generic_meta_data={**source_generic_meta_data, **generic_meta_data}, ) diff --git a/tools/data_converter/waymo/converter.py b/tools/data_converter/waymo/converter.py index 50196768..2eb47649 100644 --- a/tools/data_converter/waymo/converter.py +++ b/tools/data_converter/waymo/converter.py @@ -243,6 +243,7 @@ def convert_sequence(self, sequence_id: str) -> None: sequence_id=sequence_name, sequence_timestamp_interval_us=sequence_timestamp_interval_us, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, # no generic sequence meta data )