From f737a333401b7f468987e4986ad825b9bf74f05a Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Tue, 27 Oct 2026 15:21:39 -0700 Subject: [PATCH 01/13] dev: add ty as a manual pre-commit hook ty is not the gate, and reporting 451 diagnostics in CI would block every PR, so the hook is manual-stage only. It sits next to mypy in the same config file, pinned to the same version as the dev dependency group, and runs via: uv run prek run ty --stage manual -a --- .pre-commit-config.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f0dedd0804..e45db91ec2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,6 +47,15 @@ repos: - types-pytest-lazy-fixture - types-pytz - types-requests + # Experimental: ty is not the gate yet, so this hook is manual-stage only. + # Run it with: uv run prek run ty --stage manual -a + # The rev must stay in step with ty== in the dev dependency group. + - repo: https://github.com/astral-sh/ty-pre-commit + rev: v0.0.84 + hooks: + - id: ty + args: [--isolated] + stages: [manual] - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.45.0 hooks: From 7a33c6365e0e601d3117ff96d958cbef92c1a79d Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Tue, 27 Oct 2026 15:21:42 -0700 Subject: [PATCH 02/13] dev: pin mypy locally and exclude vendor from it The mypy pre-commit hook runs 1.18.2 in an isolated environment, so a locally installed mypy disagreed with it: 2.4.0 reported 8 errors the gate did not, and without the hook's exclusion a plain 'mypy .' picked up 9k errors from generated Thrift bindings in vendor/. Pinning the dev dependency and excluding vendor/ make 'uv run mypy pyiceberg tests' and the gate agree on the same 0. --- pyproject.toml | 2 ++ uv.lock | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8268338fa1..ebe0c0e984 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,6 +129,7 @@ dev = [ "nbformat>=5.10.0", "ipykernel>=6.29.0", "ty==0.0.84", + "mypy==1.18.2", ] # for mkdocs docs = [ @@ -262,6 +263,7 @@ filterwarnings = [ ] [tool.mypy] +exclude = ["^vendor/"] mypy_path = "python" no_implicit_optional = true namespace_packages = false diff --git a/uv.lock b/uv.lock index 7901c5ea12..1801cfe509 100644 --- a/uv.lock +++ b/uv.lock @@ -3448,6 +3448,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/59/e26cb779be4c591d1a910f59d29aca9fba4de70349840a833beba2652371/multidict-6.9.1-py3-none-any.whl", hash = "sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b", size = 19176, upload-time = "2026-09-21T17:59:03.501Z" }, ] +[[package]] +name = "mypy" +version = "1.18.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/77/8f0d0001ffad290cef2f7f216f96c814866248a0b92a722365ed54648e7e/mypy-1.18.2.tar.gz", hash = "sha256:06a398102a5f203d7477b2923dda3634c36727fa5c237d8f859ef90c42a9924b", size = 3448846, upload-time = "2025-09-19T00:11:10.519Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/87/cafd3ae563f88f94eec33f35ff722d043e09832ea8530ef149ec1efbaf08/mypy-1.18.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:807d9315ab9d464125aa9fcf6d84fde6e1dc67da0b6f80e7405506b8ac72bc7f", size = 12731198, upload-time = "2025-09-19T00:09:44.857Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e0/1e96c3d4266a06d4b0197ace5356d67d937d8358e2ee3ffac71faa843724/mypy-1.18.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:776bb00de1778caf4db739c6e83919c1d85a448f71979b6a0edd774ea8399341", size = 11817879, upload-time = "2025-09-19T00:09:47.131Z" }, + { url = "https://files.pythonhosted.org/packages/72/ef/0c9ba89eb03453e76bdac5a78b08260a848c7bfc5d6603634774d9cd9525/mypy-1.18.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1379451880512ffce14505493bd9fe469e0697543717298242574882cf8cdb8d", size = 12427292, upload-time = "2025-09-19T00:10:22.472Z" }, + { url = "https://files.pythonhosted.org/packages/1a/52/ec4a061dd599eb8179d5411d99775bec2a20542505988f40fc2fee781068/mypy-1.18.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1331eb7fd110d60c24999893320967594ff84c38ac6d19e0a76c5fd809a84c86", size = 13163750, upload-time = "2025-09-19T00:09:51.472Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5f/2cf2ceb3b36372d51568f2208c021870fe7834cf3186b653ac6446511839/mypy-1.18.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3ca30b50a51e7ba93b00422e486cbb124f1c56a535e20eff7b2d6ab72b3b2e37", size = 13351827, upload-time = "2025-09-19T00:09:58.311Z" }, + { url = "https://files.pythonhosted.org/packages/c8/7d/2697b930179e7277529eaaec1513f8de622818696857f689e4a5432e5e27/mypy-1.18.2-cp311-cp311-win_amd64.whl", hash = "sha256:664dc726e67fa54e14536f6e1224bcfce1d9e5ac02426d2326e2bb4e081d1ce8", size = 9757983, upload-time = "2025-09-19T00:10:09.071Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/dfdd2bc60c66611dd8335f463818514733bc763e4760dee289dcc33df709/mypy-1.18.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33eca32dd124b29400c31d7cf784e795b050ace0e1f91b8dc035672725617e34", size = 12908273, upload-time = "2025-09-19T00:10:58.321Z" }, + { url = "https://files.pythonhosted.org/packages/81/14/6a9de6d13a122d5608e1a04130724caf9170333ac5a924e10f670687d3eb/mypy-1.18.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a3c47adf30d65e89b2dcd2fa32f3aeb5e94ca970d2c15fcb25e297871c8e4764", size = 11920910, upload-time = "2025-09-19T00:10:20.043Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a9/b29de53e42f18e8cc547e38daa9dfa132ffdc64f7250e353f5c8cdd44bee/mypy-1.18.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d6c838e831a062f5f29d11c9057c6009f60cb294fea33a98422688181fe2893", size = 12465585, upload-time = "2025-09-19T00:10:33.005Z" }, + { url = "https://files.pythonhosted.org/packages/77/ae/6c3d2c7c61ff21f2bee938c917616c92ebf852f015fb55917fd6e2811db2/mypy-1.18.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01199871b6110a2ce984bde85acd481232d17413868c9807e95c1b0739a58914", size = 13348562, upload-time = "2025-09-19T00:10:11.51Z" }, + { url = "https://files.pythonhosted.org/packages/4d/31/aec68ab3b4aebdf8f36d191b0685d99faa899ab990753ca0fee60fb99511/mypy-1.18.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a2afc0fa0b0e91b4599ddfe0f91e2c26c2b5a5ab263737e998d6817874c5f7c8", size = 13533296, upload-time = "2025-09-19T00:10:06.568Z" }, + { url = "https://files.pythonhosted.org/packages/9f/83/abcb3ad9478fca3ebeb6a5358bb0b22c95ea42b43b7789c7fb1297ca44f4/mypy-1.18.2-cp312-cp312-win_amd64.whl", hash = "sha256:d8068d0afe682c7c4897c0f7ce84ea77f6de953262b12d07038f4d296d547074", size = 9828828, upload-time = "2025-09-19T00:10:28.203Z" }, + { url = "https://files.pythonhosted.org/packages/5f/04/7f462e6fbba87a72bc8097b93f6842499c428a6ff0c81dd46948d175afe8/mypy-1.18.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:07b8b0f580ca6d289e69209ec9d3911b4a26e5abfde32228a288eb79df129fcc", size = 12898728, upload-time = "2025-09-19T00:10:01.33Z" }, + { url = "https://files.pythonhosted.org/packages/99/5b/61ed4efb64f1871b41fd0b82d29a64640f3516078f6c7905b68ab1ad8b13/mypy-1.18.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ed4482847168439651d3feee5833ccedbf6657e964572706a2adb1f7fa4dfe2e", size = 11910758, upload-time = "2025-09-19T00:10:42.607Z" }, + { url = "https://files.pythonhosted.org/packages/3c/46/d297d4b683cc89a6e4108c4250a6a6b717f5fa96e1a30a7944a6da44da35/mypy-1.18.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3ad2afadd1e9fea5cf99a45a822346971ede8685cc581ed9cd4d42eaf940986", size = 12475342, upload-time = "2025-09-19T00:11:00.371Z" }, + { url = "https://files.pythonhosted.org/packages/83/45/4798f4d00df13eae3bfdf726c9244bcb495ab5bd588c0eed93a2f2dd67f3/mypy-1.18.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a431a6f1ef14cf8c144c6b14793a23ec4eae3db28277c358136e79d7d062f62d", size = 13338709, upload-time = "2025-09-19T00:11:03.358Z" }, + { url = "https://files.pythonhosted.org/packages/d7/09/479f7358d9625172521a87a9271ddd2441e1dab16a09708f056e97007207/mypy-1.18.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7ab28cc197f1dd77a67e1c6f35cd1f8e8b73ed2217e4fc005f9e6a504e46e7ba", size = 13529806, upload-time = "2025-09-19T00:10:26.073Z" }, + { url = "https://files.pythonhosted.org/packages/71/cf/ac0f2c7e9d0ea3c75cd99dff7aec1c9df4a1376537cb90e4c882267ee7e9/mypy-1.18.2-cp313-cp313-win_amd64.whl", hash = "sha256:0e2785a84b34a72ba55fb5daf079a1003a34c05b22238da94fcae2bbe46f3544", size = 9833262, upload-time = "2025-09-19T00:10:40.035Z" }, + { url = "https://files.pythonhosted.org/packages/5a/0c/7d5300883da16f0063ae53996358758b2a2df2a09c72a5061fa79a1f5006/mypy-1.18.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:62f0e1e988ad41c2a110edde6c398383a889d95b36b3e60bcf155f5164c4fdce", size = 12893775, upload-time = "2025-09-19T00:10:03.814Z" }, + { url = "https://files.pythonhosted.org/packages/50/df/2cffbf25737bdb236f60c973edf62e3e7b4ee1c25b6878629e88e2cde967/mypy-1.18.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8795a039bab805ff0c1dfdb8cd3344642c2b99b8e439d057aba30850b8d3423d", size = 11936852, upload-time = "2025-09-19T00:10:51.631Z" }, + { url = "https://files.pythonhosted.org/packages/be/50/34059de13dd269227fb4a03be1faee6e2a4b04a2051c82ac0a0b5a773c9a/mypy-1.18.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ca1e64b24a700ab5ce10133f7ccd956a04715463d30498e64ea8715236f9c9c", size = 12480242, upload-time = "2025-09-19T00:11:07.955Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/040983fad5132d85914c874a2836252bbc57832065548885b5bb5b0d4359/mypy-1.18.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d924eef3795cc89fecf6bedc6ed32b33ac13e8321344f6ddbf8ee89f706c05cb", size = 13326683, upload-time = "2025-09-19T00:09:55.572Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ba/89b2901dd77414dd7a8c8729985832a5735053be15b744c18e4586e506ef/mypy-1.18.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20c02215a080e3a2be3aa50506c67242df1c151eaba0dcbc1e4e557922a26075", size = 13514749, upload-time = "2025-09-19T00:10:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/25/bc/cc98767cffd6b2928ba680f3e5bc969c4152bf7c2d83f92f5a504b92b0eb/mypy-1.18.2-cp314-cp314-win_amd64.whl", hash = "sha256:749b5f83198f1ca64345603118a6f01a4e99ad4bf9d103ddc5a3200cc4614adf", size = 9982959, upload-time = "2025-09-19T00:10:37.344Z" }, + { url = "https://files.pythonhosted.org/packages/87/e3/be76d87158ebafa0309946c4a73831974d4d6ab4f4ef40c3b53a385a66fd/mypy-1.18.2-py3-none-any.whl", hash = "sha256:22a1748707dd62b58d2ae53562ffc4d7f8bcc727e8ac7cbc69c053ddc874d47e", size = 2352367, upload-time = "2025-09-19T00:10:15.489Z" }, +] + [[package]] name = "mypy-boto3-dynamodb" version = "1.43.64" @@ -3472,6 +3510,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/bf/781db91d144e0f2a4c91649499722de9b57f6a3c251b34d339820f3f7f0f/mypy_boto3_glue-1.43.100-py3-none-any.whl", hash = "sha256:104b3d95133e10b4ad825432cfeeeb97ec56d5bb7ee70509ffc2a6a749b45a03", size = 154974, upload-time = "2026-09-22T20:27:52.045Z" }, ] +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + [[package]] name = "natsort" version = "8.4.0" @@ -4602,6 +4649,7 @@ dev = [ { name = "google-cloud-bigquery" }, { name = "ipykernel" }, { name = "moto", extra = ["server"] }, + { name = "mypy" }, { name = "mypy-boto3-dynamodb" }, { name = "mypy-boto3-glue" }, { name = "nbformat" }, @@ -4697,6 +4745,7 @@ dev = [ { name = "google-cloud-bigquery", specifier = ">=3.33.0,<4" }, { name = "ipykernel", specifier = ">=6.29.0" }, { name = "moto", extras = ["server"], specifier = ">=5.0.2" }, + { name = "mypy", specifier = "==1.18.2" }, { name = "mypy-boto3-dynamodb", specifier = ">=1.28.18" }, { name = "mypy-boto3-glue", specifier = ">=1.28.18" }, { name = "nbformat", specifier = ">=5.10.0" }, From 3829df4e3419c6643069e5aca9da76d26fdc538b Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Tue, 27 Oct 2026 15:21:45 -0700 Subject: [PATCH 03/13] ty: exclude tests/ from analysis for now tests/ is 400 of the 851 diagnostics, and the package is what a gate would have to cover first. Add tests/ back before ty is considered for the gate, since mypy checks them today. --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ebe0c0e984..86a693cd6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -276,7 +276,8 @@ implicit_reexport = false ignore_missing_imports = true [tool.ty.src] -exclude = ["vendor/"] +# Start with the package only; tests/ is 400 of the 851 initial diagnostics. +exclude = ["vendor/", "tests/"] [tool.ty.environment] # The vendored Hive Metastore client is packaged from vendor/, so it must stay importable From 2acd0ddb2869351be40ef58ef341b5b803bdd822 Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Tue, 27 Oct 2026 15:28:18 -0700 Subject: [PATCH 04/13] ty: report missing type arguments like mypy does mypy runs with disallow_any_generics, so bare dicts and fields are rejected there, while ty leaves missing-type-argument at ignore by default. Enabling it surfaces 102 findings (451 -> 553 with tests/ excluded). --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 86a693cd6e..577643c9e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -288,5 +288,9 @@ extra-paths = ["vendor"] # mkdocs_gen_files are only installed with the docs dependency group allowed-unresolved-imports = ["pyiceberg_core.datafusion", "griffe", "mkdocs_gen_files"] +[tool.ty.rules] +# mypy runs with disallow_any_generics; ty silences this rule by default. +missing-type-argument = "error" + [tool.coverage.run] source = ['pyiceberg/'] From 3e5b85a8a4769b5d6a7aedcfc8545cf2eadeace1 Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Tue, 27 Oct 2026 15:28:18 -0700 Subject: [PATCH 05/13] utils: annotate Singleton instead of suppressing mypy type: ignore was covering two real gaps: an unparameterised ClassVar[dict] and an unannotated __new__. Both checkers accept the class once the field and the constructor are annotated, and ty has no rule requiring parameter annotations, so the suppressions could not be replaced by configuration. __new__ returns Self rather than Singleton: a concrete return type makes every subclass constructor resolve to Singleton and produces 340 phantom diagnostics. __deepcopy__ takes memo again, matching copy.deepcopy and pydantic's RootModel (dropping it raised TypeError in test_deepcopy_of_singleton_fixed_type). --- pyiceberg/utils/singleton.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyiceberg/utils/singleton.py b/pyiceberg/utils/singleton.py index b59f43fbcd..4df845a8e2 100644 --- a/pyiceberg/utils/singleton.py +++ b/pyiceberg/utils/singleton.py @@ -30,6 +30,8 @@ from typing import Any, ClassVar +from typing_extensions import Self + def _convert_to_hashable_type(element: Any) -> Any: if isinstance(element, dict): @@ -40,15 +42,15 @@ def _convert_to_hashable_type(element: Any) -> Any: class Singleton: - _instances: ClassVar[dict] = {} # type: ignore + _instances: ClassVar[dict[Any, Any]] = {} - def __new__(cls, *args, **kwargs): # type: ignore + def __new__(cls, *args: Any, **kwargs: Any) -> Self: key = (cls, tuple(args), _convert_to_hashable_type(kwargs)) if key not in cls._instances: cls._instances[key] = super().__new__(cls) return cls._instances[key] - def __deepcopy__(self, memo: dict[int, Any]) -> Any: + def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Self: """ Prevent deep copy operations for singletons. From 8dbcd7a569e5e29b50ea23c9cd834dd5bb94ff1c Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Tue, 27 Oct 2026 16:03:50 -0700 Subject: [PATCH 06/13] dev: give mypy suppressions their error codes ty does not parse the bracketed form at all, while a bare # type: ignore counts as a suppression it may not need. 55 of the bare ignores in pyiceberg suppressed only mypy errors, so each one produced unused-type-ignore-comment. Naming the code removes those 55 warnings (549 -> 494 ty diagnostics) and leaves mypy passing, while the comments now say which error they cover. Lines that both checkers flag would need '# ty: ignore[rule]' appended after the mypy comment; none of the 55 here do. --- pyiceberg/avro/codecs/bzip2.py | 2 +- pyiceberg/avro/codecs/snappy_codec.py | 2 +- pyiceberg/avro/codecs/zstandard_codec.py | 2 +- pyiceberg/avro/file.py | 4 ++-- pyiceberg/avro/resolver.py | 4 ++-- pyiceberg/catalog/__init__.py | 4 ++-- pyiceberg/catalog/bigquery_metastore.py | 2 +- pyiceberg/catalog/dynamodb.py | 2 +- pyiceberg/catalog/sql.py | 2 +- pyiceberg/cli/console.py | 12 ++++++------ pyiceberg/conversions.py | 12 ++++++------ pyiceberg/expressions/__init__.py | 4 ++-- pyiceberg/expressions/literals.py | 2 +- pyiceberg/io/fsspec.py | 4 ++-- pyiceberg/partitioning.py | 2 +- pyiceberg/schema.py | 10 +++++----- pyiceberg/table/name_mapping.py | 2 +- pyiceberg/table/sorting.py | 2 +- pyiceberg/table/update/__init__.py | 4 ++-- pyiceberg/table/update/schema.py | 6 +++--- pyiceberg/table/update/snapshot.py | 4 ++-- pyiceberg/transforms.py | 16 ++++++++-------- pyiceberg/types.py | 4 ++-- pyiceberg/utils/lazydict.py | 2 +- 24 files changed, 55 insertions(+), 55 deletions(-) diff --git a/pyiceberg/avro/codecs/bzip2.py b/pyiceberg/avro/codecs/bzip2.py index 24786681f2..35cd319c0b 100644 --- a/pyiceberg/avro/codecs/bzip2.py +++ b/pyiceberg/avro/codecs/bzip2.py @@ -33,7 +33,7 @@ def decompress(data: bytes) -> bytes: except ImportError: - class BZip2Codec(Codec): # type: ignore + class BZip2Codec(Codec): # type: ignore[no-redef] @staticmethod def compress(data: bytes) -> tuple[bytes, int]: raise ImportError("Python bzip2 support not installed, please install the extension") diff --git a/pyiceberg/avro/codecs/snappy_codec.py b/pyiceberg/avro/codecs/snappy_codec.py index 2ea7229772..1a14712451 100644 --- a/pyiceberg/avro/codecs/snappy_codec.py +++ b/pyiceberg/avro/codecs/snappy_codec.py @@ -59,7 +59,7 @@ def decompress(data: bytes) -> bytes: except ImportError: - class SnappyCodec(Codec): # type: ignore + class SnappyCodec(Codec): # type: ignore[no-redef] @staticmethod def compress(data: bytes) -> tuple[bytes, int]: raise ImportError("Snappy support not installed, please install using `pip install pyiceberg[snappy]`") diff --git a/pyiceberg/avro/codecs/zstandard_codec.py b/pyiceberg/avro/codecs/zstandard_codec.py index 4cc815214f..8f9b5153d6 100644 --- a/pyiceberg/avro/codecs/zstandard_codec.py +++ b/pyiceberg/avro/codecs/zstandard_codec.py @@ -43,7 +43,7 @@ def decompress(data: bytes) -> bytes: except ImportError: - class ZStandardCodec(Codec): # type: ignore + class ZStandardCodec(Codec): # type: ignore[no-redef] @staticmethod def compress(data: bytes) -> tuple[bytes, int]: raise ImportError("Zstandard support not installed, please install using `pip install pyiceberg[zstandard]`") diff --git a/pyiceberg/avro/file.py b/pyiceberg/avro/file.py index 3184e3950a..582cc32dbb 100644 --- a/pyiceberg/avro/file.py +++ b/pyiceberg/avro/file.py @@ -96,7 +96,7 @@ def compression_codec(self) -> type[Codec] | None: if codec_name not in KNOWN_CODECS: raise ValueError(f"Unsupported codec: {codec_name}") - return KNOWN_CODECS[codec_name] # type: ignore + return KNOWN_CODECS[codec_name] # type: ignore[index] def get_schema(self) -> Schema: if _SCHEMA_KEY in self.meta: @@ -308,7 +308,7 @@ def compression_codec(self) -> type[Codec] | None: if codec_name not in KNOWN_CODECS: raise ValueError(f"Unsupported codec: {codec_name}") - return KNOWN_CODECS[codec_name] # type: ignore + return KNOWN_CODECS[codec_name] # type: ignore[index] def write_block(self, objects: list[D]) -> None: for obj in objects: diff --git a/pyiceberg/avro/resolver.py b/pyiceberg/avro/resolver.py index 3f19ae9878..d8dff0e4f5 100644 --- a/pyiceberg/avro/resolver.py +++ b/pyiceberg/avro/resolver.py @@ -233,7 +233,7 @@ def resolve_writer( """ if record_schema == file_schema: return construct_writer(file_schema) - return visit_with_partner(file_schema, record_schema, WriteSchemaResolver(), SchemaPartnerAccessor()) # type: ignore + return visit_with_partner(file_schema, record_schema, WriteSchemaResolver(), SchemaPartnerAccessor()) # type: ignore[misc] def resolve_reader( @@ -253,7 +253,7 @@ def resolve_reader( Raises: NotImplementedError: If attempting to resolve an unrecognized object type. """ - return visit_with_partner(file_schema, read_schema, ReadSchemaResolver(read_types, read_enums), SchemaPartnerAccessor()) # type: ignore + return visit_with_partner(file_schema, read_schema, ReadSchemaResolver(read_types, read_enums), SchemaPartnerAccessor()) # type: ignore[misc] class EnumReader(Reader): diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index cdb4d26b3b..c81aef7a4f 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -919,7 +919,7 @@ def _convert_schema_if_needed( downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False if isinstance(schema, pa.Schema): - schema: Schema = visit_pyarrow( # type: ignore + schema: Schema = visit_pyarrow( # type: ignore[no-redef] schema, _ConvertToIcebergWithoutIDs( downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version @@ -1077,7 +1077,7 @@ def _create_staged_table( Returns: StagedTable: the created staged table instance. """ - schema: Schema = self._convert_schema_if_needed( # type: ignore + schema: Schema = self._convert_schema_if_needed( # type: ignore[no-redef] schema, int(properties.get(TableProperties.FORMAT_VERSION, TableProperties.DEFAULT_FORMAT_VERSION)), # type: ignore ) diff --git a/pyiceberg/catalog/bigquery_metastore.py b/pyiceberg/catalog/bigquery_metastore.py index 74d1c952bc..0c11418d42 100644 --- a/pyiceberg/catalog/bigquery_metastore.py +++ b/pyiceberg/catalog/bigquery_metastore.py @@ -131,7 +131,7 @@ def create_table( ValueError: If the identifier is invalid, or no path is given to store metadata. """ - schema: Schema = self._convert_schema_if_needed(schema) # type: ignore + schema: Schema = self._convert_schema_if_needed(schema) # type: ignore[no-redef] dataset_name, table_name = self.identifier_to_database_and_table(identifier) diff --git a/pyiceberg/catalog/dynamodb.py b/pyiceberg/catalog/dynamodb.py index 220313ab74..0266dd5d4a 100644 --- a/pyiceberg/catalog/dynamodb.py +++ b/pyiceberg/catalog/dynamodb.py @@ -204,7 +204,7 @@ def create_table( ValueError: If the identifier is invalid, or no path is given to store metadata. """ - schema: Schema = self._convert_schema_if_needed( # type: ignore + schema: Schema = self._convert_schema_if_needed( # type: ignore[no-redef] schema, int(properties.get(TableProperties.FORMAT_VERSION, TableProperties.DEFAULT_FORMAT_VERSION)), # type: ignore ) diff --git a/pyiceberg/catalog/sql.py b/pyiceberg/catalog/sql.py index 94c7965a88..13a71ca894 100644 --- a/pyiceberg/catalog/sql.py +++ b/pyiceberg/catalog/sql.py @@ -270,7 +270,7 @@ def create_table( ValueError: If the identifier is invalid, or no path is given to store metadata. """ - schema: Schema = self._convert_schema_if_needed( # type: ignore + schema: Schema = self._convert_schema_if_needed( # type: ignore[no-redef] schema, int(properties.get(TableProperties.FORMAT_VERSION, TableProperties.DEFAULT_FORMAT_VERSION)), # type: ignore ) diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index 9f8bed5eba..6d1add9e23 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -41,7 +41,7 @@ def catch_exception() -> Callable: # type: ignore def decorator(func: Callable) -> Callable: # type: ignore @wraps(func) - def wrapper(*args: Any, **kwargs: Any): # type: ignore + def wrapper(*args: Any, **kwargs: Any): # type: ignore[no-untyped-def] try: return func(*args, **kwargs) except Exception as e: @@ -324,7 +324,7 @@ def table(ctx: Context, identifier: str, purge: bool) -> None: # noqa: F811 output.text(f"Dropped table: {identifier}{purge_message}") -@drop.command() # type: ignore +@drop.command() # type: ignore[no-redef] @click.argument("identifier") @click.pass_context @catch_exception() @@ -422,7 +422,7 @@ def set() -> None: """Set a property on tables/namespaces.""" -@set.command() # type: ignore +@set.command() # type: ignore[no-redef] @click.argument("identifier") @click.argument("property_name") @click.argument("property_value") @@ -436,7 +436,7 @@ def namespace(ctx: Context, identifier: str, property_name: str, property_value: output.text(f"Updated {property_name} on {identifier}") -@set.command() # type: ignore +@set.command() # type: ignore[no-redef] @click.argument("identifier") @click.argument("property_name") @click.argument("property_value") @@ -458,7 +458,7 @@ def remove() -> None: """Remove a property from tables/namespaces.""" -@remove.command() # type: ignore +@remove.command() # type: ignore[no-redef] @click.argument("identifier") @click.argument("property_name") @click.pass_context @@ -475,7 +475,7 @@ def namespace(ctx: Context, identifier: str, property_name: str) -> None: # noq raise NoSuchPropertyException(f"Property {property_name} does not exist on {identifier}") -@remove.command() # type: ignore +@remove.command() # type: ignore[no-redef] @click.argument("identifier") @click.argument("property_name") @click.pass_context diff --git a/pyiceberg/conversions.py b/pyiceberg/conversions.py index fde1885443..4fc8dd0405 100644 --- a/pyiceberg/conversions.py +++ b/pyiceberg/conversions.py @@ -329,8 +329,8 @@ def _(_: UnknownType, value: Any) -> None: return None -@singledispatch # type: ignore -def from_bytes(primitive_type: PrimitiveType, b: bytes) -> L: # type: ignore +@singledispatch # type: ignore[arg-type] +def from_bytes(primitive_type: PrimitiveType, b: bytes) -> L: # type: ignore[type-var] """Convert bytes to a built-in python value. Args: @@ -406,8 +406,8 @@ def _(type_: UnknownType, buf: bytes) -> None: return None -@singledispatch # type: ignore -def to_json(primitive_type: PrimitiveType, val: Any) -> L: # type: ignore +@singledispatch # type: ignore[arg-type] +def to_json(primitive_type: PrimitiveType, val: Any) -> L: # type: ignore[type-var] """Convert built-in python values into JSON value types. https://iceberg.apache.org/spec/#json-single-value-serialization @@ -545,8 +545,8 @@ def _(_: GeographyType, val: bytes) -> str: ) -@singledispatch # type: ignore -def from_json(primitive_type: PrimitiveType, val: Any) -> L: # type: ignore +@singledispatch # type: ignore[arg-type] +def from_json(primitive_type: PrimitiveType, val: Any) -> L: # type: ignore[type-var] """Convert JSON value types into built-in python values. https://iceberg.apache.org/spec/#json-single-value-serialization diff --git a/pyiceberg/expressions/__init__.py b/pyiceberg/expressions/__init__.py index ad916171c5..51a68ab987 100644 --- a/pyiceberg/expressions/__init__.py +++ b/pyiceberg/expressions/__init__.py @@ -711,7 +711,7 @@ def __init__( def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundSetPredicate: bound_term = self.term.bind(schema, case_sensitive) literal_set = self.literals - return self.as_bound(bound_term, {lit.to(bound_term.ref().field.field_type) for lit in literal_set}) # type: ignore + return self.as_bound(bound_term, {lit.to(bound_term.ref().field.field_type) for lit in literal_set}) # type: ignore[misc] def __str__(self) -> str: """Return the string representation of the SetPredicate class.""" @@ -926,7 +926,7 @@ def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundLiteralPredi else: raise ValueError(f"Cannot bind {type(self).__name__} with BelowMin literal") - return self.as_bound(bound_term, lit) # type: ignore + return self.as_bound(bound_term, lit) # type: ignore[misc] def __eq__(self, other: Any) -> bool: """Return the equality of two instances of the LiteralPredicate class.""" diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index b91e404627..9045357331 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -87,7 +87,7 @@ class Literal(IcebergRootModel[L], Generic[L], ABC): # type: ignore root: L = Field() - def __init__(self, value: L, value_type: type[L], /, **data): # type: ignore + def __init__(self, value: L, value_type: type[L], /, **data): # type: ignore[no-untyped-def] if value is None: raise TypeError("Invalid literal value: None") diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 49f10b1310..33a605be90 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -126,7 +126,7 @@ def __init__(self, properties: Properties) -> None: self._session = requests.Session() def __call__(self, request: "AWSRequest", operation_name: str | None = None, **_: Any) -> None: - signer_url = self.properties.get(S3_SIGNER_URI, self.properties[URI]).rstrip("/") # type: ignore + signer_url = self.properties.get(S3_SIGNER_URI, self.properties[URI]).rstrip("/") # type: ignore[union-attr] signer_endpoint = self.properties.get(S3_SIGNER_ENDPOINT, S3_SIGNER_ENDPOINT_DEFAULT) signer_headers: dict[str, str] = {} @@ -299,7 +299,7 @@ async def get_token(self, *scopes: str, **kwargs: Any) -> AccessToken: if token := properties.get(ADLS_TOKEN): credential = StaticTokenCredential(token) else: - credential = properties.get(ADLS_CREDENTIAL) # type: ignore + credential = properties.get(ADLS_CREDENTIAL) # type: ignore[assignment] return AzureBlobFileSystem( connection_string=properties.get(ADLS_CONNECTION_STRING), diff --git a/pyiceberg/partitioning.py b/pyiceberg/partitioning.py index 3d06287b6c..74932b8ddb 100644 --- a/pyiceberg/partitioning.py +++ b/pyiceberg/partitioning.py @@ -80,7 +80,7 @@ class PartitionField(IcebergBaseModel): source_id: int = Field(alias="source-id") field_id: int = Field(alias="field-id") - transform: Annotated[ # type: ignore + transform: Annotated[ # type: ignore[type-arg] Transform, BeforeValidator(parse_transform), PlainSerializer(lambda c: str(c), return_type=str), # pylint: disable=W0108 diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index 99f983074b..fc6fd3823a 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -670,7 +670,7 @@ def visit_with_partner( @visit_with_partner.register(Schema) def _(schema: Schema, partner: P, visitor: SchemaWithPartnerVisitor[P, T], accessor: PartnerAccessor[P]) -> T: struct_partner = accessor.schema_partner(partner) - return visitor.schema(schema, partner, visit_with_partner(schema.as_struct(), struct_partner, visitor, accessor)) # type: ignore + return visitor.schema(schema, partner, visit_with_partner(schema.as_struct(), struct_partner, visitor, accessor)) # type: ignore[misc] @visit_with_partner.register(StructType) @@ -680,7 +680,7 @@ def _(struct: StructType, partner: P, visitor: SchemaWithPartnerVisitor[P, T], a field_partner = accessor.field_partner(partner, field.field_id, field.name) visitor.before_field(field, field_partner) try: - field_result = visit_with_partner(field.field_type, field_partner, visitor, accessor) # type: ignore + field_result = visit_with_partner(field.field_type, field_partner, visitor, accessor) # type: ignore[misc] field_results.append(visitor.field(field, field_partner, field_result)) finally: visitor.after_field(field, field_partner) @@ -693,7 +693,7 @@ def _(list_type: ListType, partner: P, visitor: SchemaWithPartnerVisitor[P, T], element_partner = accessor.list_element_partner(partner) visitor.before_list_element(list_type.element_field, element_partner) try: - element_result = visit_with_partner(list_type.element_type, element_partner, visitor, accessor) # type: ignore + element_result = visit_with_partner(list_type.element_type, element_partner, visitor, accessor) # type: ignore[misc] finally: visitor.after_list_element(list_type.element_field, element_partner) @@ -705,14 +705,14 @@ def _(map_type: MapType, partner: P, visitor: SchemaWithPartnerVisitor[P, T], ac key_partner = accessor.map_key_partner(partner) visitor.before_map_key(map_type.key_field, key_partner) try: - key_result = visit_with_partner(map_type.key_type, key_partner, visitor, accessor) # type: ignore + key_result = visit_with_partner(map_type.key_type, key_partner, visitor, accessor) # type: ignore[misc] finally: visitor.after_map_key(map_type.key_field, key_partner) value_partner = accessor.map_value_partner(partner) visitor.before_map_value(map_type.value_field, value_partner) try: - value_result = visit_with_partner(map_type.value_type, value_partner, visitor, accessor) # type: ignore + value_result = visit_with_partner(map_type.value_type, value_partner, visitor, accessor) # type: ignore[misc] finally: visitor.after_map_value(map_type.value_field, value_partner) return visitor.map(map_type, partner, key_result, value_result) diff --git a/pyiceberg/table/name_mapping.py b/pyiceberg/table/name_mapping.py index bd8b8529c5..7367ba875e 100644 --- a/pyiceberg/table/name_mapping.py +++ b/pyiceberg/table/name_mapping.py @@ -375,4 +375,4 @@ def primitive(self, primitive: PrimitiveType, primitive_partner: MappedField | N def apply_name_mapping(schema_without_ids: Schema, name_mapping: NameMapping) -> Schema: - return visit_with_partner(schema_without_ids, name_mapping, NameMappingProjectionVisitor(), NameMappingAccessor()) # type: ignore + return visit_with_partner(schema_without_ids, name_mapping, NameMappingProjectionVisitor(), NameMappingAccessor()) # type: ignore[misc] diff --git a/pyiceberg/table/sorting.py b/pyiceberg/table/sorting.py index 76bc109935..adf2cf4236 100644 --- a/pyiceberg/table/sorting.py +++ b/pyiceberg/table/sorting.py @@ -114,7 +114,7 @@ def map_source_ids_onto_source_id(cls, data: Any) -> Any: return data source_id: int = Field(alias="source-id") - transform: Annotated[ # type: ignore + transform: Annotated[ # type: ignore[type-arg] Transform, BeforeValidator(parse_transform), PlainSerializer(lambda c: str(c), return_type=str), # pylint: disable=W0108 diff --git a/pyiceberg/table/update/__init__.py b/pyiceberg/table/update/__init__.py index aa1a09ebb2..38047ef06c 100644 --- a/pyiceberg/table/update/__init__.py +++ b/pyiceberg/table/update/__init__.py @@ -786,8 +786,8 @@ def _update_table_metadata_log(base_metadata: TableMetadata, metadata_location: ), # type: ignore ) previous_metadata_log = base_metadata.metadata_log - if len(base_metadata.metadata_log) >= max_metadata_log_entries: # type: ignore - remove_index = len(base_metadata.metadata_log) - max_metadata_log_entries + 1 # type: ignore + if len(base_metadata.metadata_log) >= max_metadata_log_entries: # type: ignore[operator] + remove_index = len(base_metadata.metadata_log) - max_metadata_log_entries + 1 # type: ignore[operator] previous_metadata_log = base_metadata.metadata_log[remove_index:] metadata_updates: dict[str, Any] = { "metadata_log": previous_metadata_log + [MetadataLogEntry(metadata_file=metadata_location, timestamp_ms=last_updated_ms)] diff --git a/pyiceberg/table/update/schema.py b/pyiceberg/table/update/schema.py index 828f1e877a..af6ccc2439 100644 --- a/pyiceberg/table/update/schema.py +++ b/pyiceberg/table/update/schema.py @@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Any from pyiceberg.exceptions import ResolveError, ValidationError -from pyiceberg.expressions import literal # type: ignore +from pyiceberg.expressions import literal # type: ignore[attr-defined] from pyiceberg.schema import ( PartnerAccessor, Schema, @@ -155,7 +155,7 @@ def union_by_name( Catalog._convert_schema_if_needed(new_schema, format_version=format_version), -1, _UnionByNameVisitor(update_schema=self, existing_schema=self._schema, case_sensitive=self._case_sensitive), - # type: ignore + # type: ignore[misc] PartnerIdByNameAccessor(partner_schema=self._schema, case_sensitive=self._case_sensitive), ) return self @@ -231,7 +231,7 @@ def add_column( except ValueError as e: raise ValueError(f"Invalid default value: {e}") from e else: - initial_default = default_value # type: ignore + initial_default = default_value # type: ignore[assignment] if (required and initial_default is None) and not self._allow_incompatible_changes: # Table format version 1 and 2 cannot add required column because there is no initial value diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index 8feebb9901..e54fb4b38d 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -740,12 +740,12 @@ def __init__( table_properties, TableProperties.MANIFEST_TARGET_SIZE_BYTES, TableProperties.MANIFEST_TARGET_SIZE_BYTES_DEFAULT, - ) # type: ignore + ) # type: ignore[assignment] self._min_count_to_merge = property_as_int( table_properties, TableProperties.MANIFEST_MIN_MERGE_COUNT, TableProperties.MANIFEST_MIN_MERGE_COUNT_DEFAULT, - ) # type: ignore + ) # type: ignore[assignment] self._merge_enabled = property_as_bool( table_properties, TableProperties.MANIFEST_MERGE_ENABLED, diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index 74925c695f..560a00aa49 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -269,12 +269,12 @@ def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: elif isinstance(pred, BoundEqualTo): return pred.as_unbound(Reference(name), _transform_literal(transformer, pred.literal)) elif isinstance(pred, BoundIn): # NotIn can't be projected - return pred.as_unbound(Reference(name), {_transform_literal(transformer, literal) for literal in pred.literals}) # type: ignore + return pred.as_unbound(Reference(name), {_transform_literal(transformer, literal) for literal in pred.literals}) # type: ignore[call-arg] else: # - Comparison predicates can't be projected, notEq can't be projected # - Small ranges can be projected: # For example, (x > 0) and (x < 3) can be turned into in({1, 2}) and projected. - return None # type: ignore + return None # type: ignore[unreachable] def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: transformer = self.transform(pred.term.ref().field.field_type) @@ -286,10 +286,10 @@ def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | elif isinstance(pred, BoundNotEqualTo): return pred.as_unbound(Reference(name), _transform_literal(transformer, pred.literal)) elif isinstance(pred, BoundNotIn): - return pred.as_unbound(Reference(name), {_transform_literal(transformer, literal) for literal in pred.literals}) # type: ignore + return pred.as_unbound(Reference(name), {_transform_literal(transformer, literal) for literal in pred.literals}) # type: ignore[call-arg] else: # no strict projection for comparison or equality - return None # type: ignore + return None # type: ignore[unreachable] def can_transform(self, source: IcebergType) -> bool: return isinstance( @@ -798,7 +798,7 @@ def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: return pred.as_unbound(Reference(name)) elif isinstance(pred, BoundIn): return _set_apply_transform(name, pred, self.transform(field_type)) - elif isinstance(field_type, (IntegerType, LongType, DecimalType)): # type: ignore + elif isinstance(field_type, (IntegerType, LongType, DecimalType)): # type: ignore[unreachable] if isinstance(pred, BoundLiteralPredicate): return _truncate_number(name, pred, self.transform(field_type)) elif isinstance(field_type, (BinaryType, StringType)): @@ -829,7 +829,7 @@ def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | elif isinstance(pred, BoundNotIn): return _set_apply_transform(name, pred, self.transform(field_type)) else: - return None # type: ignore + return None # type: ignore[unreachable] if isinstance(pred, BoundLiteralPredicate): if isinstance(pred, BoundStartsWith): @@ -854,7 +854,7 @@ def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | elif isinstance(pred, BoundNotIn): return _set_apply_transform(name, pred, self.transform(field_type)) else: - return None # type: ignore + return None # type: ignore[unreachable] @property def width(self) -> int: @@ -1129,7 +1129,7 @@ def _remove_transform(partition_name: str, pred: BoundPredicate) -> UnboundPredi elif isinstance(pred, BoundLiteralPredicate): return pred.as_unbound(Reference(partition_name), pred.literal) elif isinstance(pred, (BoundIn, BoundNotIn)): - return pred.as_unbound(Reference(partition_name), pred.literals) # type: ignore + return pred.as_unbound(Reference(partition_name), pred.literals) # type: ignore[call-arg] else: raise ValueError(f"Cannot replace transform in unknown predicate: {pred}") diff --git a/pyiceberg/types.py b/pyiceberg/types.py index 3295beb666..32b7ad5689 100644 --- a/pyiceberg/types.py +++ b/pyiceberg/types.py @@ -429,8 +429,8 @@ class NestedField(IcebergType): field_type: SerializeAsAny[IcebergType] = Field(alias="type") required: bool = Field(default=False) doc: str | None = Field(default=None, repr=False) - initial_default: DefaultValue | None = Field(alias="initial-default", default=None, repr=True) # type: ignore - write_default: DefaultValue | None = Field(alias="write-default", default=None, repr=True) # type: ignore + initial_default: DefaultValue | None = Field(alias="initial-default", default=None, repr=True) # type: ignore[type-arg] + write_default: DefaultValue | None = Field(alias="write-default", default=None, repr=True) # type: ignore[type-arg] @field_validator("field_type", mode="before") def convert_field_type(cls, v: Any) -> IcebergType: diff --git a/pyiceberg/utils/lazydict.py b/pyiceberg/utils/lazydict.py index b110b6291b..6b77c7da76 100644 --- a/pyiceberg/utils/lazydict.py +++ b/pyiceberg/utils/lazydict.py @@ -62,6 +62,6 @@ def __len__(self) -> int: source = self._dict or self._build_dict() return len(source) - def __dict__(self) -> dict[K, V]: # type: ignore + def __dict__(self) -> dict[K, V]: # type: ignore[override] """Convert the lazy dict in a dict.""" return self._dict or self._build_dict() From 0bc99ebe57f0e877a1c2aebf7dbf005e41e20299 Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Tue, 27 Oct 2026 16:55:31 -0700 Subject: [PATCH 07/13] utils: type the Singleton instance cache key The registry was ClassVar[dict[Any, Any]], so a wrongly typed key or a non-Singleton value went unnoticed. The key is now (class, positional args, hashable kwargs) and the value a Singleton. The value cannot be Self: ClassVar rejects type variables, and both mypy and ty reject Generic[Self] ('not a valid argument to Generic'), so the return needs a cast. It is sound because cls is part of the key. --- pyiceberg/utils/singleton.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pyiceberg/utils/singleton.py b/pyiceberg/utils/singleton.py index 4df845a8e2..8d51bc9f08 100644 --- a/pyiceberg/utils/singleton.py +++ b/pyiceberg/utils/singleton.py @@ -28,7 +28,7 @@ More information on metaclasses: https://docs.python.org/3/reference/datamodel.html#metaclasses """ -from typing import Any, ClassVar +from typing import Any, ClassVar, cast from typing_extensions import Self @@ -41,14 +41,19 @@ def _convert_to_hashable_type(element: Any) -> Any: return element +# (class, positional args, hashable kwargs) is the cache key for instances +_InstanceKey = tuple[type["Singleton"], tuple[Any, ...], Any] + + class Singleton: - _instances: ClassVar[dict[Any, Any]] = {} + _instances: ClassVar[dict[_InstanceKey, "Singleton"]] = {} def __new__(cls, *args: Any, **kwargs: Any) -> Self: - key = (cls, tuple(args), _convert_to_hashable_type(kwargs)) + key: _InstanceKey = (cls, tuple(args), _convert_to_hashable_type(kwargs)) if key not in cls._instances: cls._instances[key] = super().__new__(cls) - return cls._instances[key] + # the key contains cls, so the cached instance is always of the calling class + return cast(Self, cls._instances[key]) def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Self: """ From cf2bed94efa3c1f1f5dcb60d74e3f97eb05a988a Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Sun, 1 Nov 2026 20:39:16 -0800 Subject: [PATCH 08/13] utils: import Self from typing requires-python is 3.11, so the typing_extensions re-export trips UP035. --- pyiceberg/utils/singleton.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyiceberg/utils/singleton.py b/pyiceberg/utils/singleton.py index 8d51bc9f08..5c91c2d612 100644 --- a/pyiceberg/utils/singleton.py +++ b/pyiceberg/utils/singleton.py @@ -28,9 +28,7 @@ More information on metaclasses: https://docs.python.org/3/reference/datamodel.html#metaclasses """ -from typing import Any, ClassVar, cast - -from typing_extensions import Self +from typing import Any, ClassVar, Self, cast def _convert_to_hashable_type(element: Any) -> Any: From dface02261820d945b192293d3b8139e2f40f044 Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Sun, 1 Nov 2026 20:39:18 -0800 Subject: [PATCH 09/13] ty: mirror the mypy configuration and scope both checkers to the package mypy checked all 214 files while ty started on the 91 in pyiceberg/, so neither result was comparable. Both now check pyiceberg/ at Python 3.11. Each ty rule that replaces a mypy flag is named next to it, and the modules that import optional extras allow their own unresolved imports, which is what ignore_missing_imports did globally. Adds a types group holding ty and the stub packages, to measure ty in the same bare environment the mypy hook runs in. --- pyproject.toml | 60 +++++++++++++++++++++++++++++++++++++++++++++----- uv.lock | 53 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 577643c9e3..01c25f0242 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -149,6 +149,16 @@ docs = [ notebook = [ "jupyterlab>=4.0.0", ] +# Only ty and the stub packages, to measure ty in the same bare environment the mypy +# pre-commit hook runs in. Not a gate: sync it with `--only-group types` into a throwaway venv. +# The ty hook in .pre-commit-config.yaml pins the same versions, so keep the two in step. +types = [ + "ty==0.0.84", + "types-cachetools==7.0.0.20260713", + "types-pytest-lazy-fixture==0.6.3.20260518", + "types-pytz==2026.5.0.20261006", + "types-requests==2.33.0.20261006", +] [tool.uv] default-groups = [ @@ -263,7 +273,9 @@ filterwarnings = [ ] [tool.mypy] -exclude = ["^vendor/"] +# Same scope as ty's [tool.ty.src] include, so both checkers see the same files. +files = ["pyiceberg"] +python_version = "3.11" mypy_path = "python" no_implicit_optional = true namespace_packages = false @@ -276,12 +288,12 @@ implicit_reexport = false ignore_missing_imports = true [tool.ty.src] -# Start with the package only; tests/ is 400 of the 851 initial diagnostics. -exclude = ["vendor/", "tests/"] +include = ["pyiceberg"] [tool.ty.environment] # The vendored Hive Metastore client is packaged from vendor/, so it must stay importable extra-paths = ["vendor"] +python-version = "3.11" [tool.ty.analysis] # pyiceberg_core.datafusion is registered at runtime by its compiled extension; griffe and @@ -289,8 +301,46 @@ extra-paths = ["vendor"] allowed-unresolved-imports = ["pyiceberg_core.datafusion", "griffe", "mkdocs_gen_files"] [tool.ty.rules] -# mypy runs with disallow_any_generics; ty silences this rule by default. -missing-type-argument = "error" +# Each rule mirrors the mypy flag it replaces. missing-type-argument is the only one ty +# silences by default; the rest already report nothing in pyiceberg/ at this level. +missing-type-argument = "error" # disallow_any_generics +invalid-parameter-default = "error" # no_implicit_optional +redundant-cast = "error" # warn_redundant_casts +redundant-condition = "error" # warn_unreachable +unused-ignore-comment = "error" # warn_unused_ignores +unused-type-ignore-comment = "error" # warn_unused_ignores +# No mypy counterpart; off until every override carries @override. +missing-override-decorator = "ignore" + +[[tool.ty.overrides]] +include = [ + "pyiceberg/avro/codecs/snappy_codec.py", + "pyiceberg/catalog/hive.py", + "pyiceberg/catalog/rest/auth.py", + "pyiceberg/io/fsspec.py", + "pyiceberg/io/pyarrow.py", + "pyiceberg/table/__init__.py", +] + +[tool.ty.overrides.analysis] +# Optional extras that only these modules import; mypy ignores missing imports globally. +allowed-unresolved-imports = [ + "snappy", + "thrift.**", + "azure.identity", + "s3fs", + "gcsfs", + "adlfs", + "azure.core.credentials", + "azure.core.credentials_async", + "huggingface_hub", + "geoarrow.pyarrow", + "bodo.pandas", + "daft", + "polars", + "ray", + "duckdb", +] [tool.coverage.run] source = ['pyiceberg/'] diff --git a/uv.lock b/uv.lock index 1801cfe509..3715685965 100644 --- a/uv.lock +++ b/uv.lock @@ -4684,6 +4684,13 @@ docs = [ notebook = [ { name = "jupyterlab" }, ] +types = [ + { name = "ty" }, + { name = "types-cachetools" }, + { name = "types-pytest-lazy-fixture" }, + { name = "types-pytz" }, + { name = "types-requests" }, +] [package.metadata] requires-dist = [ @@ -4778,6 +4785,13 @@ docs = [ { name = "mkdocstrings-python", specifier = "==2.0.9" }, ] notebook = [{ name = "jupyterlab", specifier = ">=4.0.0" }] +types = [ + { name = "ty", specifier = "==0.0.84" }, + { name = "types-cachetools", specifier = "==7.0.0.20260713" }, + { name = "types-pytest-lazy-fixture", specifier = "==0.6.3.20260518" }, + { name = "types-pytz", specifier = "==2026.5.0.20261006" }, + { name = "types-requests", specifier = "==2.33.0.20261006" }, +] [[package]] name = "pyiceberg-core" @@ -6091,6 +6105,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/1c/e6e91852b72566b1ab9aca504712857dfb89324c12092a5bd61d42d34427/ty-0.0.84-py3-none-win_arm64.whl", hash = "sha256:6c9d3718c4a0d318bc9c92af9b2aa85c3294e7dff54cf1d69ad79568f56516da", size = 13656453, upload-time = "2026-09-24T13:16:21.51Z" }, ] +[[package]] +name = "types-cachetools" +version = "7.0.0.20260713" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/64/66d7efdb36ecf6826aca5415e59fe2df96e97d24157147e53acfbe8dda11/types_cachetools-7.0.0.20260713.tar.gz", hash = "sha256:f1acf079e9c66a81e096a897ef0b261a82117cf856834e37b4bd0c9a116a076a", size = 10199, upload-time = "2026-07-13T05:22:21.845Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/c7/d3525c9dbdc1be7786bad46655ef051b6e7993f656d304719ec40079c91c/types_cachetools-7.0.0.20260713-py3-none-any.whl", hash = "sha256:6db9bcc7a3840d39e91c04117d85a9d0937eacc9d14d12a873e2b01a2d24a71d", size = 9615, upload-time = "2026-07-13T05:22:20.76Z" }, +] + +[[package]] +name = "types-pytest-lazy-fixture" +version = "0.6.3.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/46/a79ab84cdace98870440968006c86eec5cab22b64f805e1770d8b19bf8d8/types_pytest_lazy_fixture-0.6.3.20260518.tar.gz", hash = "sha256:5e4c259c3c0eddbbf7719d81109050f4e9dcf9f650291a6ff3de42b964734acd", size = 7472, upload-time = "2026-05-18T06:05:45.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/3e/bfcb69336e4a1c69b87f634a3ddebeb6c77f55932631407d191d8b394128/types_pytest_lazy_fixture-0.6.3.20260518-py3-none-any.whl", hash = "sha256:5fc2bc417c2804145ddfcff91bd3f472c8cfbe79b84a2a3abae661ef45408ec1", size = 7599, upload-time = "2026-05-18T06:05:44.471Z" }, +] + +[[package]] +name = "types-pytz" +version = "2026.5.0.20261006" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/47/b493f47f2dd0971db06459439ebdeb114cf95029188268099b331bbc4727/types_pytz-2026.5.0.20261006.tar.gz", hash = "sha256:1a522c2ec03aad8d4baaf97105958019ad51704b1e471c882d1c6ccea3e5e64b", size = 11073, upload-time = "2026-10-06T08:15:12.87Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/38/e375026fb4ff74fed6bdb84d7d336571d0603b6b9675751579cec84cfc9b/types_pytz-2026.5.0.20261006-py3-none-any.whl", hash = "sha256:9e4a893b362a8eed4e10a348c80603ade65bdb3819419e589364afafe2ea08b1", size = 10168, upload-time = "2026-10-06T08:15:11.985Z" }, +] + +[[package]] +name = "types-requests" +version = "2.33.0.20261006" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/15/9b7e2e2e7c87d01366185b198b3febc6bc0c973f2bf21a62da4ab7d3495e/types_requests-2.33.0.20261006.tar.gz", hash = "sha256:0652999e9306aea345f40732d58fa49a7f6cade6a0d74d92119c5c8d82eddaf0", size = 25316, upload-time = "2026-10-06T08:15:57.782Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/72/b82789207b3d360ce9f5a4372c790c52cc3dd928aeb7d4faeec652b651b2/types_requests-2.33.0.20261006-py3-none-any.whl", hash = "sha256:26cc8146505cab33cda9737991929e4144c559bebe05078ccc6998f27c4ca2c1", size = 21445, upload-time = "2026-10-06T08:15:56.658Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" From 62b50ec29088cc76275f1f92da5ad58938a6e374 Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Sun, 1 Nov 2026 20:39:21 -0800 Subject: [PATCH 10/13] dev: run ty in its own pre-commit environment The ty-pre-commit hook resolved the project environment, so ty saw every library while the mypy hook saw only stubs. A local hook installs ty and the stubs into the hook environment, which ty picks up through the VIRTUAL_ENV the runner exports, so both checkers now see the same types. Flags live in the entry because hook args are also passed to uv pip install when the hook environment is built. The six --config lines are the deltas this bare environment needs, since most inline directives cover diagnostics that require the real libraries. Adds make typecheck-ty, because the hook is manual-stage and make lint skips it. --- .pre-commit-config.yaml | 31 +++++++++++++++++++++++++++---- Makefile | 4 ++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e45db91ec2..46dcdfff16 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,6 +40,8 @@ repos: rev: v1.18.2 hooks: - id: mypy + # Scoped to the package while ty and mypy are compared on the same files. + files: ^pyiceberg/ args: [--config=pyproject.toml] additional_dependencies: @@ -49,12 +51,33 @@ repos: - types-requests # Experimental: ty is not the gate yet, so this hook is manual-stage only. # Run it with: uv run prek run ty --stage manual -a - # The rev must stay in step with ty== in the dev dependency group. - - repo: https://github.com/astral-sh/ty-pre-commit - rev: v0.0.84 + # Local hook so ty runs in this hook's own environment, which holds only ty and the stub + # packages, the same bare environment the mypy hook gets. The entry points at it through + # VIRTUAL_ENV, which the hook runner exports. Keep these pins in step with the types group. + - repo: local hooks: - id: ty - args: [--isolated] + name: ty + # Bare-environment deltas. Hook args are also handed to `uv pip install` when the hook + # environment is built, so the flags live in the entry instead. + entry: >- + ty check + -c 'rules.unresolved-import = "ignore"' + -c 'rules.unused-ignore-comment = "ignore"' + -c 'rules.unused-type-ignore-comment = "ignore"' + -c 'rules.invalid-assignment = "ignore"' + -c 'rules.invalid-argument-type = "ignore"' + -c 'rules.call-non-callable = "ignore"' + language: python + additional_dependencies: + - ty==0.0.84 + - types-cachetools==7.0.0.20260713 + - types-pytest-lazy-fixture==0.6.3.20260518 + - types-pytz==2026.5.0.20261006 + - types-requests==2.33.0.20261006 + always_run: true + pass_filenames: false + require_serial: true stages: [manual] - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.45.0 diff --git a/Makefile b/Makefile index 9f0282c65b..a69d8c8a3c 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ test test-integration test-integration-setup test-integration-exec test-integration-cleanup test-integration-rebuild \ test-s3 test-adls test-gcs test-coverage coverage-report test test-notebook\ docs-serve docs-build notebook notebook-infra \ + typecheck-ty \ clean .DEFAULT_GOAL := help @@ -97,6 +98,9 @@ check-license: ## Check license headers lint: ## Run code linters via prek (pre-commit hooks) uv run $(PYTHON_ARG) prek run -a +typecheck-ty: ## Type-check with ty in its isolated stub-only hook environment + uv run $(PYTHON_ARG) prek run ty --stage manual -a + # =============== # Testing Section # =============== From 12a8f3b1e777e068f3d095d11d1cea21efd9c2d3 Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Sun, 1 Nov 2026 20:39:28 -0800 Subject: [PATCH 11/13] ty: suppress the diagnostics the package reports today 470 diagnostics in 45 files, suppressed in place so each can be deleted as the underlying error is fixed. The rules stay at "error", so a new diagnostic on an unsuppressed line still fails, and unused-ignore-comment stays on, so fixing a site without deleting its directive fails too. Rules total: missing-type-argument 126, invalid-method-override 104, invalid-argument-type 82, unresolved-attribute 54, invalid-return-type 44, no-matching-overload 36, then a tail of seven. Two signatures in io/pyarrow.py and one call in table/update/__init__.py were reflowed by ruff format, because the comments pushed them past 130 characters. --- pyiceberg/avro/codecs/snappy_codec.py | 2 +- pyiceberg/avro/decoder.py | 2 +- pyiceberg/avro/decoder_fast.pyi | 4 +- pyiceberg/avro/file.py | 2 +- pyiceberg/avro/reader.py | 4 +- pyiceberg/avro/resolver.py | 32 +-- pyiceberg/avro/writer.py | 2 +- pyiceberg/catalog/dynamodb.py | 20 +- pyiceberg/catalog/glue.py | 2 +- pyiceberg/catalog/hive.py | 16 +- pyiceberg/catalog/rest/__init__.py | 72 +++--- pyiceberg/catalog/rest/auth.py | 4 +- pyiceberg/catalog/sql.py | 8 +- pyiceberg/cli/console.py | 2 +- pyiceberg/cli/output.py | 2 +- pyiceberg/conversions.py | 2 +- pyiceberg/encryption/key_metadata.py | 2 +- pyiceberg/encryption/manager.py | 2 +- pyiceberg/expressions/__init__.py | 44 ++-- pyiceberg/expressions/literals.py | 18 +- pyiceberg/expressions/parser.py | 6 +- pyiceberg/expressions/visitors.py | 2 +- pyiceberg/io/pyarrow.py | 349 +++++++++++++------------- pyiceberg/manifest.py | 6 +- pyiceberg/partitioning.py | 2 +- pyiceberg/schema.py | 8 +- pyiceberg/serializers.py | 2 +- pyiceberg/table/__init__.py | 6 +- pyiceberg/table/deletion_vector.py | 6 +- pyiceberg/table/inspect.py | 28 +-- pyiceberg/table/metadata.py | 12 +- pyiceberg/table/name_mapping.py | 12 +- pyiceberg/table/puffin.py | 2 +- pyiceberg/table/snapshots.py | 4 +- pyiceberg/table/sorting.py | 2 +- pyiceberg/table/update/__init__.py | 29 +-- pyiceberg/table/update/schema.py | 26 +- pyiceberg/table/update/snapshot.py | 8 +- pyiceberg/table/upsert_util.py | 10 +- pyiceberg/transforms.py | 36 +-- pyiceberg/typedef.py | 4 +- pyiceberg/types.py | 4 +- pyiceberg/utils/deprecated.py | 2 +- pyiceberg/utils/schema_conversion.py | 2 +- pyiceberg/view/metadata.py | 2 +- 45 files changed, 412 insertions(+), 400 deletions(-) diff --git a/pyiceberg/avro/codecs/snappy_codec.py b/pyiceberg/avro/codecs/snappy_codec.py index 1a14712451..d3553667a6 100644 --- a/pyiceberg/avro/codecs/snappy_codec.py +++ b/pyiceberg/avro/codecs/snappy_codec.py @@ -54,7 +54,7 @@ def decompress(data: bytes) -> bytes: checksum = data[-4:] # store checksum before truncating data data = data[0:-4] # remove checksum from the data uncompressed = snappy.decompress(data) - SnappyCodec._check_crc32(uncompressed, checksum) + SnappyCodec._check_crc32(uncompressed, checksum) # ty: ignore[unresolved-attribute] return uncompressed except ImportError: diff --git a/pyiceberg/avro/decoder.py b/pyiceberg/avro/decoder.py index e971d52d48..d5a2383b13 100644 --- a/pyiceberg/avro/decoder.py +++ b/pyiceberg/avro/decoder.py @@ -137,7 +137,7 @@ def __init__(self, input_stream: bytes | InputStream) -> None: """Reader is a Python object on which we can call read, seek, and tell.""" if isinstance(input_stream, bytes): # In the case of bytes, we wrap it into a BytesIO to make it a stream - self._input_stream = io.BytesIO(input_stream) + self._input_stream = io.BytesIO(input_stream) # ty: ignore[invalid-assignment] else: self._input_stream = input_stream diff --git a/pyiceberg/avro/decoder_fast.pyi b/pyiceberg/avro/decoder_fast.pyi index cc367c4764..144b158712 100644 --- a/pyiceberg/avro/decoder_fast.pyi +++ b/pyiceberg/avro/decoder_fast.pyi @@ -33,10 +33,10 @@ class CythonBinaryDecoder(BinaryDecoder): def read_int(self) -> int: pass - def read_ints(self, count: int) -> tuple[int, ...]: + def read_ints(self, count: int) -> tuple[int, ...]: # ty: ignore[invalid-method-override] pass - def read_int_bytes_dict(self, count: int, dest: dict[int, bytes]) -> None: + def read_int_bytes_dict(self, count: int, dest: dict[int, bytes]) -> None: # ty: ignore[invalid-method-override] pass def read_bytes(self) -> bytes: diff --git a/pyiceberg/avro/file.py b/pyiceberg/avro/file.py index 582cc32dbb..4905937882 100644 --- a/pyiceberg/avro/file.py +++ b/pyiceberg/avro/file.py @@ -257,7 +257,7 @@ def __init__( raise ValueError(f"Sync interval must be positive: {sync_interval}") self.sync_interval = sync_interval self._block = io.BytesIO() - self._block_encoder = BinaryEncoder(output_stream=self._block) + self._block_encoder = BinaryEncoder(output_stream=self._block) # ty: ignore[invalid-argument-type] self._block_records = 0 def __enter__(self) -> AvroOutputFile[D]: diff --git a/pyiceberg/avro/reader.py b/pyiceberg/avro/reader.py index e3ee8305f0..237b3ff8ce 100644 --- a/pyiceberg/avro/reader.py +++ b/pyiceberg/avro/reader.py @@ -93,7 +93,7 @@ def __repr__(self) -> str: class NoneReader(Reader): - def read(self, _: BinaryDecoder) -> None: + def read(self, _: BinaryDecoder) -> None: # ty: ignore[invalid-method-override] return None def skip(self, decoder: BinaryDecoder) -> None: @@ -107,7 +107,7 @@ class DefaultReader(Reader): def __init__(self, default_value: Any) -> None: self.default_value = default_value - def read(self, _: BinaryDecoder) -> Any: + def read(self, _: BinaryDecoder) -> Any: # ty: ignore[invalid-method-override] return self.default_value def skip(self, decoder: BinaryDecoder) -> None: diff --git a/pyiceberg/avro/resolver.py b/pyiceberg/avro/resolver.py index d8dff0e4f5..e89702dcaf 100644 --- a/pyiceberg/avro/resolver.py +++ b/pyiceberg/avro/resolver.py @@ -185,7 +185,7 @@ def visit_time(self, time_type: TimeType) -> Writer: def visit_timestamp(self, timestamp_type: TimestampType) -> Writer: return TimestampWriter() - def visit_timestamp_ns(self, timestamp_ns_type: TimestampNanoType) -> Writer: + def visit_timestamp_ns(self, timestamp_ns_type: TimestampNanoType) -> Writer: # ty: ignore[invalid-method-override] return TimestampNanoWriter() def visit_timestamptz(self, timestamptz_type: TimestamptzType) -> Writer: @@ -276,10 +276,10 @@ def skip(self, decoder: BinaryDecoder) -> None: class WriteSchemaResolver(PrimitiveWithPartnerVisitor[IcebergType, Writer]): - def schema(self, file_schema: Schema, record_schema: IcebergType | None, result: Writer) -> Writer: + def schema(self, file_schema: Schema, record_schema: IcebergType | None, result: Writer) -> Writer: # ty: ignore[invalid-method-override] return result - def struct(self, file_schema: StructType, record_struct: IcebergType | None, file_writers: list[Writer]) -> Writer: + def struct(self, file_schema: StructType, record_struct: IcebergType | None, file_writers: list[Writer]) -> Writer: # ty: ignore[invalid-method-override] if not isinstance(record_struct, StructType): raise ResolveError(f"File/write schema are not aligned for struct, got {record_struct}") @@ -301,16 +301,16 @@ def struct(self, file_schema: StructType, record_struct: IcebergType | None, fil return StructWriter(field_writers=tuple(results)) - def field(self, file_field: NestedField, record_type: IcebergType | None, field_writer: Writer) -> Writer: + def field(self, file_field: NestedField, record_type: IcebergType | None, field_writer: Writer) -> Writer: # ty: ignore[invalid-method-override] return field_writer if file_field.required else OptionWriter(field_writer) - def list(self, file_list_type: ListType, file_list: IcebergType | None, element_writer: Writer) -> Writer: + def list(self, file_list_type: ListType, file_list: IcebergType | None, element_writer: Writer) -> Writer: # ty: ignore[invalid-method-override] return ListWriter(element_writer if file_list_type.element_required else OptionWriter(element_writer)) - def map(self, file_map_type: MapType, file_primitive: IcebergType | None, key_writer: Writer, value_writer: Writer) -> Writer: + def map(self, file_map_type: MapType, file_primitive: IcebergType | None, key_writer: Writer, value_writer: Writer) -> Writer: # ty: ignore[invalid-method-override] return MapWriter(key_writer, value_writer if file_map_type.value_required else OptionWriter(value_writer)) - def primitive(self, file_primitive: PrimitiveType, record_primitive: IcebergType | None) -> Writer: + def primitive(self, file_primitive: PrimitiveType, record_primitive: IcebergType | None) -> Writer: # ty: ignore[invalid-method-override] if record_primitive is not None: # ensure that the type can be projected to the expected if file_primitive != record_primitive: @@ -393,16 +393,16 @@ def __init__( self.read_enums = read_enums self.context = [] - def schema(self, schema: Schema, expected_schema: IcebergType | None, result: Reader) -> Reader: + def schema(self, schema: Schema, expected_schema: IcebergType | None, result: Reader) -> Reader: # ty: ignore[invalid-method-override] return result - def before_field(self, field: NestedField, field_partner: NestedField | None) -> None: + def before_field(self, field: NestedField, field_partner: NestedField | None) -> None: # ty: ignore[invalid-method-override] self.context.append(field.field_id) - def after_field(self, field: NestedField, field_partner: NestedField | None) -> None: + def after_field(self, field: NestedField, field_partner: NestedField | None) -> None: # ty: ignore[invalid-method-override] self.context.pop() - def struct(self, struct: StructType, expected_struct: IcebergType | None, field_readers: list[Reader]) -> Reader: + def struct(self, struct: StructType, expected_struct: IcebergType | None, field_readers: list[Reader]) -> Reader: # ty: ignore[invalid-method-override] read_struct_id = self.context[STRUCT_ROOT] if len(self.context) > 0 else STRUCT_ROOT struct_callable = self.read_types.get(read_struct_id, Record) @@ -439,22 +439,22 @@ def struct(self, struct: StructType, expected_struct: IcebergType | None, field_ return StructReader(tuple(results), struct_callable, expected_struct) - def field(self, field: NestedField, expected_field: IcebergType | None, field_reader: Reader) -> Reader: + def field(self, field: NestedField, expected_field: IcebergType | None, field_reader: Reader) -> Reader: # ty: ignore[invalid-method-override] return field_reader if field.required else OptionReader(field_reader) - def list(self, list_type: ListType, expected_list: IcebergType | None, element_reader: Reader) -> Reader: + def list(self, list_type: ListType, expected_list: IcebergType | None, element_reader: Reader) -> Reader: # ty: ignore[invalid-method-override] if expected_list and not isinstance(expected_list, ListType): raise ResolveError(f"File/read schema are not aligned for list, got {expected_list}") return ListReader(element_reader if list_type.element_required else OptionReader(element_reader)) - def map(self, map_type: MapType, expected_map: IcebergType | None, key_reader: Reader, value_reader: Reader) -> Reader: + def map(self, map_type: MapType, expected_map: IcebergType | None, key_reader: Reader, value_reader: Reader) -> Reader: # ty: ignore[invalid-method-override] if expected_map and not isinstance(expected_map, MapType): raise ResolveError(f"File/read schema are not aligned for map, got {expected_map}") return MapReader(key_reader, value_reader if map_type.value_required else OptionReader(value_reader)) - def primitive(self, primitive: PrimitiveType, expected_primitive: IcebergType | None) -> Reader: + def primitive(self, primitive: PrimitiveType, expected_primitive: IcebergType | None) -> Reader: # ty: ignore[invalid-method-override] if expected_primitive is not None: if not isinstance(expected_primitive, PrimitiveType): raise ResolveError(f"File/read schema are not aligned for {primitive}, got {expected_primitive}") @@ -547,7 +547,7 @@ def schema_partner(self, partner: IcebergType | None) -> IcebergType | None: raise ResolveError(f"File/read schema are not aligned for schema, got {partner}") - def field_partner(self, partner: IcebergType | None, field_id: int, field_name: str) -> IcebergType | None: + def field_partner(self, partner: IcebergType | None, field_id: int, field_name: str) -> IcebergType | None: # ty: ignore[invalid-method-override] if isinstance(partner, StructType): field = partner.field(field_id) else: diff --git a/pyiceberg/avro/writer.py b/pyiceberg/avro/writer.py index f78d1a486e..bd7a7d8634 100644 --- a/pyiceberg/avro/writer.py +++ b/pyiceberg/avro/writer.py @@ -234,5 +234,5 @@ class DefaultWriter(Writer): writer: Writer value: Any - def write(self, encoder: BinaryEncoder, _: Any) -> None: + def write(self, encoder: BinaryEncoder, _: Any) -> None: # ty: ignore[invalid-method-override] self.writer.write(encoder, self.value) diff --git a/pyiceberg/catalog/dynamodb.py b/pyiceberg/catalog/dynamodb.py index 0266dd5d4a..3d7a644656 100644 --- a/pyiceberg/catalog/dynamodb.py +++ b/pyiceberg/catalog/dynamodb.py @@ -150,9 +150,9 @@ def _ensure_catalog_table_exists_or_create(self) -> None: try: self.dynamodb.create_table( TableName=self.dynamodb_table_name, - AttributeDefinitions=CREATE_CATALOG_ATTRIBUTE_DEFINITIONS, - KeySchema=CREATE_CATALOG_KEY_SCHEMA, - GlobalSecondaryIndexes=CREATE_CATALOG_GLOBAL_SECONDARY_INDEXES, + AttributeDefinitions=CREATE_CATALOG_ATTRIBUTE_DEFINITIONS, # ty: ignore[invalid-argument-type] + KeySchema=CREATE_CATALOG_KEY_SCHEMA, # ty: ignore[invalid-argument-type] + GlobalSecondaryIndexes=CREATE_CATALOG_GLOBAL_SECONDARY_INDEXES, # ty: ignore[invalid-argument-type] BillingMode=DYNAMODB_PAY_PER_REQUEST, ) except ( @@ -160,7 +160,7 @@ def _ensure_catalog_table_exists_or_create(self) -> None: self.dynamodb.exceptions.LimitExceededException, self.dynamodb.exceptions.InternalServerError, ) as e: - raise GenericDynamoDbError(e.message) from e + raise GenericDynamoDbError(e.message) from e # ty: ignore[unresolved-attribute] def _dynamodb_table_exists(self) -> bool: try: @@ -168,7 +168,7 @@ def _dynamodb_table_exists(self) -> bool: except self.dynamodb.exceptions.ResourceNotFoundException: return False except self.dynamodb.exceptions.InternalServerError as e: - raise GenericDynamoDbError(e.message) from e + raise GenericDynamoDbError(e.message) from e # ty: ignore[unresolved-attribute] if response["Table"]["TableStatus"] != ACTIVE: raise GenericDynamoDbError(f"DynamoDB table for catalog {self.dynamodb_table_name} is not {ACTIVE}") @@ -462,7 +462,7 @@ def list_tables(self, namespace: str | Identifier) -> list[Identifier]: self.dynamodb.exceptions.InternalServerError, self.dynamodb.exceptions.ResourceNotFoundException, ) as e: - raise GenericDynamoDbError(e.message) from e + raise GenericDynamoDbError(e.message) from e # ty: ignore[unresolved-attribute] table_identifiers = [] for page in page_iterator: @@ -508,7 +508,7 @@ def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]: self.dynamodb.exceptions.InternalServerError, self.dynamodb.exceptions.ResourceNotFoundException, ) as e: - raise GenericDynamoDbError(e.message) from e + raise GenericDynamoDbError(e.message) from e # ty: ignore[unresolved-attribute] database_identifiers = [] for page in page_iterator: @@ -651,7 +651,7 @@ def _get_dynamo_item(self, identifier: str, namespace: str) -> dict[str, Any]: self.dynamodb.exceptions.RequestLimitExceeded, self.dynamodb.exceptions.InternalServerError, ) as e: - raise GenericDynamoDbError(e.message) from e + raise GenericDynamoDbError(e.message) from e # ty: ignore[unresolved-attribute] def _put_dynamo_item(self, item: dict[str, Any], condition_expression: str) -> None: try: @@ -666,7 +666,7 @@ def _put_dynamo_item(self, item: dict[str, Any], condition_expression: str) -> N self.dynamodb.exceptions.ItemCollectionSizeLimitExceededException, self.dynamodb.exceptions.TransactionConflictException, ) as e: - raise GenericDynamoDbError(e.message) from e + raise GenericDynamoDbError(e.message) from e # ty: ignore[unresolved-attribute] def _delete_dynamo_item(self, namespace: str, identifier: str, condition_expression: str) -> None: try: @@ -694,7 +694,7 @@ def _delete_dynamo_item(self, namespace: str, identifier: str, condition_express self.dynamodb.exceptions.ItemCollectionSizeLimitExceededException, self.dynamodb.exceptions.TransactionConflictException, ) as e: - raise GenericDynamoDbError(e.message) from e + raise GenericDynamoDbError(e.message) from e # ty: ignore[unresolved-attribute] def _convert_dynamo_table_item_to_iceberg_table(self, dynamo_table_item: dict[str, Any]) -> Table: table_dict = _convert_dynamo_item_to_regular_dict(dynamo_table_item) diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py index bbb954fcda..c07fc57faf 100644 --- a/pyiceberg/catalog/glue.py +++ b/pyiceberg/catalog/glue.py @@ -219,7 +219,7 @@ def primitive(self, primitive: PrimitiveType) -> str: return f"decimal({primitive.precision},{primitive.scale})" if (primitive_type := type(primitive)) not in GLUE_PRIMITIVE_TYPES: return str(primitive) - return GLUE_PRIMITIVE_TYPES[primitive_type] + return GLUE_PRIMITIVE_TYPES[primitive_type] # ty: ignore[invalid-argument-type] def _to_columns(metadata: TableMetadata) -> list["ColumnTypeDef"]: diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 0f43bbfb81..52b2c24455 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -294,7 +294,7 @@ def primitive(self, primitive: PrimitiveType) -> str: # Hive2 doesn't support timestamp with local time zone return "timestamp" else: - return HIVE_PRIMITIVE_TYPES[type(primitive)] + return HIVE_PRIMITIVE_TYPES[type(primitive)] # ty: ignore[invalid-argument-type] class HiveCatalog(MetastoreCatalog): @@ -333,7 +333,7 @@ def _create_hive_client(properties: dict[str, str]) -> _HiveClient: raise ValueError(f"Unable to connect to hive using uri: {properties[URI]}") def _convert_hive_into_iceberg(self, table: HiveTable) -> Table: - properties: dict[str, str] = table.parameters + properties: dict[str, str] = table.parameters # ty: ignore[invalid-assignment] if TABLE_TYPE not in properties: raise NoSuchPropertyException( f"Property table_type missing, could not determine type: {table.dbName}.{table.tableName}" @@ -354,7 +354,7 @@ def _convert_hive_into_iceberg(self, table: HiveTable) -> Table: file = io.new_input(metadata_location) metadata = FromInputFile.table_metadata(file) return Table( - identifier=(table.dbName, table.tableName), + identifier=(table.dbName, table.tableName), # ty: ignore[invalid-argument-type] metadata=metadata, metadata_location=metadata_location, io=self._load_file_io(location=metadata_location, table_properties=metadata.properties), @@ -536,8 +536,8 @@ def _create_lock_request(self, database_name: str, table_name: str) -> LockReque def _wait_for_lock(self, database_name: str, table_name: str, lockid: int, open_client: Client) -> LockResponse: @retry( retry=retry_if_exception_type(WaitingForLockException), - wait=wait_exponential(multiplier=2, min=self._lock_check_min_wait_time, max=self._lock_check_max_wait_time), - stop=stop_after_attempt(self._lock_check_retries), + wait=wait_exponential(multiplier=2, min=self._lock_check_min_wait_time, max=self._lock_check_max_wait_time), # ty: ignore[invalid-argument-type] + stop=stop_after_attempt(self._lock_check_retries), # ty: ignore[invalid-argument-type] reraise=True, ) def _do_wait_for_lock() -> LockResponse: @@ -581,7 +581,7 @@ def commit_table( try: if lock.state != LockState.ACQUIRED: if lock.state == LockState.WAITING: - self._wait_for_lock(database_name, table_name, lock.lockid, open_client) + self._wait_for_lock(database_name, table_name, lock.lockid, open_client) # ty: ignore[invalid-argument-type] else: raise CommitFailedException(f"Failed to acquire lock for {table_identifier}, state: {lock.state}") @@ -810,10 +810,10 @@ def list_tables(self, namespace: str | Identifier) -> list[Identifier]: """ database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) with self._client as open_client: - return [ + return [ # ty: ignore[invalid-return-type] (database_name, table.tableName) for table in self._fetch_hive_tables(open_client, database_name) - if table.parameters.get(TABLE_TYPE, "").lower() == ICEBERG + if table.parameters.get(TABLE_TYPE, "").lower() == ICEBERG # ty: ignore[unresolved-attribute] ] @override diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index c8e2399fe8..b0853bdf83 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -488,7 +488,7 @@ def __init__(self, timeout: float | None = None, max_retries: Retry | int = DEFA self._timeout = timeout super().__init__(max_retries=max_retries) - def send( + def send( # ty: ignore[invalid-method-override] self, request: PreparedRequest, stream: bool = False, @@ -499,7 +499,7 @@ def send( ) -> Response: if timeout is None: timeout = self._timeout - return super().send(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) + return super().send(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) # ty: ignore[invalid-argument-type] def _create_connection_adapter(properties: Properties) -> _RetryTimeoutHTTPAdapter | None: @@ -678,7 +678,7 @@ def supports_server_side_planning(self, table_config: Properties = EMPTY_DICT) - return False return self._effective_scan_planning_mode(table_config) == ScanPlanningMode.SERVER - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def _plan_table_scan(self, identifier: str | Identifier, request: PlanTableScanRequest) -> PlanningResponse: """Submit a scan plan request to the REST server. @@ -704,7 +704,7 @@ def _plan_table_scan(self, identifier: str | Identifier, request: PlanTableScanR return _PLANNING_RESPONSE_ADAPTER.validate_json(response.text) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def _fetch_scan_tasks(self, identifier: str | Identifier, plan_task: str) -> ScanTasks: """Fetch additional scan tasks using a plan task token. @@ -731,7 +731,7 @@ def _fetch_scan_tasks(self, identifier: str | Identifier, plan_task: str) -> Sca return ScanTasks.model_validate_json(response.text) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def _fetch_planning_result(self, identifier: str | Identifier, plan_id: str) -> PlanningResponse: """Fetch the result of an async scan plan by plan-id. @@ -1045,12 +1045,12 @@ def _fetch_config(self) -> None: config_response = ConfigResponse.model_validate_json(response.text) config = config_response.defaults - config.update(self.properties) - config.update(config_response.overrides) - self.properties = config + config.update(self.properties) # ty: ignore[unresolved-attribute] + config.update(config_response.overrides) # ty: ignore[no-matching-overload, unresolved-attribute] + self.properties = config # ty: ignore[invalid-assignment] # Update URI based on overrides - self.uri = config[URI] + self.uri = config[URI] # ty: ignore[not-subscriptable] # Determine supported endpoints endpoints = config_response.endpoints @@ -1105,7 +1105,7 @@ class SigV4Adapter(HTTPAdapter): def __init__(self, **properties: str): self._properties = properties max_retries = property_as_int(self._properties, SIGV4_MAX_RETRIES, SIGV4_MAX_RETRIES_DEFAULT) - super().__init__(max_retries=max_retries) + super().__init__(max_retries=max_retries) # ty: ignore[invalid-argument-type] self._boto_session = boto3.Session( profile_name=get_first_property_value(self._properties, AWS_PROFILE_NAME), region_name=get_first_property_value(self._properties, AWS_REGION), @@ -1197,7 +1197,7 @@ def _refresh_token(self) -> None: # Reactive token refresh is atypical - we should proactively refresh tokens in a separate thread # instead of retrying on Auth Exceptions. Keeping refresh behavior for the LegacyOAuth2AuthManager # for backward compatibility - auth_manager = self._session.auth.auth_manager # type: ignore[union-attr] + auth_manager = self._session.auth.auth_manager # type: ignore[union-attr] # ty: ignore[unresolved-attribute] if isinstance(auth_manager, LegacyOAuth2AuthManager): auth_manager._refresh_token() @@ -1252,7 +1252,7 @@ def _create_table( return TableResponse.model_validate_json(response.text) @override - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def create_table( self, identifier: str | Identifier, @@ -1274,7 +1274,7 @@ def create_table( return self._response_to_table(self.identifier_to_tuple(identifier), table_response) @override - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def create_table_transaction( self, identifier: str | Identifier, @@ -1297,7 +1297,7 @@ def create_table_transaction( return CreateTableTransaction(staged_table) @override - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def create_view( self, identifier: str | Identifier, @@ -1336,7 +1336,7 @@ def create_view( view_response = ViewResponse.model_validate_json(response.text) return self._response_to_view(self.identifier_to_tuple(identifier), view_response) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def register_table(self, identifier: str | Identifier, metadata_location: str, overwrite: bool = False) -> Table: """Register a new table using existing metadata. @@ -1372,7 +1372,7 @@ def register_table(self, identifier: str | Identifier, metadata_location: str, o table_response = TableResponse.model_validate_json(response.text) return self._response_to_table(self.identifier_to_tuple(identifier), table_response) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def unregister_table(self, identifier: str | Identifier) -> tuple[str, TableMetadata]: """Unregister a table from the catalog without removing data or metadata files. @@ -1398,7 +1398,7 @@ def unregister_table(self, identifier: str | Identifier) -> tuple[str, TableMeta result = UnregisterTableResult.model_validate_json(response.content) return (result.metadata_location, result.metadata) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def list_tables(self, namespace: str | Identifier) -> list[Identifier]: self._check_endpoint(Capability.V1_LIST_TABLES) @@ -1434,7 +1434,7 @@ def list_tables(self, namespace: str | Identifier) -> list[Identifier]: return tables - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def load_table(self, identifier: str | Identifier) -> Table: self._check_endpoint(Capability.V1_LOAD_TABLE) @@ -1456,7 +1456,7 @@ def load_table(self, identifier: str | Identifier) -> Table: table_response = TableResponse.model_validate_json(response.text) return self._response_to_table(self.identifier_to_tuple(identifier), table_response) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def _load_credentials( self, identifier: str | Identifier, @@ -1482,7 +1482,7 @@ def load_credentials( credentials_response = self._load_credentials(identifier) return self._resolve_storage_credentials(credentials_response.storage_credentials, location) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def drop_table(self, identifier: str | Identifier, purge_requested: bool = False) -> None: self._check_endpoint(Capability.V1_DELETE_TABLE) @@ -1495,12 +1495,12 @@ def drop_table(self, identifier: str | Identifier, purge_requested: bool = False except HTTPError as exc: _handle_non_200_response(exc, {404: NoSuchTableError}) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def purge_table(self, identifier: str | Identifier) -> None: self.drop_table(identifier=identifier, purge_requested=True) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def rename_table(self, from_identifier: str | Identifier, to_identifier: str | Identifier) -> Table: self._check_endpoint(Capability.V1_RENAME_TABLE) @@ -1537,7 +1537,7 @@ def _remove_catalog_name_from_table_request_identifier(self, table_request: Comm ) return table_request - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def list_views(self, namespace: str | Identifier) -> list[Identifier]: if Capability.V1_LIST_VIEWS not in self._supported_endpoints: @@ -1575,7 +1575,7 @@ def list_views(self, namespace: str | Identifier) -> list[Identifier]: return views - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def load_view(self, identifier: str | Identifier) -> View: self._check_endpoint(Capability.V1_LOAD_VIEW) @@ -1590,7 +1590,7 @@ def load_view(self, identifier: str | Identifier) -> View: view_response = ViewResponse.model_validate_json(response.text) return self._response_to_view(self.identifier_to_tuple(identifier), view_response) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def commit_table( self, table: Table, requirements: tuple[TableRequirement, ...], updates: tuple[TableUpdate, ...] @@ -1638,7 +1638,7 @@ def commit_table( ) return CommitTableResponse.model_validate_json(response.text) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def create_namespace(self, namespace: str | Identifier, properties: Properties = EMPTY_DICT) -> None: self._check_endpoint(Capability.V1_CREATE_NAMESPACE) @@ -1650,7 +1650,7 @@ def create_namespace(self, namespace: str | Identifier, properties: Properties = except HTTPError as exc: _handle_non_200_response(exc, {409: NamespaceAlreadyExistsError}) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def drop_namespace(self, namespace: str | Identifier) -> None: self._check_endpoint(Capability.V1_DELETE_NAMESPACE) @@ -1662,7 +1662,7 @@ def drop_namespace(self, namespace: str | Identifier) -> None: except HTTPError as exc: _handle_non_200_response(exc, {404: NoSuchNamespaceError, 409: NamespaceNotEmptyError}) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]: self._check_endpoint(Capability.V1_LIST_NAMESPACES) @@ -1700,7 +1700,7 @@ def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]: return namespaces - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def load_namespace_properties(self, namespace: str | Identifier) -> Properties: self._check_endpoint(Capability.V1_LOAD_NAMESPACE) @@ -1714,7 +1714,7 @@ def load_namespace_properties(self, namespace: str | Identifier) -> Properties: return NamespaceResponse.model_validate_json(response.text).properties - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def update_namespace_properties( self, namespace: str | Identifier, removals: set[str] | None = None, updates: Properties = EMPTY_DICT @@ -1735,7 +1735,7 @@ def update_namespace_properties( missing=parsed_response.missing, ) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def namespace_exists(self, namespace: str | Identifier) -> bool: namespace_tuple = self._check_valid_namespace_identifier(namespace) @@ -1763,7 +1763,7 @@ def namespace_exists(self, namespace: str | Identifier) -> bool: return False - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def table_exists(self, identifier: str | Identifier) -> bool: """Check if a table exists. @@ -1798,7 +1798,7 @@ def table_exists(self, identifier: str | Identifier) -> bool: return False - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def view_exists(self, identifier: str | Identifier) -> bool: """Check if a view exists. @@ -1824,7 +1824,7 @@ def view_exists(self, identifier: str | Identifier) -> bool: return False - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def register_view(self, identifier: str | Identifier, metadata_location: str) -> View: self._check_endpoint(Capability.V1_REGISTER_VIEW) @@ -1847,7 +1847,7 @@ def register_view(self, identifier: str | Identifier, metadata_location: str) -> view_response = ViewResponse.model_validate_json(response.text) return self._response_to_view(self.identifier_to_tuple(identifier), view_response) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def drop_view(self, identifier: str | Identifier) -> None: self._check_endpoint(Capability.V1_DELETE_VIEW) @@ -1859,7 +1859,7 @@ def drop_view(self, identifier: str | Identifier) -> None: except HTTPError as exc: _handle_non_200_response(exc, {404: NoSuchViewError}) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def rename_view(self, from_identifier: str | Identifier, to_identifier: str | Identifier) -> None: self._check_endpoint(Capability.V1_RENAME_VIEW) diff --git a/pyiceberg/catalog/rest/auth.py b/pyiceberg/catalog/rest/auth.py index 602074282c..4e4f4b8d75 100644 --- a/pyiceberg/catalog/rest/auth.py +++ b/pyiceberg/catalog/rest/auth.py @@ -184,7 +184,7 @@ def _refresh_token(self) -> None: "The expiration time of the Token must be provided by the Server in the Access Token Response " "in `expires_in` field, or by the PyIceberg Client." ) - self._expires_at = time.monotonic() + expires_in - self.refresh_margin + self._expires_at = time.monotonic() + expires_in - self.refresh_margin # ty: ignore[invalid-assignment] def get_token(self) -> str: with self._lock: @@ -330,7 +330,7 @@ def __init__(self, auth_manager: AuthManager): """ self.auth_manager = auth_manager - def __call__(self, request: PreparedRequest) -> PreparedRequest: + def __call__(self, request: PreparedRequest) -> PreparedRequest: # ty: ignore[invalid-method-override] """ Modify the outgoing request to include the Authorization header. diff --git a/pyiceberg/catalog/sql.py b/pyiceberg/catalog/sql.py index 13a71ca894..0467f736a9 100644 --- a/pyiceberg/catalog/sql.py +++ b/pyiceberg/catalog/sql.py @@ -389,7 +389,7 @@ def drop_table(self, identifier: str | Identifier) -> None: if type_filter is not None: stmt = stmt.where(type_filter) res = session.execute(stmt) - if res.rowcount < 1: + if res.rowcount < 1: # ty: ignore[unresolved-attribute] raise NoSuchTableError(f"Table does not exist: {namespace}.{table_name}") else: try: @@ -450,7 +450,7 @@ def rename_table(self, from_identifier: str | Identifier, to_identifier: str | I if type_filter is not None: stmt = stmt.where(type_filter) result = session.execute(stmt) - if result.rowcount < 1: + if result.rowcount < 1: # ty: ignore[unresolved-attribute] raise NoSuchTableError(f"Table does not exist: {from_table_name}") else: try: @@ -532,7 +532,7 @@ def commit_table( ) ) result = session.execute(stmt) - if result.rowcount < 1: + if result.rowcount < 1: # ty: ignore[unresolved-attribute] raise CommitFailedException(f"Table has been updated by another process: {namespace}.{table_name}") else: try: @@ -566,7 +566,7 @@ def commit_table( ) @override - def namespace_exists(self, identifier: str | Identifier) -> bool: + def namespace_exists(self, identifier: str | Identifier) -> bool: # ty: ignore[invalid-method-override] namespace_tuple = Catalog.identifier_to_tuple(identifier) namespace = Catalog.namespace_to_string(namespace_tuple, NoSuchNamespaceError) namespace_starts_with = namespace.replace("!", "!!").replace("_", "!_").replace("%", "!%") + ".%" diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index 6d1add9e23..cd9cdddf5f 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -45,7 +45,7 @@ def wrapper(*args: Any, **kwargs: Any): # type: ignore[no-untyped-def] try: return func(*args, **kwargs) except Exception as e: - ctx: Context = click.get_current_context(silent=True) + ctx: Context = click.get_current_context(silent=True) # ty: ignore[invalid-assignment] _, output = _catalog_and_output(ctx) output.exception(e) ctx.exit(1) diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py index cfb00abb78..df8d700ed5 100644 --- a/pyiceberg/cli/output.py +++ b/pyiceberg/cli/output.py @@ -202,7 +202,7 @@ def uuid(self, uuid: UUID | None) -> None: def version(self, version: str) -> None: Console().print(version) - def describe_refs(self, ref_details: list[tuple[str, SnapshotRefType, dict[str, str]]]) -> None: + def describe_refs(self, ref_details: list[tuple[str, SnapshotRefType, dict[str, str]]]) -> None: # ty: ignore[invalid-method-override] refs_table = RichTable(title="Snapshot Refs") refs_table.add_column("Ref") refs_table.add_column("Type") diff --git a/pyiceberg/conversions.py b/pyiceberg/conversions.py index 4fc8dd0405..f30a0fda60 100644 --- a/pyiceberg/conversions.py +++ b/pyiceberg/conversions.py @@ -323,7 +323,7 @@ def _(primitive_type: DecimalType, value: Decimal) -> bytes: return decimal_to_bytes(value) -@to_bytes.register(UnknownType) +@to_bytes.register(UnknownType) # ty: ignore[invalid-argument-type] def _(_: UnknownType, value: Any) -> None: """Return None since the unknown type has no binary representation.""" return None diff --git a/pyiceberg/encryption/key_metadata.py b/pyiceberg/encryption/key_metadata.py index e6546be26f..b897621bd4 100644 --- a/pyiceberg/encryption/key_metadata.py +++ b/pyiceberg/encryption/key_metadata.py @@ -82,7 +82,7 @@ def from_bytes(cls, data: bytes) -> StandardKeyMetadata: def to_bytes(self) -> bytes: """Encode key metadata to its wire format.""" output = io.BytesIO() - encoder = BinaryEncoder(output) + encoder = BinaryEncoder(output) # ty: ignore[invalid-argument-type] encoder.write(bytes([KEY_METADATA_V1])) record = Record(self.encryption_key, self.aad_prefix, self.file_length) construct_writer(KEY_METADATA_SCHEMA_V1).write(encoder, record) diff --git a/pyiceberg/encryption/manager.py b/pyiceberg/encryption/manager.py index 73c57dfd5b..b9376cd390 100644 --- a/pyiceberg/encryption/manager.py +++ b/pyiceberg/encryption/manager.py @@ -156,7 +156,7 @@ def _find_active_kek(self) -> EncryptedKey | None: for key in self._encryption_keys.values() if key.encrypted_by_id == self._table_key_id and not _is_kek_expired(key) ] - return max(keks, key=lambda kek: _kek_created_at(kek) or 0, default=None) + return max(keks, key=lambda kek: _kek_created_at(kek) or 0, default=None) # ty: ignore[invalid-argument-type] def _unwrap_kek(self, kek: EncryptedKey) -> SecureKey: if (cached := self._kek_cache.get(kek.key_id)) is not None and time.monotonic() - cached[1] < KEK_CACHE_TTL_SECONDS: diff --git a/pyiceberg/expressions/__init__.py b/pyiceberg/expressions/__init__.py index 51a68ab987..ff96d8860e 100644 --- a/pyiceberg/expressions/__init__.py +++ b/pyiceberg/expressions/__init__.py @@ -341,9 +341,9 @@ def __invert__(self) -> BooleanExpression: # De Morgan's law: not (A and B) = (not A) or (not B) return Or(~self.left, ~self.right) - def __deepcopy__(self, memo: dict[int, Any]) -> And: + def __deepcopy__(self, memo: dict[int, Any]) -> And: # ty: ignore[invalid-method-override] """Return a deep copy of the And expression.""" - return And(copy.deepcopy(self.left, memo), copy.deepcopy(self.right, memo)) + return And(copy.deepcopy(self.left, memo), copy.deepcopy(self.right, memo)) # ty: ignore[invalid-return-type] def __getnewargs__(self) -> tuple[BooleanExpression, BooleanExpression]: """Pickle the And class.""" @@ -392,9 +392,9 @@ def __invert__(self) -> BooleanExpression: # De Morgan's law: not (A or B) = (not A) and (not B) return And(~self.left, ~self.right) - def __deepcopy__(self, memo: dict[int, Any]) -> Or: + def __deepcopy__(self, memo: dict[int, Any]) -> Or: # ty: ignore[invalid-method-override] """Return a deep copy of the Or expression.""" - return Or(copy.deepcopy(self.left, memo), copy.deepcopy(self.right, memo)) + return Or(copy.deepcopy(self.left, memo), copy.deepcopy(self.right, memo)) # ty: ignore[invalid-return-type] def __getnewargs__(self) -> tuple[BooleanExpression, BooleanExpression]: """Pickle the Or class.""" @@ -438,16 +438,16 @@ def __invert__(self) -> BooleanExpression: """Transform the Expression into its negated version.""" return self.child - def __deepcopy__(self, memo: dict[int, Any]) -> Not: + def __deepcopy__(self, memo: dict[int, Any]) -> Not: # ty: ignore[invalid-method-override] """Return a deep copy of the Not expression.""" - return Not(copy.deepcopy(self.child, memo)) + return Not(copy.deepcopy(self.child, memo)) # ty: ignore[invalid-return-type] def __getnewargs__(self) -> tuple[BooleanExpression]: """Pickle the Not class.""" return (self.child,) -class AlwaysTrue(BooleanExpression, Singleton, IcebergRootModel[bool]): +class AlwaysTrue(BooleanExpression, Singleton, IcebergRootModel[bool]): # ty: ignore[invalid-method-override] """TRUE expression.""" root: bool = True @@ -465,7 +465,7 @@ def __repr__(self) -> str: return "AlwaysTrue()" -class AlwaysFalse(BooleanExpression, Singleton, IcebergRootModel[bool]): +class AlwaysFalse(BooleanExpression, Singleton, IcebergRootModel[bool]): # ty: ignore[invalid-method-override] """FALSE expression.""" root: bool = False @@ -576,7 +576,7 @@ def __new__(cls, term: BoundTerm) -> BooleanExpression: # pylint: disable=W0221 def __invert__(self) -> BoundNotNull: """Transform the Expression into its negated version.""" - return BoundNotNull(self.term) + return BoundNotNull(self.term) # ty: ignore[invalid-return-type] @property def as_unbound(self) -> type[IsNull]: @@ -591,7 +591,7 @@ def __new__(cls, term: BoundTerm) -> BooleanExpression: # pylint: disable=W0221 def __invert__(self) -> BoundIsNull: """Transform the Expression into its negated version.""" - return BoundIsNull(self.term) + return BoundIsNull(self.term) # ty: ignore[invalid-return-type] @property def as_unbound(self) -> type[NotNull]: @@ -637,7 +637,7 @@ def __new__(cls, term: BoundTerm) -> BooleanExpression: # pylint: disable=W0221 def __invert__(self) -> BoundNotNaN: """Transform the Expression into its negated version.""" - return BoundNotNaN(self.term) + return BoundNotNaN(self.term) # ty: ignore[invalid-return-type] @property def as_unbound(self) -> type[IsNaN]: @@ -653,7 +653,7 @@ def __new__(cls, term: BoundTerm) -> BooleanExpression: # pylint: disable=W0221 def __invert__(self) -> BoundIsNaN: """Transform the Expression into its negated version.""" - return BoundIsNaN(self.term) + return BoundIsNaN(self.term) # ty: ignore[invalid-return-type] @property def as_unbound(self) -> type[NotNaN]: @@ -787,7 +787,7 @@ def __new__(cls, term: BoundTerm, literals: set[LiteralValue]) -> BooleanExpress def __invert__(self) -> BoundNotIn: """Transform the Expression into its negated version.""" - return BoundNotIn(self.term, self.literals) + return BoundNotIn(self.term, self.literals) # ty: ignore[invalid-return-type] def __eq__(self, other: Any) -> bool: """Return the equality of two instances of the BoundIn class.""" @@ -814,7 +814,7 @@ def __new__( # pylint: disable=W0221 def __invert__(self) -> BoundIn: """Transform the Expression into its negated version.""" - return BoundIn(self.term, self.literals) + return BoundIn(self.term, self.literals) # ty: ignore[invalid-return-type] @property def as_unbound(self) -> type[NotIn]: @@ -841,9 +841,9 @@ def __new__( # pylint: disable=W0221 literals_set = _to_literal_set(literals) count = len(literals_set) if count == 0: - return AlwaysFalse() + return AlwaysFalse() # ty: ignore[invalid-return-type] elif count == 1: - return EqualTo(term, next(iter(literals_set))) + return EqualTo(term, next(iter(literals_set))) # ty: ignore[invalid-return-type] else: return super().__new__(cls) @@ -876,9 +876,9 @@ def __new__( # pylint: disable=W0221 literals_set = _to_literal_set(literals) count = len(literals_set) if count == 0: - return AlwaysTrue() + return AlwaysTrue() # ty: ignore[invalid-return-type] elif count == 1: - return NotEqualTo(term, next(iter(literals_set))) + return NotEqualTo(term, next(iter(literals_set))) # ty: ignore[invalid-return-type] else: return super().__new__(cls) @@ -913,16 +913,16 @@ def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundLiteralPredi if isinstance(lit, AboveMax): if isinstance(self, (LessThan, LessThanOrEqual, NotEqualTo)): - return AlwaysTrue() + return AlwaysTrue() # ty: ignore[invalid-return-type] elif isinstance(self, (GreaterThan, GreaterThanOrEqual, EqualTo)): - return AlwaysFalse() + return AlwaysFalse() # ty: ignore[invalid-return-type] else: raise ValueError(f"Cannot bind {type(self).__name__} with AboveMax literal") elif isinstance(lit, BelowMin): if isinstance(self, (GreaterThan, GreaterThanOrEqual, NotEqualTo)): - return AlwaysTrue() + return AlwaysTrue() # ty: ignore[invalid-return-type] elif isinstance(self, (LessThan, LessThanOrEqual, EqualTo)): - return AlwaysFalse() + return AlwaysFalse() # ty: ignore[invalid-return-type] else: raise ValueError(f"Cannot bind {type(self).__name__} with BelowMin literal") diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index 9045357331..d120652045 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -157,25 +157,25 @@ def literal(value: L) -> Literal[L]: LongLiteral(123) """ if isinstance(value, float): - return DoubleLiteral(value) + return DoubleLiteral(value) # ty: ignore[invalid-return-type] elif isinstance(value, bool): - return BooleanLiteral(value) + return BooleanLiteral(value) # ty: ignore[invalid-return-type] elif isinstance(value, int): return LongLiteral(value) elif isinstance(value, str): - return StringLiteral(value) + return StringLiteral(value) # ty: ignore[invalid-return-type] elif isinstance(value, UUID): - return UUIDLiteral(value.bytes) + return UUIDLiteral(value.bytes) # ty: ignore[invalid-return-type] elif isinstance(value, bytes): - return BinaryLiteral(value) + return BinaryLiteral(value) # ty: ignore[invalid-return-type] elif isinstance(value, Decimal): - return DecimalLiteral(value) + return DecimalLiteral(value) # ty: ignore[invalid-return-type] elif isinstance(value, datetime): - return TimestampLiteral(datetime_to_micros(value)) + return TimestampLiteral(datetime_to_micros(value)) # ty: ignore[invalid-return-type] elif isinstance(value, date): return DateLiteral(date_to_days(value)) elif isinstance(value, time): - return TimeLiteral(time_to_micros(value)) + return TimeLiteral(time_to_micros(value)) # ty: ignore[invalid-return-type] else: raise TypeError(f"Invalid literal value: {repr(value)}") @@ -374,7 +374,7 @@ def _(self, type_var: DecimalType) -> Literal[Decimal]: class FloatLiteral(Literal[float]): def __init__(self, value: float) -> None: super().__init__(value, float) - self._value32 = struct.unpack(" bool: """Return the equality of two instances of the FloatLiteral class.""" diff --git a/pyiceberg/expressions/parser.py b/pyiceberg/expressions/parser.py index d813bcaa07..08cf61cd30 100644 --- a/pyiceberg/expressions/parser.py +++ b/pyiceberg/expressions/parser.py @@ -278,15 +278,15 @@ def _(result: ParseResults) -> BooleanExpression: def handle_not(result: ParseResults) -> Not: - return Not(result[0][0]) + return Not(result[0][0]) # ty: ignore[invalid-return-type] def handle_and(result: ParseResults) -> And: - return And(*result[0]) + return And(*result[0]) # ty: ignore[invalid-return-type] def handle_or(result: ParseResults) -> Or: - return Or(*result[0]) + return Or(*result[0]) # ty: ignore[invalid-return-type] boolean_expression = infix_notation( diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index a8de088669..e66f3b7652 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -1033,7 +1033,7 @@ def _cast_if_necessary(self, iceberg_type: IcebergType, literal: L | set[L]) -> iceberg_type_class = type(iceberg_type) conversions = {TimestampType: micros_to_timestamp, TimestamptzType: micros_to_timestamptz} if iceberg_type_class in conversions: - conversion_function = conversions[iceberg_type_class] + conversion_function = conversions[iceberg_type_class] # ty: ignore[invalid-argument-type] if isinstance(literal, set): return {conversion_function(lit) for lit in literal} # type: ignore else: diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 1dd816d5d3..34b965e8d4 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -245,12 +245,12 @@ def _import_retry_strategy(impl: str) -> S3RetryStrategy | None: class UnsupportedPyArrowTypeException(Exception): """Cannot convert PyArrow type to corresponding Iceberg type.""" - def __init__(self, field: pa.Field, *args: Any): + def __init__(self, field: pa.Field, *args: Any): # ty: ignore[missing-type-argument] self.field = field super().__init__(*args) -class PyArrowLocalFileSystem(pyarrow.fs.LocalFileSystem): +class PyArrowLocalFileSystem(pyarrow.fs.LocalFileSystem): # ty: ignore[possibly-missing-submodule] def open_output_stream(self, path: str, *args: Any, **kwargs: Any) -> pyarrow.NativeFile: # In LocalFileSystem, parent directories must be first created before opening an output stream self.create_dir(os.path.dirname(path), recursive=True) @@ -352,7 +352,7 @@ def open(self, seekable: bool = True) -> InputStream: elif e.errno == 13 or "AWS Error [code 15]" in str(e): raise PermissionError(f"Cannot open file, access denied: {self.location}") from e raise # pragma: no cover - If some other kind of OSError, raise the raw error - return input_file + return input_file # ty: ignore[invalid-return-type] @override def create(self, overwrite: bool = False) -> OutputStream: @@ -384,7 +384,7 @@ def create(self, overwrite: bool = False) -> OutputStream: if e.errno == 13 or "AWS Error [code 15]" in str(e): raise PermissionError(f"Cannot create file, access denied: {self.location}") from e raise # pragma: no cover - If some other kind of OSError, raise the raw error - return output_file + return output_file # ty: ignore[invalid-return-type] @override def to_input_file(self) -> PyArrowFile: @@ -618,7 +618,7 @@ def _initialize_azure_fs(self, netloc: str | None = None) -> FileSystem: f"Provided: {provided_keys}, Missing: {missing_keys}" ) - return AzureFileSystem(**client_kwargs) + return AzureFileSystem(**client_kwargs) # ty: ignore[invalid-argument-type] def _initialize_hdfs_fs(self, scheme: str, netloc: str | None) -> FileSystem: from pyarrow.fs import HadoopFileSystem @@ -756,13 +756,13 @@ def __init__( self._include_field_ids = include_field_ids self._file_format = file_format - def schema(self, _: Schema, struct_result: pa.StructType) -> pa.Schema: - return pa.schema(list(struct_result), metadata=self._metadata) + def schema(self, _: Schema, struct_result: pa.StructType) -> pa.Schema: # ty: ignore[invalid-method-override] + return pa.schema(list(struct_result), metadata=self._metadata) # ty: ignore[invalid-argument-type] - def struct(self, _: StructType, field_results: builtins.list[pa.DataType]) -> pa.DataType: - return pa.struct(field_results) + def struct(self, _: StructType, field_results: builtins.list[pa.DataType]) -> pa.DataType: # ty: ignore[invalid-method-override] + return pa.struct(field_results) # ty: ignore[invalid-argument-type] - def field(self, field: NestedField, field_result: pa.DataType) -> pa.Field: + def field(self, field: NestedField, field_result: pa.DataType) -> pa.Field: # ty: ignore[invalid-method-override, missing-type-argument] metadata = {} if field.doc: metadata[PYARROW_FIELD_DOC_KEY] = field.doc @@ -790,7 +790,7 @@ def list(self, list_type: ListType, element_result: pa.DataType) -> pa.DataType: def map(self, map_type: MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: key_field = self.field(map_type.key_field, key_result) value_field = self.field(map_type.value_field, value_result) - return pa.map_(key_type=key_field, item_type=value_field) + return pa.map_(key_type=key_field, item_type=value_field) # ty: ignore[invalid-argument-type] def visit_fixed(self, fixed_type: FixedType) -> pa.DataType: return pa.binary(len(fixed_type)) @@ -804,53 +804,53 @@ def visit_decimal(self, decimal_type: DecimalType) -> pa.DataType: # https://github.com/apache/arrow/blob/598938711a8376cbfdceaf5c77ab0fd5057e6c02/cpp/src/parquet/arrow/schema.cc#L380-L392 return pa.decimal128(decimal_type.precision, decimal_type.scale) - def visit_boolean(self, _: BooleanType) -> pa.DataType: + def visit_boolean(self, _: BooleanType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.bool_() - def visit_integer(self, _: IntegerType) -> pa.DataType: + def visit_integer(self, _: IntegerType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.int32() - def visit_long(self, _: LongType) -> pa.DataType: + def visit_long(self, _: LongType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.int64() - def visit_float(self, _: FloatType) -> pa.DataType: + def visit_float(self, _: FloatType) -> pa.DataType: # ty: ignore[invalid-method-override] # 32-bit IEEE 754 floating point return pa.float32() - def visit_double(self, _: DoubleType) -> pa.DataType: + def visit_double(self, _: DoubleType) -> pa.DataType: # ty: ignore[invalid-method-override] # 64-bit IEEE 754 floating point return pa.float64() - def visit_date(self, _: DateType) -> pa.DataType: + def visit_date(self, _: DateType) -> pa.DataType: # ty: ignore[invalid-method-override] # Date encoded as an int return pa.date32() - def visit_time(self, _: TimeType) -> pa.DataType: + def visit_time(self, _: TimeType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.time64("us") - def visit_timestamp(self, _: TimestampType) -> pa.DataType: + def visit_timestamp(self, _: TimestampType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.timestamp(unit="us") - def visit_timestamp_ns(self, _: TimestampNanoType) -> pa.DataType: + def visit_timestamp_ns(self, _: TimestampNanoType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.timestamp(unit="ns") - def visit_timestamptz(self, _: TimestamptzType) -> pa.DataType: + def visit_timestamptz(self, _: TimestamptzType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.timestamp(unit="us", tz="UTC") - def visit_timestamptz_ns(self, _: TimestamptzNanoType) -> pa.DataType: + def visit_timestamptz_ns(self, _: TimestamptzNanoType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.timestamp(unit="ns", tz="UTC") - def visit_string(self, _: StringType) -> pa.DataType: + def visit_string(self, _: StringType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.large_string() - def visit_uuid(self, _: UUIDType) -> pa.DataType: + def visit_uuid(self, _: UUIDType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.uuid() - def visit_unknown(self, _: UnknownType) -> pa.DataType: + def visit_unknown(self, _: UnknownType) -> pa.DataType: # ty: ignore[invalid-method-override] """Type `UnknownType` can be promoted to any primitive type in V3+ tables per the Iceberg spec.""" return pa.null() - def visit_binary(self, _: BinaryType) -> pa.DataType: + def visit_binary(self, _: BinaryType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.large_binary() def visit_geometry(self, geometry_type: GeometryType) -> pa.DataType: @@ -885,10 +885,10 @@ def visit_geography(self, geography_type: GeographyType) -> pa.DataType: return pa.large_binary() -def _convert_scalar(value: Any, iceberg_type: IcebergType) -> pa.scalar: +def _convert_scalar(value: Any, iceberg_type: IcebergType) -> pa.scalar: # ty: ignore[invalid-type-form] if not isinstance(iceberg_type, PrimitiveType): raise ValueError(f"Expected primitive type, got: {iceberg_type}") - return pa.scalar(value=value, type=schema_to_pyarrow(iceberg_type)) + return pa.scalar(value=value, type=schema_to_pyarrow(iceberg_type)) # ty: ignore[no-matching-overload] class _ConvertToArrowExpression(BoundBooleanExpressionVisitor[pc.Expression]): @@ -925,26 +925,26 @@ def _get_field_name(self, term: BoundTerm) -> str | tuple[str, ...]: return term.ref().field.name def visit_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression: - pyarrow_literals = pa.array(literals, type=schema_to_pyarrow(term.ref().field.field_type)) + pyarrow_literals = pa.array(literals, type=schema_to_pyarrow(term.ref().field.field_type)) # ty: ignore[no-matching-overload] return pc.field(self._get_field_name(term)).isin(pyarrow_literals) def visit_not_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression: - pyarrow_literals = pa.array(literals, type=schema_to_pyarrow(term.ref().field.field_type)) + pyarrow_literals = pa.array(literals, type=schema_to_pyarrow(term.ref().field.field_type)) # ty: ignore[no-matching-overload] return ~pc.field(self._get_field_name(term)).isin(pyarrow_literals) def visit_is_nan(self, term: BoundTerm) -> pc.Expression: ref = pc.field(self._get_field_name(term)) - return pc.is_nan(ref) + return pc.is_nan(ref) # ty: ignore[invalid-return-type] def visit_not_nan(self, term: BoundTerm) -> pc.Expression: ref = pc.field(self._get_field_name(term)) - return ~pc.is_nan(ref) + return ~pc.is_nan(ref) # ty: ignore[unsupported-operator] def visit_is_null(self, term: BoundTerm) -> pc.Expression: return pc.field(self._get_field_name(term)).is_null(nan_is_null=False) def visit_not_null(self, term: BoundTerm) -> pc.Expression: - return pc.field(self._get_field_name(term)).is_valid() + return pc.field(self._get_field_name(term)).is_valid() # ty: ignore[invalid-return-type] def visit_equal(self, term: BoundTerm, literal: Literal[Any]) -> pc.Expression: return pc.field(self._get_field_name(term)) == _convert_scalar(literal.value, term.ref().field.field_type) @@ -965,10 +965,10 @@ def visit_less_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> pc return pc.field(self._get_field_name(term)) <= _convert_scalar(literal.value, term.ref().field.field_type) def visit_starts_with(self, term: BoundTerm, literal: Literal[Any]) -> pc.Expression: - return pc.starts_with(pc.field(self._get_field_name(term)), literal.value) + return pc.starts_with(pc.field(self._get_field_name(term)), literal.value) # ty: ignore[invalid-return-type] def visit_not_starts_with(self, term: BoundTerm, literal: Literal[Any]) -> pc.Expression: - return ~pc.starts_with(pc.field(self._get_field_name(term)), literal.value) + return ~pc.starts_with(pc.field(self._get_field_name(term)), literal.value) # ty: ignore[unsupported-operator] def visit_true(self) -> pc.Expression: return pc.scalar(True) @@ -1149,7 +1149,7 @@ def _expression_to_complementary_pyarrow(expr: BooleanExpression, schema: Schema @lru_cache def _get_file_format(file_format: FileFormat, **kwargs: dict[str, Any]) -> ds.FileFormat: if file_format == FileFormat.PARQUET: - return ds.ParquetFileFormat(**kwargs) + return ds.ParquetFileFormat(**kwargs) # ty: ignore[invalid-argument-type] elif file_format == FileFormat.ORC: # ORC doesn't support pre_buffer and buffer_size parameters orc_kwargs = {k: v for k, v in kwargs.items() if k not in ["pre_buffer", "buffer_size"]} @@ -1158,21 +1158,21 @@ def _get_file_format(file_format: FileFormat, **kwargs: dict[str, Any]) -> ds.Fi raise ValueError(f"Unsupported file format: {file_format}") -def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]: +def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]: # ty: ignore[missing-type-argument] if data_file.file_format == FileFormat.PARQUET: with io.new_input(data_file.file_path).open() as fi: delete_fragment = _get_file_format( data_file.file_format, dictionary_columns=("file_path",), pre_buffer=True, buffer_size=ONE_MEGABYTE - ).make_fragment(fi) + ).make_fragment(fi) # ty: ignore[invalid-argument-type] table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table() table = table.unify_dictionaries() return { file.as_py(): table.filter(pc.field("file_path") == file).column("pos") - for file in table.column("file_path").chunks[0].dictionary + for file in table.column("file_path").chunks[0].dictionary # ty: ignore[unresolved-attribute] } elif data_file.file_format == FileFormat.ORC: with io.new_input(data_file.file_path).open() as fi: - delete_fragment = _get_file_format(data_file.file_format).make_fragment(fi) + delete_fragment = _get_file_format(data_file.file_format).make_fragment(fi) # ty: ignore[invalid-argument-type] table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table() # For ORC, file_path columns are not dictionary-encoded, so we use unique() directly return { @@ -1185,7 +1185,7 @@ def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray] raise ValueError(f"Delete file format not supported: {data_file.file_format}") -def _combine_positional_deletes(positional_deletes: list[pa.ChunkedArray], start_index: int, end_index: int) -> pa.Array: +def _combine_positional_deletes(positional_deletes: list[pa.ChunkedArray], start_index: int, end_index: int) -> pa.Array: # ty: ignore[missing-type-argument] if len(positional_deletes) == 1: all_chunks = positional_deletes[0] else: @@ -1200,7 +1200,7 @@ def _combine_positional_deletes(positional_deletes: list[pa.ChunkedArray], start result = pc.filter(full_range, pc.invert(pc.is_in(full_range, value_set=all_chunks))) # Subtract the start_index from each element in the result - return pc.subtract(result, pa.scalar(start_index)) + return pc.subtract(result, pa.scalar(start_index)) # ty: ignore[invalid-return-type] def pyarrow_to_schema( @@ -1275,15 +1275,15 @@ def _(obj: pa.StructType, visitor: PyArrowSchemaVisitor[T]) -> T: @visit_pyarrow.register(pa.ListType) @visit_pyarrow.register(pa.FixedSizeListType) @visit_pyarrow.register(pa.LargeListType) -def _(obj: pa.ListType | pa.LargeListType | pa.FixedSizeListType, visitor: PyArrowSchemaVisitor[T]) -> T: +def _(obj: pa.ListType | pa.LargeListType | pa.FixedSizeListType, visitor: PyArrowSchemaVisitor[T]) -> T: # ty: ignore[missing-type-argument] visitor.before_list_element(obj.value_field) result = visit_pyarrow(obj.value_type, visitor) visitor.after_list_element(obj.value_field) - return visitor.list(obj, result) + return visitor.list(obj, result) # ty: ignore[invalid-argument-type] @visit_pyarrow.register(pa.MapType) -def _(obj: pa.MapType, visitor: PyArrowSchemaVisitor[T]) -> T: +def _(obj: pa.MapType, visitor: PyArrowSchemaVisitor[T]) -> T: # ty: ignore[missing-type-argument] visitor.before_map_key(obj.key_field) key_result = visit_pyarrow(obj.key_type, visitor) visitor.after_map_key(obj.key_field) @@ -1296,7 +1296,7 @@ def _(obj: pa.MapType, visitor: PyArrowSchemaVisitor[T]) -> T: @visit_pyarrow.register(pa.DictionaryType) -def _(obj: pa.DictionaryType, visitor: PyArrowSchemaVisitor[T]) -> T: +def _(obj: pa.DictionaryType, visitor: PyArrowSchemaVisitor[T]) -> T: # ty: ignore[missing-type-argument] # Parquet has no dictionary type. dictionary-encoding is handled # as an encoding detail, not as a separate type. # We will follow this approach in determining the Iceberg Type, @@ -1306,7 +1306,7 @@ def _(obj: pa.DictionaryType, visitor: PyArrowSchemaVisitor[T]) -> T: @visit_pyarrow.register(pa.Field) -def _(obj: pa.Field, visitor: PyArrowSchemaVisitor[T]) -> T: +def _(obj: pa.Field, visitor: PyArrowSchemaVisitor[T]) -> T: # ty: ignore[missing-type-argument] field_type = obj.type visitor.before_field(obj) @@ -1327,28 +1327,28 @@ def _(obj: pa.DataType, visitor: PyArrowSchemaVisitor[T]) -> T: class PyArrowSchemaVisitor(Generic[T], ABC): - def before_field(self, field: pa.Field) -> None: + def before_field(self, field: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately before visiting a field.""" - def after_field(self, field: pa.Field) -> None: + def after_field(self, field: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately after visiting a field.""" - def before_list_element(self, element: pa.Field) -> None: + def before_list_element(self, element: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately before visiting an element within a ListType.""" - def after_list_element(self, element: pa.Field) -> None: + def after_list_element(self, element: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately after visiting an element within a ListType.""" - def before_map_key(self, key: pa.Field) -> None: + def before_map_key(self, key: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately before visiting a key within a MapType.""" - def after_map_key(self, key: pa.Field) -> None: + def after_map_key(self, key: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately after visiting a key within a MapType.""" - def before_map_value(self, value: pa.Field) -> None: + def before_map_value(self, value: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately before visiting a value within a MapType.""" - def after_map_value(self, value: pa.Field) -> None: + def after_map_value(self, value: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately after visiting a value within a MapType.""" @abstractmethod @@ -1360,15 +1360,15 @@ def struct(self, struct: pa.StructType, field_results: builtins.list[T]) -> T: """Visit a struct.""" @abstractmethod - def field(self, field: pa.Field, field_result: T) -> T: + def field(self, field: pa.Field, field_result: T) -> T: # ty: ignore[missing-type-argument] """Visit a field.""" @abstractmethod - def list(self, list_type: pa.ListType, element_result: T) -> T: + def list(self, list_type: pa.ListType, element_result: T) -> T: # ty: ignore[missing-type-argument] """Visit a list.""" @abstractmethod - def map(self, map_type: pa.MapType, key_result: T, value_result: T) -> T: + def map(self, map_type: pa.MapType, key_result: T, value_result: T) -> T: # ty: ignore[missing-type-argument] """Visit a map.""" @abstractmethod @@ -1376,7 +1376,7 @@ def primitive(self, primitive: pa.DataType) -> T: """Visit a primitive type.""" -def _get_field_id(field: pa.Field) -> int | None: +def _get_field_id(field: pa.Field) -> int | None: # ty: ignore[missing-type-argument] """Return the Iceberg field ID from Parquet or ORC metadata if available.""" if field.metadata: # Try Parquet field ID first @@ -1397,15 +1397,15 @@ def schema(self, schema: pa.Schema, struct_result: bool) -> bool: def struct(self, struct: pa.StructType, field_results: builtins.list[bool]) -> bool: return all(field_results) - def field(self, field: pa.Field, field_result: bool) -> bool: + def field(self, field: pa.Field, field_result: bool) -> bool: # ty: ignore[missing-type-argument] return all([_get_field_id(field) is not None, field_result]) - def list(self, list_type: pa.ListType, element_result: bool) -> bool: + def list(self, list_type: pa.ListType, element_result: bool) -> bool: # ty: ignore[missing-type-argument] element_field = list_type.value_field element_id = _get_field_id(element_field) return element_result and element_id is not None - def map(self, map_type: pa.MapType, key_result: bool, value_result: bool) -> bool: + def map(self, map_type: pa.MapType, key_result: bool, value_result: bool) -> bool: # ty: ignore[missing-type-argument] key_field = map_type.key_field key_id = _get_field_id(key_field) value_field = map_type.item_field @@ -1428,32 +1428,32 @@ def __init__( self._downcast_ns_timestamp_to_us = downcast_ns_timestamp_to_us self._format_version = format_version - def _field_id(self, field: pa.Field) -> int: + def _field_id(self, field: pa.Field) -> int: # ty: ignore[missing-type-argument] if (field_id := _get_field_id(field)) is not None: return field_id else: raise ValueError(f"Cannot convert {field} to Iceberg Field as field_id is empty.") - def schema(self, schema: pa.Schema, struct_result: StructType) -> Schema: + def schema(self, schema: pa.Schema, struct_result: StructType) -> Schema: # ty: ignore[invalid-method-override] return Schema(*struct_result.fields) - def struct(self, struct: pa.StructType, field_results: builtins.list[NestedField]) -> StructType: + def struct(self, struct: pa.StructType, field_results: builtins.list[NestedField]) -> StructType: # ty: ignore[invalid-method-override] return StructType(*field_results) - def field(self, field: pa.Field, field_result: IcebergType) -> NestedField: + def field(self, field: pa.Field, field_result: IcebergType) -> NestedField: # ty: ignore[invalid-method-override, missing-type-argument] field_id = self._field_id(field) field_doc = doc_str.decode() if (field.metadata and (doc_str := field.metadata.get(PYARROW_FIELD_DOC_KEY))) else None field_type = field_result return NestedField(field_id, field.name, field_type, required=not field.nullable, doc=field_doc) - def list(self, list_type: pa.ListType, element_result: IcebergType) -> ListType: + def list(self, list_type: pa.ListType, element_result: IcebergType) -> ListType: # ty: ignore[invalid-method-override, missing-type-argument] element_field = list_type.value_field self._field_names.append(LIST_ELEMENT_NAME) element_id = self._field_id(element_field) self._field_names.pop() return ListType(element_id, element_result, element_required=not element_field.nullable) - def map(self, map_type: pa.MapType, key_result: IcebergType, value_result: IcebergType) -> MapType: + def map(self, map_type: pa.MapType, key_result: IcebergType, value_result: IcebergType) -> MapType: # ty: ignore[invalid-method-override, missing-type-argument] key_field = map_type.key_field self._field_names.append(MAP_KEY_NAME) key_id = self._field_id(key_field) @@ -1540,48 +1540,48 @@ def primitive(self, primitive: pa.DataType) -> PrimitiveType: raise TypeError(f"Unsupported type: {primitive}") - def before_field(self, field: pa.Field) -> None: + def before_field(self, field: pa.Field) -> None: # ty: ignore[missing-type-argument] self._field_names.append(field.name) - def after_field(self, field: pa.Field) -> None: + def after_field(self, field: pa.Field) -> None: # ty: ignore[missing-type-argument] self._field_names.pop() - def before_list_element(self, element: pa.Field) -> None: + def before_list_element(self, element: pa.Field) -> None: # ty: ignore[missing-type-argument] self._field_names.append(LIST_ELEMENT_NAME) - def after_list_element(self, element: pa.Field) -> None: + def after_list_element(self, element: pa.Field) -> None: # ty: ignore[missing-type-argument] self._field_names.pop() - def before_map_key(self, key: pa.Field) -> None: + def before_map_key(self, key: pa.Field) -> None: # ty: ignore[missing-type-argument] self._field_names.append(MAP_KEY_NAME) - def after_map_key(self, element: pa.Field) -> None: + def after_map_key(self, element: pa.Field) -> None: # ty: ignore[invalid-method-override, missing-type-argument] self._field_names.pop() - def before_map_value(self, value: pa.Field) -> None: + def before_map_value(self, value: pa.Field) -> None: # ty: ignore[missing-type-argument] self._field_names.append(MAP_VALUE_NAME) - def after_map_value(self, element: pa.Field) -> None: + def after_map_value(self, element: pa.Field) -> None: # ty: ignore[invalid-method-override, missing-type-argument] self._field_names.pop() class _ConvertToLargeTypes(PyArrowSchemaVisitor[IcebergType | pa.Schema]): - def schema(self, schema: pa.Schema, struct_result: pa.StructType) -> pa.Schema: + def schema(self, schema: pa.Schema, struct_result: pa.StructType) -> pa.Schema: # ty: ignore[invalid-method-override] return pa.schema(struct_result) - def struct(self, struct: pa.StructType, field_results: builtins.list[pa.Field]) -> pa.StructType: + def struct(self, struct: pa.StructType, field_results: builtins.list[pa.Field]) -> pa.StructType: # ty: ignore[invalid-method-override, missing-type-argument] return pa.struct(field_results) - def field(self, field: pa.Field, field_result: pa.DataType) -> pa.Field: + def field(self, field: pa.Field, field_result: pa.DataType) -> pa.Field: # ty: ignore[invalid-method-override, missing-type-argument] return field.with_type(field_result) - def list(self, list_type: pa.ListType, element_result: pa.DataType) -> pa.DataType: + def list(self, list_type: pa.ListType, element_result: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override, missing-type-argument] return pa.large_list(element_result) - def map(self, map_type: pa.MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: + def map(self, map_type: pa.MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override, missing-type-argument] return pa.map_(key_result, value_result) - def primitive(self, primitive: pa.DataType) -> pa.DataType: + def primitive(self, primitive: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override] if primitive == pa.string(): return pa.large_string() elif primitive == pa.binary(): @@ -1590,22 +1590,22 @@ def primitive(self, primitive: pa.DataType) -> pa.DataType: class _ConvertToSmallTypes(PyArrowSchemaVisitor[IcebergType | pa.Schema]): - def schema(self, schema: pa.Schema, struct_result: pa.StructType) -> pa.Schema: + def schema(self, schema: pa.Schema, struct_result: pa.StructType) -> pa.Schema: # ty: ignore[invalid-method-override] return pa.schema(struct_result) - def struct(self, struct: pa.StructType, field_results: builtins.list[pa.Field]) -> pa.StructType: + def struct(self, struct: pa.StructType, field_results: builtins.list[pa.Field]) -> pa.StructType: # ty: ignore[invalid-method-override, missing-type-argument] return pa.struct(field_results) - def field(self, field: pa.Field, field_result: pa.DataType) -> pa.Field: + def field(self, field: pa.Field, field_result: pa.DataType) -> pa.Field: # ty: ignore[invalid-method-override, missing-type-argument] return field.with_type(field_result) - def list(self, list_type: pa.ListType, element_result: pa.DataType) -> pa.DataType: + def list(self, list_type: pa.ListType, element_result: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override, missing-type-argument] return pa.list_(element_result) - def map(self, map_type: pa.MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: + def map(self, map_type: pa.MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override, missing-type-argument] return pa.map_(key_result, value_result) - def primitive(self, primitive: pa.DataType) -> pa.DataType: + def primitive(self, primitive: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override] if primitive == pa.large_string(): return pa.string() elif primitive == pa.large_binary(): @@ -1619,16 +1619,16 @@ class _ConvertViewTypes(PyArrowSchemaVisitor[pa.Schema | pa.DataType]): Everything else is kept as is: list flavour, field names, nullability, metadata and dictionary encoding. """ - def schema(self, schema: pa.Schema, struct_result: pa.StructType) -> pa.Schema: - return pa.schema(struct_result, metadata=schema.metadata) + def schema(self, schema: pa.Schema, struct_result: pa.StructType) -> pa.Schema: # ty: ignore[invalid-method-override] + return pa.schema(struct_result, metadata=schema.metadata) # ty: ignore[invalid-argument-type] - def struct(self, struct: pa.StructType, field_results: builtins.list[pa.Field]) -> pa.StructType: + def struct(self, struct: pa.StructType, field_results: builtins.list[pa.Field]) -> pa.StructType: # ty: ignore[invalid-method-override, missing-type-argument] return pa.struct(field_results) - def field(self, field: pa.Field, field_result: pa.DataType) -> pa.Field: + def field(self, field: pa.Field, field_result: pa.DataType) -> pa.Field: # ty: ignore[invalid-method-override, missing-type-argument] return field.with_type(_keep_dictionary(field.type, field_result)) - def list(self, list_type: pa.ListType | pa.LargeListType | pa.FixedSizeListType, element_result: pa.DataType) -> pa.DataType: + def list(self, list_type: pa.ListType | pa.LargeListType | pa.FixedSizeListType, element_result: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override, missing-type-argument] value_field = list_type.value_field.with_type(_keep_dictionary(list_type.value_type, element_result)) if isinstance(list_type, pa.FixedSizeListType): return pa.list_(value_field, list_type.list_size) @@ -1636,8 +1636,8 @@ def list(self, list_type: pa.ListType | pa.LargeListType | pa.FixedSizeListType, return pa.large_list(value_field) return pa.list_(value_field) - def map(self, map_type: pa.MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: - return pa.map_( + def map(self, map_type: pa.MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override, missing-type-argument] + return pa.map_( # ty: ignore[no-matching-overload] map_type.key_field.with_type(_keep_dictionary(map_type.key_type, key_result)), map_type.item_field.with_type(_keep_dictionary(map_type.item_type, value_result)), keys_sorted=map_type.keys_sorted, @@ -1654,7 +1654,7 @@ def primitive(self, primitive: pa.DataType) -> pa.DataType: def _keep_dictionary(original: pa.DataType, result: pa.DataType) -> pa.DataType: """Re-apply dictionary encoding that visit_pyarrow strips before visiting the value type.""" if isinstance(original, pa.DictionaryType): - return pa.dictionary(original.index_type, result, ordered=original.ordered) + return pa.dictionary(original.index_type, result, ordered=original.ordered) # ty: ignore[invalid-argument-type] return result @@ -1669,7 +1669,7 @@ class _ConvertToIcebergWithoutIDs(_ConvertToIceberg): creating a new Iceberg table. """ - def _field_id(self, field: pa.Field) -> int: + def _field_id(self, field: pa.Field) -> int: # ty: ignore[missing-type-argument] return -1 @@ -1704,7 +1704,7 @@ def _task_to_record_batches( projected_schema: Schema, table_schema: Schema, projected_field_ids: set[int], - positional_deletes: list[ChunkedArray] | None, + positional_deletes: list[ChunkedArray] | None, # ty: ignore[missing-type-argument] case_sensitive: bool, name_mapping: NameMapping | None = None, partition_spec: PartitionSpec | None = None, @@ -1717,7 +1717,7 @@ def _task_to_record_batches( format_kwargs["dictionary_columns"] = dictionary_columns arrow_format = _get_file_format(task.file.file_format, **format_kwargs) with io.new_input(task.file.file_path).open() as fin: - fragment = arrow_format.make_fragment(fin) + fragment = arrow_format.make_fragment(fin) # ty: ignore[invalid-argument-type] physical_schema = fragment.physical_schema # For V1 and V2, we only support Timestamp 'us' in Iceberg Schema, @@ -1799,8 +1799,8 @@ class _DeleteFileKey: content_size_in_bytes: int | None -def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[str, list[ChunkedArray]]: - deletes_per_file: dict[str, list[ChunkedArray]] = {} +def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[str, list[ChunkedArray]]: # ty: ignore[missing-type-argument] + deletes_per_file: dict[str, list[ChunkedArray]] = {} # ty: ignore[missing-type-argument] # DataFile equality uses only the path; DVs from different tasks may share a file. unique_deletes: dict[_DeleteFileKey, DataFile] = {} for task in tasks: @@ -1809,7 +1809,7 @@ def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[st unique_deletes.setdefault(key, delete_file) if len(unique_deletes) > 0: executor = ExecutorFactory.get_or_create() - deletes_per_files: Iterator[dict[str, ChunkedArray]] = executor.map( + deletes_per_files: Iterator[dict[str, ChunkedArray]] = executor.map( # ty: ignore[missing-type-argument] lambda args: _read_deletes(*args), [(io, delete_file) for delete_file in unique_deletes.values()], ) @@ -1955,7 +1955,9 @@ def batches_for_task(task: FileScanTask) -> list[pa.RecordBatch]: break def _record_batches_from_scan_tasks_and_deletes( - self, tasks: Iterable[FileScanTask], deletes_per_file: dict[str, list[ChunkedArray]] + self, + tasks: Iterable[FileScanTask], + deletes_per_file: dict[str, list[ChunkedArray]], # ty: ignore[missing-type-argument] ) -> Iterator[pa.RecordBatch]: total_row_count = 0 for task in tasks: @@ -2013,7 +2015,7 @@ def _to_requested_schema( return pa.RecordBatch.from_struct_array(struct_array) -class ArrowProjectionVisitor(SchemaWithPartnerVisitor[pa.Array, pa.Array | None]): +class ArrowProjectionVisitor(SchemaWithPartnerVisitor[pa.Array, pa.Array | None]): # ty: ignore[missing-type-argument] _file_schema: Schema _include_field_ids: bool _downcast_ns_timestamp_to_us: bool @@ -2041,7 +2043,7 @@ def __init__( self._allow_timestamp_tz_mismatch = allow_timestamp_tz_mismatch self._format_model = format_model - def _cast_if_needed(self, field: NestedField, values: pa.Array) -> pa.Array: + def _cast_if_needed(self, field: NestedField, values: pa.Array) -> pa.Array: # ty: ignore[missing-type-argument] file_field = self._file_schema.find_field(field.field_id) if field.field_type.is_primitive: @@ -2051,29 +2053,29 @@ def _cast_if_needed(self, field: NestedField, values: pa.Array) -> pa.Array: self._allow_timestamp_tz_mismatch and values.type.tz in UTC_ALIASES ) if ( - pa.types.is_timestamp(target_type) - and not target_type.tz + pa.types.is_timestamp(target_type) # ty: ignore[invalid-argument-type] + and not target_type.tz # ty: ignore[unresolved-attribute] and pa.types.is_timestamp(values.type) and source_tz_compatible ): # Downcasting of nanoseconds to microseconds - if target_type.unit == "us" and values.type.unit == "ns" and self._downcast_ns_timestamp_to_us: - return values.cast(target_type, safe=False) - elif target_type.unit == "us" and values.type.unit in {"s", "ms", "us"}: - return values.cast(target_type) + if target_type.unit == "us" and values.type.unit == "ns" and self._downcast_ns_timestamp_to_us: # ty: ignore[unresolved-attribute] + return values.cast(target_type, safe=False) # ty: ignore[invalid-argument-type] + elif target_type.unit == "us" and values.type.unit in {"s", "ms", "us"}: # ty: ignore[unresolved-attribute] + return values.cast(target_type) # ty: ignore[invalid-argument-type] raise ValueError(f"Unsupported schema projection from {values.type} to {target_type}") elif field.field_type == TimestamptzType(): if ( - pa.types.is_timestamp(target_type) - and target_type.tz == "UTC" + pa.types.is_timestamp(target_type) # ty: ignore[invalid-argument-type] + and target_type.tz == "UTC" # ty: ignore[unresolved-attribute] and pa.types.is_timestamp(values.type) and (values.type.tz in UTC_ALIASES or values.type.tz is None) ): # Downcasting of nanoseconds to microseconds - if target_type.unit == "us" and values.type.unit == "ns" and self._downcast_ns_timestamp_to_us: - return values.cast(target_type, safe=False) - elif target_type.unit == "us" and values.type.unit in {"s", "ms", "us"}: - return values.cast(target_type) + if target_type.unit == "us" and values.type.unit == "ns" and self._downcast_ns_timestamp_to_us: # ty: ignore[unresolved-attribute] + return values.cast(target_type, safe=False) # ty: ignore[invalid-argument-type] + elif target_type.unit == "us" and values.type.unit in {"s", "ms", "us"}: # ty: ignore[unresolved-attribute] + return values.cast(target_type) # ty: ignore[invalid-argument-type] raise ValueError(f"Unsupported schema projection from {values.type} to {target_type}") elif isinstance(field.field_type, (IntegerType, LongType)): # Cast smaller integer types to target type for cross-platform compatibility @@ -2081,28 +2083,28 @@ def _cast_if_needed(self, field: NestedField, values: pa.Array) -> pa.Array: # Narrowing conversions fall through to promote() handling below if pa.types.is_integer(values.type): source_width = values.type.bit_width - target_width = target_type.bit_width + target_width = target_type.bit_width # ty: ignore[unresolved-attribute] if source_width < target_width: - return values.cast(target_type) + return values.cast(target_type) # ty: ignore[invalid-argument-type] elif isinstance(field.field_type, (FloatType, DoubleType)): # Cast smaller float types to target type for cross-platform compatibility # Only allow widening conversions (smaller bit width to larger), e.g. float16 -> float32 # Narrowing conversions fall through to promote() handling below if pa.types.is_floating(values.type): source_width = values.type.bit_width - target_width = target_type.bit_width + target_width = target_type.bit_width # ty: ignore[unresolved-attribute] if source_width < target_width: - return values.cast(target_type) + return values.cast(target_type) # ty: ignore[invalid-argument-type] if field.field_type != file_field.field_type: target_schema = schema_to_pyarrow( promote(file_field.field_type, field.field_type), include_field_ids=self._include_field_ids ) - return values.cast(target_schema) + return values.cast(target_schema) # ty: ignore[invalid-argument-type] return values - def _construct_field(self, field: NestedField, arrow_type: pa.DataType) -> pa.Field: + def _construct_field(self, field: NestedField, arrow_type: pa.DataType) -> pa.Field: # ty: ignore[missing-type-argument] metadata: dict[bytes, bytes] = {} if field.doc: metadata[PYARROW_FIELD_DOC_KEY] = field.doc.encode() @@ -2116,16 +2118,19 @@ def _construct_field(self, field: NestedField, arrow_type: pa.DataType) -> pa.Fi metadata=metadata, ) - def schema(self, schema: Schema, schema_partner: pa.Array | None, struct_result: pa.Array | None) -> pa.Array | None: + def schema(self, schema: Schema, schema_partner: pa.Array | None, struct_result: pa.Array | None) -> pa.Array | None: # ty: ignore[missing-type-argument] return struct_result - def struct( - self, struct: StructType, struct_array: pa.Array | None, field_results: builtins.list[pa.Array | None] - ) -> pa.Array | None: + def struct( # ty: ignore[invalid-method-override] + self, + struct: StructType, + struct_array: pa.Array | None, # ty: ignore[missing-type-argument] + field_results: builtins.list[pa.Array | None], # ty: ignore[missing-type-argument] + ) -> pa.Array | None: # ty: ignore[missing-type-argument] if struct_array is None: return None - field_arrays: list[pa.Array] = [] - fields: list[pa.Field] = [] + field_arrays: list[pa.Array] = [] # ty: ignore[missing-type-argument] + fields: list[pa.Field] = [] # ty: ignore[missing-type-argument] for field, field_array in zip(struct.fields, field_results, strict=True): if field_array is not None: array = self._cast_if_needed(field, field_array) @@ -2136,46 +2141,50 @@ def struct( arrow_type = schema_to_pyarrow(field.field_type, include_field_ids=self._include_field_ids) if field.field_id in self._projected_missing_fields: projected_value = self._projected_missing_fields[field.field_id] - field_arrays.append(pa.repeat(pa.scalar(projected_value, type=arrow_type), len(struct_array))) + field_arrays.append(pa.repeat(pa.scalar(projected_value, type=arrow_type), len(struct_array))) # ty: ignore[no-matching-overload] elif field.initial_default is None: - field_arrays.append(pa.nulls(len(struct_array), type=arrow_type)) + field_arrays.append(pa.nulls(len(struct_array), type=arrow_type)) # ty: ignore[no-matching-overload] else: - field_arrays.append(pa.repeat(pa.scalar(field.initial_default, type=arrow_type), len(struct_array))) - fields.append(self._construct_field(field, arrow_type)) + field_arrays.append(pa.repeat(pa.scalar(field.initial_default, type=arrow_type), len(struct_array))) # ty: ignore[no-matching-overload] + fields.append(self._construct_field(field, arrow_type)) # ty: ignore[invalid-argument-type] else: raise ResolveError(f"Field is required, and could not be found in the file: {field}") return pa.StructArray.from_arrays( arrays=field_arrays, - fields=pa.struct(fields), + fields=pa.struct(fields), # ty: ignore[invalid-argument-type] mask=struct_array.is_null() if isinstance(struct_array, pa.StructArray) else None, ) - def field(self, field: NestedField, _: pa.Array | None, field_array: pa.Array | None) -> pa.Array | None: + def field(self, field: NestedField, _: pa.Array | None, field_array: pa.Array | None) -> pa.Array | None: # ty: ignore[invalid-method-override, missing-type-argument] return field_array - def list(self, list_type: ListType, list_array: pa.Array | None, value_array: pa.Array | None) -> pa.Array | None: + def list(self, list_type: ListType, list_array: pa.Array | None, value_array: pa.Array | None) -> pa.Array | None: # ty: ignore[invalid-method-override, missing-type-argument] if isinstance(list_array, (pa.ListArray, pa.LargeListArray, pa.FixedSizeListArray)) and value_array is not None: list_initializer = pa.large_list if isinstance(list_array, pa.LargeListArray) else pa.list_ if isinstance(value_array, pa.StructArray): # This can be removed once this has been fixed: # https://github.com/apache/arrow/issues/38809 - list_array = pa.LargeListArray.from_arrays(list_array.offsets, value_array) + list_array = pa.LargeListArray.from_arrays(list_array.offsets, value_array) # ty: ignore[invalid-argument-type, unresolved-attribute] value_array = self._cast_if_needed(list_type.element_field, value_array) arrow_field = list_initializer(self._construct_field(list_type.element_field, value_array.type)) return list_array.cast(arrow_field) else: return None - def map( - self, map_type: MapType, map_array: pa.Array | None, key_result: pa.Array | None, value_result: pa.Array | None - ) -> pa.Array | None: + def map( # ty: ignore[invalid-method-override] + self, + map_type: MapType, + map_array: pa.Array | None, # ty: ignore[missing-type-argument] + key_result: pa.Array | None, # ty: ignore[missing-type-argument] + value_result: pa.Array | None, # ty: ignore[missing-type-argument] + ) -> pa.Array | None: # ty: ignore[missing-type-argument] if isinstance(map_array, pa.MapArray) and key_result is not None and value_result is not None: key_result = self._cast_if_needed(map_type.key_field, key_result) value_result = self._cast_if_needed(map_type.value_field, value_result) arrow_field = pa.map_( - self._construct_field(map_type.key_field, key_result.type), - self._construct_field(map_type.value_field, value_result.type), + self._construct_field(map_type.key_field, key_result.type), # ty: ignore[invalid-argument-type] + self._construct_field(map_type.value_field, value_result.type), # ty: ignore[invalid-argument-type] ) if isinstance(value_result, pa.StructArray): # Arrow does not allow reordering of fields, therefore we have to copy the array :( @@ -2196,20 +2205,20 @@ def map( else: return None - def primitive(self, _: PrimitiveType, array: pa.Array | None) -> pa.Array | None: + def primitive(self, _: PrimitiveType, array: pa.Array | None) -> pa.Array | None: # ty: ignore[invalid-method-override, missing-type-argument] return array -class ArrowAccessor(PartnerAccessor[pa.Array]): +class ArrowAccessor(PartnerAccessor[pa.Array]): # ty: ignore[missing-type-argument] file_schema: Schema def __init__(self, file_schema: Schema): self.file_schema = file_schema - def schema_partner(self, partner: pa.Array | None) -> pa.Array | None: + def schema_partner(self, partner: pa.Array | None) -> pa.Array | None: # ty: ignore[missing-type-argument] return partner - def field_partner(self, partner_struct: pa.Array | None, field_id: int, _: str) -> pa.Array | None: + def field_partner(self, partner_struct: pa.Array | None, field_id: int, _: str) -> pa.Array | None: # ty: ignore[invalid-method-override, missing-type-argument] if partner_struct is not None: # use the field name from the file schema try: @@ -2228,13 +2237,13 @@ def field_partner(self, partner_struct: pa.Array | None, field_id: int, _: str) return None - def list_element_partner(self, partner_list: pa.Array | None) -> pa.Array | None: + def list_element_partner(self, partner_list: pa.Array | None) -> pa.Array | None: # ty: ignore[missing-type-argument] return partner_list.values if isinstance(partner_list, (pa.ListArray, pa.LargeListArray, pa.FixedSizeListArray)) else None - def map_key_partner(self, partner_map: pa.Array | None) -> pa.Array | None: + def map_key_partner(self, partner_map: pa.Array | None) -> pa.Array | None: # ty: ignore[missing-type-argument] return partner_map.keys if isinstance(partner_map, pa.MapArray) else None - def map_value_partner(self, partner_map: pa.Array | None) -> pa.Array | None: + def map_value_partner(self, partner_map: pa.Array | None) -> pa.Array | None: # ty: ignore[missing-type-argument] return partner_map.items if isinstance(partner_map, pa.MapArray) else None @@ -2661,8 +2670,8 @@ def data_file_statistics_from_parquet_metadata( data_offset = row_group.column(0).data_page_offset dictionary_offset = row_group.column(0).dictionary_page_offset - if row_group.column(0).has_dictionary_page and dictionary_offset < data_offset: - split_offsets.append(dictionary_offset) + if row_group.column(0).has_dictionary_page and dictionary_offset < data_offset: # ty: ignore[unsupported-operator] + split_offsets.append(dictionary_offset) # ty: ignore[invalid-argument-type] else: split_offsets.append(data_offset) @@ -2684,8 +2693,8 @@ def data_file_statistics_from_parquet_metadata( try: statistics = column.statistics - if statistics.has_null_count: - null_value_counts[field_id] = null_value_counts.get(field_id, 0) + statistics.null_count + if statistics.has_null_count: # ty: ignore[unresolved-attribute] + null_value_counts[field_id] = null_value_counts.get(field_id, 0) + statistics.null_count # ty: ignore[unresolved-attribute, unsupported-operator] if stats_col.mode == MetricsMode(MetricModeTypes.COUNTS): continue @@ -2693,26 +2702,28 @@ def data_file_statistics_from_parquet_metadata( if field_id not in col_aggs: try: col_aggs[field_id] = StatsAggregator( - stats_col.iceberg_type, statistics.physical_type, stats_col.mode.length + stats_col.iceberg_type, + statistics.physical_type, # ty: ignore[unresolved-attribute] + stats_col.mode.length, ) except ValueError as e: raise ValueError(f"{e} for column '{stats_col.column_name}'") from e - if isinstance(stats_col.iceberg_type, DecimalType) and statistics.physical_type != "FIXED_LEN_BYTE_ARRAY": + if isinstance(stats_col.iceberg_type, DecimalType) and statistics.physical_type != "FIXED_LEN_BYTE_ARRAY": # ty: ignore[unresolved-attribute] scale = stats_col.iceberg_type.scale ( - col_aggs[field_id].update_min(unscaled_to_decimal(statistics.min_raw, scale)) - if statistics.min_raw is not None + col_aggs[field_id].update_min(unscaled_to_decimal(statistics.min_raw, scale)) # ty: ignore[unresolved-attribute] + if statistics.min_raw is not None # ty: ignore[unresolved-attribute] else None ) ( - col_aggs[field_id].update_max(unscaled_to_decimal(statistics.max_raw, scale)) - if statistics.max_raw is not None + col_aggs[field_id].update_max(unscaled_to_decimal(statistics.max_raw, scale)) # ty: ignore[unresolved-attribute] + if statistics.max_raw is not None # ty: ignore[unresolved-attribute] else None ) else: - col_aggs[field_id].update_min(statistics.min) - col_aggs[field_id].update_max(statistics.max) + col_aggs[field_id].update_min(statistics.min) # ty: ignore[unresolved-attribute] + col_aggs[field_id].update_max(statistics.max) # ty: ignore[unresolved-attribute] except pyarrow.lib.ArrowNotImplementedError as e: invalidate_col.add(field_id) @@ -2956,22 +2967,22 @@ def _check_pyarrow_schema_compatible( name_mapping = requested_schema.name_mapping try: - provided_schema = pyarrow_to_schema( + provided_schema = pyarrow_to_schema( # ty: ignore[invalid-assignment] provided_schema, name_mapping=name_mapping, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version, ) except ValueError as e: - provided_schema = _pyarrow_to_schema_without_ids( + provided_schema = _pyarrow_to_schema_without_ids( # ty: ignore[invalid-assignment] provided_schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version ) - additional_names = set(provided_schema._name_to_id.keys()) - set(requested_schema._name_to_id.keys()) + additional_names = set(provided_schema._name_to_id.keys()) - set(requested_schema._name_to_id.keys()) # ty: ignore[unresolved-attribute] raise ValueError( f"PyArrow table contains more columns: {', '.join(sorted(additional_names))}. " "Update the schema first (hint, use union_by_name)." ) from e - _check_schema_compatible(requested_schema, provided_schema) + _check_schema_compatible(requested_schema, provided_schema) # ty: ignore[invalid-argument-type] def parquet_files_to_data_files(io: FileIO, table_metadata: TableMetadata, file_paths: Iterator[str]) -> Iterator[DataFile]: @@ -2983,7 +2994,7 @@ def parquet_files_to_data_files(io: FileIO, table_metadata: TableMetadata, file_ def parquet_file_to_data_file(io: FileIO, table_metadata: TableMetadata, file_path: str) -> DataFile: input_file = io.new_input(file_path) with input_file.open() as input_stream: - parquet_metadata = pq.read_metadata(input_stream) + parquet_metadata = pq.read_metadata(input_stream) # ty: ignore[invalid-argument-type] arrow_schema = parquet_metadata.schema.to_arrow_schema() @@ -3217,11 +3228,11 @@ def _partition_value_filter(partition_field_name: str, value: Any) -> pc.Express if value is None: return pc.field(partition_field_name).is_null() elif isinstance(value, float) and math.isnan(value): - return pc.is_nan(pc.field(partition_field_name)) + return pc.is_nan(pc.field(partition_field_name)) # ty: ignore[invalid-return-type] return pc.field(partition_field_name) == value -def _get_field_from_arrow_table(arrow_table: pa.Table, field_path: str) -> pa.Array: +def _get_field_from_arrow_table(arrow_table: pa.Table, field_path: str) -> pa.Array: # ty: ignore[missing-type-argument] """Get a field from an Arrow table, supporting both literal field names and nested field paths. This function handles two cases: @@ -3240,7 +3251,7 @@ def _get_field_from_arrow_table(arrow_table: pa.Table, field_path: str) -> pa.Ar """ # Try exact column name match (handles field names containing literal dots) if field_path in arrow_table.column_names: - return arrow_table[field_path] + return arrow_table[field_path] # ty: ignore[invalid-return-type] # If not found as exact name, treat as nested field path path_parts = field_path.split(".") diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index 0881273c45..f44ef7dd05 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -732,13 +732,13 @@ def update(self, value: Any) -> None: self._min = value self._max = value else: - self._max = max(self._max, value) + self._max = max(self._max, value) # ty: ignore[invalid-argument-type] self._min = min(self._min, value) def construct_partition_summaries(spec: PartitionSpec, schema: Schema, partitions: list[Record]) -> list[PartitionFieldSummary]: types = [field.field_type for field in spec.partition_type(schema).fields] - field_stats = [PartitionFieldStats(field_type) for field_type in types] + field_stats = [PartitionFieldStats(field_type) for field_type in types] # ty: ignore[invalid-argument-type] for partition_keys in partitions: for i, field_type in enumerate(types): if not isinstance(field_type, PrimitiveType): @@ -811,7 +811,7 @@ def construct_partition_summaries(spec: PartitionSpec, schema: Schema, partition class ManifestFile(Record): @classmethod def from_args(cls, _table_format_version: TableVersion = DEFAULT_READ_VERSION, **arguments: Any) -> ManifestFile: - return super()._bind(**arguments, struct=MANIFEST_LIST_FILE_SCHEMAS[_table_format_version]) + return super()._bind(**arguments, struct=MANIFEST_LIST_FILE_SCHEMAS[_table_format_version]) # ty: ignore[invalid-argument-type] @property def manifest_path(self) -> str: diff --git a/pyiceberg/partitioning.py b/pyiceberg/partitioning.py index 74932b8ddb..373bc74427 100644 --- a/pyiceberg/partitioning.py +++ b/pyiceberg/partitioning.py @@ -81,7 +81,7 @@ class PartitionField(IcebergBaseModel): source_id: int = Field(alias="source-id") field_id: int = Field(alias="field-id") transform: Annotated[ # type: ignore[type-arg] - Transform, + Transform, # ty: ignore[missing-type-argument] BeforeValidator(parse_transform), PlainSerializer(lambda c: str(c), return_type=str), # pylint: disable=W0108 WithJsonSchema({"type": "string"}, mode="serialization"), diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index fc6fd3823a..c9dd167f5c 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -1211,7 +1211,7 @@ def index_by_name(schema_or_type: Schema | IcebergType) -> dict[str, int]: Returns: Dict[str, int]: An index of field names to field IDs. """ - if len(schema_or_type.fields) > 0: + if len(schema_or_type.fields) > 0: # ty: ignore[unresolved-attribute] indexer = _IndexByName() visit(schema_or_type, indexer) return indexer.by_name() @@ -1330,7 +1330,7 @@ def _get_and_increment(self, current_id: int) -> int: self.old_id_to_new_id[current_id] = new_id return new_id - def schema(self, schema: Schema, struct_result: Callable[[], StructType]) -> Schema: + def schema(self, schema: Schema, struct_result: Callable[[], StructType]) -> Schema: # ty: ignore[invalid-method-override] return Schema( *struct_result().fields, identifier_field_ids=[self.old_id_to_new_id[field_id] for field_id in schema.identifier_field_ids], @@ -1468,7 +1468,7 @@ def field(self, field: NestedField, field_result: IcebergType | None) -> Iceberg ) def struct(self, struct: StructType, field_results: builtins.list[IcebergType | None]) -> IcebergType | None: - return StructType(*[field for field in field_results if field is not None]) + return StructType(*[field for field in field_results if field is not None]) # ty: ignore[invalid-argument-type] def list(self, list_type: ListType, element_result: IcebergType | None) -> IcebergType | None: return ListType(element_id=list_type.element_id, element_type=element_result, element_required=list_type.element_required) @@ -1615,7 +1615,7 @@ def _project_selected_struct(projected_field: IcebergType | None) -> StructType: if projected_field is None: return StructType() else: - return projected_field + return projected_field # ty: ignore[invalid-return-type] @staticmethod def _project_list(list_type: ListType, element_result: IcebergType) -> ListType: diff --git a/pyiceberg/serializers.py b/pyiceberg/serializers.py index 726e6b5f62..d9be90a84b 100644 --- a/pyiceberg/serializers.py +++ b/pyiceberg/serializers.py @@ -67,7 +67,7 @@ def bytes_compressor(self) -> Callable[[bytes], bytes]: class GzipCompressor(Compressor): def stream_decompressor(self, inp: InputStream) -> InputStream: - return gzip.open(inp) + return gzip.open(inp) # ty: ignore[invalid-return-type] def bytes_compressor(self) -> Callable[[bytes], bytes]: return gzip.compress diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 8d79472af5..fd3e16f284 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -1052,7 +1052,7 @@ def add_files( if self.table_metadata.name_mapping() is None: self.set_properties( - **{TableProperties.DEFAULT_NAME_MAPPING: self.table_metadata.schema().name_mapping.model_dump_json()} + **{TableProperties.DEFAULT_NAME_MAPPING: self.table_metadata.schema().name_mapping.model_dump_json()} # ty: ignore[invalid-argument-type] ) with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files: data_files = _parquet_files_to_data_files( @@ -1559,7 +1559,7 @@ def location_provider(self) -> LocationProvider: @property def last_sequence_number(self) -> int: - return self.metadata.last_sequence_number + return self.metadata.last_sequence_number # ty: ignore[unresolved-attribute] def current_snapshot(self) -> Snapshot | None: """Get the current snapshot for this table, or None if there is no current snapshot.""" @@ -2423,7 +2423,7 @@ def _to_arrow_batch_reader_via_file_scan_tasks( batches = chain([first_batch], batches) target_schema = pa.schema( [first_batch.schema.field(field.name) if field.name in dict_col_set else field for field in target_schema], - metadata=target_schema.metadata, + metadata=target_schema.metadata, # ty: ignore[invalid-argument-type] ) return pa.RecordBatchReader.from_batches(target_schema, batches).cast(target_schema) diff --git a/pyiceberg/table/deletion_vector.py b/pyiceberg/table/deletion_vector.py index 9c56af24a2..0e9bb40b7d 100644 --- a/pyiceberg/table/deletion_vector.py +++ b/pyiceberg/table/deletion_vector.py @@ -76,15 +76,15 @@ def _deserialize_bitmap(pl: bytes) -> list[BitMap]: last_key = key - return bitmaps + return bitmaps # ty: ignore[invalid-return-type] @staticmethod - def _bitmaps_to_chunked_array(bitmaps: list[BitMap]) -> "pa.ChunkedArray": + def _bitmaps_to_chunked_array(bitmaps: list[BitMap]) -> "pa.ChunkedArray": # ty: ignore[missing-type-argument] import pyarrow as pa return pa.chunked_array([(key_pos << 32) + pos for pos in bitmap] for key_pos, bitmap in enumerate(bitmaps)) - def to_vector(self) -> "pa.ChunkedArray": + def to_vector(self) -> "pa.ChunkedArray": # ty: ignore[missing-type-argument] return self._bitmaps_to_chunked_array(self._bitmaps) diff --git a/pyiceberg/table/inspect.py b/pyiceberg/table/inspect.py index e580b66dc1..628eca745c 100644 --- a/pyiceberg/table/inspect.py +++ b/pyiceberg/table/inspect.py @@ -183,14 +183,14 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: pa.field("value_count", pa.int64(), nullable=True), pa.field("null_value_count", pa.int64(), nullable=True), pa.field("nan_value_count", pa.int64(), nullable=True), - pa.field("lower_bound", pa_bound_type, nullable=True), - pa.field("upper_bound", pa_bound_type, nullable=True), + pa.field("lower_bound", pa_bound_type, nullable=True), # ty: ignore[invalid-argument-type] + pa.field("upper_bound", pa_bound_type, nullable=True), # ty: ignore[invalid-argument-type] ] ) for field in self.tbl.metadata.schema().fields: readable_metrics_struct.append( - pa.field(schema.find_column_name(field.field_id), _readable_metrics_struct(field.field_type), nullable=False) + pa.field(schema.find_column_name(field.field_id), _readable_metrics_struct(field.field_type), nullable=False) # ty: ignore[invalid-argument-type] ) partition_record = self.tbl.metadata.specs_struct() @@ -209,7 +209,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: pa.field("content", pa.int8(), nullable=False), pa.field("file_path", pa.string(), nullable=False), pa.field("file_format", pa.string(), nullable=False), - pa.field("partition", pa_record_struct, nullable=False), + pa.field("partition", pa_record_struct, nullable=False), # ty: ignore[invalid-argument-type] pa.field("record_count", pa.int64(), nullable=False), pa.field("file_size_in_bytes", pa.int64(), nullable=False), pa.field("column_sizes", pa.map_(pa.int32(), pa.int64()), nullable=True), @@ -247,8 +247,8 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: "null_value_count": null_value_counts.get(field.field_id), "nan_value_count": nan_value_counts.get(field.field_id), # Makes them readable - "lower_bound": _readable_bound(field.field_type, lower_bounds.get(field.field_id)), - "upper_bound": _readable_bound(field.field_type, upper_bounds.get(field.field_id)), + "lower_bound": _readable_bound(field.field_type, lower_bounds.get(field.field_id)), # ty: ignore[invalid-argument-type] + "upper_bound": _readable_bound(field.field_type, upper_bounds.get(field.field_id)), # ty: ignore[invalid-argument-type] } for field in self.tbl.metadata.schema().fields } @@ -391,7 +391,7 @@ def partitions( pa_record_struct = schema_to_pyarrow(partition_record) partitions_schema = pa.schema( [ - pa.field("partition", pa_record_struct, nullable=False), + pa.field("partition", pa_record_struct, nullable=False), # ty: ignore[invalid-argument-type] pa.field("spec_id", pa.int32(), nullable=False), ] ) @@ -420,7 +420,7 @@ def partitions( ) return pa.Table.from_pylist( - partitions_map.values(), + partitions_map.values(), # ty: ignore[invalid-argument-type] schema=table_schema, ) @@ -754,8 +754,8 @@ def _get_files_from_manifest( "value_count": value_counts.get(field.field_id), "null_value_count": null_value_counts.get(field.field_id), "nan_value_count": nan_value_counts.get(field.field_id), - "lower_bound": _readable_bound(field.field_type, lower_bounds.get(field.field_id)), - "upper_bound": _readable_bound(field.field_type, upper_bounds.get(field.field_id)), + "lower_bound": _readable_bound(field.field_type, lower_bounds.get(field.field_id)), # ty: ignore[invalid-argument-type] + "upper_bound": _readable_bound(field.field_type, upper_bounds.get(field.field_id)), # ty: ignore[invalid-argument-type] } for field in self.tbl.metadata.schema().fields } @@ -820,8 +820,8 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: pa.field("value_count", pa.int64(), nullable=True), pa.field("null_value_count", pa.int64(), nullable=True), pa.field("nan_value_count", pa.int64(), nullable=True), - pa.field("lower_bound", pa_bound_type, nullable=True), - pa.field("upper_bound", pa_bound_type, nullable=True), + pa.field("lower_bound", pa_bound_type, nullable=True), # ty: ignore[invalid-argument-type] + pa.field("upper_bound", pa_bound_type, nullable=True), # ty: ignore[invalid-argument-type] ] ) @@ -830,7 +830,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: for field in self.tbl.metadata.schema().fields: readable_metrics_struct.append( - pa.field(schema.find_column_name(field.field_id), _readable_metrics_struct(field.field_type), nullable=False) + pa.field(schema.find_column_name(field.field_id), _readable_metrics_struct(field.field_type), nullable=False) # ty: ignore[invalid-argument-type] ) files_schema = pa.schema( @@ -839,7 +839,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: pa.field("file_path", pa.string(), nullable=False), pa.field("file_format", pa.dictionary(pa.int32(), pa.string()), nullable=False), pa.field("spec_id", pa.int32(), nullable=False), - pa.field("partition", pa_record_struct, nullable=False), + pa.field("partition", pa_record_struct, nullable=False), # ty: ignore[invalid-argument-type] pa.field("record_count", pa.int64(), nullable=False), pa.field("file_size_in_bytes", pa.int64(), nullable=False), pa.field("column_sizes", pa.map_(pa.int32(), pa.int64()), nullable=True), diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index 8236f12229..70f57ed82f 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -278,7 +278,7 @@ def schema_by_id(self, schema_id: int) -> Schema | None: """Get the schema by schema_id.""" return next((schema for schema in self.schemas if schema.schema_id == schema_id), None) - def schema(self) -> Schema: + def schema(self) -> Schema: # ty: ignore[invalid-method-override] """Return the schema for this table.""" return next(schema for schema in self.schemas if schema.schema_id == self.current_schema_id) @@ -360,7 +360,7 @@ def current_snapshot(self) -> Snapshot | None: return None def next_sequence_number(self) -> int: - return self.last_sequence_number + 1 if self.format_version > 1 else INITIAL_SEQUENCE_NUMBER + return self.last_sequence_number + 1 if self.format_version > 1 else INITIAL_SEQUENCE_NUMBER # ty: ignore[unresolved-attribute] def sort_order(self) -> SortOrder: """Get the current sort order for this table, or UNSORTED_SORT_ORDER if there is no sort order.""" @@ -379,7 +379,7 @@ def serialize_current_snapshot_id(self, current_snapshot_id: int | None) -> int @field_serializer("snapshots") def serialize_snapshots(self, snapshots: list[Snapshot]) -> list[Snapshot]: # Snapshot field `sequence-number` should not be written for v1 metadata - if self.format_version == 1: + if self.format_version == 1: # ty: ignore[unresolved-attribute] return [snapshot.model_copy(update={"sequence_number": None}) for snapshot in snapshots] return snapshots @@ -420,7 +420,7 @@ def cleanup_snapshot_id(cls, data: dict[str, Any]) -> dict[str, Any]: @model_validator(mode="after") def construct_refs(self) -> TableMetadataV1: - return construct_refs(self) + return construct_refs(self) # ty: ignore[invalid-return-type] @model_validator(mode="before") def set_v2_compatible_defaults(cls, data: dict[str, Any]) -> dict[str, Any]: @@ -621,10 +621,10 @@ def construct_refs(self) -> TableMetadata: encryption_keys: list[EncryptedKey] = Field(alias="encryption-keys", default_factory=list) """An optional list of encryption keys used for table encryption.""" - @model_serializer(mode="wrap") + @model_serializer(mode="wrap") # ty: ignore[invalid-argument-type] def serialize_model(self, handler: ModelWrapSerializerWithoutInfo) -> dict[str, Any]: """Set custom serializer to leave out `encryption-keys` when it is empty.""" - serialized: dict[str, Any] = handler(self) + serialized: dict[str, Any] = handler(self) # ty: ignore[missing-argument] if not self.encryption_keys: serialized.pop("encryption-keys", None) return serialized diff --git a/pyiceberg/table/name_mapping.py b/pyiceberg/table/name_mapping.py index 7367ba875e..7af874f128 100644 --- a/pyiceberg/table/name_mapping.py +++ b/pyiceberg/table/name_mapping.py @@ -39,7 +39,7 @@ class MappedField(IcebergBaseModel): field_id: int | None = Field(alias="field-id", default=None) - names: list[str] = conlist(str) + names: list[str] = conlist(str) # ty: ignore[invalid-assignment] fields: list[MappedField] = Field(default_factory=list) @field_validator("fields", mode="before") @@ -81,7 +81,7 @@ def __len__(self) -> int: """Return the number of mappings.""" return len(self.root) - def __iter__(self) -> Iterator[MappedField]: + def __iter__(self) -> Iterator[MappedField]: # ty: ignore[invalid-method-override] """Iterate over the mapped fields.""" return iter(self.root) @@ -256,7 +256,7 @@ class NameMappingAccessor(PartnerAccessor[MappedField]): def schema_partner(self, partner: MappedField | None) -> MappedField | None: return partner - def field_partner( + def field_partner( # ty: ignore[invalid-method-override] self, partner_struct: list[MappedField] | MappedField | None, _: int, field_name: str ) -> MappedField | None: if partner_struct is not None: @@ -322,10 +322,10 @@ def before_map_value(self, value: NestedField, value_partner: P | None) -> None: def after_map_value(self, value: NestedField, value_partner: P | None) -> None: self.current_path.pop() - def schema(self, schema: Schema, schema_partner: MappedField | None, struct_result: StructType) -> IcebergType: - return Schema(*struct_result.fields, schema_id=schema.schema_id) + def schema(self, schema: Schema, schema_partner: MappedField | None, struct_result: StructType) -> IcebergType: # ty: ignore[invalid-method-override] + return Schema(*struct_result.fields, schema_id=schema.schema_id) # ty: ignore[invalid-return-type] - def struct( + def struct( # ty: ignore[invalid-method-override] self, struct: StructType, struct_partner: MappedField | None, field_results: builtins.list[NestedField] ) -> IcebergType: return StructType(*field_results) diff --git a/pyiceberg/table/puffin.py b/pyiceberg/table/puffin.py index 13315bf802..2ee4987f42 100644 --- a/pyiceberg/table/puffin.py +++ b/pyiceberg/table/puffin.py @@ -78,7 +78,7 @@ def get_blob_payload(self, blob: PuffinBlobMetadata) -> bytes: raise ValueError(f"Unsupported puffin compression codec: {blob.compression_codec!r}") @deprecated(deprecated_in="0.12.0", removed_in="0.13.0", help_message="Use deletion_vectors_from_puffin_file(...) instead") - def to_vector(self) -> dict[str, "pa.ChunkedArray"]: + def to_vector(self) -> dict[str, "pa.ChunkedArray"]: # ty: ignore[missing-type-argument] from pyiceberg.table.deletion_vector import deletion_vectors_from_puffin_file # local import avoids the cycle return {dv.referenced_data_file: dv.to_vector() for dv in deletion_vectors_from_puffin_file(self)} diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index 66d3d7f239..3ddc601b07 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -185,7 +185,7 @@ def to_dict(self) -> dict[str, str]: return properties -class Summary(IcebergBaseModel, Mapping[str, str]): +class Summary(IcebergBaseModel, Mapping[str, str]): # ty: ignore[invalid-method-override] """A class that stores the summary information for a Snapshot. The snapshot summary’s operation field is used by some operations, @@ -212,7 +212,7 @@ def __getitem__(self, __key: str) -> Any | None: # type: ignore def __setitem__(self, key: str, value: Any) -> None: """Set a key as it is a map.""" if key.lower() == "operation": - self.operation = value + self.operation = value # ty: ignore[invalid-assignment] else: self._additional_properties[key] = value diff --git a/pyiceberg/table/sorting.py b/pyiceberg/table/sorting.py index adf2cf4236..63b67be3b9 100644 --- a/pyiceberg/table/sorting.py +++ b/pyiceberg/table/sorting.py @@ -115,7 +115,7 @@ def map_source_ids_onto_source_id(cls, data: Any) -> Any: source_id: int = Field(alias="source-id") transform: Annotated[ # type: ignore[type-arg] - Transform, + Transform, # ty: ignore[missing-type-argument] BeforeValidator(parse_transform), PlainSerializer(lambda c: str(c), return_type=str), # pylint: disable=W0108 WithJsonSchema({"type": "string"}, mode="serialization"), diff --git a/pyiceberg/table/update/__init__.py b/pyiceberg/table/update/__init__.py index 38047ef06c..cf993f9038 100644 --- a/pyiceberg/table/update/__init__.py +++ b/pyiceberg/table/update/__init__.py @@ -455,24 +455,24 @@ def _(update: AddSnapshotUpdate, base_metadata: TableMetadata, context: _TableMe elif ( base_metadata.format_version >= 2 and update.snapshot.sequence_number is not None - and update.snapshot.sequence_number <= base_metadata.last_sequence_number + and update.snapshot.sequence_number <= base_metadata.last_sequence_number # ty: ignore[unresolved-attribute] and update.snapshot.parent_snapshot_id is not None ): raise ValueError( f"Cannot add snapshot with sequence number {update.snapshot.sequence_number} " - f"older than last sequence number {base_metadata.last_sequence_number}" + f"older than last sequence number {base_metadata.last_sequence_number}" # ty: ignore[unresolved-attribute] ) elif base_metadata.format_version >= 3 and update.snapshot.first_row_id is None: raise ValueError("Cannot add snapshot without first row id") elif ( base_metadata.format_version >= 3 and update.snapshot.first_row_id is not None - and base_metadata.next_row_id is not None - and update.snapshot.first_row_id < base_metadata.next_row_id + and base_metadata.next_row_id is not None # ty: ignore[unresolved-attribute] + and update.snapshot.first_row_id < base_metadata.next_row_id # ty: ignore[unresolved-attribute] ): raise ValueError( f"Cannot add a snapshot with first row id smaller than the table's next-row-id " - f"{update.snapshot.first_row_id} < {base_metadata.next_row_id}" + f"{update.snapshot.first_row_id} < {base_metadata.next_row_id}" # ty: ignore[unresolved-attribute] ) context.add_update(update) @@ -481,9 +481,9 @@ def _(update: AddSnapshotUpdate, base_metadata: TableMetadata, context: _TableMe "last_updated_ms": update.snapshot.timestamp_ms, "last_sequence_number": update.snapshot.sequence_number, "snapshots": base_metadata.snapshots + [update.snapshot], - "next_row_id": base_metadata.next_row_id + update.snapshot.added_rows + "next_row_id": base_metadata.next_row_id + update.snapshot.added_rows # ty: ignore[unresolved-attribute] if base_metadata.format_version >= 3 - and base_metadata.next_row_id is not None + and base_metadata.next_row_id is not None # ty: ignore[unresolved-attribute] and update.snapshot.added_rows is not None else None, } @@ -626,7 +626,7 @@ def _( @_apply_table_update.register(SetStatisticsUpdate) def _(update: SetStatisticsUpdate, base_metadata: TableMetadata, context: _TableMetadataUpdateContext) -> TableMetadata: - statistics = filter_statistics_by_snapshot_id(base_metadata.statistics, update.statistics.snapshot_id) + statistics = filter_statistics_by_snapshot_id(base_metadata.statistics, update.statistics.snapshot_id) # ty: ignore[invalid-argument-type] context.add_update(update) return base_metadata.model_copy(update={"statistics": statistics + [update.statistics]}) @@ -637,7 +637,7 @@ def _(update: RemoveStatisticsUpdate, base_metadata: TableMetadata, context: _Ta if not any(stat.snapshot_id == update.snapshot_id for stat in base_metadata.statistics): raise ValueError(f"Statistics with snapshot id {update.snapshot_id} does not exist") - statistics = filter_statistics_by_snapshot_id(base_metadata.statistics, update.snapshot_id) + statistics = filter_statistics_by_snapshot_id(base_metadata.statistics, update.snapshot_id) # ty: ignore[invalid-argument-type] context.add_update(update) return base_metadata.model_copy(update={"statistics": statistics}) @@ -678,7 +678,8 @@ def _(update: RemoveSchemasUpdate, base_metadata: TableMetadata, context: _Table @_apply_table_update.register(SetPartitionStatisticsUpdate) def _(update: SetPartitionStatisticsUpdate, base_metadata: TableMetadata, context: _TableMetadataUpdateContext) -> TableMetadata: partition_statistics = filter_statistics_by_snapshot_id( - base_metadata.partition_statistics, update.partition_statistics.snapshot_id + base_metadata.partition_statistics, # ty: ignore[invalid-argument-type] + update.partition_statistics.snapshot_id, ) context.add_update(update) @@ -692,7 +693,7 @@ def _( if not any(part_stat.snapshot_id == update.snapshot_id for part_stat in base_metadata.partition_statistics): raise ValueError(f"Partition Statistics with snapshot id {update.snapshot_id} does not exist") - statistics = filter_statistics_by_snapshot_id(base_metadata.partition_statistics, update.snapshot_id) + statistics = filter_statistics_by_snapshot_id(base_metadata.partition_statistics, update.snapshot_id) # ty: ignore[invalid-argument-type] context.add_update(update) return base_metadata.model_copy(update={"partition_statistics": statistics}) @@ -799,7 +800,7 @@ class ValidatableTableRequirement(IcebergBaseModel): type: str @abstractmethod - def validate(self, base_metadata: TableMetadata | None) -> None: + def validate(self, base_metadata: TableMetadata | None) -> None: # ty: ignore[invalid-method-override] """Validate the requirement against the base metadata. Args: @@ -844,9 +845,9 @@ class AssertRefSnapshotId(ValidatableTableRequirement): ref: str = Field(...) snapshot_id: int | None = Field(default=None, alias="snapshot-id") - @model_serializer(mode="wrap") + @model_serializer(mode="wrap") # ty: ignore[invalid-argument-type] def serialize_model(self, handler: ModelWrapSerializerWithoutInfo) -> dict[str, Any]: - partial_result = handler(self) + partial_result = handler(self) # ty: ignore[missing-argument] # Ensure "snapshot-id" is always present, even if value is None return {**partial_result, "snapshot-id": self.snapshot_id} diff --git a/pyiceberg/table/update/schema.py b/pyiceberg/table/update/schema.py index af6ccc2439..6c33d4bb81 100644 --- a/pyiceberg/table/update/schema.py +++ b/pyiceberg/table/update/schema.py @@ -244,7 +244,7 @@ def add_column( field = NestedField( field_id=new_id, name=name, - field_type=new_type, + field_type=new_type, # ty: ignore[invalid-argument-type] required=required, doc=doc, initial_default=initial_default, @@ -857,10 +857,10 @@ def __init__(self, update_schema: UpdateSchema, existing_schema: Schema, case_se self.existing_schema = existing_schema self.case_sensitive = case_sensitive - def schema(self, schema: Schema, partner_id: int | None, struct_result: bool) -> bool: + def schema(self, schema: Schema, partner_id: int | None, struct_result: bool) -> bool: # ty: ignore[invalid-method-override] return struct_result - def struct(self, struct: StructType, partner_id: int | None, missing_positions: builtins.list[bool]) -> bool: + def struct(self, struct: StructType, partner_id: int | None, missing_positions: builtins.list[bool]) -> bool: # ty: ignore[invalid-method-override] if partner_id is None: return True @@ -875,7 +875,7 @@ def struct(self, struct: StructType, partner_id: int | None, missing_positions: self._add_column(partner_id, fields[pos]) else: field = fields[pos] - if nested_field := partner_struct.field_by_name(field.name, case_sensitive=self.case_sensitive): + if nested_field := partner_struct.field_by_name(field.name, case_sensitive=self.case_sensitive): # ty: ignore[unresolved-attribute] self._update_column(field, nested_field) return False @@ -915,10 +915,10 @@ def _find_field_type(self, field_id: int) -> IcebergType: else: return self.existing_schema.find_field(field_id).field_type - def field(self, field: NestedField, partner_id: int | None, field_result: bool) -> bool: + def field(self, field: NestedField, partner_id: int | None, field_result: bool) -> bool: # ty: ignore[invalid-method-override] return partner_id is None - def list(self, list_type: ListType, list_partner_id: int | None, element_missing: bool) -> bool: + def list(self, list_type: ListType, list_partner_id: int | None, element_missing: bool) -> bool: # ty: ignore[invalid-method-override] if list_partner_id is None: return True @@ -933,7 +933,7 @@ def list(self, list_type: ListType, list_partner_id: int | None, element_missing return False - def map(self, map_type: MapType, map_partner_id: int | None, key_missing: bool, value_missing: bool) -> bool: + def map(self, map_type: MapType, map_partner_id: int | None, key_missing: bool, value_missing: bool) -> bool: # ty: ignore[invalid-method-override] if map_partner_id is None: return True @@ -952,7 +952,7 @@ def map(self, map_type: MapType, map_partner_id: int | None, key_missing: bool, return False - def primitive(self, primitive: PrimitiveType, primitive_partner_id: int | None) -> bool: + def primitive(self, primitive: PrimitiveType, primitive_partner_id: int | None) -> bool: # ty: ignore[invalid-method-override] return primitive_partner_id is None @@ -967,7 +967,7 @@ def __init__(self, partner_schema: Schema, case_sensitive: bool) -> None: def schema_partner(self, partner: int | None) -> int | None: return -1 - def field_partner(self, partner_field_id: int | None, field_id: int, field_name: str) -> int | None: + def field_partner(self, partner_field_id: int | None, field_id: int, field_name: str) -> int | None: # ty: ignore[invalid-method-override] if partner_field_id is not None: if partner_field_id == -1: struct = self.partner_schema.as_struct() @@ -976,12 +976,12 @@ def field_partner(self, partner_field_id: int | None, field_id: int, field_name: if not struct.is_struct: raise ValueError(f"Expected StructType: {struct}") - if field := struct.field_by_name(name=field_name, case_sensitive=self.case_sensitive): + if field := struct.field_by_name(name=field_name, case_sensitive=self.case_sensitive): # ty: ignore[unresolved-attribute] return field.field_id return None - def list_element_partner(self, partner_list_id: int | None) -> int | None: + def list_element_partner(self, partner_list_id: int | None) -> int | None: # ty: ignore[invalid-method-override] if partner_list_id is not None and (field := self.partner_schema.find_field(partner_list_id)): if not isinstance(field.field_type, ListType): raise ValueError(f"Expected ListType: {field}") @@ -989,7 +989,7 @@ def list_element_partner(self, partner_list_id: int | None) -> int | None: else: return None - def map_key_partner(self, partner_map_id: int | None) -> int | None: + def map_key_partner(self, partner_map_id: int | None) -> int | None: # ty: ignore[invalid-method-override] if partner_map_id is not None and (field := self.partner_schema.find_field(partner_map_id)): if not isinstance(field.field_type, MapType): raise ValueError(f"Expected MapType: {field}") @@ -997,7 +997,7 @@ def map_key_partner(self, partner_map_id: int | None) -> int | None: else: return None - def map_value_partner(self, partner_map_id: int | None) -> int | None: + def map_value_partner(self, partner_map_id: int | None) -> int | None: # ty: ignore[invalid-method-override] if partner_map_id is not None and (field := self.partner_schema.find_field(partner_map_id)): if not isinstance(field.field_type, MapType): raise ValueError(f"Expected MapType: {field}") diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index e54fb4b38d..d3e8bb5901 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -347,7 +347,7 @@ def _commit(self) -> UpdatesAndRequirements: first_row_id: int | None = None if self._transaction.table_metadata.format_version >= 3: - first_row_id = self._transaction.table_metadata.next_row_id + first_row_id = self._transaction.table_metadata.next_row_id # ty: ignore[unresolved-attribute] snapshot = Snapshot( snapshot_id=self._snapshot_id, @@ -370,7 +370,7 @@ def _commit(self) -> UpdatesAndRequirements: return ( ( add_snapshot_update, - SetSnapshotRefUpdate( + SetSnapshotRefUpdate( # ty: ignore[pydantic-discarded-extra-argument] snapshot_id=self._snapshot_id, parent_snapshot_id=self._parent_snapshot_id, ref_name=self._target_branch, @@ -736,12 +736,12 @@ def __init__( super().__init__(operation, transaction, io, commit_uuid, snapshot_properties, branch) table_properties = self._transaction.table_metadata.properties - self._target_size_bytes = property_as_int( + self._target_size_bytes = property_as_int( # ty: ignore[invalid-assignment] table_properties, TableProperties.MANIFEST_TARGET_SIZE_BYTES, TableProperties.MANIFEST_TARGET_SIZE_BYTES_DEFAULT, ) # type: ignore[assignment] - self._min_count_to_merge = property_as_int( + self._min_count_to_merge = property_as_int( # ty: ignore[invalid-assignment] table_properties, TableProperties.MANIFEST_MIN_MERGE_COUNT, TableProperties.MANIFEST_MIN_MERGE_COUNT_DEFAULT, diff --git a/pyiceberg/table/upsert_util.py b/pyiceberg/table/upsert_util.py index 6f32826eb0..b7bf8ece5e 100644 --- a/pyiceberg/table/upsert_util.py +++ b/pyiceberg/table/upsert_util.py @@ -107,18 +107,18 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols matching_indices[TARGET_INDEX_COLUMN_NAME].to_pylist(), strict=True, ): - source_row = source_table.slice(source_idx, 1) - target_row = target_table.slice(target_idx, 1) + source_row = source_table.slice(source_idx, 1) # ty: ignore[invalid-argument-type] + target_row = target_table.slice(target_idx, 1) # ty: ignore[invalid-argument-type] for key in non_key_cols: - source_val = source_row.column(key)[0].as_py() - target_val = target_row.column(key)[0].as_py() + source_val = source_row.column(key)[0].as_py() # ty: ignore[unresolved-attribute] + target_val = target_row.column(key)[0].as_py() # ty: ignore[unresolved-attribute] if source_val != target_val: to_update_indices.append(source_idx) break # Step 5: Take rows from source table using the indices and cast to target schema if to_update_indices: - return source_table.take(to_update_indices) + return source_table.take(to_update_indices) # ty: ignore[invalid-argument-type] else: return source_table.schema.empty_table() diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index 560a00aa49..2837f25c74 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -92,7 +92,7 @@ if TYPE_CHECKING: import pyarrow as pa - ArrayLike = TypeVar("ArrayLike", pa.Array, pa.ChunkedArray) + ArrayLike = TypeVar("ArrayLike", pa.Array, pa.ChunkedArray) # ty: ignore[missing-type-argument] S = TypeVar("S") T = TypeVar("T") @@ -128,7 +128,7 @@ def _pyiceberg_transform_wrapper( def _transform(array: "ArrayLike") -> "ArrayLike": def _cast_if_needed(arr: "ArrayLike") -> "ArrayLike": if expected_type is not None: - return arr.cast(expected_type) + return arr.cast(expected_type) # ty: ignore[invalid-return-type] else: return arr @@ -137,8 +137,8 @@ def _cast_if_needed(arr: "ArrayLike") -> "ArrayLike": elif isinstance(array, pa.ChunkedArray): result_chunks = [] for arr in array.iterchunks(): - result_chunks.append(_cast_if_needed(transform_func(arr, *args))) - return pa.chunked_array(result_chunks) + result_chunks.append(_cast_if_needed(transform_func(arr, *args))) # ty: ignore[invalid-argument-type] + return pa.chunked_array(result_chunks) # ty: ignore[invalid-return-type] else: raise ValueError(f"PyArrow array can only be of type pa.Array or pa.ChunkedArray, but found {type(array)}") @@ -203,7 +203,7 @@ def __eq__(self, other: Any) -> bool: return False @abstractmethod - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": ... + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": ... # ty: ignore[missing-type-argument] def parse_transform(v: Any) -> Transform[Any, Any]: @@ -381,7 +381,7 @@ def __repr__(self) -> str: """Return the string representation of the BucketTransform class.""" return f"BucketTransform(num_buckets={self._num_buckets})" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform return _pyiceberg_transform_wrapper(pyiceberg_core_transform.bucket, self._num_buckets) @@ -402,7 +402,7 @@ class TimeTransform(Transform[S, int], Generic[S], Singleton): def granularity(self) -> TimeResolution: ... def satisfies_order_of(self, other: Transform[S, T]) -> bool: - return self.granularity <= other.granularity if hasattr(other, "granularity") else False + return self.granularity <= other.granularity if hasattr(other, "granularity") else False # ty: ignore[unsupported-operator] def result_type(self, source: IcebergType) -> IntegerType: return IntegerType() @@ -495,7 +495,7 @@ def __repr__(self) -> str: """Return the string representation of the YearTransform class.""" return "YearTransform()" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] pa = try_import("pyarrow") pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform return _pyiceberg_transform_wrapper(pyiceberg_core_transform.year, expected_type=pa.int32()) @@ -555,7 +555,7 @@ def __repr__(self) -> str: """Return the string representation of the MonthTransform class.""" return "MonthTransform()" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] pa = try_import("pyarrow") pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform @@ -605,7 +605,7 @@ def day_func(v: Any) -> int: def can_transform(self, source: IcebergType) -> bool: return isinstance(source, (DateType, TimestampType, TimestamptzType, TimestampNanoType, TimestamptzNanoType)) - def result_type(self, source: IcebergType) -> IcebergType: + def result_type(self, source: IcebergType) -> IcebergType: # ty: ignore[invalid-method-override] """Return the result type of a day transform. The physical representation conforms to the Iceberg spec as DateType is internally converted to int. @@ -624,7 +624,7 @@ def __repr__(self) -> str: """Return the string representation of the DayTransform class.""" return "DayTransform()" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] pa = try_import("pyarrow", extras_name="pyarrow") pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform @@ -677,7 +677,7 @@ def __repr__(self) -> str: """Return the string representation of the HourTransform class.""" return "HourTransform()" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform return _pyiceberg_transform_wrapper(pyiceberg_core_transform.hour) @@ -741,7 +741,7 @@ def satisfies_order_of(self, other: Transform[S, T]) -> bool: """Ordering by value is the same as long as the other preserves order.""" return other.preserves_order - def to_human_string(self, source_type: IcebergType, value: S | None) -> str: + def to_human_string(self, source_type: IcebergType, value: S | None) -> str: # ty: ignore[invalid-method-override] return _human_string(value, source_type) if value is not None else "null" def __str__(self) -> str: @@ -752,7 +752,7 @@ def __repr__(self) -> str: """Return the string representation of the IdentityTransform class.""" return "IdentityTransform()" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] return lambda v: v @@ -905,7 +905,7 @@ def __repr__(self) -> str: """Return the string representation of the TruncateTransform class.""" return f"TruncateTransform(width={self._width})" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform return _pyiceberg_transform_wrapper(pyiceberg_core_transform.truncate, self._width) @@ -992,7 +992,7 @@ def __repr__(self) -> str: """Return the string representation of the UnknownTransform class.""" return f"UnknownTransform(transform={repr(self._transform)})" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] raise NotImplementedError() @@ -1004,7 +1004,7 @@ class VoidTransform(Transform[S, None], Singleton): def transform(self, source: IcebergType) -> Callable[[S | None], T | None]: return lambda v: None - def can_transform(self, _: IcebergType) -> bool: + def can_transform(self, _: IcebergType) -> bool: # ty: ignore[invalid-method-override] return True def result_type(self, source: IcebergType) -> IcebergType: @@ -1023,7 +1023,7 @@ def __repr__(self) -> str: """Return the string representation of the VoidTransform class.""" return "VoidTransform()" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] try: import pyarrow as pa except ModuleNotFoundError as e: diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index 59bb5e259f..1f1f029a33 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -137,14 +137,14 @@ def _exclude_private_properties(self, exclude: set[str] | None = None) -> set[st {field for field in self.__dict__ if field.startswith("_") and not field == "__root__"}, exclude or set() ) - def model_dump( + def model_dump( # ty: ignore[invalid-method-override] self, exclude_none: bool = True, exclude: set[str] | None = None, by_alias: bool = True, **kwargs: Any ) -> dict[str, Any]: return super().model_dump( exclude_none=exclude_none, exclude=self._exclude_private_properties(exclude), by_alias=by_alias, **kwargs ) - def model_dump_json( + def model_dump_json( # ty: ignore[invalid-method-override] self, exclude_none: bool = True, exclude: set[str] | None = None, by_alias: bool = True, **kwargs: Any ) -> str: return super().model_dump_json( diff --git a/pyiceberg/types.py b/pyiceberg/types.py index 32b7ad5689..0d80b554ad 100644 --- a/pyiceberg/types.py +++ b/pyiceberg/types.py @@ -261,7 +261,7 @@ def minimum_format_version(self) -> TableVersion: return 1 -class PrimitiveType(Singleton, IcebergRootModel[str], IcebergType): +class PrimitiveType(Singleton, IcebergRootModel[str], IcebergType): # ty: ignore[invalid-method-override] """Base class for all Iceberg Primitive Types.""" root: Any = Field() @@ -437,7 +437,7 @@ def convert_field_type(cls, v: Any) -> IcebergType: """Convert string values into IcebergType instances.""" if isinstance(v, str): try: - return IcebergType.handle_primitive_type(v, None) + return IcebergType.handle_primitive_type(v, None) # ty: ignore[call-non-callable] except ValueError as e: raise ValueError(f"Unsupported field type: '{v}'") from e return v diff --git a/pyiceberg/utils/deprecated.py b/pyiceberg/utils/deprecated.py index b5bb6d01cd..9d8e9f5593 100644 --- a/pyiceberg/utils/deprecated.py +++ b/pyiceberg/utils/deprecated.py @@ -31,7 +31,7 @@ def deprecated(deprecated_in: str, removed_in: str, help_message: str | None = N def decorator(func: Callable): # type: ignore @functools.wraps(func) def new_func(*args: Any, **kwargs: Any) -> Any: - message = f"Call to {func.__name__}, deprecated in {deprecated_in}, will be removed in {removed_in}.{help_message}" + message = f"Call to {func.__name__}, deprecated in {deprecated_in}, will be removed in {removed_in}.{help_message}" # ty: ignore[unresolved-attribute] _deprecation_warning(message) diff --git a/pyiceberg/utils/schema_conversion.py b/pyiceberg/utils/schema_conversion.py index 732439d34b..280e44d291 100644 --- a/pyiceberg/utils/schema_conversion.py +++ b/pyiceberg/utils/schema_conversion.py @@ -624,7 +624,7 @@ def visit_time(self, time_type: TimeType) -> AvroType: def visit_timestamp(self, timestamp_type: TimestampType) -> AvroType: return {"type": "long", "logicalType": "timestamp-micros", "adjust-to-utc": False} - def visit_timestamp_ns(self, timestamp_ns_type: TimestampNanoType) -> AvroType: + def visit_timestamp_ns(self, timestamp_ns_type: TimestampNanoType) -> AvroType: # ty: ignore[invalid-method-override] return {"type": "long", "logicalType": "timestamp-nanos", "adjust-to-utc": False} def visit_timestamptz(self, timestamptz_type: TimestamptzType) -> AvroType: diff --git a/pyiceberg/view/metadata.py b/pyiceberg/view/metadata.py index 33766040e3..1a0859ad48 100644 --- a/pyiceberg/view/metadata.py +++ b/pyiceberg/view/metadata.py @@ -38,7 +38,7 @@ class SQLViewRepresentation(IcebergBaseModel): """The dialect of the SQL, e.g. `spark`, `trino`, `presto`.""" -class ViewRepresentation(IcebergBaseModel, RootModel): +class ViewRepresentation(IcebergBaseModel, RootModel): # ty: ignore[invalid-method-override, missing-type-argument] root: SQLViewRepresentation From 040369e90e4b9681ced20fff5580802f3c6a96f7 Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Mon, 2 Nov 2026 15:28:18 -0800 Subject: [PATCH 12/13] ci: report ty in the existing lint job Runs ty in the project environment next to the mypy hook, as a non-blocking continue-on-error step on the 3.11 leg, so the 470 diagnostics that the inline comments suppress are what CI measures rather than the stub-only hook. No new job and no new required check, so python-ci-required and .asf.yaml stay as they are. --- .github/workflows/python-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 1abd4c8600..0362653106 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -61,6 +61,11 @@ jobs: - name: Run linters if: matrix.python == '3.11' run: make lint + - name: Type-check with ty + # Not the gate yet, so a failure reports without failing the job. + if: matrix.python == '3.11' + continue-on-error: true + run: uv run ty check - name: Run unit tests with coverage run: COVERAGE=1 make test - name: Generate coverage report (85%) # Coverage threshold should only increase over time — never decrease it! From ce941cedc6561ce951ed11543eec71f17dfefc19 Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Mon, 2 Nov 2026 15:29:02 -0800 Subject: [PATCH 13/13] dev: trim a redundant comment on the ty hook --- .pre-commit-config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46dcdfff16..d52e7bb369 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,8 +58,6 @@ repos: hooks: - id: ty name: ty - # Bare-environment deltas. Hook args are also handed to `uv pip install` when the hook - # environment is built, so the flags live in the entry instead. entry: >- ty check -c 'rules.unresolved-import = "ignore"'