diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 1abd4c8600..0362653106 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -61,6 +61,11 @@ jobs: - name: Run linters if: matrix.python == '3.11' run: make lint + - name: Type-check with ty + # Not the gate yet, so a failure reports without failing the job. + if: matrix.python == '3.11' + continue-on-error: true + run: uv run ty check - name: Run unit tests with coverage run: COVERAGE=1 make test - name: Generate coverage report (85%) # Coverage threshold should only increase over time — never decrease it! diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f0dedd0804..d52e7bb369 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,6 +40,8 @@ repos: rev: v1.18.2 hooks: - id: mypy + # Scoped to the package while ty and mypy are compared on the same files. + files: ^pyiceberg/ args: [--config=pyproject.toml] additional_dependencies: @@ -47,6 +49,34 @@ repos: - types-pytest-lazy-fixture - types-pytz - types-requests + # Experimental: ty is not the gate yet, so this hook is manual-stage only. + # Run it with: uv run prek run ty --stage manual -a + # Local hook so ty runs in this hook's own environment, which holds only ty and the stub + # packages, the same bare environment the mypy hook gets. The entry points at it through + # VIRTUAL_ENV, which the hook runner exports. Keep these pins in step with the types group. + - repo: local + hooks: + - id: ty + name: ty + entry: >- + ty check + -c 'rules.unresolved-import = "ignore"' + -c 'rules.unused-ignore-comment = "ignore"' + -c 'rules.unused-type-ignore-comment = "ignore"' + -c 'rules.invalid-assignment = "ignore"' + -c 'rules.invalid-argument-type = "ignore"' + -c 'rules.call-non-callable = "ignore"' + language: python + additional_dependencies: + - ty==0.0.84 + - types-cachetools==7.0.0.20260713 + - types-pytest-lazy-fixture==0.6.3.20260518 + - types-pytz==2026.5.0.20261006 + - types-requests==2.33.0.20261006 + always_run: true + pass_filenames: false + require_serial: true + stages: [manual] - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.45.0 hooks: diff --git a/Makefile b/Makefile index 9f0282c65b..a69d8c8a3c 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ test test-integration test-integration-setup test-integration-exec test-integration-cleanup test-integration-rebuild \ test-s3 test-adls test-gcs test-coverage coverage-report test test-notebook\ docs-serve docs-build notebook notebook-infra \ + typecheck-ty \ clean .DEFAULT_GOAL := help @@ -97,6 +98,9 @@ check-license: ## Check license headers lint: ## Run code linters via prek (pre-commit hooks) uv run $(PYTHON_ARG) prek run -a +typecheck-ty: ## Type-check with ty in its isolated stub-only hook environment + uv run $(PYTHON_ARG) prek run ty --stage manual -a + # =============== # Testing Section # =============== diff --git a/pyiceberg/avro/codecs/bzip2.py b/pyiceberg/avro/codecs/bzip2.py index 24786681f2..35cd319c0b 100644 --- a/pyiceberg/avro/codecs/bzip2.py +++ b/pyiceberg/avro/codecs/bzip2.py @@ -33,7 +33,7 @@ def decompress(data: bytes) -> bytes: except ImportError: - class BZip2Codec(Codec): # type: ignore + class BZip2Codec(Codec): # type: ignore[no-redef] @staticmethod def compress(data: bytes) -> tuple[bytes, int]: raise ImportError("Python bzip2 support not installed, please install the extension") diff --git a/pyiceberg/avro/codecs/snappy_codec.py b/pyiceberg/avro/codecs/snappy_codec.py index 2ea7229772..d3553667a6 100644 --- a/pyiceberg/avro/codecs/snappy_codec.py +++ b/pyiceberg/avro/codecs/snappy_codec.py @@ -54,12 +54,12 @@ def decompress(data: bytes) -> bytes: checksum = data[-4:] # store checksum before truncating data data = data[0:-4] # remove checksum from the data uncompressed = snappy.decompress(data) - SnappyCodec._check_crc32(uncompressed, checksum) + SnappyCodec._check_crc32(uncompressed, checksum) # ty: ignore[unresolved-attribute] return uncompressed except ImportError: - class SnappyCodec(Codec): # type: ignore + class SnappyCodec(Codec): # type: ignore[no-redef] @staticmethod def compress(data: bytes) -> tuple[bytes, int]: raise ImportError("Snappy support not installed, please install using `pip install pyiceberg[snappy]`") diff --git a/pyiceberg/avro/codecs/zstandard_codec.py b/pyiceberg/avro/codecs/zstandard_codec.py index 4cc815214f..8f9b5153d6 100644 --- a/pyiceberg/avro/codecs/zstandard_codec.py +++ b/pyiceberg/avro/codecs/zstandard_codec.py @@ -43,7 +43,7 @@ def decompress(data: bytes) -> bytes: except ImportError: - class ZStandardCodec(Codec): # type: ignore + class ZStandardCodec(Codec): # type: ignore[no-redef] @staticmethod def compress(data: bytes) -> tuple[bytes, int]: raise ImportError("Zstandard support not installed, please install using `pip install pyiceberg[zstandard]`") diff --git a/pyiceberg/avro/decoder.py b/pyiceberg/avro/decoder.py index e971d52d48..d5a2383b13 100644 --- a/pyiceberg/avro/decoder.py +++ b/pyiceberg/avro/decoder.py @@ -137,7 +137,7 @@ def __init__(self, input_stream: bytes | InputStream) -> None: """Reader is a Python object on which we can call read, seek, and tell.""" if isinstance(input_stream, bytes): # In the case of bytes, we wrap it into a BytesIO to make it a stream - self._input_stream = io.BytesIO(input_stream) + self._input_stream = io.BytesIO(input_stream) # ty: ignore[invalid-assignment] else: self._input_stream = input_stream diff --git a/pyiceberg/avro/decoder_fast.pyi b/pyiceberg/avro/decoder_fast.pyi index cc367c4764..144b158712 100644 --- a/pyiceberg/avro/decoder_fast.pyi +++ b/pyiceberg/avro/decoder_fast.pyi @@ -33,10 +33,10 @@ class CythonBinaryDecoder(BinaryDecoder): def read_int(self) -> int: pass - def read_ints(self, count: int) -> tuple[int, ...]: + def read_ints(self, count: int) -> tuple[int, ...]: # ty: ignore[invalid-method-override] pass - def read_int_bytes_dict(self, count: int, dest: dict[int, bytes]) -> None: + def read_int_bytes_dict(self, count: int, dest: dict[int, bytes]) -> None: # ty: ignore[invalid-method-override] pass def read_bytes(self) -> bytes: diff --git a/pyiceberg/avro/file.py b/pyiceberg/avro/file.py index 3184e3950a..4905937882 100644 --- a/pyiceberg/avro/file.py +++ b/pyiceberg/avro/file.py @@ -96,7 +96,7 @@ def compression_codec(self) -> type[Codec] | None: if codec_name not in KNOWN_CODECS: raise ValueError(f"Unsupported codec: {codec_name}") - return KNOWN_CODECS[codec_name] # type: ignore + return KNOWN_CODECS[codec_name] # type: ignore[index] def get_schema(self) -> Schema: if _SCHEMA_KEY in self.meta: @@ -257,7 +257,7 @@ def __init__( raise ValueError(f"Sync interval must be positive: {sync_interval}") self.sync_interval = sync_interval self._block = io.BytesIO() - self._block_encoder = BinaryEncoder(output_stream=self._block) + self._block_encoder = BinaryEncoder(output_stream=self._block) # ty: ignore[invalid-argument-type] self._block_records = 0 def __enter__(self) -> AvroOutputFile[D]: @@ -308,7 +308,7 @@ def compression_codec(self) -> type[Codec] | None: if codec_name not in KNOWN_CODECS: raise ValueError(f"Unsupported codec: {codec_name}") - return KNOWN_CODECS[codec_name] # type: ignore + return KNOWN_CODECS[codec_name] # type: ignore[index] def write_block(self, objects: list[D]) -> None: for obj in objects: diff --git a/pyiceberg/avro/reader.py b/pyiceberg/avro/reader.py index e3ee8305f0..237b3ff8ce 100644 --- a/pyiceberg/avro/reader.py +++ b/pyiceberg/avro/reader.py @@ -93,7 +93,7 @@ def __repr__(self) -> str: class NoneReader(Reader): - def read(self, _: BinaryDecoder) -> None: + def read(self, _: BinaryDecoder) -> None: # ty: ignore[invalid-method-override] return None def skip(self, decoder: BinaryDecoder) -> None: @@ -107,7 +107,7 @@ class DefaultReader(Reader): def __init__(self, default_value: Any) -> None: self.default_value = default_value - def read(self, _: BinaryDecoder) -> Any: + def read(self, _: BinaryDecoder) -> Any: # ty: ignore[invalid-method-override] return self.default_value def skip(self, decoder: BinaryDecoder) -> None: diff --git a/pyiceberg/avro/resolver.py b/pyiceberg/avro/resolver.py index 3f19ae9878..e89702dcaf 100644 --- a/pyiceberg/avro/resolver.py +++ b/pyiceberg/avro/resolver.py @@ -185,7 +185,7 @@ def visit_time(self, time_type: TimeType) -> Writer: def visit_timestamp(self, timestamp_type: TimestampType) -> Writer: return TimestampWriter() - def visit_timestamp_ns(self, timestamp_ns_type: TimestampNanoType) -> Writer: + def visit_timestamp_ns(self, timestamp_ns_type: TimestampNanoType) -> Writer: # ty: ignore[invalid-method-override] return TimestampNanoWriter() def visit_timestamptz(self, timestamptz_type: TimestamptzType) -> Writer: @@ -233,7 +233,7 @@ def resolve_writer( """ if record_schema == file_schema: return construct_writer(file_schema) - return visit_with_partner(file_schema, record_schema, WriteSchemaResolver(), SchemaPartnerAccessor()) # type: ignore + return visit_with_partner(file_schema, record_schema, WriteSchemaResolver(), SchemaPartnerAccessor()) # type: ignore[misc] def resolve_reader( @@ -253,7 +253,7 @@ def resolve_reader( Raises: NotImplementedError: If attempting to resolve an unrecognized object type. """ - return visit_with_partner(file_schema, read_schema, ReadSchemaResolver(read_types, read_enums), SchemaPartnerAccessor()) # type: ignore + return visit_with_partner(file_schema, read_schema, ReadSchemaResolver(read_types, read_enums), SchemaPartnerAccessor()) # type: ignore[misc] class EnumReader(Reader): @@ -276,10 +276,10 @@ def skip(self, decoder: BinaryDecoder) -> None: class WriteSchemaResolver(PrimitiveWithPartnerVisitor[IcebergType, Writer]): - def schema(self, file_schema: Schema, record_schema: IcebergType | None, result: Writer) -> Writer: + def schema(self, file_schema: Schema, record_schema: IcebergType | None, result: Writer) -> Writer: # ty: ignore[invalid-method-override] return result - def struct(self, file_schema: StructType, record_struct: IcebergType | None, file_writers: list[Writer]) -> Writer: + def struct(self, file_schema: StructType, record_struct: IcebergType | None, file_writers: list[Writer]) -> Writer: # ty: ignore[invalid-method-override] if not isinstance(record_struct, StructType): raise ResolveError(f"File/write schema are not aligned for struct, got {record_struct}") @@ -301,16 +301,16 @@ def struct(self, file_schema: StructType, record_struct: IcebergType | None, fil return StructWriter(field_writers=tuple(results)) - def field(self, file_field: NestedField, record_type: IcebergType | None, field_writer: Writer) -> Writer: + def field(self, file_field: NestedField, record_type: IcebergType | None, field_writer: Writer) -> Writer: # ty: ignore[invalid-method-override] return field_writer if file_field.required else OptionWriter(field_writer) - def list(self, file_list_type: ListType, file_list: IcebergType | None, element_writer: Writer) -> Writer: + def list(self, file_list_type: ListType, file_list: IcebergType | None, element_writer: Writer) -> Writer: # ty: ignore[invalid-method-override] return ListWriter(element_writer if file_list_type.element_required else OptionWriter(element_writer)) - def map(self, file_map_type: MapType, file_primitive: IcebergType | None, key_writer: Writer, value_writer: Writer) -> Writer: + def map(self, file_map_type: MapType, file_primitive: IcebergType | None, key_writer: Writer, value_writer: Writer) -> Writer: # ty: ignore[invalid-method-override] return MapWriter(key_writer, value_writer if file_map_type.value_required else OptionWriter(value_writer)) - def primitive(self, file_primitive: PrimitiveType, record_primitive: IcebergType | None) -> Writer: + def primitive(self, file_primitive: PrimitiveType, record_primitive: IcebergType | None) -> Writer: # ty: ignore[invalid-method-override] if record_primitive is not None: # ensure that the type can be projected to the expected if file_primitive != record_primitive: @@ -393,16 +393,16 @@ def __init__( self.read_enums = read_enums self.context = [] - def schema(self, schema: Schema, expected_schema: IcebergType | None, result: Reader) -> Reader: + def schema(self, schema: Schema, expected_schema: IcebergType | None, result: Reader) -> Reader: # ty: ignore[invalid-method-override] return result - def before_field(self, field: NestedField, field_partner: NestedField | None) -> None: + def before_field(self, field: NestedField, field_partner: NestedField | None) -> None: # ty: ignore[invalid-method-override] self.context.append(field.field_id) - def after_field(self, field: NestedField, field_partner: NestedField | None) -> None: + def after_field(self, field: NestedField, field_partner: NestedField | None) -> None: # ty: ignore[invalid-method-override] self.context.pop() - def struct(self, struct: StructType, expected_struct: IcebergType | None, field_readers: list[Reader]) -> Reader: + def struct(self, struct: StructType, expected_struct: IcebergType | None, field_readers: list[Reader]) -> Reader: # ty: ignore[invalid-method-override] read_struct_id = self.context[STRUCT_ROOT] if len(self.context) > 0 else STRUCT_ROOT struct_callable = self.read_types.get(read_struct_id, Record) @@ -439,22 +439,22 @@ def struct(self, struct: StructType, expected_struct: IcebergType | None, field_ return StructReader(tuple(results), struct_callable, expected_struct) - def field(self, field: NestedField, expected_field: IcebergType | None, field_reader: Reader) -> Reader: + def field(self, field: NestedField, expected_field: IcebergType | None, field_reader: Reader) -> Reader: # ty: ignore[invalid-method-override] return field_reader if field.required else OptionReader(field_reader) - def list(self, list_type: ListType, expected_list: IcebergType | None, element_reader: Reader) -> Reader: + def list(self, list_type: ListType, expected_list: IcebergType | None, element_reader: Reader) -> Reader: # ty: ignore[invalid-method-override] if expected_list and not isinstance(expected_list, ListType): raise ResolveError(f"File/read schema are not aligned for list, got {expected_list}") return ListReader(element_reader if list_type.element_required else OptionReader(element_reader)) - def map(self, map_type: MapType, expected_map: IcebergType | None, key_reader: Reader, value_reader: Reader) -> Reader: + def map(self, map_type: MapType, expected_map: IcebergType | None, key_reader: Reader, value_reader: Reader) -> Reader: # ty: ignore[invalid-method-override] if expected_map and not isinstance(expected_map, MapType): raise ResolveError(f"File/read schema are not aligned for map, got {expected_map}") return MapReader(key_reader, value_reader if map_type.value_required else OptionReader(value_reader)) - def primitive(self, primitive: PrimitiveType, expected_primitive: IcebergType | None) -> Reader: + def primitive(self, primitive: PrimitiveType, expected_primitive: IcebergType | None) -> Reader: # ty: ignore[invalid-method-override] if expected_primitive is not None: if not isinstance(expected_primitive, PrimitiveType): raise ResolveError(f"File/read schema are not aligned for {primitive}, got {expected_primitive}") @@ -547,7 +547,7 @@ def schema_partner(self, partner: IcebergType | None) -> IcebergType | None: raise ResolveError(f"File/read schema are not aligned for schema, got {partner}") - def field_partner(self, partner: IcebergType | None, field_id: int, field_name: str) -> IcebergType | None: + def field_partner(self, partner: IcebergType | None, field_id: int, field_name: str) -> IcebergType | None: # ty: ignore[invalid-method-override] if isinstance(partner, StructType): field = partner.field(field_id) else: diff --git a/pyiceberg/avro/writer.py b/pyiceberg/avro/writer.py index f78d1a486e..bd7a7d8634 100644 --- a/pyiceberg/avro/writer.py +++ b/pyiceberg/avro/writer.py @@ -234,5 +234,5 @@ class DefaultWriter(Writer): writer: Writer value: Any - def write(self, encoder: BinaryEncoder, _: Any) -> None: + def write(self, encoder: BinaryEncoder, _: Any) -> None: # ty: ignore[invalid-method-override] self.writer.write(encoder, self.value) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index cdb4d26b3b..c81aef7a4f 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -919,7 +919,7 @@ def _convert_schema_if_needed( downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False if isinstance(schema, pa.Schema): - schema: Schema = visit_pyarrow( # type: ignore + schema: Schema = visit_pyarrow( # type: ignore[no-redef] schema, _ConvertToIcebergWithoutIDs( downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version @@ -1077,7 +1077,7 @@ def _create_staged_table( Returns: StagedTable: the created staged table instance. """ - schema: Schema = self._convert_schema_if_needed( # type: ignore + schema: Schema = self._convert_schema_if_needed( # type: ignore[no-redef] schema, int(properties.get(TableProperties.FORMAT_VERSION, TableProperties.DEFAULT_FORMAT_VERSION)), # type: ignore ) diff --git a/pyiceberg/catalog/bigquery_metastore.py b/pyiceberg/catalog/bigquery_metastore.py index 74d1c952bc..0c11418d42 100644 --- a/pyiceberg/catalog/bigquery_metastore.py +++ b/pyiceberg/catalog/bigquery_metastore.py @@ -131,7 +131,7 @@ def create_table( ValueError: If the identifier is invalid, or no path is given to store metadata. """ - schema: Schema = self._convert_schema_if_needed(schema) # type: ignore + schema: Schema = self._convert_schema_if_needed(schema) # type: ignore[no-redef] dataset_name, table_name = self.identifier_to_database_and_table(identifier) diff --git a/pyiceberg/catalog/dynamodb.py b/pyiceberg/catalog/dynamodb.py index 220313ab74..3d7a644656 100644 --- a/pyiceberg/catalog/dynamodb.py +++ b/pyiceberg/catalog/dynamodb.py @@ -150,9 +150,9 @@ def _ensure_catalog_table_exists_or_create(self) -> None: try: self.dynamodb.create_table( TableName=self.dynamodb_table_name, - AttributeDefinitions=CREATE_CATALOG_ATTRIBUTE_DEFINITIONS, - KeySchema=CREATE_CATALOG_KEY_SCHEMA, - GlobalSecondaryIndexes=CREATE_CATALOG_GLOBAL_SECONDARY_INDEXES, + AttributeDefinitions=CREATE_CATALOG_ATTRIBUTE_DEFINITIONS, # ty: ignore[invalid-argument-type] + KeySchema=CREATE_CATALOG_KEY_SCHEMA, # ty: ignore[invalid-argument-type] + GlobalSecondaryIndexes=CREATE_CATALOG_GLOBAL_SECONDARY_INDEXES, # ty: ignore[invalid-argument-type] BillingMode=DYNAMODB_PAY_PER_REQUEST, ) except ( @@ -160,7 +160,7 @@ def _ensure_catalog_table_exists_or_create(self) -> None: self.dynamodb.exceptions.LimitExceededException, self.dynamodb.exceptions.InternalServerError, ) as e: - raise GenericDynamoDbError(e.message) from e + raise GenericDynamoDbError(e.message) from e # ty: ignore[unresolved-attribute] def _dynamodb_table_exists(self) -> bool: try: @@ -168,7 +168,7 @@ def _dynamodb_table_exists(self) -> bool: except self.dynamodb.exceptions.ResourceNotFoundException: return False except self.dynamodb.exceptions.InternalServerError as e: - raise GenericDynamoDbError(e.message) from e + raise GenericDynamoDbError(e.message) from e # ty: ignore[unresolved-attribute] if response["Table"]["TableStatus"] != ACTIVE: raise GenericDynamoDbError(f"DynamoDB table for catalog {self.dynamodb_table_name} is not {ACTIVE}") @@ -204,7 +204,7 @@ def create_table( ValueError: If the identifier is invalid, or no path is given to store metadata. """ - schema: Schema = self._convert_schema_if_needed( # type: ignore + schema: Schema = self._convert_schema_if_needed( # type: ignore[no-redef] schema, int(properties.get(TableProperties.FORMAT_VERSION, TableProperties.DEFAULT_FORMAT_VERSION)), # type: ignore ) @@ -462,7 +462,7 @@ def list_tables(self, namespace: str | Identifier) -> list[Identifier]: self.dynamodb.exceptions.InternalServerError, self.dynamodb.exceptions.ResourceNotFoundException, ) as e: - raise GenericDynamoDbError(e.message) from e + raise GenericDynamoDbError(e.message) from e # ty: ignore[unresolved-attribute] table_identifiers = [] for page in page_iterator: @@ -508,7 +508,7 @@ def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]: self.dynamodb.exceptions.InternalServerError, self.dynamodb.exceptions.ResourceNotFoundException, ) as e: - raise GenericDynamoDbError(e.message) from e + raise GenericDynamoDbError(e.message) from e # ty: ignore[unresolved-attribute] database_identifiers = [] for page in page_iterator: @@ -651,7 +651,7 @@ def _get_dynamo_item(self, identifier: str, namespace: str) -> dict[str, Any]: self.dynamodb.exceptions.RequestLimitExceeded, self.dynamodb.exceptions.InternalServerError, ) as e: - raise GenericDynamoDbError(e.message) from e + raise GenericDynamoDbError(e.message) from e # ty: ignore[unresolved-attribute] def _put_dynamo_item(self, item: dict[str, Any], condition_expression: str) -> None: try: @@ -666,7 +666,7 @@ def _put_dynamo_item(self, item: dict[str, Any], condition_expression: str) -> N self.dynamodb.exceptions.ItemCollectionSizeLimitExceededException, self.dynamodb.exceptions.TransactionConflictException, ) as e: - raise GenericDynamoDbError(e.message) from e + raise GenericDynamoDbError(e.message) from e # ty: ignore[unresolved-attribute] def _delete_dynamo_item(self, namespace: str, identifier: str, condition_expression: str) -> None: try: @@ -694,7 +694,7 @@ def _delete_dynamo_item(self, namespace: str, identifier: str, condition_express self.dynamodb.exceptions.ItemCollectionSizeLimitExceededException, self.dynamodb.exceptions.TransactionConflictException, ) as e: - raise GenericDynamoDbError(e.message) from e + raise GenericDynamoDbError(e.message) from e # ty: ignore[unresolved-attribute] def _convert_dynamo_table_item_to_iceberg_table(self, dynamo_table_item: dict[str, Any]) -> Table: table_dict = _convert_dynamo_item_to_regular_dict(dynamo_table_item) diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py index bbb954fcda..c07fc57faf 100644 --- a/pyiceberg/catalog/glue.py +++ b/pyiceberg/catalog/glue.py @@ -219,7 +219,7 @@ def primitive(self, primitive: PrimitiveType) -> str: return f"decimal({primitive.precision},{primitive.scale})" if (primitive_type := type(primitive)) not in GLUE_PRIMITIVE_TYPES: return str(primitive) - return GLUE_PRIMITIVE_TYPES[primitive_type] + return GLUE_PRIMITIVE_TYPES[primitive_type] # ty: ignore[invalid-argument-type] def _to_columns(metadata: TableMetadata) -> list["ColumnTypeDef"]: diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 0f43bbfb81..52b2c24455 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -294,7 +294,7 @@ def primitive(self, primitive: PrimitiveType) -> str: # Hive2 doesn't support timestamp with local time zone return "timestamp" else: - return HIVE_PRIMITIVE_TYPES[type(primitive)] + return HIVE_PRIMITIVE_TYPES[type(primitive)] # ty: ignore[invalid-argument-type] class HiveCatalog(MetastoreCatalog): @@ -333,7 +333,7 @@ def _create_hive_client(properties: dict[str, str]) -> _HiveClient: raise ValueError(f"Unable to connect to hive using uri: {properties[URI]}") def _convert_hive_into_iceberg(self, table: HiveTable) -> Table: - properties: dict[str, str] = table.parameters + properties: dict[str, str] = table.parameters # ty: ignore[invalid-assignment] if TABLE_TYPE not in properties: raise NoSuchPropertyException( f"Property table_type missing, could not determine type: {table.dbName}.{table.tableName}" @@ -354,7 +354,7 @@ def _convert_hive_into_iceberg(self, table: HiveTable) -> Table: file = io.new_input(metadata_location) metadata = FromInputFile.table_metadata(file) return Table( - identifier=(table.dbName, table.tableName), + identifier=(table.dbName, table.tableName), # ty: ignore[invalid-argument-type] metadata=metadata, metadata_location=metadata_location, io=self._load_file_io(location=metadata_location, table_properties=metadata.properties), @@ -536,8 +536,8 @@ def _create_lock_request(self, database_name: str, table_name: str) -> LockReque def _wait_for_lock(self, database_name: str, table_name: str, lockid: int, open_client: Client) -> LockResponse: @retry( retry=retry_if_exception_type(WaitingForLockException), - wait=wait_exponential(multiplier=2, min=self._lock_check_min_wait_time, max=self._lock_check_max_wait_time), - stop=stop_after_attempt(self._lock_check_retries), + wait=wait_exponential(multiplier=2, min=self._lock_check_min_wait_time, max=self._lock_check_max_wait_time), # ty: ignore[invalid-argument-type] + stop=stop_after_attempt(self._lock_check_retries), # ty: ignore[invalid-argument-type] reraise=True, ) def _do_wait_for_lock() -> LockResponse: @@ -581,7 +581,7 @@ def commit_table( try: if lock.state != LockState.ACQUIRED: if lock.state == LockState.WAITING: - self._wait_for_lock(database_name, table_name, lock.lockid, open_client) + self._wait_for_lock(database_name, table_name, lock.lockid, open_client) # ty: ignore[invalid-argument-type] else: raise CommitFailedException(f"Failed to acquire lock for {table_identifier}, state: {lock.state}") @@ -810,10 +810,10 @@ def list_tables(self, namespace: str | Identifier) -> list[Identifier]: """ database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) with self._client as open_client: - return [ + return [ # ty: ignore[invalid-return-type] (database_name, table.tableName) for table in self._fetch_hive_tables(open_client, database_name) - if table.parameters.get(TABLE_TYPE, "").lower() == ICEBERG + if table.parameters.get(TABLE_TYPE, "").lower() == ICEBERG # ty: ignore[unresolved-attribute] ] @override diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index c8e2399fe8..b0853bdf83 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -488,7 +488,7 @@ def __init__(self, timeout: float | None = None, max_retries: Retry | int = DEFA self._timeout = timeout super().__init__(max_retries=max_retries) - def send( + def send( # ty: ignore[invalid-method-override] self, request: PreparedRequest, stream: bool = False, @@ -499,7 +499,7 @@ def send( ) -> Response: if timeout is None: timeout = self._timeout - return super().send(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) + return super().send(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) # ty: ignore[invalid-argument-type] def _create_connection_adapter(properties: Properties) -> _RetryTimeoutHTTPAdapter | None: @@ -678,7 +678,7 @@ def supports_server_side_planning(self, table_config: Properties = EMPTY_DICT) - return False return self._effective_scan_planning_mode(table_config) == ScanPlanningMode.SERVER - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def _plan_table_scan(self, identifier: str | Identifier, request: PlanTableScanRequest) -> PlanningResponse: """Submit a scan plan request to the REST server. @@ -704,7 +704,7 @@ def _plan_table_scan(self, identifier: str | Identifier, request: PlanTableScanR return _PLANNING_RESPONSE_ADAPTER.validate_json(response.text) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def _fetch_scan_tasks(self, identifier: str | Identifier, plan_task: str) -> ScanTasks: """Fetch additional scan tasks using a plan task token. @@ -731,7 +731,7 @@ def _fetch_scan_tasks(self, identifier: str | Identifier, plan_task: str) -> Sca return ScanTasks.model_validate_json(response.text) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def _fetch_planning_result(self, identifier: str | Identifier, plan_id: str) -> PlanningResponse: """Fetch the result of an async scan plan by plan-id. @@ -1045,12 +1045,12 @@ def _fetch_config(self) -> None: config_response = ConfigResponse.model_validate_json(response.text) config = config_response.defaults - config.update(self.properties) - config.update(config_response.overrides) - self.properties = config + config.update(self.properties) # ty: ignore[unresolved-attribute] + config.update(config_response.overrides) # ty: ignore[no-matching-overload, unresolved-attribute] + self.properties = config # ty: ignore[invalid-assignment] # Update URI based on overrides - self.uri = config[URI] + self.uri = config[URI] # ty: ignore[not-subscriptable] # Determine supported endpoints endpoints = config_response.endpoints @@ -1105,7 +1105,7 @@ class SigV4Adapter(HTTPAdapter): def __init__(self, **properties: str): self._properties = properties max_retries = property_as_int(self._properties, SIGV4_MAX_RETRIES, SIGV4_MAX_RETRIES_DEFAULT) - super().__init__(max_retries=max_retries) + super().__init__(max_retries=max_retries) # ty: ignore[invalid-argument-type] self._boto_session = boto3.Session( profile_name=get_first_property_value(self._properties, AWS_PROFILE_NAME), region_name=get_first_property_value(self._properties, AWS_REGION), @@ -1197,7 +1197,7 @@ def _refresh_token(self) -> None: # Reactive token refresh is atypical - we should proactively refresh tokens in a separate thread # instead of retrying on Auth Exceptions. Keeping refresh behavior for the LegacyOAuth2AuthManager # for backward compatibility - auth_manager = self._session.auth.auth_manager # type: ignore[union-attr] + auth_manager = self._session.auth.auth_manager # type: ignore[union-attr] # ty: ignore[unresolved-attribute] if isinstance(auth_manager, LegacyOAuth2AuthManager): auth_manager._refresh_token() @@ -1252,7 +1252,7 @@ def _create_table( return TableResponse.model_validate_json(response.text) @override - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def create_table( self, identifier: str | Identifier, @@ -1274,7 +1274,7 @@ def create_table( return self._response_to_table(self.identifier_to_tuple(identifier), table_response) @override - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def create_table_transaction( self, identifier: str | Identifier, @@ -1297,7 +1297,7 @@ def create_table_transaction( return CreateTableTransaction(staged_table) @override - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def create_view( self, identifier: str | Identifier, @@ -1336,7 +1336,7 @@ def create_view( view_response = ViewResponse.model_validate_json(response.text) return self._response_to_view(self.identifier_to_tuple(identifier), view_response) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def register_table(self, identifier: str | Identifier, metadata_location: str, overwrite: bool = False) -> Table: """Register a new table using existing metadata. @@ -1372,7 +1372,7 @@ def register_table(self, identifier: str | Identifier, metadata_location: str, o table_response = TableResponse.model_validate_json(response.text) return self._response_to_table(self.identifier_to_tuple(identifier), table_response) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def unregister_table(self, identifier: str | Identifier) -> tuple[str, TableMetadata]: """Unregister a table from the catalog without removing data or metadata files. @@ -1398,7 +1398,7 @@ def unregister_table(self, identifier: str | Identifier) -> tuple[str, TableMeta result = UnregisterTableResult.model_validate_json(response.content) return (result.metadata_location, result.metadata) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def list_tables(self, namespace: str | Identifier) -> list[Identifier]: self._check_endpoint(Capability.V1_LIST_TABLES) @@ -1434,7 +1434,7 @@ def list_tables(self, namespace: str | Identifier) -> list[Identifier]: return tables - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def load_table(self, identifier: str | Identifier) -> Table: self._check_endpoint(Capability.V1_LOAD_TABLE) @@ -1456,7 +1456,7 @@ def load_table(self, identifier: str | Identifier) -> Table: table_response = TableResponse.model_validate_json(response.text) return self._response_to_table(self.identifier_to_tuple(identifier), table_response) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] def _load_credentials( self, identifier: str | Identifier, @@ -1482,7 +1482,7 @@ def load_credentials( credentials_response = self._load_credentials(identifier) return self._resolve_storage_credentials(credentials_response.storage_credentials, location) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def drop_table(self, identifier: str | Identifier, purge_requested: bool = False) -> None: self._check_endpoint(Capability.V1_DELETE_TABLE) @@ -1495,12 +1495,12 @@ def drop_table(self, identifier: str | Identifier, purge_requested: bool = False except HTTPError as exc: _handle_non_200_response(exc, {404: NoSuchTableError}) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def purge_table(self, identifier: str | Identifier) -> None: self.drop_table(identifier=identifier, purge_requested=True) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def rename_table(self, from_identifier: str | Identifier, to_identifier: str | Identifier) -> Table: self._check_endpoint(Capability.V1_RENAME_TABLE) @@ -1537,7 +1537,7 @@ def _remove_catalog_name_from_table_request_identifier(self, table_request: Comm ) return table_request - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def list_views(self, namespace: str | Identifier) -> list[Identifier]: if Capability.V1_LIST_VIEWS not in self._supported_endpoints: @@ -1575,7 +1575,7 @@ def list_views(self, namespace: str | Identifier) -> list[Identifier]: return views - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def load_view(self, identifier: str | Identifier) -> View: self._check_endpoint(Capability.V1_LOAD_VIEW) @@ -1590,7 +1590,7 @@ def load_view(self, identifier: str | Identifier) -> View: view_response = ViewResponse.model_validate_json(response.text) return self._response_to_view(self.identifier_to_tuple(identifier), view_response) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def commit_table( self, table: Table, requirements: tuple[TableRequirement, ...], updates: tuple[TableUpdate, ...] @@ -1638,7 +1638,7 @@ def commit_table( ) return CommitTableResponse.model_validate_json(response.text) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def create_namespace(self, namespace: str | Identifier, properties: Properties = EMPTY_DICT) -> None: self._check_endpoint(Capability.V1_CREATE_NAMESPACE) @@ -1650,7 +1650,7 @@ def create_namespace(self, namespace: str | Identifier, properties: Properties = except HTTPError as exc: _handle_non_200_response(exc, {409: NamespaceAlreadyExistsError}) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def drop_namespace(self, namespace: str | Identifier) -> None: self._check_endpoint(Capability.V1_DELETE_NAMESPACE) @@ -1662,7 +1662,7 @@ def drop_namespace(self, namespace: str | Identifier) -> None: except HTTPError as exc: _handle_non_200_response(exc, {404: NoSuchNamespaceError, 409: NamespaceNotEmptyError}) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]: self._check_endpoint(Capability.V1_LIST_NAMESPACES) @@ -1700,7 +1700,7 @@ def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]: return namespaces - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def load_namespace_properties(self, namespace: str | Identifier) -> Properties: self._check_endpoint(Capability.V1_LOAD_NAMESPACE) @@ -1714,7 +1714,7 @@ def load_namespace_properties(self, namespace: str | Identifier) -> Properties: return NamespaceResponse.model_validate_json(response.text).properties - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def update_namespace_properties( self, namespace: str | Identifier, removals: set[str] | None = None, updates: Properties = EMPTY_DICT @@ -1735,7 +1735,7 @@ def update_namespace_properties( missing=parsed_response.missing, ) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def namespace_exists(self, namespace: str | Identifier) -> bool: namespace_tuple = self._check_valid_namespace_identifier(namespace) @@ -1763,7 +1763,7 @@ def namespace_exists(self, namespace: str | Identifier) -> bool: return False - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def table_exists(self, identifier: str | Identifier) -> bool: """Check if a table exists. @@ -1798,7 +1798,7 @@ def table_exists(self, identifier: str | Identifier) -> bool: return False - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def view_exists(self, identifier: str | Identifier) -> bool: """Check if a view exists. @@ -1824,7 +1824,7 @@ def view_exists(self, identifier: str | Identifier) -> bool: return False - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def register_view(self, identifier: str | Identifier, metadata_location: str) -> View: self._check_endpoint(Capability.V1_REGISTER_VIEW) @@ -1847,7 +1847,7 @@ def register_view(self, identifier: str | Identifier, metadata_location: str) -> view_response = ViewResponse.model_validate_json(response.text) return self._response_to_view(self.identifier_to_tuple(identifier), view_response) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def drop_view(self, identifier: str | Identifier) -> None: self._check_endpoint(Capability.V1_DELETE_VIEW) @@ -1859,7 +1859,7 @@ def drop_view(self, identifier: str | Identifier) -> None: except HTTPError as exc: _handle_non_200_response(exc, {404: NoSuchViewError}) - @retry(**_RETRY_ARGS) + @retry(**_RETRY_ARGS) # ty: ignore[no-matching-overload] @override def rename_view(self, from_identifier: str | Identifier, to_identifier: str | Identifier) -> None: self._check_endpoint(Capability.V1_RENAME_VIEW) diff --git a/pyiceberg/catalog/rest/auth.py b/pyiceberg/catalog/rest/auth.py index 602074282c..4e4f4b8d75 100644 --- a/pyiceberg/catalog/rest/auth.py +++ b/pyiceberg/catalog/rest/auth.py @@ -184,7 +184,7 @@ def _refresh_token(self) -> None: "The expiration time of the Token must be provided by the Server in the Access Token Response " "in `expires_in` field, or by the PyIceberg Client." ) - self._expires_at = time.monotonic() + expires_in - self.refresh_margin + self._expires_at = time.monotonic() + expires_in - self.refresh_margin # ty: ignore[invalid-assignment] def get_token(self) -> str: with self._lock: @@ -330,7 +330,7 @@ def __init__(self, auth_manager: AuthManager): """ self.auth_manager = auth_manager - def __call__(self, request: PreparedRequest) -> PreparedRequest: + def __call__(self, request: PreparedRequest) -> PreparedRequest: # ty: ignore[invalid-method-override] """ Modify the outgoing request to include the Authorization header. diff --git a/pyiceberg/catalog/sql.py b/pyiceberg/catalog/sql.py index 94c7965a88..0467f736a9 100644 --- a/pyiceberg/catalog/sql.py +++ b/pyiceberg/catalog/sql.py @@ -270,7 +270,7 @@ def create_table( ValueError: If the identifier is invalid, or no path is given to store metadata. """ - schema: Schema = self._convert_schema_if_needed( # type: ignore + schema: Schema = self._convert_schema_if_needed( # type: ignore[no-redef] schema, int(properties.get(TableProperties.FORMAT_VERSION, TableProperties.DEFAULT_FORMAT_VERSION)), # type: ignore ) @@ -389,7 +389,7 @@ def drop_table(self, identifier: str | Identifier) -> None: if type_filter is not None: stmt = stmt.where(type_filter) res = session.execute(stmt) - if res.rowcount < 1: + if res.rowcount < 1: # ty: ignore[unresolved-attribute] raise NoSuchTableError(f"Table does not exist: {namespace}.{table_name}") else: try: @@ -450,7 +450,7 @@ def rename_table(self, from_identifier: str | Identifier, to_identifier: str | I if type_filter is not None: stmt = stmt.where(type_filter) result = session.execute(stmt) - if result.rowcount < 1: + if result.rowcount < 1: # ty: ignore[unresolved-attribute] raise NoSuchTableError(f"Table does not exist: {from_table_name}") else: try: @@ -532,7 +532,7 @@ def commit_table( ) ) result = session.execute(stmt) - if result.rowcount < 1: + if result.rowcount < 1: # ty: ignore[unresolved-attribute] raise CommitFailedException(f"Table has been updated by another process: {namespace}.{table_name}") else: try: @@ -566,7 +566,7 @@ def commit_table( ) @override - def namespace_exists(self, identifier: str | Identifier) -> bool: + def namespace_exists(self, identifier: str | Identifier) -> bool: # ty: ignore[invalid-method-override] namespace_tuple = Catalog.identifier_to_tuple(identifier) namespace = Catalog.namespace_to_string(namespace_tuple, NoSuchNamespaceError) namespace_starts_with = namespace.replace("!", "!!").replace("_", "!_").replace("%", "!%") + ".%" diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index 9f8bed5eba..cd9cdddf5f 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -41,11 +41,11 @@ def catch_exception() -> Callable: # type: ignore def decorator(func: Callable) -> Callable: # type: ignore @wraps(func) - def wrapper(*args: Any, **kwargs: Any): # type: ignore + def wrapper(*args: Any, **kwargs: Any): # type: ignore[no-untyped-def] try: return func(*args, **kwargs) except Exception as e: - ctx: Context = click.get_current_context(silent=True) + ctx: Context = click.get_current_context(silent=True) # ty: ignore[invalid-assignment] _, output = _catalog_and_output(ctx) output.exception(e) ctx.exit(1) @@ -324,7 +324,7 @@ def table(ctx: Context, identifier: str, purge: bool) -> None: # noqa: F811 output.text(f"Dropped table: {identifier}{purge_message}") -@drop.command() # type: ignore +@drop.command() # type: ignore[no-redef] @click.argument("identifier") @click.pass_context @catch_exception() @@ -422,7 +422,7 @@ def set() -> None: """Set a property on tables/namespaces.""" -@set.command() # type: ignore +@set.command() # type: ignore[no-redef] @click.argument("identifier") @click.argument("property_name") @click.argument("property_value") @@ -436,7 +436,7 @@ def namespace(ctx: Context, identifier: str, property_name: str, property_value: output.text(f"Updated {property_name} on {identifier}") -@set.command() # type: ignore +@set.command() # type: ignore[no-redef] @click.argument("identifier") @click.argument("property_name") @click.argument("property_value") @@ -458,7 +458,7 @@ def remove() -> None: """Remove a property from tables/namespaces.""" -@remove.command() # type: ignore +@remove.command() # type: ignore[no-redef] @click.argument("identifier") @click.argument("property_name") @click.pass_context @@ -475,7 +475,7 @@ def namespace(ctx: Context, identifier: str, property_name: str) -> None: # noq raise NoSuchPropertyException(f"Property {property_name} does not exist on {identifier}") -@remove.command() # type: ignore +@remove.command() # type: ignore[no-redef] @click.argument("identifier") @click.argument("property_name") @click.pass_context diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py index cfb00abb78..df8d700ed5 100644 --- a/pyiceberg/cli/output.py +++ b/pyiceberg/cli/output.py @@ -202,7 +202,7 @@ def uuid(self, uuid: UUID | None) -> None: def version(self, version: str) -> None: Console().print(version) - def describe_refs(self, ref_details: list[tuple[str, SnapshotRefType, dict[str, str]]]) -> None: + def describe_refs(self, ref_details: list[tuple[str, SnapshotRefType, dict[str, str]]]) -> None: # ty: ignore[invalid-method-override] refs_table = RichTable(title="Snapshot Refs") refs_table.add_column("Ref") refs_table.add_column("Type") diff --git a/pyiceberg/conversions.py b/pyiceberg/conversions.py index fde1885443..f30a0fda60 100644 --- a/pyiceberg/conversions.py +++ b/pyiceberg/conversions.py @@ -323,14 +323,14 @@ def _(primitive_type: DecimalType, value: Decimal) -> bytes: return decimal_to_bytes(value) -@to_bytes.register(UnknownType) +@to_bytes.register(UnknownType) # ty: ignore[invalid-argument-type] def _(_: UnknownType, value: Any) -> None: """Return None since the unknown type has no binary representation.""" return None -@singledispatch # type: ignore -def from_bytes(primitive_type: PrimitiveType, b: bytes) -> L: # type: ignore +@singledispatch # type: ignore[arg-type] +def from_bytes(primitive_type: PrimitiveType, b: bytes) -> L: # type: ignore[type-var] """Convert bytes to a built-in python value. Args: @@ -406,8 +406,8 @@ def _(type_: UnknownType, buf: bytes) -> None: return None -@singledispatch # type: ignore -def to_json(primitive_type: PrimitiveType, val: Any) -> L: # type: ignore +@singledispatch # type: ignore[arg-type] +def to_json(primitive_type: PrimitiveType, val: Any) -> L: # type: ignore[type-var] """Convert built-in python values into JSON value types. https://iceberg.apache.org/spec/#json-single-value-serialization @@ -545,8 +545,8 @@ def _(_: GeographyType, val: bytes) -> str: ) -@singledispatch # type: ignore -def from_json(primitive_type: PrimitiveType, val: Any) -> L: # type: ignore +@singledispatch # type: ignore[arg-type] +def from_json(primitive_type: PrimitiveType, val: Any) -> L: # type: ignore[type-var] """Convert JSON value types into built-in python values. https://iceberg.apache.org/spec/#json-single-value-serialization diff --git a/pyiceberg/encryption/key_metadata.py b/pyiceberg/encryption/key_metadata.py index e6546be26f..b897621bd4 100644 --- a/pyiceberg/encryption/key_metadata.py +++ b/pyiceberg/encryption/key_metadata.py @@ -82,7 +82,7 @@ def from_bytes(cls, data: bytes) -> StandardKeyMetadata: def to_bytes(self) -> bytes: """Encode key metadata to its wire format.""" output = io.BytesIO() - encoder = BinaryEncoder(output) + encoder = BinaryEncoder(output) # ty: ignore[invalid-argument-type] encoder.write(bytes([KEY_METADATA_V1])) record = Record(self.encryption_key, self.aad_prefix, self.file_length) construct_writer(KEY_METADATA_SCHEMA_V1).write(encoder, record) diff --git a/pyiceberg/encryption/manager.py b/pyiceberg/encryption/manager.py index 73c57dfd5b..b9376cd390 100644 --- a/pyiceberg/encryption/manager.py +++ b/pyiceberg/encryption/manager.py @@ -156,7 +156,7 @@ def _find_active_kek(self) -> EncryptedKey | None: for key in self._encryption_keys.values() if key.encrypted_by_id == self._table_key_id and not _is_kek_expired(key) ] - return max(keks, key=lambda kek: _kek_created_at(kek) or 0, default=None) + return max(keks, key=lambda kek: _kek_created_at(kek) or 0, default=None) # ty: ignore[invalid-argument-type] def _unwrap_kek(self, kek: EncryptedKey) -> SecureKey: if (cached := self._kek_cache.get(kek.key_id)) is not None and time.monotonic() - cached[1] < KEK_CACHE_TTL_SECONDS: diff --git a/pyiceberg/expressions/__init__.py b/pyiceberg/expressions/__init__.py index ad916171c5..ff96d8860e 100644 --- a/pyiceberg/expressions/__init__.py +++ b/pyiceberg/expressions/__init__.py @@ -341,9 +341,9 @@ def __invert__(self) -> BooleanExpression: # De Morgan's law: not (A and B) = (not A) or (not B) return Or(~self.left, ~self.right) - def __deepcopy__(self, memo: dict[int, Any]) -> And: + def __deepcopy__(self, memo: dict[int, Any]) -> And: # ty: ignore[invalid-method-override] """Return a deep copy of the And expression.""" - return And(copy.deepcopy(self.left, memo), copy.deepcopy(self.right, memo)) + return And(copy.deepcopy(self.left, memo), copy.deepcopy(self.right, memo)) # ty: ignore[invalid-return-type] def __getnewargs__(self) -> tuple[BooleanExpression, BooleanExpression]: """Pickle the And class.""" @@ -392,9 +392,9 @@ def __invert__(self) -> BooleanExpression: # De Morgan's law: not (A or B) = (not A) and (not B) return And(~self.left, ~self.right) - def __deepcopy__(self, memo: dict[int, Any]) -> Or: + def __deepcopy__(self, memo: dict[int, Any]) -> Or: # ty: ignore[invalid-method-override] """Return a deep copy of the Or expression.""" - return Or(copy.deepcopy(self.left, memo), copy.deepcopy(self.right, memo)) + return Or(copy.deepcopy(self.left, memo), copy.deepcopy(self.right, memo)) # ty: ignore[invalid-return-type] def __getnewargs__(self) -> tuple[BooleanExpression, BooleanExpression]: """Pickle the Or class.""" @@ -438,16 +438,16 @@ def __invert__(self) -> BooleanExpression: """Transform the Expression into its negated version.""" return self.child - def __deepcopy__(self, memo: dict[int, Any]) -> Not: + def __deepcopy__(self, memo: dict[int, Any]) -> Not: # ty: ignore[invalid-method-override] """Return a deep copy of the Not expression.""" - return Not(copy.deepcopy(self.child, memo)) + return Not(copy.deepcopy(self.child, memo)) # ty: ignore[invalid-return-type] def __getnewargs__(self) -> tuple[BooleanExpression]: """Pickle the Not class.""" return (self.child,) -class AlwaysTrue(BooleanExpression, Singleton, IcebergRootModel[bool]): +class AlwaysTrue(BooleanExpression, Singleton, IcebergRootModel[bool]): # ty: ignore[invalid-method-override] """TRUE expression.""" root: bool = True @@ -465,7 +465,7 @@ def __repr__(self) -> str: return "AlwaysTrue()" -class AlwaysFalse(BooleanExpression, Singleton, IcebergRootModel[bool]): +class AlwaysFalse(BooleanExpression, Singleton, IcebergRootModel[bool]): # ty: ignore[invalid-method-override] """FALSE expression.""" root: bool = False @@ -576,7 +576,7 @@ def __new__(cls, term: BoundTerm) -> BooleanExpression: # pylint: disable=W0221 def __invert__(self) -> BoundNotNull: """Transform the Expression into its negated version.""" - return BoundNotNull(self.term) + return BoundNotNull(self.term) # ty: ignore[invalid-return-type] @property def as_unbound(self) -> type[IsNull]: @@ -591,7 +591,7 @@ def __new__(cls, term: BoundTerm) -> BooleanExpression: # pylint: disable=W0221 def __invert__(self) -> BoundIsNull: """Transform the Expression into its negated version.""" - return BoundIsNull(self.term) + return BoundIsNull(self.term) # ty: ignore[invalid-return-type] @property def as_unbound(self) -> type[NotNull]: @@ -637,7 +637,7 @@ def __new__(cls, term: BoundTerm) -> BooleanExpression: # pylint: disable=W0221 def __invert__(self) -> BoundNotNaN: """Transform the Expression into its negated version.""" - return BoundNotNaN(self.term) + return BoundNotNaN(self.term) # ty: ignore[invalid-return-type] @property def as_unbound(self) -> type[IsNaN]: @@ -653,7 +653,7 @@ def __new__(cls, term: BoundTerm) -> BooleanExpression: # pylint: disable=W0221 def __invert__(self) -> BoundIsNaN: """Transform the Expression into its negated version.""" - return BoundIsNaN(self.term) + return BoundIsNaN(self.term) # ty: ignore[invalid-return-type] @property def as_unbound(self) -> type[NotNaN]: @@ -711,7 +711,7 @@ def __init__( def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundSetPredicate: bound_term = self.term.bind(schema, case_sensitive) literal_set = self.literals - return self.as_bound(bound_term, {lit.to(bound_term.ref().field.field_type) for lit in literal_set}) # type: ignore + return self.as_bound(bound_term, {lit.to(bound_term.ref().field.field_type) for lit in literal_set}) # type: ignore[misc] def __str__(self) -> str: """Return the string representation of the SetPredicate class.""" @@ -787,7 +787,7 @@ def __new__(cls, term: BoundTerm, literals: set[LiteralValue]) -> BooleanExpress def __invert__(self) -> BoundNotIn: """Transform the Expression into its negated version.""" - return BoundNotIn(self.term, self.literals) + return BoundNotIn(self.term, self.literals) # ty: ignore[invalid-return-type] def __eq__(self, other: Any) -> bool: """Return the equality of two instances of the BoundIn class.""" @@ -814,7 +814,7 @@ def __new__( # pylint: disable=W0221 def __invert__(self) -> BoundIn: """Transform the Expression into its negated version.""" - return BoundIn(self.term, self.literals) + return BoundIn(self.term, self.literals) # ty: ignore[invalid-return-type] @property def as_unbound(self) -> type[NotIn]: @@ -841,9 +841,9 @@ def __new__( # pylint: disable=W0221 literals_set = _to_literal_set(literals) count = len(literals_set) if count == 0: - return AlwaysFalse() + return AlwaysFalse() # ty: ignore[invalid-return-type] elif count == 1: - return EqualTo(term, next(iter(literals_set))) + return EqualTo(term, next(iter(literals_set))) # ty: ignore[invalid-return-type] else: return super().__new__(cls) @@ -876,9 +876,9 @@ def __new__( # pylint: disable=W0221 literals_set = _to_literal_set(literals) count = len(literals_set) if count == 0: - return AlwaysTrue() + return AlwaysTrue() # ty: ignore[invalid-return-type] elif count == 1: - return NotEqualTo(term, next(iter(literals_set))) + return NotEqualTo(term, next(iter(literals_set))) # ty: ignore[invalid-return-type] else: return super().__new__(cls) @@ -913,20 +913,20 @@ def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundLiteralPredi if isinstance(lit, AboveMax): if isinstance(self, (LessThan, LessThanOrEqual, NotEqualTo)): - return AlwaysTrue() + return AlwaysTrue() # ty: ignore[invalid-return-type] elif isinstance(self, (GreaterThan, GreaterThanOrEqual, EqualTo)): - return AlwaysFalse() + return AlwaysFalse() # ty: ignore[invalid-return-type] else: raise ValueError(f"Cannot bind {type(self).__name__} with AboveMax literal") elif isinstance(lit, BelowMin): if isinstance(self, (GreaterThan, GreaterThanOrEqual, NotEqualTo)): - return AlwaysTrue() + return AlwaysTrue() # ty: ignore[invalid-return-type] elif isinstance(self, (LessThan, LessThanOrEqual, EqualTo)): - return AlwaysFalse() + return AlwaysFalse() # ty: ignore[invalid-return-type] else: raise ValueError(f"Cannot bind {type(self).__name__} with BelowMin literal") - return self.as_bound(bound_term, lit) # type: ignore + return self.as_bound(bound_term, lit) # type: ignore[misc] def __eq__(self, other: Any) -> bool: """Return the equality of two instances of the LiteralPredicate class.""" diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index b91e404627..d120652045 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -87,7 +87,7 @@ class Literal(IcebergRootModel[L], Generic[L], ABC): # type: ignore root: L = Field() - def __init__(self, value: L, value_type: type[L], /, **data): # type: ignore + def __init__(self, value: L, value_type: type[L], /, **data): # type: ignore[no-untyped-def] if value is None: raise TypeError("Invalid literal value: None") @@ -157,25 +157,25 @@ def literal(value: L) -> Literal[L]: LongLiteral(123) """ if isinstance(value, float): - return DoubleLiteral(value) + return DoubleLiteral(value) # ty: ignore[invalid-return-type] elif isinstance(value, bool): - return BooleanLiteral(value) + return BooleanLiteral(value) # ty: ignore[invalid-return-type] elif isinstance(value, int): return LongLiteral(value) elif isinstance(value, str): - return StringLiteral(value) + return StringLiteral(value) # ty: ignore[invalid-return-type] elif isinstance(value, UUID): - return UUIDLiteral(value.bytes) + return UUIDLiteral(value.bytes) # ty: ignore[invalid-return-type] elif isinstance(value, bytes): - return BinaryLiteral(value) + return BinaryLiteral(value) # ty: ignore[invalid-return-type] elif isinstance(value, Decimal): - return DecimalLiteral(value) + return DecimalLiteral(value) # ty: ignore[invalid-return-type] elif isinstance(value, datetime): - return TimestampLiteral(datetime_to_micros(value)) + return TimestampLiteral(datetime_to_micros(value)) # ty: ignore[invalid-return-type] elif isinstance(value, date): return DateLiteral(date_to_days(value)) elif isinstance(value, time): - return TimeLiteral(time_to_micros(value)) + return TimeLiteral(time_to_micros(value)) # ty: ignore[invalid-return-type] else: raise TypeError(f"Invalid literal value: {repr(value)}") @@ -374,7 +374,7 @@ def _(self, type_var: DecimalType) -> Literal[Decimal]: class FloatLiteral(Literal[float]): def __init__(self, value: float) -> None: super().__init__(value, float) - self._value32 = struct.unpack(" bool: """Return the equality of two instances of the FloatLiteral class.""" diff --git a/pyiceberg/expressions/parser.py b/pyiceberg/expressions/parser.py index d813bcaa07..08cf61cd30 100644 --- a/pyiceberg/expressions/parser.py +++ b/pyiceberg/expressions/parser.py @@ -278,15 +278,15 @@ def _(result: ParseResults) -> BooleanExpression: def handle_not(result: ParseResults) -> Not: - return Not(result[0][0]) + return Not(result[0][0]) # ty: ignore[invalid-return-type] def handle_and(result: ParseResults) -> And: - return And(*result[0]) + return And(*result[0]) # ty: ignore[invalid-return-type] def handle_or(result: ParseResults) -> Or: - return Or(*result[0]) + return Or(*result[0]) # ty: ignore[invalid-return-type] boolean_expression = infix_notation( diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index a8de088669..e66f3b7652 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -1033,7 +1033,7 @@ def _cast_if_necessary(self, iceberg_type: IcebergType, literal: L | set[L]) -> iceberg_type_class = type(iceberg_type) conversions = {TimestampType: micros_to_timestamp, TimestamptzType: micros_to_timestamptz} if iceberg_type_class in conversions: - conversion_function = conversions[iceberg_type_class] + conversion_function = conversions[iceberg_type_class] # ty: ignore[invalid-argument-type] if isinstance(literal, set): return {conversion_function(lit) for lit in literal} # type: ignore else: diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 49f10b1310..33a605be90 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -126,7 +126,7 @@ def __init__(self, properties: Properties) -> None: self._session = requests.Session() def __call__(self, request: "AWSRequest", operation_name: str | None = None, **_: Any) -> None: - signer_url = self.properties.get(S3_SIGNER_URI, self.properties[URI]).rstrip("/") # type: ignore + signer_url = self.properties.get(S3_SIGNER_URI, self.properties[URI]).rstrip("/") # type: ignore[union-attr] signer_endpoint = self.properties.get(S3_SIGNER_ENDPOINT, S3_SIGNER_ENDPOINT_DEFAULT) signer_headers: dict[str, str] = {} @@ -299,7 +299,7 @@ async def get_token(self, *scopes: str, **kwargs: Any) -> AccessToken: if token := properties.get(ADLS_TOKEN): credential = StaticTokenCredential(token) else: - credential = properties.get(ADLS_CREDENTIAL) # type: ignore + credential = properties.get(ADLS_CREDENTIAL) # type: ignore[assignment] return AzureBlobFileSystem( connection_string=properties.get(ADLS_CONNECTION_STRING), diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 1dd816d5d3..34b965e8d4 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -245,12 +245,12 @@ def _import_retry_strategy(impl: str) -> S3RetryStrategy | None: class UnsupportedPyArrowTypeException(Exception): """Cannot convert PyArrow type to corresponding Iceberg type.""" - def __init__(self, field: pa.Field, *args: Any): + def __init__(self, field: pa.Field, *args: Any): # ty: ignore[missing-type-argument] self.field = field super().__init__(*args) -class PyArrowLocalFileSystem(pyarrow.fs.LocalFileSystem): +class PyArrowLocalFileSystem(pyarrow.fs.LocalFileSystem): # ty: ignore[possibly-missing-submodule] def open_output_stream(self, path: str, *args: Any, **kwargs: Any) -> pyarrow.NativeFile: # In LocalFileSystem, parent directories must be first created before opening an output stream self.create_dir(os.path.dirname(path), recursive=True) @@ -352,7 +352,7 @@ def open(self, seekable: bool = True) -> InputStream: elif e.errno == 13 or "AWS Error [code 15]" in str(e): raise PermissionError(f"Cannot open file, access denied: {self.location}") from e raise # pragma: no cover - If some other kind of OSError, raise the raw error - return input_file + return input_file # ty: ignore[invalid-return-type] @override def create(self, overwrite: bool = False) -> OutputStream: @@ -384,7 +384,7 @@ def create(self, overwrite: bool = False) -> OutputStream: if e.errno == 13 or "AWS Error [code 15]" in str(e): raise PermissionError(f"Cannot create file, access denied: {self.location}") from e raise # pragma: no cover - If some other kind of OSError, raise the raw error - return output_file + return output_file # ty: ignore[invalid-return-type] @override def to_input_file(self) -> PyArrowFile: @@ -618,7 +618,7 @@ def _initialize_azure_fs(self, netloc: str | None = None) -> FileSystem: f"Provided: {provided_keys}, Missing: {missing_keys}" ) - return AzureFileSystem(**client_kwargs) + return AzureFileSystem(**client_kwargs) # ty: ignore[invalid-argument-type] def _initialize_hdfs_fs(self, scheme: str, netloc: str | None) -> FileSystem: from pyarrow.fs import HadoopFileSystem @@ -756,13 +756,13 @@ def __init__( self._include_field_ids = include_field_ids self._file_format = file_format - def schema(self, _: Schema, struct_result: pa.StructType) -> pa.Schema: - return pa.schema(list(struct_result), metadata=self._metadata) + def schema(self, _: Schema, struct_result: pa.StructType) -> pa.Schema: # ty: ignore[invalid-method-override] + return pa.schema(list(struct_result), metadata=self._metadata) # ty: ignore[invalid-argument-type] - def struct(self, _: StructType, field_results: builtins.list[pa.DataType]) -> pa.DataType: - return pa.struct(field_results) + def struct(self, _: StructType, field_results: builtins.list[pa.DataType]) -> pa.DataType: # ty: ignore[invalid-method-override] + return pa.struct(field_results) # ty: ignore[invalid-argument-type] - def field(self, field: NestedField, field_result: pa.DataType) -> pa.Field: + def field(self, field: NestedField, field_result: pa.DataType) -> pa.Field: # ty: ignore[invalid-method-override, missing-type-argument] metadata = {} if field.doc: metadata[PYARROW_FIELD_DOC_KEY] = field.doc @@ -790,7 +790,7 @@ def list(self, list_type: ListType, element_result: pa.DataType) -> pa.DataType: def map(self, map_type: MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: key_field = self.field(map_type.key_field, key_result) value_field = self.field(map_type.value_field, value_result) - return pa.map_(key_type=key_field, item_type=value_field) + return pa.map_(key_type=key_field, item_type=value_field) # ty: ignore[invalid-argument-type] def visit_fixed(self, fixed_type: FixedType) -> pa.DataType: return pa.binary(len(fixed_type)) @@ -804,53 +804,53 @@ def visit_decimal(self, decimal_type: DecimalType) -> pa.DataType: # https://github.com/apache/arrow/blob/598938711a8376cbfdceaf5c77ab0fd5057e6c02/cpp/src/parquet/arrow/schema.cc#L380-L392 return pa.decimal128(decimal_type.precision, decimal_type.scale) - def visit_boolean(self, _: BooleanType) -> pa.DataType: + def visit_boolean(self, _: BooleanType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.bool_() - def visit_integer(self, _: IntegerType) -> pa.DataType: + def visit_integer(self, _: IntegerType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.int32() - def visit_long(self, _: LongType) -> pa.DataType: + def visit_long(self, _: LongType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.int64() - def visit_float(self, _: FloatType) -> pa.DataType: + def visit_float(self, _: FloatType) -> pa.DataType: # ty: ignore[invalid-method-override] # 32-bit IEEE 754 floating point return pa.float32() - def visit_double(self, _: DoubleType) -> pa.DataType: + def visit_double(self, _: DoubleType) -> pa.DataType: # ty: ignore[invalid-method-override] # 64-bit IEEE 754 floating point return pa.float64() - def visit_date(self, _: DateType) -> pa.DataType: + def visit_date(self, _: DateType) -> pa.DataType: # ty: ignore[invalid-method-override] # Date encoded as an int return pa.date32() - def visit_time(self, _: TimeType) -> pa.DataType: + def visit_time(self, _: TimeType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.time64("us") - def visit_timestamp(self, _: TimestampType) -> pa.DataType: + def visit_timestamp(self, _: TimestampType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.timestamp(unit="us") - def visit_timestamp_ns(self, _: TimestampNanoType) -> pa.DataType: + def visit_timestamp_ns(self, _: TimestampNanoType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.timestamp(unit="ns") - def visit_timestamptz(self, _: TimestamptzType) -> pa.DataType: + def visit_timestamptz(self, _: TimestamptzType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.timestamp(unit="us", tz="UTC") - def visit_timestamptz_ns(self, _: TimestamptzNanoType) -> pa.DataType: + def visit_timestamptz_ns(self, _: TimestamptzNanoType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.timestamp(unit="ns", tz="UTC") - def visit_string(self, _: StringType) -> pa.DataType: + def visit_string(self, _: StringType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.large_string() - def visit_uuid(self, _: UUIDType) -> pa.DataType: + def visit_uuid(self, _: UUIDType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.uuid() - def visit_unknown(self, _: UnknownType) -> pa.DataType: + def visit_unknown(self, _: UnknownType) -> pa.DataType: # ty: ignore[invalid-method-override] """Type `UnknownType` can be promoted to any primitive type in V3+ tables per the Iceberg spec.""" return pa.null() - def visit_binary(self, _: BinaryType) -> pa.DataType: + def visit_binary(self, _: BinaryType) -> pa.DataType: # ty: ignore[invalid-method-override] return pa.large_binary() def visit_geometry(self, geometry_type: GeometryType) -> pa.DataType: @@ -885,10 +885,10 @@ def visit_geography(self, geography_type: GeographyType) -> pa.DataType: return pa.large_binary() -def _convert_scalar(value: Any, iceberg_type: IcebergType) -> pa.scalar: +def _convert_scalar(value: Any, iceberg_type: IcebergType) -> pa.scalar: # ty: ignore[invalid-type-form] if not isinstance(iceberg_type, PrimitiveType): raise ValueError(f"Expected primitive type, got: {iceberg_type}") - return pa.scalar(value=value, type=schema_to_pyarrow(iceberg_type)) + return pa.scalar(value=value, type=schema_to_pyarrow(iceberg_type)) # ty: ignore[no-matching-overload] class _ConvertToArrowExpression(BoundBooleanExpressionVisitor[pc.Expression]): @@ -925,26 +925,26 @@ def _get_field_name(self, term: BoundTerm) -> str | tuple[str, ...]: return term.ref().field.name def visit_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression: - pyarrow_literals = pa.array(literals, type=schema_to_pyarrow(term.ref().field.field_type)) + pyarrow_literals = pa.array(literals, type=schema_to_pyarrow(term.ref().field.field_type)) # ty: ignore[no-matching-overload] return pc.field(self._get_field_name(term)).isin(pyarrow_literals) def visit_not_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression: - pyarrow_literals = pa.array(literals, type=schema_to_pyarrow(term.ref().field.field_type)) + pyarrow_literals = pa.array(literals, type=schema_to_pyarrow(term.ref().field.field_type)) # ty: ignore[no-matching-overload] return ~pc.field(self._get_field_name(term)).isin(pyarrow_literals) def visit_is_nan(self, term: BoundTerm) -> pc.Expression: ref = pc.field(self._get_field_name(term)) - return pc.is_nan(ref) + return pc.is_nan(ref) # ty: ignore[invalid-return-type] def visit_not_nan(self, term: BoundTerm) -> pc.Expression: ref = pc.field(self._get_field_name(term)) - return ~pc.is_nan(ref) + return ~pc.is_nan(ref) # ty: ignore[unsupported-operator] def visit_is_null(self, term: BoundTerm) -> pc.Expression: return pc.field(self._get_field_name(term)).is_null(nan_is_null=False) def visit_not_null(self, term: BoundTerm) -> pc.Expression: - return pc.field(self._get_field_name(term)).is_valid() + return pc.field(self._get_field_name(term)).is_valid() # ty: ignore[invalid-return-type] def visit_equal(self, term: BoundTerm, literal: Literal[Any]) -> pc.Expression: return pc.field(self._get_field_name(term)) == _convert_scalar(literal.value, term.ref().field.field_type) @@ -965,10 +965,10 @@ def visit_less_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> pc return pc.field(self._get_field_name(term)) <= _convert_scalar(literal.value, term.ref().field.field_type) def visit_starts_with(self, term: BoundTerm, literal: Literal[Any]) -> pc.Expression: - return pc.starts_with(pc.field(self._get_field_name(term)), literal.value) + return pc.starts_with(pc.field(self._get_field_name(term)), literal.value) # ty: ignore[invalid-return-type] def visit_not_starts_with(self, term: BoundTerm, literal: Literal[Any]) -> pc.Expression: - return ~pc.starts_with(pc.field(self._get_field_name(term)), literal.value) + return ~pc.starts_with(pc.field(self._get_field_name(term)), literal.value) # ty: ignore[unsupported-operator] def visit_true(self) -> pc.Expression: return pc.scalar(True) @@ -1149,7 +1149,7 @@ def _expression_to_complementary_pyarrow(expr: BooleanExpression, schema: Schema @lru_cache def _get_file_format(file_format: FileFormat, **kwargs: dict[str, Any]) -> ds.FileFormat: if file_format == FileFormat.PARQUET: - return ds.ParquetFileFormat(**kwargs) + return ds.ParquetFileFormat(**kwargs) # ty: ignore[invalid-argument-type] elif file_format == FileFormat.ORC: # ORC doesn't support pre_buffer and buffer_size parameters orc_kwargs = {k: v for k, v in kwargs.items() if k not in ["pre_buffer", "buffer_size"]} @@ -1158,21 +1158,21 @@ def _get_file_format(file_format: FileFormat, **kwargs: dict[str, Any]) -> ds.Fi raise ValueError(f"Unsupported file format: {file_format}") -def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]: +def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]: # ty: ignore[missing-type-argument] if data_file.file_format == FileFormat.PARQUET: with io.new_input(data_file.file_path).open() as fi: delete_fragment = _get_file_format( data_file.file_format, dictionary_columns=("file_path",), pre_buffer=True, buffer_size=ONE_MEGABYTE - ).make_fragment(fi) + ).make_fragment(fi) # ty: ignore[invalid-argument-type] table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table() table = table.unify_dictionaries() return { file.as_py(): table.filter(pc.field("file_path") == file).column("pos") - for file in table.column("file_path").chunks[0].dictionary + for file in table.column("file_path").chunks[0].dictionary # ty: ignore[unresolved-attribute] } elif data_file.file_format == FileFormat.ORC: with io.new_input(data_file.file_path).open() as fi: - delete_fragment = _get_file_format(data_file.file_format).make_fragment(fi) + delete_fragment = _get_file_format(data_file.file_format).make_fragment(fi) # ty: ignore[invalid-argument-type] table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table() # For ORC, file_path columns are not dictionary-encoded, so we use unique() directly return { @@ -1185,7 +1185,7 @@ def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray] raise ValueError(f"Delete file format not supported: {data_file.file_format}") -def _combine_positional_deletes(positional_deletes: list[pa.ChunkedArray], start_index: int, end_index: int) -> pa.Array: +def _combine_positional_deletes(positional_deletes: list[pa.ChunkedArray], start_index: int, end_index: int) -> pa.Array: # ty: ignore[missing-type-argument] if len(positional_deletes) == 1: all_chunks = positional_deletes[0] else: @@ -1200,7 +1200,7 @@ def _combine_positional_deletes(positional_deletes: list[pa.ChunkedArray], start result = pc.filter(full_range, pc.invert(pc.is_in(full_range, value_set=all_chunks))) # Subtract the start_index from each element in the result - return pc.subtract(result, pa.scalar(start_index)) + return pc.subtract(result, pa.scalar(start_index)) # ty: ignore[invalid-return-type] def pyarrow_to_schema( @@ -1275,15 +1275,15 @@ def _(obj: pa.StructType, visitor: PyArrowSchemaVisitor[T]) -> T: @visit_pyarrow.register(pa.ListType) @visit_pyarrow.register(pa.FixedSizeListType) @visit_pyarrow.register(pa.LargeListType) -def _(obj: pa.ListType | pa.LargeListType | pa.FixedSizeListType, visitor: PyArrowSchemaVisitor[T]) -> T: +def _(obj: pa.ListType | pa.LargeListType | pa.FixedSizeListType, visitor: PyArrowSchemaVisitor[T]) -> T: # ty: ignore[missing-type-argument] visitor.before_list_element(obj.value_field) result = visit_pyarrow(obj.value_type, visitor) visitor.after_list_element(obj.value_field) - return visitor.list(obj, result) + return visitor.list(obj, result) # ty: ignore[invalid-argument-type] @visit_pyarrow.register(pa.MapType) -def _(obj: pa.MapType, visitor: PyArrowSchemaVisitor[T]) -> T: +def _(obj: pa.MapType, visitor: PyArrowSchemaVisitor[T]) -> T: # ty: ignore[missing-type-argument] visitor.before_map_key(obj.key_field) key_result = visit_pyarrow(obj.key_type, visitor) visitor.after_map_key(obj.key_field) @@ -1296,7 +1296,7 @@ def _(obj: pa.MapType, visitor: PyArrowSchemaVisitor[T]) -> T: @visit_pyarrow.register(pa.DictionaryType) -def _(obj: pa.DictionaryType, visitor: PyArrowSchemaVisitor[T]) -> T: +def _(obj: pa.DictionaryType, visitor: PyArrowSchemaVisitor[T]) -> T: # ty: ignore[missing-type-argument] # Parquet has no dictionary type. dictionary-encoding is handled # as an encoding detail, not as a separate type. # We will follow this approach in determining the Iceberg Type, @@ -1306,7 +1306,7 @@ def _(obj: pa.DictionaryType, visitor: PyArrowSchemaVisitor[T]) -> T: @visit_pyarrow.register(pa.Field) -def _(obj: pa.Field, visitor: PyArrowSchemaVisitor[T]) -> T: +def _(obj: pa.Field, visitor: PyArrowSchemaVisitor[T]) -> T: # ty: ignore[missing-type-argument] field_type = obj.type visitor.before_field(obj) @@ -1327,28 +1327,28 @@ def _(obj: pa.DataType, visitor: PyArrowSchemaVisitor[T]) -> T: class PyArrowSchemaVisitor(Generic[T], ABC): - def before_field(self, field: pa.Field) -> None: + def before_field(self, field: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately before visiting a field.""" - def after_field(self, field: pa.Field) -> None: + def after_field(self, field: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately after visiting a field.""" - def before_list_element(self, element: pa.Field) -> None: + def before_list_element(self, element: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately before visiting an element within a ListType.""" - def after_list_element(self, element: pa.Field) -> None: + def after_list_element(self, element: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately after visiting an element within a ListType.""" - def before_map_key(self, key: pa.Field) -> None: + def before_map_key(self, key: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately before visiting a key within a MapType.""" - def after_map_key(self, key: pa.Field) -> None: + def after_map_key(self, key: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately after visiting a key within a MapType.""" - def before_map_value(self, value: pa.Field) -> None: + def before_map_value(self, value: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately before visiting a value within a MapType.""" - def after_map_value(self, value: pa.Field) -> None: + def after_map_value(self, value: pa.Field) -> None: # ty: ignore[missing-type-argument] """Override this method to perform an action immediately after visiting a value within a MapType.""" @abstractmethod @@ -1360,15 +1360,15 @@ def struct(self, struct: pa.StructType, field_results: builtins.list[T]) -> T: """Visit a struct.""" @abstractmethod - def field(self, field: pa.Field, field_result: T) -> T: + def field(self, field: pa.Field, field_result: T) -> T: # ty: ignore[missing-type-argument] """Visit a field.""" @abstractmethod - def list(self, list_type: pa.ListType, element_result: T) -> T: + def list(self, list_type: pa.ListType, element_result: T) -> T: # ty: ignore[missing-type-argument] """Visit a list.""" @abstractmethod - def map(self, map_type: pa.MapType, key_result: T, value_result: T) -> T: + def map(self, map_type: pa.MapType, key_result: T, value_result: T) -> T: # ty: ignore[missing-type-argument] """Visit a map.""" @abstractmethod @@ -1376,7 +1376,7 @@ def primitive(self, primitive: pa.DataType) -> T: """Visit a primitive type.""" -def _get_field_id(field: pa.Field) -> int | None: +def _get_field_id(field: pa.Field) -> int | None: # ty: ignore[missing-type-argument] """Return the Iceberg field ID from Parquet or ORC metadata if available.""" if field.metadata: # Try Parquet field ID first @@ -1397,15 +1397,15 @@ def schema(self, schema: pa.Schema, struct_result: bool) -> bool: def struct(self, struct: pa.StructType, field_results: builtins.list[bool]) -> bool: return all(field_results) - def field(self, field: pa.Field, field_result: bool) -> bool: + def field(self, field: pa.Field, field_result: bool) -> bool: # ty: ignore[missing-type-argument] return all([_get_field_id(field) is not None, field_result]) - def list(self, list_type: pa.ListType, element_result: bool) -> bool: + def list(self, list_type: pa.ListType, element_result: bool) -> bool: # ty: ignore[missing-type-argument] element_field = list_type.value_field element_id = _get_field_id(element_field) return element_result and element_id is not None - def map(self, map_type: pa.MapType, key_result: bool, value_result: bool) -> bool: + def map(self, map_type: pa.MapType, key_result: bool, value_result: bool) -> bool: # ty: ignore[missing-type-argument] key_field = map_type.key_field key_id = _get_field_id(key_field) value_field = map_type.item_field @@ -1428,32 +1428,32 @@ def __init__( self._downcast_ns_timestamp_to_us = downcast_ns_timestamp_to_us self._format_version = format_version - def _field_id(self, field: pa.Field) -> int: + def _field_id(self, field: pa.Field) -> int: # ty: ignore[missing-type-argument] if (field_id := _get_field_id(field)) is not None: return field_id else: raise ValueError(f"Cannot convert {field} to Iceberg Field as field_id is empty.") - def schema(self, schema: pa.Schema, struct_result: StructType) -> Schema: + def schema(self, schema: pa.Schema, struct_result: StructType) -> Schema: # ty: ignore[invalid-method-override] return Schema(*struct_result.fields) - def struct(self, struct: pa.StructType, field_results: builtins.list[NestedField]) -> StructType: + def struct(self, struct: pa.StructType, field_results: builtins.list[NestedField]) -> StructType: # ty: ignore[invalid-method-override] return StructType(*field_results) - def field(self, field: pa.Field, field_result: IcebergType) -> NestedField: + def field(self, field: pa.Field, field_result: IcebergType) -> NestedField: # ty: ignore[invalid-method-override, missing-type-argument] field_id = self._field_id(field) field_doc = doc_str.decode() if (field.metadata and (doc_str := field.metadata.get(PYARROW_FIELD_DOC_KEY))) else None field_type = field_result return NestedField(field_id, field.name, field_type, required=not field.nullable, doc=field_doc) - def list(self, list_type: pa.ListType, element_result: IcebergType) -> ListType: + def list(self, list_type: pa.ListType, element_result: IcebergType) -> ListType: # ty: ignore[invalid-method-override, missing-type-argument] element_field = list_type.value_field self._field_names.append(LIST_ELEMENT_NAME) element_id = self._field_id(element_field) self._field_names.pop() return ListType(element_id, element_result, element_required=not element_field.nullable) - def map(self, map_type: pa.MapType, key_result: IcebergType, value_result: IcebergType) -> MapType: + def map(self, map_type: pa.MapType, key_result: IcebergType, value_result: IcebergType) -> MapType: # ty: ignore[invalid-method-override, missing-type-argument] key_field = map_type.key_field self._field_names.append(MAP_KEY_NAME) key_id = self._field_id(key_field) @@ -1540,48 +1540,48 @@ def primitive(self, primitive: pa.DataType) -> PrimitiveType: raise TypeError(f"Unsupported type: {primitive}") - def before_field(self, field: pa.Field) -> None: + def before_field(self, field: pa.Field) -> None: # ty: ignore[missing-type-argument] self._field_names.append(field.name) - def after_field(self, field: pa.Field) -> None: + def after_field(self, field: pa.Field) -> None: # ty: ignore[missing-type-argument] self._field_names.pop() - def before_list_element(self, element: pa.Field) -> None: + def before_list_element(self, element: pa.Field) -> None: # ty: ignore[missing-type-argument] self._field_names.append(LIST_ELEMENT_NAME) - def after_list_element(self, element: pa.Field) -> None: + def after_list_element(self, element: pa.Field) -> None: # ty: ignore[missing-type-argument] self._field_names.pop() - def before_map_key(self, key: pa.Field) -> None: + def before_map_key(self, key: pa.Field) -> None: # ty: ignore[missing-type-argument] self._field_names.append(MAP_KEY_NAME) - def after_map_key(self, element: pa.Field) -> None: + def after_map_key(self, element: pa.Field) -> None: # ty: ignore[invalid-method-override, missing-type-argument] self._field_names.pop() - def before_map_value(self, value: pa.Field) -> None: + def before_map_value(self, value: pa.Field) -> None: # ty: ignore[missing-type-argument] self._field_names.append(MAP_VALUE_NAME) - def after_map_value(self, element: pa.Field) -> None: + def after_map_value(self, element: pa.Field) -> None: # ty: ignore[invalid-method-override, missing-type-argument] self._field_names.pop() class _ConvertToLargeTypes(PyArrowSchemaVisitor[IcebergType | pa.Schema]): - def schema(self, schema: pa.Schema, struct_result: pa.StructType) -> pa.Schema: + def schema(self, schema: pa.Schema, struct_result: pa.StructType) -> pa.Schema: # ty: ignore[invalid-method-override] return pa.schema(struct_result) - def struct(self, struct: pa.StructType, field_results: builtins.list[pa.Field]) -> pa.StructType: + def struct(self, struct: pa.StructType, field_results: builtins.list[pa.Field]) -> pa.StructType: # ty: ignore[invalid-method-override, missing-type-argument] return pa.struct(field_results) - def field(self, field: pa.Field, field_result: pa.DataType) -> pa.Field: + def field(self, field: pa.Field, field_result: pa.DataType) -> pa.Field: # ty: ignore[invalid-method-override, missing-type-argument] return field.with_type(field_result) - def list(self, list_type: pa.ListType, element_result: pa.DataType) -> pa.DataType: + def list(self, list_type: pa.ListType, element_result: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override, missing-type-argument] return pa.large_list(element_result) - def map(self, map_type: pa.MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: + def map(self, map_type: pa.MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override, missing-type-argument] return pa.map_(key_result, value_result) - def primitive(self, primitive: pa.DataType) -> pa.DataType: + def primitive(self, primitive: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override] if primitive == pa.string(): return pa.large_string() elif primitive == pa.binary(): @@ -1590,22 +1590,22 @@ def primitive(self, primitive: pa.DataType) -> pa.DataType: class _ConvertToSmallTypes(PyArrowSchemaVisitor[IcebergType | pa.Schema]): - def schema(self, schema: pa.Schema, struct_result: pa.StructType) -> pa.Schema: + def schema(self, schema: pa.Schema, struct_result: pa.StructType) -> pa.Schema: # ty: ignore[invalid-method-override] return pa.schema(struct_result) - def struct(self, struct: pa.StructType, field_results: builtins.list[pa.Field]) -> pa.StructType: + def struct(self, struct: pa.StructType, field_results: builtins.list[pa.Field]) -> pa.StructType: # ty: ignore[invalid-method-override, missing-type-argument] return pa.struct(field_results) - def field(self, field: pa.Field, field_result: pa.DataType) -> pa.Field: + def field(self, field: pa.Field, field_result: pa.DataType) -> pa.Field: # ty: ignore[invalid-method-override, missing-type-argument] return field.with_type(field_result) - def list(self, list_type: pa.ListType, element_result: pa.DataType) -> pa.DataType: + def list(self, list_type: pa.ListType, element_result: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override, missing-type-argument] return pa.list_(element_result) - def map(self, map_type: pa.MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: + def map(self, map_type: pa.MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override, missing-type-argument] return pa.map_(key_result, value_result) - def primitive(self, primitive: pa.DataType) -> pa.DataType: + def primitive(self, primitive: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override] if primitive == pa.large_string(): return pa.string() elif primitive == pa.large_binary(): @@ -1619,16 +1619,16 @@ class _ConvertViewTypes(PyArrowSchemaVisitor[pa.Schema | pa.DataType]): Everything else is kept as is: list flavour, field names, nullability, metadata and dictionary encoding. """ - def schema(self, schema: pa.Schema, struct_result: pa.StructType) -> pa.Schema: - return pa.schema(struct_result, metadata=schema.metadata) + def schema(self, schema: pa.Schema, struct_result: pa.StructType) -> pa.Schema: # ty: ignore[invalid-method-override] + return pa.schema(struct_result, metadata=schema.metadata) # ty: ignore[invalid-argument-type] - def struct(self, struct: pa.StructType, field_results: builtins.list[pa.Field]) -> pa.StructType: + def struct(self, struct: pa.StructType, field_results: builtins.list[pa.Field]) -> pa.StructType: # ty: ignore[invalid-method-override, missing-type-argument] return pa.struct(field_results) - def field(self, field: pa.Field, field_result: pa.DataType) -> pa.Field: + def field(self, field: pa.Field, field_result: pa.DataType) -> pa.Field: # ty: ignore[invalid-method-override, missing-type-argument] return field.with_type(_keep_dictionary(field.type, field_result)) - def list(self, list_type: pa.ListType | pa.LargeListType | pa.FixedSizeListType, element_result: pa.DataType) -> pa.DataType: + def list(self, list_type: pa.ListType | pa.LargeListType | pa.FixedSizeListType, element_result: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override, missing-type-argument] value_field = list_type.value_field.with_type(_keep_dictionary(list_type.value_type, element_result)) if isinstance(list_type, pa.FixedSizeListType): return pa.list_(value_field, list_type.list_size) @@ -1636,8 +1636,8 @@ def list(self, list_type: pa.ListType | pa.LargeListType | pa.FixedSizeListType, return pa.large_list(value_field) return pa.list_(value_field) - def map(self, map_type: pa.MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: - return pa.map_( + def map(self, map_type: pa.MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: # ty: ignore[invalid-method-override, missing-type-argument] + return pa.map_( # ty: ignore[no-matching-overload] map_type.key_field.with_type(_keep_dictionary(map_type.key_type, key_result)), map_type.item_field.with_type(_keep_dictionary(map_type.item_type, value_result)), keys_sorted=map_type.keys_sorted, @@ -1654,7 +1654,7 @@ def primitive(self, primitive: pa.DataType) -> pa.DataType: def _keep_dictionary(original: pa.DataType, result: pa.DataType) -> pa.DataType: """Re-apply dictionary encoding that visit_pyarrow strips before visiting the value type.""" if isinstance(original, pa.DictionaryType): - return pa.dictionary(original.index_type, result, ordered=original.ordered) + return pa.dictionary(original.index_type, result, ordered=original.ordered) # ty: ignore[invalid-argument-type] return result @@ -1669,7 +1669,7 @@ class _ConvertToIcebergWithoutIDs(_ConvertToIceberg): creating a new Iceberg table. """ - def _field_id(self, field: pa.Field) -> int: + def _field_id(self, field: pa.Field) -> int: # ty: ignore[missing-type-argument] return -1 @@ -1704,7 +1704,7 @@ def _task_to_record_batches( projected_schema: Schema, table_schema: Schema, projected_field_ids: set[int], - positional_deletes: list[ChunkedArray] | None, + positional_deletes: list[ChunkedArray] | None, # ty: ignore[missing-type-argument] case_sensitive: bool, name_mapping: NameMapping | None = None, partition_spec: PartitionSpec | None = None, @@ -1717,7 +1717,7 @@ def _task_to_record_batches( format_kwargs["dictionary_columns"] = dictionary_columns arrow_format = _get_file_format(task.file.file_format, **format_kwargs) with io.new_input(task.file.file_path).open() as fin: - fragment = arrow_format.make_fragment(fin) + fragment = arrow_format.make_fragment(fin) # ty: ignore[invalid-argument-type] physical_schema = fragment.physical_schema # For V1 and V2, we only support Timestamp 'us' in Iceberg Schema, @@ -1799,8 +1799,8 @@ class _DeleteFileKey: content_size_in_bytes: int | None -def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[str, list[ChunkedArray]]: - deletes_per_file: dict[str, list[ChunkedArray]] = {} +def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[str, list[ChunkedArray]]: # ty: ignore[missing-type-argument] + deletes_per_file: dict[str, list[ChunkedArray]] = {} # ty: ignore[missing-type-argument] # DataFile equality uses only the path; DVs from different tasks may share a file. unique_deletes: dict[_DeleteFileKey, DataFile] = {} for task in tasks: @@ -1809,7 +1809,7 @@ def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[st unique_deletes.setdefault(key, delete_file) if len(unique_deletes) > 0: executor = ExecutorFactory.get_or_create() - deletes_per_files: Iterator[dict[str, ChunkedArray]] = executor.map( + deletes_per_files: Iterator[dict[str, ChunkedArray]] = executor.map( # ty: ignore[missing-type-argument] lambda args: _read_deletes(*args), [(io, delete_file) for delete_file in unique_deletes.values()], ) @@ -1955,7 +1955,9 @@ def batches_for_task(task: FileScanTask) -> list[pa.RecordBatch]: break def _record_batches_from_scan_tasks_and_deletes( - self, tasks: Iterable[FileScanTask], deletes_per_file: dict[str, list[ChunkedArray]] + self, + tasks: Iterable[FileScanTask], + deletes_per_file: dict[str, list[ChunkedArray]], # ty: ignore[missing-type-argument] ) -> Iterator[pa.RecordBatch]: total_row_count = 0 for task in tasks: @@ -2013,7 +2015,7 @@ def _to_requested_schema( return pa.RecordBatch.from_struct_array(struct_array) -class ArrowProjectionVisitor(SchemaWithPartnerVisitor[pa.Array, pa.Array | None]): +class ArrowProjectionVisitor(SchemaWithPartnerVisitor[pa.Array, pa.Array | None]): # ty: ignore[missing-type-argument] _file_schema: Schema _include_field_ids: bool _downcast_ns_timestamp_to_us: bool @@ -2041,7 +2043,7 @@ def __init__( self._allow_timestamp_tz_mismatch = allow_timestamp_tz_mismatch self._format_model = format_model - def _cast_if_needed(self, field: NestedField, values: pa.Array) -> pa.Array: + def _cast_if_needed(self, field: NestedField, values: pa.Array) -> pa.Array: # ty: ignore[missing-type-argument] file_field = self._file_schema.find_field(field.field_id) if field.field_type.is_primitive: @@ -2051,29 +2053,29 @@ def _cast_if_needed(self, field: NestedField, values: pa.Array) -> pa.Array: self._allow_timestamp_tz_mismatch and values.type.tz in UTC_ALIASES ) if ( - pa.types.is_timestamp(target_type) - and not target_type.tz + pa.types.is_timestamp(target_type) # ty: ignore[invalid-argument-type] + and not target_type.tz # ty: ignore[unresolved-attribute] and pa.types.is_timestamp(values.type) and source_tz_compatible ): # Downcasting of nanoseconds to microseconds - if target_type.unit == "us" and values.type.unit == "ns" and self._downcast_ns_timestamp_to_us: - return values.cast(target_type, safe=False) - elif target_type.unit == "us" and values.type.unit in {"s", "ms", "us"}: - return values.cast(target_type) + if target_type.unit == "us" and values.type.unit == "ns" and self._downcast_ns_timestamp_to_us: # ty: ignore[unresolved-attribute] + return values.cast(target_type, safe=False) # ty: ignore[invalid-argument-type] + elif target_type.unit == "us" and values.type.unit in {"s", "ms", "us"}: # ty: ignore[unresolved-attribute] + return values.cast(target_type) # ty: ignore[invalid-argument-type] raise ValueError(f"Unsupported schema projection from {values.type} to {target_type}") elif field.field_type == TimestamptzType(): if ( - pa.types.is_timestamp(target_type) - and target_type.tz == "UTC" + pa.types.is_timestamp(target_type) # ty: ignore[invalid-argument-type] + and target_type.tz == "UTC" # ty: ignore[unresolved-attribute] and pa.types.is_timestamp(values.type) and (values.type.tz in UTC_ALIASES or values.type.tz is None) ): # Downcasting of nanoseconds to microseconds - if target_type.unit == "us" and values.type.unit == "ns" and self._downcast_ns_timestamp_to_us: - return values.cast(target_type, safe=False) - elif target_type.unit == "us" and values.type.unit in {"s", "ms", "us"}: - return values.cast(target_type) + if target_type.unit == "us" and values.type.unit == "ns" and self._downcast_ns_timestamp_to_us: # ty: ignore[unresolved-attribute] + return values.cast(target_type, safe=False) # ty: ignore[invalid-argument-type] + elif target_type.unit == "us" and values.type.unit in {"s", "ms", "us"}: # ty: ignore[unresolved-attribute] + return values.cast(target_type) # ty: ignore[invalid-argument-type] raise ValueError(f"Unsupported schema projection from {values.type} to {target_type}") elif isinstance(field.field_type, (IntegerType, LongType)): # Cast smaller integer types to target type for cross-platform compatibility @@ -2081,28 +2083,28 @@ def _cast_if_needed(self, field: NestedField, values: pa.Array) -> pa.Array: # Narrowing conversions fall through to promote() handling below if pa.types.is_integer(values.type): source_width = values.type.bit_width - target_width = target_type.bit_width + target_width = target_type.bit_width # ty: ignore[unresolved-attribute] if source_width < target_width: - return values.cast(target_type) + return values.cast(target_type) # ty: ignore[invalid-argument-type] elif isinstance(field.field_type, (FloatType, DoubleType)): # Cast smaller float types to target type for cross-platform compatibility # Only allow widening conversions (smaller bit width to larger), e.g. float16 -> float32 # Narrowing conversions fall through to promote() handling below if pa.types.is_floating(values.type): source_width = values.type.bit_width - target_width = target_type.bit_width + target_width = target_type.bit_width # ty: ignore[unresolved-attribute] if source_width < target_width: - return values.cast(target_type) + return values.cast(target_type) # ty: ignore[invalid-argument-type] if field.field_type != file_field.field_type: target_schema = schema_to_pyarrow( promote(file_field.field_type, field.field_type), include_field_ids=self._include_field_ids ) - return values.cast(target_schema) + return values.cast(target_schema) # ty: ignore[invalid-argument-type] return values - def _construct_field(self, field: NestedField, arrow_type: pa.DataType) -> pa.Field: + def _construct_field(self, field: NestedField, arrow_type: pa.DataType) -> pa.Field: # ty: ignore[missing-type-argument] metadata: dict[bytes, bytes] = {} if field.doc: metadata[PYARROW_FIELD_DOC_KEY] = field.doc.encode() @@ -2116,16 +2118,19 @@ def _construct_field(self, field: NestedField, arrow_type: pa.DataType) -> pa.Fi metadata=metadata, ) - def schema(self, schema: Schema, schema_partner: pa.Array | None, struct_result: pa.Array | None) -> pa.Array | None: + def schema(self, schema: Schema, schema_partner: pa.Array | None, struct_result: pa.Array | None) -> pa.Array | None: # ty: ignore[missing-type-argument] return struct_result - def struct( - self, struct: StructType, struct_array: pa.Array | None, field_results: builtins.list[pa.Array | None] - ) -> pa.Array | None: + def struct( # ty: ignore[invalid-method-override] + self, + struct: StructType, + struct_array: pa.Array | None, # ty: ignore[missing-type-argument] + field_results: builtins.list[pa.Array | None], # ty: ignore[missing-type-argument] + ) -> pa.Array | None: # ty: ignore[missing-type-argument] if struct_array is None: return None - field_arrays: list[pa.Array] = [] - fields: list[pa.Field] = [] + field_arrays: list[pa.Array] = [] # ty: ignore[missing-type-argument] + fields: list[pa.Field] = [] # ty: ignore[missing-type-argument] for field, field_array in zip(struct.fields, field_results, strict=True): if field_array is not None: array = self._cast_if_needed(field, field_array) @@ -2136,46 +2141,50 @@ def struct( arrow_type = schema_to_pyarrow(field.field_type, include_field_ids=self._include_field_ids) if field.field_id in self._projected_missing_fields: projected_value = self._projected_missing_fields[field.field_id] - field_arrays.append(pa.repeat(pa.scalar(projected_value, type=arrow_type), len(struct_array))) + field_arrays.append(pa.repeat(pa.scalar(projected_value, type=arrow_type), len(struct_array))) # ty: ignore[no-matching-overload] elif field.initial_default is None: - field_arrays.append(pa.nulls(len(struct_array), type=arrow_type)) + field_arrays.append(pa.nulls(len(struct_array), type=arrow_type)) # ty: ignore[no-matching-overload] else: - field_arrays.append(pa.repeat(pa.scalar(field.initial_default, type=arrow_type), len(struct_array))) - fields.append(self._construct_field(field, arrow_type)) + field_arrays.append(pa.repeat(pa.scalar(field.initial_default, type=arrow_type), len(struct_array))) # ty: ignore[no-matching-overload] + fields.append(self._construct_field(field, arrow_type)) # ty: ignore[invalid-argument-type] else: raise ResolveError(f"Field is required, and could not be found in the file: {field}") return pa.StructArray.from_arrays( arrays=field_arrays, - fields=pa.struct(fields), + fields=pa.struct(fields), # ty: ignore[invalid-argument-type] mask=struct_array.is_null() if isinstance(struct_array, pa.StructArray) else None, ) - def field(self, field: NestedField, _: pa.Array | None, field_array: pa.Array | None) -> pa.Array | None: + def field(self, field: NestedField, _: pa.Array | None, field_array: pa.Array | None) -> pa.Array | None: # ty: ignore[invalid-method-override, missing-type-argument] return field_array - def list(self, list_type: ListType, list_array: pa.Array | None, value_array: pa.Array | None) -> pa.Array | None: + def list(self, list_type: ListType, list_array: pa.Array | None, value_array: pa.Array | None) -> pa.Array | None: # ty: ignore[invalid-method-override, missing-type-argument] if isinstance(list_array, (pa.ListArray, pa.LargeListArray, pa.FixedSizeListArray)) and value_array is not None: list_initializer = pa.large_list if isinstance(list_array, pa.LargeListArray) else pa.list_ if isinstance(value_array, pa.StructArray): # This can be removed once this has been fixed: # https://github.com/apache/arrow/issues/38809 - list_array = pa.LargeListArray.from_arrays(list_array.offsets, value_array) + list_array = pa.LargeListArray.from_arrays(list_array.offsets, value_array) # ty: ignore[invalid-argument-type, unresolved-attribute] value_array = self._cast_if_needed(list_type.element_field, value_array) arrow_field = list_initializer(self._construct_field(list_type.element_field, value_array.type)) return list_array.cast(arrow_field) else: return None - def map( - self, map_type: MapType, map_array: pa.Array | None, key_result: pa.Array | None, value_result: pa.Array | None - ) -> pa.Array | None: + def map( # ty: ignore[invalid-method-override] + self, + map_type: MapType, + map_array: pa.Array | None, # ty: ignore[missing-type-argument] + key_result: pa.Array | None, # ty: ignore[missing-type-argument] + value_result: pa.Array | None, # ty: ignore[missing-type-argument] + ) -> pa.Array | None: # ty: ignore[missing-type-argument] if isinstance(map_array, pa.MapArray) and key_result is not None and value_result is not None: key_result = self._cast_if_needed(map_type.key_field, key_result) value_result = self._cast_if_needed(map_type.value_field, value_result) arrow_field = pa.map_( - self._construct_field(map_type.key_field, key_result.type), - self._construct_field(map_type.value_field, value_result.type), + self._construct_field(map_type.key_field, key_result.type), # ty: ignore[invalid-argument-type] + self._construct_field(map_type.value_field, value_result.type), # ty: ignore[invalid-argument-type] ) if isinstance(value_result, pa.StructArray): # Arrow does not allow reordering of fields, therefore we have to copy the array :( @@ -2196,20 +2205,20 @@ def map( else: return None - def primitive(self, _: PrimitiveType, array: pa.Array | None) -> pa.Array | None: + def primitive(self, _: PrimitiveType, array: pa.Array | None) -> pa.Array | None: # ty: ignore[invalid-method-override, missing-type-argument] return array -class ArrowAccessor(PartnerAccessor[pa.Array]): +class ArrowAccessor(PartnerAccessor[pa.Array]): # ty: ignore[missing-type-argument] file_schema: Schema def __init__(self, file_schema: Schema): self.file_schema = file_schema - def schema_partner(self, partner: pa.Array | None) -> pa.Array | None: + def schema_partner(self, partner: pa.Array | None) -> pa.Array | None: # ty: ignore[missing-type-argument] return partner - def field_partner(self, partner_struct: pa.Array | None, field_id: int, _: str) -> pa.Array | None: + def field_partner(self, partner_struct: pa.Array | None, field_id: int, _: str) -> pa.Array | None: # ty: ignore[invalid-method-override, missing-type-argument] if partner_struct is not None: # use the field name from the file schema try: @@ -2228,13 +2237,13 @@ def field_partner(self, partner_struct: pa.Array | None, field_id: int, _: str) return None - def list_element_partner(self, partner_list: pa.Array | None) -> pa.Array | None: + def list_element_partner(self, partner_list: pa.Array | None) -> pa.Array | None: # ty: ignore[missing-type-argument] return partner_list.values if isinstance(partner_list, (pa.ListArray, pa.LargeListArray, pa.FixedSizeListArray)) else None - def map_key_partner(self, partner_map: pa.Array | None) -> pa.Array | None: + def map_key_partner(self, partner_map: pa.Array | None) -> pa.Array | None: # ty: ignore[missing-type-argument] return partner_map.keys if isinstance(partner_map, pa.MapArray) else None - def map_value_partner(self, partner_map: pa.Array | None) -> pa.Array | None: + def map_value_partner(self, partner_map: pa.Array | None) -> pa.Array | None: # ty: ignore[missing-type-argument] return partner_map.items if isinstance(partner_map, pa.MapArray) else None @@ -2661,8 +2670,8 @@ def data_file_statistics_from_parquet_metadata( data_offset = row_group.column(0).data_page_offset dictionary_offset = row_group.column(0).dictionary_page_offset - if row_group.column(0).has_dictionary_page and dictionary_offset < data_offset: - split_offsets.append(dictionary_offset) + if row_group.column(0).has_dictionary_page and dictionary_offset < data_offset: # ty: ignore[unsupported-operator] + split_offsets.append(dictionary_offset) # ty: ignore[invalid-argument-type] else: split_offsets.append(data_offset) @@ -2684,8 +2693,8 @@ def data_file_statistics_from_parquet_metadata( try: statistics = column.statistics - if statistics.has_null_count: - null_value_counts[field_id] = null_value_counts.get(field_id, 0) + statistics.null_count + if statistics.has_null_count: # ty: ignore[unresolved-attribute] + null_value_counts[field_id] = null_value_counts.get(field_id, 0) + statistics.null_count # ty: ignore[unresolved-attribute, unsupported-operator] if stats_col.mode == MetricsMode(MetricModeTypes.COUNTS): continue @@ -2693,26 +2702,28 @@ def data_file_statistics_from_parquet_metadata( if field_id not in col_aggs: try: col_aggs[field_id] = StatsAggregator( - stats_col.iceberg_type, statistics.physical_type, stats_col.mode.length + stats_col.iceberg_type, + statistics.physical_type, # ty: ignore[unresolved-attribute] + stats_col.mode.length, ) except ValueError as e: raise ValueError(f"{e} for column '{stats_col.column_name}'") from e - if isinstance(stats_col.iceberg_type, DecimalType) and statistics.physical_type != "FIXED_LEN_BYTE_ARRAY": + if isinstance(stats_col.iceberg_type, DecimalType) and statistics.physical_type != "FIXED_LEN_BYTE_ARRAY": # ty: ignore[unresolved-attribute] scale = stats_col.iceberg_type.scale ( - col_aggs[field_id].update_min(unscaled_to_decimal(statistics.min_raw, scale)) - if statistics.min_raw is not None + col_aggs[field_id].update_min(unscaled_to_decimal(statistics.min_raw, scale)) # ty: ignore[unresolved-attribute] + if statistics.min_raw is not None # ty: ignore[unresolved-attribute] else None ) ( - col_aggs[field_id].update_max(unscaled_to_decimal(statistics.max_raw, scale)) - if statistics.max_raw is not None + col_aggs[field_id].update_max(unscaled_to_decimal(statistics.max_raw, scale)) # ty: ignore[unresolved-attribute] + if statistics.max_raw is not None # ty: ignore[unresolved-attribute] else None ) else: - col_aggs[field_id].update_min(statistics.min) - col_aggs[field_id].update_max(statistics.max) + col_aggs[field_id].update_min(statistics.min) # ty: ignore[unresolved-attribute] + col_aggs[field_id].update_max(statistics.max) # ty: ignore[unresolved-attribute] except pyarrow.lib.ArrowNotImplementedError as e: invalidate_col.add(field_id) @@ -2956,22 +2967,22 @@ def _check_pyarrow_schema_compatible( name_mapping = requested_schema.name_mapping try: - provided_schema = pyarrow_to_schema( + provided_schema = pyarrow_to_schema( # ty: ignore[invalid-assignment] provided_schema, name_mapping=name_mapping, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version, ) except ValueError as e: - provided_schema = _pyarrow_to_schema_without_ids( + provided_schema = _pyarrow_to_schema_without_ids( # ty: ignore[invalid-assignment] provided_schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version ) - additional_names = set(provided_schema._name_to_id.keys()) - set(requested_schema._name_to_id.keys()) + additional_names = set(provided_schema._name_to_id.keys()) - set(requested_schema._name_to_id.keys()) # ty: ignore[unresolved-attribute] raise ValueError( f"PyArrow table contains more columns: {', '.join(sorted(additional_names))}. " "Update the schema first (hint, use union_by_name)." ) from e - _check_schema_compatible(requested_schema, provided_schema) + _check_schema_compatible(requested_schema, provided_schema) # ty: ignore[invalid-argument-type] def parquet_files_to_data_files(io: FileIO, table_metadata: TableMetadata, file_paths: Iterator[str]) -> Iterator[DataFile]: @@ -2983,7 +2994,7 @@ def parquet_files_to_data_files(io: FileIO, table_metadata: TableMetadata, file_ def parquet_file_to_data_file(io: FileIO, table_metadata: TableMetadata, file_path: str) -> DataFile: input_file = io.new_input(file_path) with input_file.open() as input_stream: - parquet_metadata = pq.read_metadata(input_stream) + parquet_metadata = pq.read_metadata(input_stream) # ty: ignore[invalid-argument-type] arrow_schema = parquet_metadata.schema.to_arrow_schema() @@ -3217,11 +3228,11 @@ def _partition_value_filter(partition_field_name: str, value: Any) -> pc.Express if value is None: return pc.field(partition_field_name).is_null() elif isinstance(value, float) and math.isnan(value): - return pc.is_nan(pc.field(partition_field_name)) + return pc.is_nan(pc.field(partition_field_name)) # ty: ignore[invalid-return-type] return pc.field(partition_field_name) == value -def _get_field_from_arrow_table(arrow_table: pa.Table, field_path: str) -> pa.Array: +def _get_field_from_arrow_table(arrow_table: pa.Table, field_path: str) -> pa.Array: # ty: ignore[missing-type-argument] """Get a field from an Arrow table, supporting both literal field names and nested field paths. This function handles two cases: @@ -3240,7 +3251,7 @@ def _get_field_from_arrow_table(arrow_table: pa.Table, field_path: str) -> pa.Ar """ # Try exact column name match (handles field names containing literal dots) if field_path in arrow_table.column_names: - return arrow_table[field_path] + return arrow_table[field_path] # ty: ignore[invalid-return-type] # If not found as exact name, treat as nested field path path_parts = field_path.split(".") diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index 0881273c45..f44ef7dd05 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -732,13 +732,13 @@ def update(self, value: Any) -> None: self._min = value self._max = value else: - self._max = max(self._max, value) + self._max = max(self._max, value) # ty: ignore[invalid-argument-type] self._min = min(self._min, value) def construct_partition_summaries(spec: PartitionSpec, schema: Schema, partitions: list[Record]) -> list[PartitionFieldSummary]: types = [field.field_type for field in spec.partition_type(schema).fields] - field_stats = [PartitionFieldStats(field_type) for field_type in types] + field_stats = [PartitionFieldStats(field_type) for field_type in types] # ty: ignore[invalid-argument-type] for partition_keys in partitions: for i, field_type in enumerate(types): if not isinstance(field_type, PrimitiveType): @@ -811,7 +811,7 @@ def construct_partition_summaries(spec: PartitionSpec, schema: Schema, partition class ManifestFile(Record): @classmethod def from_args(cls, _table_format_version: TableVersion = DEFAULT_READ_VERSION, **arguments: Any) -> ManifestFile: - return super()._bind(**arguments, struct=MANIFEST_LIST_FILE_SCHEMAS[_table_format_version]) + return super()._bind(**arguments, struct=MANIFEST_LIST_FILE_SCHEMAS[_table_format_version]) # ty: ignore[invalid-argument-type] @property def manifest_path(self) -> str: diff --git a/pyiceberg/partitioning.py b/pyiceberg/partitioning.py index 3d06287b6c..373bc74427 100644 --- a/pyiceberg/partitioning.py +++ b/pyiceberg/partitioning.py @@ -80,8 +80,8 @@ class PartitionField(IcebergBaseModel): source_id: int = Field(alias="source-id") field_id: int = Field(alias="field-id") - transform: Annotated[ # type: ignore - Transform, + transform: Annotated[ # type: ignore[type-arg] + Transform, # ty: ignore[missing-type-argument] BeforeValidator(parse_transform), PlainSerializer(lambda c: str(c), return_type=str), # pylint: disable=W0108 WithJsonSchema({"type": "string"}, mode="serialization"), diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index 99f983074b..c9dd167f5c 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -670,7 +670,7 @@ def visit_with_partner( @visit_with_partner.register(Schema) def _(schema: Schema, partner: P, visitor: SchemaWithPartnerVisitor[P, T], accessor: PartnerAccessor[P]) -> T: struct_partner = accessor.schema_partner(partner) - return visitor.schema(schema, partner, visit_with_partner(schema.as_struct(), struct_partner, visitor, accessor)) # type: ignore + return visitor.schema(schema, partner, visit_with_partner(schema.as_struct(), struct_partner, visitor, accessor)) # type: ignore[misc] @visit_with_partner.register(StructType) @@ -680,7 +680,7 @@ def _(struct: StructType, partner: P, visitor: SchemaWithPartnerVisitor[P, T], a field_partner = accessor.field_partner(partner, field.field_id, field.name) visitor.before_field(field, field_partner) try: - field_result = visit_with_partner(field.field_type, field_partner, visitor, accessor) # type: ignore + field_result = visit_with_partner(field.field_type, field_partner, visitor, accessor) # type: ignore[misc] field_results.append(visitor.field(field, field_partner, field_result)) finally: visitor.after_field(field, field_partner) @@ -693,7 +693,7 @@ def _(list_type: ListType, partner: P, visitor: SchemaWithPartnerVisitor[P, T], element_partner = accessor.list_element_partner(partner) visitor.before_list_element(list_type.element_field, element_partner) try: - element_result = visit_with_partner(list_type.element_type, element_partner, visitor, accessor) # type: ignore + element_result = visit_with_partner(list_type.element_type, element_partner, visitor, accessor) # type: ignore[misc] finally: visitor.after_list_element(list_type.element_field, element_partner) @@ -705,14 +705,14 @@ def _(map_type: MapType, partner: P, visitor: SchemaWithPartnerVisitor[P, T], ac key_partner = accessor.map_key_partner(partner) visitor.before_map_key(map_type.key_field, key_partner) try: - key_result = visit_with_partner(map_type.key_type, key_partner, visitor, accessor) # type: ignore + key_result = visit_with_partner(map_type.key_type, key_partner, visitor, accessor) # type: ignore[misc] finally: visitor.after_map_key(map_type.key_field, key_partner) value_partner = accessor.map_value_partner(partner) visitor.before_map_value(map_type.value_field, value_partner) try: - value_result = visit_with_partner(map_type.value_type, value_partner, visitor, accessor) # type: ignore + value_result = visit_with_partner(map_type.value_type, value_partner, visitor, accessor) # type: ignore[misc] finally: visitor.after_map_value(map_type.value_field, value_partner) return visitor.map(map_type, partner, key_result, value_result) @@ -1211,7 +1211,7 @@ def index_by_name(schema_or_type: Schema | IcebergType) -> dict[str, int]: Returns: Dict[str, int]: An index of field names to field IDs. """ - if len(schema_or_type.fields) > 0: + if len(schema_or_type.fields) > 0: # ty: ignore[unresolved-attribute] indexer = _IndexByName() visit(schema_or_type, indexer) return indexer.by_name() @@ -1330,7 +1330,7 @@ def _get_and_increment(self, current_id: int) -> int: self.old_id_to_new_id[current_id] = new_id return new_id - def schema(self, schema: Schema, struct_result: Callable[[], StructType]) -> Schema: + def schema(self, schema: Schema, struct_result: Callable[[], StructType]) -> Schema: # ty: ignore[invalid-method-override] return Schema( *struct_result().fields, identifier_field_ids=[self.old_id_to_new_id[field_id] for field_id in schema.identifier_field_ids], @@ -1468,7 +1468,7 @@ def field(self, field: NestedField, field_result: IcebergType | None) -> Iceberg ) def struct(self, struct: StructType, field_results: builtins.list[IcebergType | None]) -> IcebergType | None: - return StructType(*[field for field in field_results if field is not None]) + return StructType(*[field for field in field_results if field is not None]) # ty: ignore[invalid-argument-type] def list(self, list_type: ListType, element_result: IcebergType | None) -> IcebergType | None: return ListType(element_id=list_type.element_id, element_type=element_result, element_required=list_type.element_required) @@ -1615,7 +1615,7 @@ def _project_selected_struct(projected_field: IcebergType | None) -> StructType: if projected_field is None: return StructType() else: - return projected_field + return projected_field # ty: ignore[invalid-return-type] @staticmethod def _project_list(list_type: ListType, element_result: IcebergType) -> ListType: diff --git a/pyiceberg/serializers.py b/pyiceberg/serializers.py index 726e6b5f62..d9be90a84b 100644 --- a/pyiceberg/serializers.py +++ b/pyiceberg/serializers.py @@ -67,7 +67,7 @@ def bytes_compressor(self) -> Callable[[bytes], bytes]: class GzipCompressor(Compressor): def stream_decompressor(self, inp: InputStream) -> InputStream: - return gzip.open(inp) + return gzip.open(inp) # ty: ignore[invalid-return-type] def bytes_compressor(self) -> Callable[[bytes], bytes]: return gzip.compress diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 8d79472af5..fd3e16f284 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -1052,7 +1052,7 @@ def add_files( if self.table_metadata.name_mapping() is None: self.set_properties( - **{TableProperties.DEFAULT_NAME_MAPPING: self.table_metadata.schema().name_mapping.model_dump_json()} + **{TableProperties.DEFAULT_NAME_MAPPING: self.table_metadata.schema().name_mapping.model_dump_json()} # ty: ignore[invalid-argument-type] ) with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files: data_files = _parquet_files_to_data_files( @@ -1559,7 +1559,7 @@ def location_provider(self) -> LocationProvider: @property def last_sequence_number(self) -> int: - return self.metadata.last_sequence_number + return self.metadata.last_sequence_number # ty: ignore[unresolved-attribute] def current_snapshot(self) -> Snapshot | None: """Get the current snapshot for this table, or None if there is no current snapshot.""" @@ -2423,7 +2423,7 @@ def _to_arrow_batch_reader_via_file_scan_tasks( batches = chain([first_batch], batches) target_schema = pa.schema( [first_batch.schema.field(field.name) if field.name in dict_col_set else field for field in target_schema], - metadata=target_schema.metadata, + metadata=target_schema.metadata, # ty: ignore[invalid-argument-type] ) return pa.RecordBatchReader.from_batches(target_schema, batches).cast(target_schema) diff --git a/pyiceberg/table/deletion_vector.py b/pyiceberg/table/deletion_vector.py index 9c56af24a2..0e9bb40b7d 100644 --- a/pyiceberg/table/deletion_vector.py +++ b/pyiceberg/table/deletion_vector.py @@ -76,15 +76,15 @@ def _deserialize_bitmap(pl: bytes) -> list[BitMap]: last_key = key - return bitmaps + return bitmaps # ty: ignore[invalid-return-type] @staticmethod - def _bitmaps_to_chunked_array(bitmaps: list[BitMap]) -> "pa.ChunkedArray": + def _bitmaps_to_chunked_array(bitmaps: list[BitMap]) -> "pa.ChunkedArray": # ty: ignore[missing-type-argument] import pyarrow as pa return pa.chunked_array([(key_pos << 32) + pos for pos in bitmap] for key_pos, bitmap in enumerate(bitmaps)) - def to_vector(self) -> "pa.ChunkedArray": + def to_vector(self) -> "pa.ChunkedArray": # ty: ignore[missing-type-argument] return self._bitmaps_to_chunked_array(self._bitmaps) diff --git a/pyiceberg/table/inspect.py b/pyiceberg/table/inspect.py index e580b66dc1..628eca745c 100644 --- a/pyiceberg/table/inspect.py +++ b/pyiceberg/table/inspect.py @@ -183,14 +183,14 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: pa.field("value_count", pa.int64(), nullable=True), pa.field("null_value_count", pa.int64(), nullable=True), pa.field("nan_value_count", pa.int64(), nullable=True), - pa.field("lower_bound", pa_bound_type, nullable=True), - pa.field("upper_bound", pa_bound_type, nullable=True), + pa.field("lower_bound", pa_bound_type, nullable=True), # ty: ignore[invalid-argument-type] + pa.field("upper_bound", pa_bound_type, nullable=True), # ty: ignore[invalid-argument-type] ] ) for field in self.tbl.metadata.schema().fields: readable_metrics_struct.append( - pa.field(schema.find_column_name(field.field_id), _readable_metrics_struct(field.field_type), nullable=False) + pa.field(schema.find_column_name(field.field_id), _readable_metrics_struct(field.field_type), nullable=False) # ty: ignore[invalid-argument-type] ) partition_record = self.tbl.metadata.specs_struct() @@ -209,7 +209,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: pa.field("content", pa.int8(), nullable=False), pa.field("file_path", pa.string(), nullable=False), pa.field("file_format", pa.string(), nullable=False), - pa.field("partition", pa_record_struct, nullable=False), + pa.field("partition", pa_record_struct, nullable=False), # ty: ignore[invalid-argument-type] pa.field("record_count", pa.int64(), nullable=False), pa.field("file_size_in_bytes", pa.int64(), nullable=False), pa.field("column_sizes", pa.map_(pa.int32(), pa.int64()), nullable=True), @@ -247,8 +247,8 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: "null_value_count": null_value_counts.get(field.field_id), "nan_value_count": nan_value_counts.get(field.field_id), # Makes them readable - "lower_bound": _readable_bound(field.field_type, lower_bounds.get(field.field_id)), - "upper_bound": _readable_bound(field.field_type, upper_bounds.get(field.field_id)), + "lower_bound": _readable_bound(field.field_type, lower_bounds.get(field.field_id)), # ty: ignore[invalid-argument-type] + "upper_bound": _readable_bound(field.field_type, upper_bounds.get(field.field_id)), # ty: ignore[invalid-argument-type] } for field in self.tbl.metadata.schema().fields } @@ -391,7 +391,7 @@ def partitions( pa_record_struct = schema_to_pyarrow(partition_record) partitions_schema = pa.schema( [ - pa.field("partition", pa_record_struct, nullable=False), + pa.field("partition", pa_record_struct, nullable=False), # ty: ignore[invalid-argument-type] pa.field("spec_id", pa.int32(), nullable=False), ] ) @@ -420,7 +420,7 @@ def partitions( ) return pa.Table.from_pylist( - partitions_map.values(), + partitions_map.values(), # ty: ignore[invalid-argument-type] schema=table_schema, ) @@ -754,8 +754,8 @@ def _get_files_from_manifest( "value_count": value_counts.get(field.field_id), "null_value_count": null_value_counts.get(field.field_id), "nan_value_count": nan_value_counts.get(field.field_id), - "lower_bound": _readable_bound(field.field_type, lower_bounds.get(field.field_id)), - "upper_bound": _readable_bound(field.field_type, upper_bounds.get(field.field_id)), + "lower_bound": _readable_bound(field.field_type, lower_bounds.get(field.field_id)), # ty: ignore[invalid-argument-type] + "upper_bound": _readable_bound(field.field_type, upper_bounds.get(field.field_id)), # ty: ignore[invalid-argument-type] } for field in self.tbl.metadata.schema().fields } @@ -820,8 +820,8 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: pa.field("value_count", pa.int64(), nullable=True), pa.field("null_value_count", pa.int64(), nullable=True), pa.field("nan_value_count", pa.int64(), nullable=True), - pa.field("lower_bound", pa_bound_type, nullable=True), - pa.field("upper_bound", pa_bound_type, nullable=True), + pa.field("lower_bound", pa_bound_type, nullable=True), # ty: ignore[invalid-argument-type] + pa.field("upper_bound", pa_bound_type, nullable=True), # ty: ignore[invalid-argument-type] ] ) @@ -830,7 +830,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: for field in self.tbl.metadata.schema().fields: readable_metrics_struct.append( - pa.field(schema.find_column_name(field.field_id), _readable_metrics_struct(field.field_type), nullable=False) + pa.field(schema.find_column_name(field.field_id), _readable_metrics_struct(field.field_type), nullable=False) # ty: ignore[invalid-argument-type] ) files_schema = pa.schema( @@ -839,7 +839,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: pa.field("file_path", pa.string(), nullable=False), pa.field("file_format", pa.dictionary(pa.int32(), pa.string()), nullable=False), pa.field("spec_id", pa.int32(), nullable=False), - pa.field("partition", pa_record_struct, nullable=False), + pa.field("partition", pa_record_struct, nullable=False), # ty: ignore[invalid-argument-type] pa.field("record_count", pa.int64(), nullable=False), pa.field("file_size_in_bytes", pa.int64(), nullable=False), pa.field("column_sizes", pa.map_(pa.int32(), pa.int64()), nullable=True), diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index 8236f12229..70f57ed82f 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -278,7 +278,7 @@ def schema_by_id(self, schema_id: int) -> Schema | None: """Get the schema by schema_id.""" return next((schema for schema in self.schemas if schema.schema_id == schema_id), None) - def schema(self) -> Schema: + def schema(self) -> Schema: # ty: ignore[invalid-method-override] """Return the schema for this table.""" return next(schema for schema in self.schemas if schema.schema_id == self.current_schema_id) @@ -360,7 +360,7 @@ def current_snapshot(self) -> Snapshot | None: return None def next_sequence_number(self) -> int: - return self.last_sequence_number + 1 if self.format_version > 1 else INITIAL_SEQUENCE_NUMBER + return self.last_sequence_number + 1 if self.format_version > 1 else INITIAL_SEQUENCE_NUMBER # ty: ignore[unresolved-attribute] def sort_order(self) -> SortOrder: """Get the current sort order for this table, or UNSORTED_SORT_ORDER if there is no sort order.""" @@ -379,7 +379,7 @@ def serialize_current_snapshot_id(self, current_snapshot_id: int | None) -> int @field_serializer("snapshots") def serialize_snapshots(self, snapshots: list[Snapshot]) -> list[Snapshot]: # Snapshot field `sequence-number` should not be written for v1 metadata - if self.format_version == 1: + if self.format_version == 1: # ty: ignore[unresolved-attribute] return [snapshot.model_copy(update={"sequence_number": None}) for snapshot in snapshots] return snapshots @@ -420,7 +420,7 @@ def cleanup_snapshot_id(cls, data: dict[str, Any]) -> dict[str, Any]: @model_validator(mode="after") def construct_refs(self) -> TableMetadataV1: - return construct_refs(self) + return construct_refs(self) # ty: ignore[invalid-return-type] @model_validator(mode="before") def set_v2_compatible_defaults(cls, data: dict[str, Any]) -> dict[str, Any]: @@ -621,10 +621,10 @@ def construct_refs(self) -> TableMetadata: encryption_keys: list[EncryptedKey] = Field(alias="encryption-keys", default_factory=list) """An optional list of encryption keys used for table encryption.""" - @model_serializer(mode="wrap") + @model_serializer(mode="wrap") # ty: ignore[invalid-argument-type] def serialize_model(self, handler: ModelWrapSerializerWithoutInfo) -> dict[str, Any]: """Set custom serializer to leave out `encryption-keys` when it is empty.""" - serialized: dict[str, Any] = handler(self) + serialized: dict[str, Any] = handler(self) # ty: ignore[missing-argument] if not self.encryption_keys: serialized.pop("encryption-keys", None) return serialized diff --git a/pyiceberg/table/name_mapping.py b/pyiceberg/table/name_mapping.py index bd8b8529c5..7af874f128 100644 --- a/pyiceberg/table/name_mapping.py +++ b/pyiceberg/table/name_mapping.py @@ -39,7 +39,7 @@ class MappedField(IcebergBaseModel): field_id: int | None = Field(alias="field-id", default=None) - names: list[str] = conlist(str) + names: list[str] = conlist(str) # ty: ignore[invalid-assignment] fields: list[MappedField] = Field(default_factory=list) @field_validator("fields", mode="before") @@ -81,7 +81,7 @@ def __len__(self) -> int: """Return the number of mappings.""" return len(self.root) - def __iter__(self) -> Iterator[MappedField]: + def __iter__(self) -> Iterator[MappedField]: # ty: ignore[invalid-method-override] """Iterate over the mapped fields.""" return iter(self.root) @@ -256,7 +256,7 @@ class NameMappingAccessor(PartnerAccessor[MappedField]): def schema_partner(self, partner: MappedField | None) -> MappedField | None: return partner - def field_partner( + def field_partner( # ty: ignore[invalid-method-override] self, partner_struct: list[MappedField] | MappedField | None, _: int, field_name: str ) -> MappedField | None: if partner_struct is not None: @@ -322,10 +322,10 @@ def before_map_value(self, value: NestedField, value_partner: P | None) -> None: def after_map_value(self, value: NestedField, value_partner: P | None) -> None: self.current_path.pop() - def schema(self, schema: Schema, schema_partner: MappedField | None, struct_result: StructType) -> IcebergType: - return Schema(*struct_result.fields, schema_id=schema.schema_id) + def schema(self, schema: Schema, schema_partner: MappedField | None, struct_result: StructType) -> IcebergType: # ty: ignore[invalid-method-override] + return Schema(*struct_result.fields, schema_id=schema.schema_id) # ty: ignore[invalid-return-type] - def struct( + def struct( # ty: ignore[invalid-method-override] self, struct: StructType, struct_partner: MappedField | None, field_results: builtins.list[NestedField] ) -> IcebergType: return StructType(*field_results) @@ -375,4 +375,4 @@ def primitive(self, primitive: PrimitiveType, primitive_partner: MappedField | N def apply_name_mapping(schema_without_ids: Schema, name_mapping: NameMapping) -> Schema: - return visit_with_partner(schema_without_ids, name_mapping, NameMappingProjectionVisitor(), NameMappingAccessor()) # type: ignore + return visit_with_partner(schema_without_ids, name_mapping, NameMappingProjectionVisitor(), NameMappingAccessor()) # type: ignore[misc] diff --git a/pyiceberg/table/puffin.py b/pyiceberg/table/puffin.py index 13315bf802..2ee4987f42 100644 --- a/pyiceberg/table/puffin.py +++ b/pyiceberg/table/puffin.py @@ -78,7 +78,7 @@ def get_blob_payload(self, blob: PuffinBlobMetadata) -> bytes: raise ValueError(f"Unsupported puffin compression codec: {blob.compression_codec!r}") @deprecated(deprecated_in="0.12.0", removed_in="0.13.0", help_message="Use deletion_vectors_from_puffin_file(...) instead") - def to_vector(self) -> dict[str, "pa.ChunkedArray"]: + def to_vector(self) -> dict[str, "pa.ChunkedArray"]: # ty: ignore[missing-type-argument] from pyiceberg.table.deletion_vector import deletion_vectors_from_puffin_file # local import avoids the cycle return {dv.referenced_data_file: dv.to_vector() for dv in deletion_vectors_from_puffin_file(self)} diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index 66d3d7f239..3ddc601b07 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -185,7 +185,7 @@ def to_dict(self) -> dict[str, str]: return properties -class Summary(IcebergBaseModel, Mapping[str, str]): +class Summary(IcebergBaseModel, Mapping[str, str]): # ty: ignore[invalid-method-override] """A class that stores the summary information for a Snapshot. The snapshot summary’s operation field is used by some operations, @@ -212,7 +212,7 @@ def __getitem__(self, __key: str) -> Any | None: # type: ignore def __setitem__(self, key: str, value: Any) -> None: """Set a key as it is a map.""" if key.lower() == "operation": - self.operation = value + self.operation = value # ty: ignore[invalid-assignment] else: self._additional_properties[key] = value diff --git a/pyiceberg/table/sorting.py b/pyiceberg/table/sorting.py index 76bc109935..63b67be3b9 100644 --- a/pyiceberg/table/sorting.py +++ b/pyiceberg/table/sorting.py @@ -114,8 +114,8 @@ def map_source_ids_onto_source_id(cls, data: Any) -> Any: return data source_id: int = Field(alias="source-id") - transform: Annotated[ # type: ignore - Transform, + transform: Annotated[ # type: ignore[type-arg] + Transform, # ty: ignore[missing-type-argument] BeforeValidator(parse_transform), PlainSerializer(lambda c: str(c), return_type=str), # pylint: disable=W0108 WithJsonSchema({"type": "string"}, mode="serialization"), diff --git a/pyiceberg/table/update/__init__.py b/pyiceberg/table/update/__init__.py index aa1a09ebb2..cf993f9038 100644 --- a/pyiceberg/table/update/__init__.py +++ b/pyiceberg/table/update/__init__.py @@ -455,24 +455,24 @@ def _(update: AddSnapshotUpdate, base_metadata: TableMetadata, context: _TableMe elif ( base_metadata.format_version >= 2 and update.snapshot.sequence_number is not None - and update.snapshot.sequence_number <= base_metadata.last_sequence_number + and update.snapshot.sequence_number <= base_metadata.last_sequence_number # ty: ignore[unresolved-attribute] and update.snapshot.parent_snapshot_id is not None ): raise ValueError( f"Cannot add snapshot with sequence number {update.snapshot.sequence_number} " - f"older than last sequence number {base_metadata.last_sequence_number}" + f"older than last sequence number {base_metadata.last_sequence_number}" # ty: ignore[unresolved-attribute] ) elif base_metadata.format_version >= 3 and update.snapshot.first_row_id is None: raise ValueError("Cannot add snapshot without first row id") elif ( base_metadata.format_version >= 3 and update.snapshot.first_row_id is not None - and base_metadata.next_row_id is not None - and update.snapshot.first_row_id < base_metadata.next_row_id + and base_metadata.next_row_id is not None # ty: ignore[unresolved-attribute] + and update.snapshot.first_row_id < base_metadata.next_row_id # ty: ignore[unresolved-attribute] ): raise ValueError( f"Cannot add a snapshot with first row id smaller than the table's next-row-id " - f"{update.snapshot.first_row_id} < {base_metadata.next_row_id}" + f"{update.snapshot.first_row_id} < {base_metadata.next_row_id}" # ty: ignore[unresolved-attribute] ) context.add_update(update) @@ -481,9 +481,9 @@ def _(update: AddSnapshotUpdate, base_metadata: TableMetadata, context: _TableMe "last_updated_ms": update.snapshot.timestamp_ms, "last_sequence_number": update.snapshot.sequence_number, "snapshots": base_metadata.snapshots + [update.snapshot], - "next_row_id": base_metadata.next_row_id + update.snapshot.added_rows + "next_row_id": base_metadata.next_row_id + update.snapshot.added_rows # ty: ignore[unresolved-attribute] if base_metadata.format_version >= 3 - and base_metadata.next_row_id is not None + and base_metadata.next_row_id is not None # ty: ignore[unresolved-attribute] and update.snapshot.added_rows is not None else None, } @@ -626,7 +626,7 @@ def _( @_apply_table_update.register(SetStatisticsUpdate) def _(update: SetStatisticsUpdate, base_metadata: TableMetadata, context: _TableMetadataUpdateContext) -> TableMetadata: - statistics = filter_statistics_by_snapshot_id(base_metadata.statistics, update.statistics.snapshot_id) + statistics = filter_statistics_by_snapshot_id(base_metadata.statistics, update.statistics.snapshot_id) # ty: ignore[invalid-argument-type] context.add_update(update) return base_metadata.model_copy(update={"statistics": statistics + [update.statistics]}) @@ -637,7 +637,7 @@ def _(update: RemoveStatisticsUpdate, base_metadata: TableMetadata, context: _Ta if not any(stat.snapshot_id == update.snapshot_id for stat in base_metadata.statistics): raise ValueError(f"Statistics with snapshot id {update.snapshot_id} does not exist") - statistics = filter_statistics_by_snapshot_id(base_metadata.statistics, update.snapshot_id) + statistics = filter_statistics_by_snapshot_id(base_metadata.statistics, update.snapshot_id) # ty: ignore[invalid-argument-type] context.add_update(update) return base_metadata.model_copy(update={"statistics": statistics}) @@ -678,7 +678,8 @@ def _(update: RemoveSchemasUpdate, base_metadata: TableMetadata, context: _Table @_apply_table_update.register(SetPartitionStatisticsUpdate) def _(update: SetPartitionStatisticsUpdate, base_metadata: TableMetadata, context: _TableMetadataUpdateContext) -> TableMetadata: partition_statistics = filter_statistics_by_snapshot_id( - base_metadata.partition_statistics, update.partition_statistics.snapshot_id + base_metadata.partition_statistics, # ty: ignore[invalid-argument-type] + update.partition_statistics.snapshot_id, ) context.add_update(update) @@ -692,7 +693,7 @@ def _( if not any(part_stat.snapshot_id == update.snapshot_id for part_stat in base_metadata.partition_statistics): raise ValueError(f"Partition Statistics with snapshot id {update.snapshot_id} does not exist") - statistics = filter_statistics_by_snapshot_id(base_metadata.partition_statistics, update.snapshot_id) + statistics = filter_statistics_by_snapshot_id(base_metadata.partition_statistics, update.snapshot_id) # ty: ignore[invalid-argument-type] context.add_update(update) return base_metadata.model_copy(update={"partition_statistics": statistics}) @@ -786,8 +787,8 @@ def _update_table_metadata_log(base_metadata: TableMetadata, metadata_location: ), # type: ignore ) previous_metadata_log = base_metadata.metadata_log - if len(base_metadata.metadata_log) >= max_metadata_log_entries: # type: ignore - remove_index = len(base_metadata.metadata_log) - max_metadata_log_entries + 1 # type: ignore + if len(base_metadata.metadata_log) >= max_metadata_log_entries: # type: ignore[operator] + remove_index = len(base_metadata.metadata_log) - max_metadata_log_entries + 1 # type: ignore[operator] previous_metadata_log = base_metadata.metadata_log[remove_index:] metadata_updates: dict[str, Any] = { "metadata_log": previous_metadata_log + [MetadataLogEntry(metadata_file=metadata_location, timestamp_ms=last_updated_ms)] @@ -799,7 +800,7 @@ class ValidatableTableRequirement(IcebergBaseModel): type: str @abstractmethod - def validate(self, base_metadata: TableMetadata | None) -> None: + def validate(self, base_metadata: TableMetadata | None) -> None: # ty: ignore[invalid-method-override] """Validate the requirement against the base metadata. Args: @@ -844,9 +845,9 @@ class AssertRefSnapshotId(ValidatableTableRequirement): ref: str = Field(...) snapshot_id: int | None = Field(default=None, alias="snapshot-id") - @model_serializer(mode="wrap") + @model_serializer(mode="wrap") # ty: ignore[invalid-argument-type] def serialize_model(self, handler: ModelWrapSerializerWithoutInfo) -> dict[str, Any]: - partial_result = handler(self) + partial_result = handler(self) # ty: ignore[missing-argument] # Ensure "snapshot-id" is always present, even if value is None return {**partial_result, "snapshot-id": self.snapshot_id} diff --git a/pyiceberg/table/update/schema.py b/pyiceberg/table/update/schema.py index 828f1e877a..6c33d4bb81 100644 --- a/pyiceberg/table/update/schema.py +++ b/pyiceberg/table/update/schema.py @@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Any from pyiceberg.exceptions import ResolveError, ValidationError -from pyiceberg.expressions import literal # type: ignore +from pyiceberg.expressions import literal # type: ignore[attr-defined] from pyiceberg.schema import ( PartnerAccessor, Schema, @@ -155,7 +155,7 @@ def union_by_name( Catalog._convert_schema_if_needed(new_schema, format_version=format_version), -1, _UnionByNameVisitor(update_schema=self, existing_schema=self._schema, case_sensitive=self._case_sensitive), - # type: ignore + # type: ignore[misc] PartnerIdByNameAccessor(partner_schema=self._schema, case_sensitive=self._case_sensitive), ) return self @@ -231,7 +231,7 @@ def add_column( except ValueError as e: raise ValueError(f"Invalid default value: {e}") from e else: - initial_default = default_value # type: ignore + initial_default = default_value # type: ignore[assignment] if (required and initial_default is None) and not self._allow_incompatible_changes: # Table format version 1 and 2 cannot add required column because there is no initial value @@ -244,7 +244,7 @@ def add_column( field = NestedField( field_id=new_id, name=name, - field_type=new_type, + field_type=new_type, # ty: ignore[invalid-argument-type] required=required, doc=doc, initial_default=initial_default, @@ -857,10 +857,10 @@ def __init__(self, update_schema: UpdateSchema, existing_schema: Schema, case_se self.existing_schema = existing_schema self.case_sensitive = case_sensitive - def schema(self, schema: Schema, partner_id: int | None, struct_result: bool) -> bool: + def schema(self, schema: Schema, partner_id: int | None, struct_result: bool) -> bool: # ty: ignore[invalid-method-override] return struct_result - def struct(self, struct: StructType, partner_id: int | None, missing_positions: builtins.list[bool]) -> bool: + def struct(self, struct: StructType, partner_id: int | None, missing_positions: builtins.list[bool]) -> bool: # ty: ignore[invalid-method-override] if partner_id is None: return True @@ -875,7 +875,7 @@ def struct(self, struct: StructType, partner_id: int | None, missing_positions: self._add_column(partner_id, fields[pos]) else: field = fields[pos] - if nested_field := partner_struct.field_by_name(field.name, case_sensitive=self.case_sensitive): + if nested_field := partner_struct.field_by_name(field.name, case_sensitive=self.case_sensitive): # ty: ignore[unresolved-attribute] self._update_column(field, nested_field) return False @@ -915,10 +915,10 @@ def _find_field_type(self, field_id: int) -> IcebergType: else: return self.existing_schema.find_field(field_id).field_type - def field(self, field: NestedField, partner_id: int | None, field_result: bool) -> bool: + def field(self, field: NestedField, partner_id: int | None, field_result: bool) -> bool: # ty: ignore[invalid-method-override] return partner_id is None - def list(self, list_type: ListType, list_partner_id: int | None, element_missing: bool) -> bool: + def list(self, list_type: ListType, list_partner_id: int | None, element_missing: bool) -> bool: # ty: ignore[invalid-method-override] if list_partner_id is None: return True @@ -933,7 +933,7 @@ def list(self, list_type: ListType, list_partner_id: int | None, element_missing return False - def map(self, map_type: MapType, map_partner_id: int | None, key_missing: bool, value_missing: bool) -> bool: + def map(self, map_type: MapType, map_partner_id: int | None, key_missing: bool, value_missing: bool) -> bool: # ty: ignore[invalid-method-override] if map_partner_id is None: return True @@ -952,7 +952,7 @@ def map(self, map_type: MapType, map_partner_id: int | None, key_missing: bool, return False - def primitive(self, primitive: PrimitiveType, primitive_partner_id: int | None) -> bool: + def primitive(self, primitive: PrimitiveType, primitive_partner_id: int | None) -> bool: # ty: ignore[invalid-method-override] return primitive_partner_id is None @@ -967,7 +967,7 @@ def __init__(self, partner_schema: Schema, case_sensitive: bool) -> None: def schema_partner(self, partner: int | None) -> int | None: return -1 - def field_partner(self, partner_field_id: int | None, field_id: int, field_name: str) -> int | None: + def field_partner(self, partner_field_id: int | None, field_id: int, field_name: str) -> int | None: # ty: ignore[invalid-method-override] if partner_field_id is not None: if partner_field_id == -1: struct = self.partner_schema.as_struct() @@ -976,12 +976,12 @@ def field_partner(self, partner_field_id: int | None, field_id: int, field_name: if not struct.is_struct: raise ValueError(f"Expected StructType: {struct}") - if field := struct.field_by_name(name=field_name, case_sensitive=self.case_sensitive): + if field := struct.field_by_name(name=field_name, case_sensitive=self.case_sensitive): # ty: ignore[unresolved-attribute] return field.field_id return None - def list_element_partner(self, partner_list_id: int | None) -> int | None: + def list_element_partner(self, partner_list_id: int | None) -> int | None: # ty: ignore[invalid-method-override] if partner_list_id is not None and (field := self.partner_schema.find_field(partner_list_id)): if not isinstance(field.field_type, ListType): raise ValueError(f"Expected ListType: {field}") @@ -989,7 +989,7 @@ def list_element_partner(self, partner_list_id: int | None) -> int | None: else: return None - def map_key_partner(self, partner_map_id: int | None) -> int | None: + def map_key_partner(self, partner_map_id: int | None) -> int | None: # ty: ignore[invalid-method-override] if partner_map_id is not None and (field := self.partner_schema.find_field(partner_map_id)): if not isinstance(field.field_type, MapType): raise ValueError(f"Expected MapType: {field}") @@ -997,7 +997,7 @@ def map_key_partner(self, partner_map_id: int | None) -> int | None: else: return None - def map_value_partner(self, partner_map_id: int | None) -> int | None: + def map_value_partner(self, partner_map_id: int | None) -> int | None: # ty: ignore[invalid-method-override] if partner_map_id is not None and (field := self.partner_schema.find_field(partner_map_id)): if not isinstance(field.field_type, MapType): raise ValueError(f"Expected MapType: {field}") diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index 8feebb9901..d3e8bb5901 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -347,7 +347,7 @@ def _commit(self) -> UpdatesAndRequirements: first_row_id: int | None = None if self._transaction.table_metadata.format_version >= 3: - first_row_id = self._transaction.table_metadata.next_row_id + first_row_id = self._transaction.table_metadata.next_row_id # ty: ignore[unresolved-attribute] snapshot = Snapshot( snapshot_id=self._snapshot_id, @@ -370,7 +370,7 @@ def _commit(self) -> UpdatesAndRequirements: return ( ( add_snapshot_update, - SetSnapshotRefUpdate( + SetSnapshotRefUpdate( # ty: ignore[pydantic-discarded-extra-argument] snapshot_id=self._snapshot_id, parent_snapshot_id=self._parent_snapshot_id, ref_name=self._target_branch, @@ -736,16 +736,16 @@ def __init__( super().__init__(operation, transaction, io, commit_uuid, snapshot_properties, branch) table_properties = self._transaction.table_metadata.properties - self._target_size_bytes = property_as_int( + self._target_size_bytes = property_as_int( # ty: ignore[invalid-assignment] table_properties, TableProperties.MANIFEST_TARGET_SIZE_BYTES, TableProperties.MANIFEST_TARGET_SIZE_BYTES_DEFAULT, - ) # type: ignore - self._min_count_to_merge = property_as_int( + ) # type: ignore[assignment] + self._min_count_to_merge = property_as_int( # ty: ignore[invalid-assignment] table_properties, TableProperties.MANIFEST_MIN_MERGE_COUNT, TableProperties.MANIFEST_MIN_MERGE_COUNT_DEFAULT, - ) # type: ignore + ) # type: ignore[assignment] self._merge_enabled = property_as_bool( table_properties, TableProperties.MANIFEST_MERGE_ENABLED, diff --git a/pyiceberg/table/upsert_util.py b/pyiceberg/table/upsert_util.py index 6f32826eb0..b7bf8ece5e 100644 --- a/pyiceberg/table/upsert_util.py +++ b/pyiceberg/table/upsert_util.py @@ -107,18 +107,18 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols matching_indices[TARGET_INDEX_COLUMN_NAME].to_pylist(), strict=True, ): - source_row = source_table.slice(source_idx, 1) - target_row = target_table.slice(target_idx, 1) + source_row = source_table.slice(source_idx, 1) # ty: ignore[invalid-argument-type] + target_row = target_table.slice(target_idx, 1) # ty: ignore[invalid-argument-type] for key in non_key_cols: - source_val = source_row.column(key)[0].as_py() - target_val = target_row.column(key)[0].as_py() + source_val = source_row.column(key)[0].as_py() # ty: ignore[unresolved-attribute] + target_val = target_row.column(key)[0].as_py() # ty: ignore[unresolved-attribute] if source_val != target_val: to_update_indices.append(source_idx) break # Step 5: Take rows from source table using the indices and cast to target schema if to_update_indices: - return source_table.take(to_update_indices) + return source_table.take(to_update_indices) # ty: ignore[invalid-argument-type] else: return source_table.schema.empty_table() diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index 74925c695f..2837f25c74 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -92,7 +92,7 @@ if TYPE_CHECKING: import pyarrow as pa - ArrayLike = TypeVar("ArrayLike", pa.Array, pa.ChunkedArray) + ArrayLike = TypeVar("ArrayLike", pa.Array, pa.ChunkedArray) # ty: ignore[missing-type-argument] S = TypeVar("S") T = TypeVar("T") @@ -128,7 +128,7 @@ def _pyiceberg_transform_wrapper( def _transform(array: "ArrayLike") -> "ArrayLike": def _cast_if_needed(arr: "ArrayLike") -> "ArrayLike": if expected_type is not None: - return arr.cast(expected_type) + return arr.cast(expected_type) # ty: ignore[invalid-return-type] else: return arr @@ -137,8 +137,8 @@ def _cast_if_needed(arr: "ArrayLike") -> "ArrayLike": elif isinstance(array, pa.ChunkedArray): result_chunks = [] for arr in array.iterchunks(): - result_chunks.append(_cast_if_needed(transform_func(arr, *args))) - return pa.chunked_array(result_chunks) + result_chunks.append(_cast_if_needed(transform_func(arr, *args))) # ty: ignore[invalid-argument-type] + return pa.chunked_array(result_chunks) # ty: ignore[invalid-return-type] else: raise ValueError(f"PyArrow array can only be of type pa.Array or pa.ChunkedArray, but found {type(array)}") @@ -203,7 +203,7 @@ def __eq__(self, other: Any) -> bool: return False @abstractmethod - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": ... + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": ... # ty: ignore[missing-type-argument] def parse_transform(v: Any) -> Transform[Any, Any]: @@ -269,12 +269,12 @@ def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: elif isinstance(pred, BoundEqualTo): return pred.as_unbound(Reference(name), _transform_literal(transformer, pred.literal)) elif isinstance(pred, BoundIn): # NotIn can't be projected - return pred.as_unbound(Reference(name), {_transform_literal(transformer, literal) for literal in pred.literals}) # type: ignore + return pred.as_unbound(Reference(name), {_transform_literal(transformer, literal) for literal in pred.literals}) # type: ignore[call-arg] else: # - Comparison predicates can't be projected, notEq can't be projected # - Small ranges can be projected: # For example, (x > 0) and (x < 3) can be turned into in({1, 2}) and projected. - return None # type: ignore + return None # type: ignore[unreachable] def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: transformer = self.transform(pred.term.ref().field.field_type) @@ -286,10 +286,10 @@ def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | elif isinstance(pred, BoundNotEqualTo): return pred.as_unbound(Reference(name), _transform_literal(transformer, pred.literal)) elif isinstance(pred, BoundNotIn): - return pred.as_unbound(Reference(name), {_transform_literal(transformer, literal) for literal in pred.literals}) # type: ignore + return pred.as_unbound(Reference(name), {_transform_literal(transformer, literal) for literal in pred.literals}) # type: ignore[call-arg] else: # no strict projection for comparison or equality - return None # type: ignore + return None # type: ignore[unreachable] def can_transform(self, source: IcebergType) -> bool: return isinstance( @@ -381,7 +381,7 @@ def __repr__(self) -> str: """Return the string representation of the BucketTransform class.""" return f"BucketTransform(num_buckets={self._num_buckets})" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform return _pyiceberg_transform_wrapper(pyiceberg_core_transform.bucket, self._num_buckets) @@ -402,7 +402,7 @@ class TimeTransform(Transform[S, int], Generic[S], Singleton): def granularity(self) -> TimeResolution: ... def satisfies_order_of(self, other: Transform[S, T]) -> bool: - return self.granularity <= other.granularity if hasattr(other, "granularity") else False + return self.granularity <= other.granularity if hasattr(other, "granularity") else False # ty: ignore[unsupported-operator] def result_type(self, source: IcebergType) -> IntegerType: return IntegerType() @@ -495,7 +495,7 @@ def __repr__(self) -> str: """Return the string representation of the YearTransform class.""" return "YearTransform()" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] pa = try_import("pyarrow") pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform return _pyiceberg_transform_wrapper(pyiceberg_core_transform.year, expected_type=pa.int32()) @@ -555,7 +555,7 @@ def __repr__(self) -> str: """Return the string representation of the MonthTransform class.""" return "MonthTransform()" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] pa = try_import("pyarrow") pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform @@ -605,7 +605,7 @@ def day_func(v: Any) -> int: def can_transform(self, source: IcebergType) -> bool: return isinstance(source, (DateType, TimestampType, TimestamptzType, TimestampNanoType, TimestamptzNanoType)) - def result_type(self, source: IcebergType) -> IcebergType: + def result_type(self, source: IcebergType) -> IcebergType: # ty: ignore[invalid-method-override] """Return the result type of a day transform. The physical representation conforms to the Iceberg spec as DateType is internally converted to int. @@ -624,7 +624,7 @@ def __repr__(self) -> str: """Return the string representation of the DayTransform class.""" return "DayTransform()" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] pa = try_import("pyarrow", extras_name="pyarrow") pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform @@ -677,7 +677,7 @@ def __repr__(self) -> str: """Return the string representation of the HourTransform class.""" return "HourTransform()" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform return _pyiceberg_transform_wrapper(pyiceberg_core_transform.hour) @@ -741,7 +741,7 @@ def satisfies_order_of(self, other: Transform[S, T]) -> bool: """Ordering by value is the same as long as the other preserves order.""" return other.preserves_order - def to_human_string(self, source_type: IcebergType, value: S | None) -> str: + def to_human_string(self, source_type: IcebergType, value: S | None) -> str: # ty: ignore[invalid-method-override] return _human_string(value, source_type) if value is not None else "null" def __str__(self) -> str: @@ -752,7 +752,7 @@ def __repr__(self) -> str: """Return the string representation of the IdentityTransform class.""" return "IdentityTransform()" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] return lambda v: v @@ -798,7 +798,7 @@ def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: return pred.as_unbound(Reference(name)) elif isinstance(pred, BoundIn): return _set_apply_transform(name, pred, self.transform(field_type)) - elif isinstance(field_type, (IntegerType, LongType, DecimalType)): # type: ignore + elif isinstance(field_type, (IntegerType, LongType, DecimalType)): # type: ignore[unreachable] if isinstance(pred, BoundLiteralPredicate): return _truncate_number(name, pred, self.transform(field_type)) elif isinstance(field_type, (BinaryType, StringType)): @@ -829,7 +829,7 @@ def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | elif isinstance(pred, BoundNotIn): return _set_apply_transform(name, pred, self.transform(field_type)) else: - return None # type: ignore + return None # type: ignore[unreachable] if isinstance(pred, BoundLiteralPredicate): if isinstance(pred, BoundStartsWith): @@ -854,7 +854,7 @@ def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | elif isinstance(pred, BoundNotIn): return _set_apply_transform(name, pred, self.transform(field_type)) else: - return None # type: ignore + return None # type: ignore[unreachable] @property def width(self) -> int: @@ -905,7 +905,7 @@ def __repr__(self) -> str: """Return the string representation of the TruncateTransform class.""" return f"TruncateTransform(width={self._width})" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform return _pyiceberg_transform_wrapper(pyiceberg_core_transform.truncate, self._width) @@ -992,7 +992,7 @@ def __repr__(self) -> str: """Return the string representation of the UnknownTransform class.""" return f"UnknownTransform(transform={repr(self._transform)})" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] raise NotImplementedError() @@ -1004,7 +1004,7 @@ class VoidTransform(Transform[S, None], Singleton): def transform(self, source: IcebergType) -> Callable[[S | None], T | None]: return lambda v: None - def can_transform(self, _: IcebergType) -> bool: + def can_transform(self, _: IcebergType) -> bool: # ty: ignore[invalid-method-override] return True def result_type(self, source: IcebergType) -> IcebergType: @@ -1023,7 +1023,7 @@ def __repr__(self) -> str: """Return the string representation of the VoidTransform class.""" return "VoidTransform()" - def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": + def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": # ty: ignore[missing-type-argument] try: import pyarrow as pa except ModuleNotFoundError as e: @@ -1129,7 +1129,7 @@ def _remove_transform(partition_name: str, pred: BoundPredicate) -> UnboundPredi elif isinstance(pred, BoundLiteralPredicate): return pred.as_unbound(Reference(partition_name), pred.literal) elif isinstance(pred, (BoundIn, BoundNotIn)): - return pred.as_unbound(Reference(partition_name), pred.literals) # type: ignore + return pred.as_unbound(Reference(partition_name), pred.literals) # type: ignore[call-arg] else: raise ValueError(f"Cannot replace transform in unknown predicate: {pred}") diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index 59bb5e259f..1f1f029a33 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -137,14 +137,14 @@ def _exclude_private_properties(self, exclude: set[str] | None = None) -> set[st {field for field in self.__dict__ if field.startswith("_") and not field == "__root__"}, exclude or set() ) - def model_dump( + def model_dump( # ty: ignore[invalid-method-override] self, exclude_none: bool = True, exclude: set[str] | None = None, by_alias: bool = True, **kwargs: Any ) -> dict[str, Any]: return super().model_dump( exclude_none=exclude_none, exclude=self._exclude_private_properties(exclude), by_alias=by_alias, **kwargs ) - def model_dump_json( + def model_dump_json( # ty: ignore[invalid-method-override] self, exclude_none: bool = True, exclude: set[str] | None = None, by_alias: bool = True, **kwargs: Any ) -> str: return super().model_dump_json( diff --git a/pyiceberg/types.py b/pyiceberg/types.py index 3295beb666..0d80b554ad 100644 --- a/pyiceberg/types.py +++ b/pyiceberg/types.py @@ -261,7 +261,7 @@ def minimum_format_version(self) -> TableVersion: return 1 -class PrimitiveType(Singleton, IcebergRootModel[str], IcebergType): +class PrimitiveType(Singleton, IcebergRootModel[str], IcebergType): # ty: ignore[invalid-method-override] """Base class for all Iceberg Primitive Types.""" root: Any = Field() @@ -429,15 +429,15 @@ class NestedField(IcebergType): field_type: SerializeAsAny[IcebergType] = Field(alias="type") required: bool = Field(default=False) doc: str | None = Field(default=None, repr=False) - initial_default: DefaultValue | None = Field(alias="initial-default", default=None, repr=True) # type: ignore - write_default: DefaultValue | None = Field(alias="write-default", default=None, repr=True) # type: ignore + initial_default: DefaultValue | None = Field(alias="initial-default", default=None, repr=True) # type: ignore[type-arg] + write_default: DefaultValue | None = Field(alias="write-default", default=None, repr=True) # type: ignore[type-arg] @field_validator("field_type", mode="before") def convert_field_type(cls, v: Any) -> IcebergType: """Convert string values into IcebergType instances.""" if isinstance(v, str): try: - return IcebergType.handle_primitive_type(v, None) + return IcebergType.handle_primitive_type(v, None) # ty: ignore[call-non-callable] except ValueError as e: raise ValueError(f"Unsupported field type: '{v}'") from e return v diff --git a/pyiceberg/utils/deprecated.py b/pyiceberg/utils/deprecated.py index b5bb6d01cd..9d8e9f5593 100644 --- a/pyiceberg/utils/deprecated.py +++ b/pyiceberg/utils/deprecated.py @@ -31,7 +31,7 @@ def deprecated(deprecated_in: str, removed_in: str, help_message: str | None = N def decorator(func: Callable): # type: ignore @functools.wraps(func) def new_func(*args: Any, **kwargs: Any) -> Any: - message = f"Call to {func.__name__}, deprecated in {deprecated_in}, will be removed in {removed_in}.{help_message}" + message = f"Call to {func.__name__}, deprecated in {deprecated_in}, will be removed in {removed_in}.{help_message}" # ty: ignore[unresolved-attribute] _deprecation_warning(message) diff --git a/pyiceberg/utils/lazydict.py b/pyiceberg/utils/lazydict.py index b110b6291b..6b77c7da76 100644 --- a/pyiceberg/utils/lazydict.py +++ b/pyiceberg/utils/lazydict.py @@ -62,6 +62,6 @@ def __len__(self) -> int: source = self._dict or self._build_dict() return len(source) - def __dict__(self) -> dict[K, V]: # type: ignore + def __dict__(self) -> dict[K, V]: # type: ignore[override] """Convert the lazy dict in a dict.""" return self._dict or self._build_dict() diff --git a/pyiceberg/utils/schema_conversion.py b/pyiceberg/utils/schema_conversion.py index 732439d34b..280e44d291 100644 --- a/pyiceberg/utils/schema_conversion.py +++ b/pyiceberg/utils/schema_conversion.py @@ -624,7 +624,7 @@ def visit_time(self, time_type: TimeType) -> AvroType: def visit_timestamp(self, timestamp_type: TimestampType) -> AvroType: return {"type": "long", "logicalType": "timestamp-micros", "adjust-to-utc": False} - def visit_timestamp_ns(self, timestamp_ns_type: TimestampNanoType) -> AvroType: + def visit_timestamp_ns(self, timestamp_ns_type: TimestampNanoType) -> AvroType: # ty: ignore[invalid-method-override] return {"type": "long", "logicalType": "timestamp-nanos", "adjust-to-utc": False} def visit_timestamptz(self, timestamptz_type: TimestamptzType) -> AvroType: diff --git a/pyiceberg/utils/singleton.py b/pyiceberg/utils/singleton.py index b59f43fbcd..5c91c2d612 100644 --- a/pyiceberg/utils/singleton.py +++ b/pyiceberg/utils/singleton.py @@ -28,7 +28,7 @@ More information on metaclasses: https://docs.python.org/3/reference/datamodel.html#metaclasses """ -from typing import Any, ClassVar +from typing import Any, ClassVar, Self, cast def _convert_to_hashable_type(element: Any) -> Any: @@ -39,16 +39,21 @@ def _convert_to_hashable_type(element: Any) -> Any: return element +# (class, positional args, hashable kwargs) is the cache key for instances +_InstanceKey = tuple[type["Singleton"], tuple[Any, ...], Any] + + class Singleton: - _instances: ClassVar[dict] = {} # type: ignore + _instances: ClassVar[dict[_InstanceKey, "Singleton"]] = {} - def __new__(cls, *args, **kwargs): # type: ignore - key = (cls, tuple(args), _convert_to_hashable_type(kwargs)) + def __new__(cls, *args: Any, **kwargs: Any) -> Self: + key: _InstanceKey = (cls, tuple(args), _convert_to_hashable_type(kwargs)) if key not in cls._instances: cls._instances[key] = super().__new__(cls) - return cls._instances[key] + # the key contains cls, so the cached instance is always of the calling class + return cast(Self, cls._instances[key]) - def __deepcopy__(self, memo: dict[int, Any]) -> Any: + def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Self: """ Prevent deep copy operations for singletons. diff --git a/pyiceberg/view/metadata.py b/pyiceberg/view/metadata.py index 33766040e3..1a0859ad48 100644 --- a/pyiceberg/view/metadata.py +++ b/pyiceberg/view/metadata.py @@ -38,7 +38,7 @@ class SQLViewRepresentation(IcebergBaseModel): """The dialect of the SQL, e.g. `spark`, `trino`, `presto`.""" -class ViewRepresentation(IcebergBaseModel, RootModel): +class ViewRepresentation(IcebergBaseModel, RootModel): # ty: ignore[invalid-method-override, missing-type-argument] root: SQLViewRepresentation diff --git a/pyproject.toml b/pyproject.toml index 8268338fa1..01c25f0242 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,6 +129,7 @@ dev = [ "nbformat>=5.10.0", "ipykernel>=6.29.0", "ty==0.0.84", + "mypy==1.18.2", ] # for mkdocs docs = [ @@ -148,6 +149,16 @@ docs = [ notebook = [ "jupyterlab>=4.0.0", ] +# Only ty and the stub packages, to measure ty in the same bare environment the mypy +# pre-commit hook runs in. Not a gate: sync it with `--only-group types` into a throwaway venv. +# The ty hook in .pre-commit-config.yaml pins the same versions, so keep the two in step. +types = [ + "ty==0.0.84", + "types-cachetools==7.0.0.20260713", + "types-pytest-lazy-fixture==0.6.3.20260518", + "types-pytz==2026.5.0.20261006", + "types-requests==2.33.0.20261006", +] [tool.uv] default-groups = [ @@ -262,6 +273,9 @@ filterwarnings = [ ] [tool.mypy] +# Same scope as ty's [tool.ty.src] include, so both checkers see the same files. +files = ["pyiceberg"] +python_version = "3.11" mypy_path = "python" no_implicit_optional = true namespace_packages = false @@ -274,16 +288,59 @@ implicit_reexport = false ignore_missing_imports = true [tool.ty.src] -exclude = ["vendor/"] +include = ["pyiceberg"] [tool.ty.environment] # The vendored Hive Metastore client is packaged from vendor/, so it must stay importable extra-paths = ["vendor"] +python-version = "3.11" [tool.ty.analysis] # pyiceberg_core.datafusion is registered at runtime by its compiled extension; griffe and # mkdocs_gen_files are only installed with the docs dependency group allowed-unresolved-imports = ["pyiceberg_core.datafusion", "griffe", "mkdocs_gen_files"] +[tool.ty.rules] +# Each rule mirrors the mypy flag it replaces. missing-type-argument is the only one ty +# silences by default; the rest already report nothing in pyiceberg/ at this level. +missing-type-argument = "error" # disallow_any_generics +invalid-parameter-default = "error" # no_implicit_optional +redundant-cast = "error" # warn_redundant_casts +redundant-condition = "error" # warn_unreachable +unused-ignore-comment = "error" # warn_unused_ignores +unused-type-ignore-comment = "error" # warn_unused_ignores +# No mypy counterpart; off until every override carries @override. +missing-override-decorator = "ignore" + +[[tool.ty.overrides]] +include = [ + "pyiceberg/avro/codecs/snappy_codec.py", + "pyiceberg/catalog/hive.py", + "pyiceberg/catalog/rest/auth.py", + "pyiceberg/io/fsspec.py", + "pyiceberg/io/pyarrow.py", + "pyiceberg/table/__init__.py", +] + +[tool.ty.overrides.analysis] +# Optional extras that only these modules import; mypy ignores missing imports globally. +allowed-unresolved-imports = [ + "snappy", + "thrift.**", + "azure.identity", + "s3fs", + "gcsfs", + "adlfs", + "azure.core.credentials", + "azure.core.credentials_async", + "huggingface_hub", + "geoarrow.pyarrow", + "bodo.pandas", + "daft", + "polars", + "ray", + "duckdb", +] + [tool.coverage.run] source = ['pyiceberg/'] diff --git a/uv.lock b/uv.lock index 7901c5ea12..3715685965 100644 --- a/uv.lock +++ b/uv.lock @@ -3448,6 +3448,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/59/e26cb779be4c591d1a910f59d29aca9fba4de70349840a833beba2652371/multidict-6.9.1-py3-none-any.whl", hash = "sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b", size = 19176, upload-time = "2026-09-21T17:59:03.501Z" }, ] +[[package]] +name = "mypy" +version = "1.18.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/77/8f0d0001ffad290cef2f7f216f96c814866248a0b92a722365ed54648e7e/mypy-1.18.2.tar.gz", hash = "sha256:06a398102a5f203d7477b2923dda3634c36727fa5c237d8f859ef90c42a9924b", size = 3448846, upload-time = "2025-09-19T00:11:10.519Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/87/cafd3ae563f88f94eec33f35ff722d043e09832ea8530ef149ec1efbaf08/mypy-1.18.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:807d9315ab9d464125aa9fcf6d84fde6e1dc67da0b6f80e7405506b8ac72bc7f", size = 12731198, upload-time = "2025-09-19T00:09:44.857Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e0/1e96c3d4266a06d4b0197ace5356d67d937d8358e2ee3ffac71faa843724/mypy-1.18.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:776bb00de1778caf4db739c6e83919c1d85a448f71979b6a0edd774ea8399341", size = 11817879, upload-time = "2025-09-19T00:09:47.131Z" }, + { url = "https://files.pythonhosted.org/packages/72/ef/0c9ba89eb03453e76bdac5a78b08260a848c7bfc5d6603634774d9cd9525/mypy-1.18.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1379451880512ffce14505493bd9fe469e0697543717298242574882cf8cdb8d", size = 12427292, upload-time = "2025-09-19T00:10:22.472Z" }, + { url = "https://files.pythonhosted.org/packages/1a/52/ec4a061dd599eb8179d5411d99775bec2a20542505988f40fc2fee781068/mypy-1.18.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1331eb7fd110d60c24999893320967594ff84c38ac6d19e0a76c5fd809a84c86", size = 13163750, upload-time = "2025-09-19T00:09:51.472Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5f/2cf2ceb3b36372d51568f2208c021870fe7834cf3186b653ac6446511839/mypy-1.18.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3ca30b50a51e7ba93b00422e486cbb124f1c56a535e20eff7b2d6ab72b3b2e37", size = 13351827, upload-time = "2025-09-19T00:09:58.311Z" }, + { url = "https://files.pythonhosted.org/packages/c8/7d/2697b930179e7277529eaaec1513f8de622818696857f689e4a5432e5e27/mypy-1.18.2-cp311-cp311-win_amd64.whl", hash = "sha256:664dc726e67fa54e14536f6e1224bcfce1d9e5ac02426d2326e2bb4e081d1ce8", size = 9757983, upload-time = "2025-09-19T00:10:09.071Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/dfdd2bc60c66611dd8335f463818514733bc763e4760dee289dcc33df709/mypy-1.18.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33eca32dd124b29400c31d7cf784e795b050ace0e1f91b8dc035672725617e34", size = 12908273, upload-time = "2025-09-19T00:10:58.321Z" }, + { url = "https://files.pythonhosted.org/packages/81/14/6a9de6d13a122d5608e1a04130724caf9170333ac5a924e10f670687d3eb/mypy-1.18.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a3c47adf30d65e89b2dcd2fa32f3aeb5e94ca970d2c15fcb25e297871c8e4764", size = 11920910, upload-time = "2025-09-19T00:10:20.043Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a9/b29de53e42f18e8cc547e38daa9dfa132ffdc64f7250e353f5c8cdd44bee/mypy-1.18.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d6c838e831a062f5f29d11c9057c6009f60cb294fea33a98422688181fe2893", size = 12465585, upload-time = "2025-09-19T00:10:33.005Z" }, + { url = "https://files.pythonhosted.org/packages/77/ae/6c3d2c7c61ff21f2bee938c917616c92ebf852f015fb55917fd6e2811db2/mypy-1.18.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01199871b6110a2ce984bde85acd481232d17413868c9807e95c1b0739a58914", size = 13348562, upload-time = "2025-09-19T00:10:11.51Z" }, + { url = "https://files.pythonhosted.org/packages/4d/31/aec68ab3b4aebdf8f36d191b0685d99faa899ab990753ca0fee60fb99511/mypy-1.18.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a2afc0fa0b0e91b4599ddfe0f91e2c26c2b5a5ab263737e998d6817874c5f7c8", size = 13533296, upload-time = "2025-09-19T00:10:06.568Z" }, + { url = "https://files.pythonhosted.org/packages/9f/83/abcb3ad9478fca3ebeb6a5358bb0b22c95ea42b43b7789c7fb1297ca44f4/mypy-1.18.2-cp312-cp312-win_amd64.whl", hash = "sha256:d8068d0afe682c7c4897c0f7ce84ea77f6de953262b12d07038f4d296d547074", size = 9828828, upload-time = "2025-09-19T00:10:28.203Z" }, + { url = "https://files.pythonhosted.org/packages/5f/04/7f462e6fbba87a72bc8097b93f6842499c428a6ff0c81dd46948d175afe8/mypy-1.18.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:07b8b0f580ca6d289e69209ec9d3911b4a26e5abfde32228a288eb79df129fcc", size = 12898728, upload-time = "2025-09-19T00:10:01.33Z" }, + { url = "https://files.pythonhosted.org/packages/99/5b/61ed4efb64f1871b41fd0b82d29a64640f3516078f6c7905b68ab1ad8b13/mypy-1.18.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ed4482847168439651d3feee5833ccedbf6657e964572706a2adb1f7fa4dfe2e", size = 11910758, upload-time = "2025-09-19T00:10:42.607Z" }, + { url = "https://files.pythonhosted.org/packages/3c/46/d297d4b683cc89a6e4108c4250a6a6b717f5fa96e1a30a7944a6da44da35/mypy-1.18.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3ad2afadd1e9fea5cf99a45a822346971ede8685cc581ed9cd4d42eaf940986", size = 12475342, upload-time = "2025-09-19T00:11:00.371Z" }, + { url = "https://files.pythonhosted.org/packages/83/45/4798f4d00df13eae3bfdf726c9244bcb495ab5bd588c0eed93a2f2dd67f3/mypy-1.18.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a431a6f1ef14cf8c144c6b14793a23ec4eae3db28277c358136e79d7d062f62d", size = 13338709, upload-time = "2025-09-19T00:11:03.358Z" }, + { url = "https://files.pythonhosted.org/packages/d7/09/479f7358d9625172521a87a9271ddd2441e1dab16a09708f056e97007207/mypy-1.18.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7ab28cc197f1dd77a67e1c6f35cd1f8e8b73ed2217e4fc005f9e6a504e46e7ba", size = 13529806, upload-time = "2025-09-19T00:10:26.073Z" }, + { url = "https://files.pythonhosted.org/packages/71/cf/ac0f2c7e9d0ea3c75cd99dff7aec1c9df4a1376537cb90e4c882267ee7e9/mypy-1.18.2-cp313-cp313-win_amd64.whl", hash = "sha256:0e2785a84b34a72ba55fb5daf079a1003a34c05b22238da94fcae2bbe46f3544", size = 9833262, upload-time = "2025-09-19T00:10:40.035Z" }, + { url = "https://files.pythonhosted.org/packages/5a/0c/7d5300883da16f0063ae53996358758b2a2df2a09c72a5061fa79a1f5006/mypy-1.18.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:62f0e1e988ad41c2a110edde6c398383a889d95b36b3e60bcf155f5164c4fdce", size = 12893775, upload-time = "2025-09-19T00:10:03.814Z" }, + { url = "https://files.pythonhosted.org/packages/50/df/2cffbf25737bdb236f60c973edf62e3e7b4ee1c25b6878629e88e2cde967/mypy-1.18.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8795a039bab805ff0c1dfdb8cd3344642c2b99b8e439d057aba30850b8d3423d", size = 11936852, upload-time = "2025-09-19T00:10:51.631Z" }, + { url = "https://files.pythonhosted.org/packages/be/50/34059de13dd269227fb4a03be1faee6e2a4b04a2051c82ac0a0b5a773c9a/mypy-1.18.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ca1e64b24a700ab5ce10133f7ccd956a04715463d30498e64ea8715236f9c9c", size = 12480242, upload-time = "2025-09-19T00:11:07.955Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/040983fad5132d85914c874a2836252bbc57832065548885b5bb5b0d4359/mypy-1.18.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d924eef3795cc89fecf6bedc6ed32b33ac13e8321344f6ddbf8ee89f706c05cb", size = 13326683, upload-time = "2025-09-19T00:09:55.572Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ba/89b2901dd77414dd7a8c8729985832a5735053be15b744c18e4586e506ef/mypy-1.18.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20c02215a080e3a2be3aa50506c67242df1c151eaba0dcbc1e4e557922a26075", size = 13514749, upload-time = "2025-09-19T00:10:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/25/bc/cc98767cffd6b2928ba680f3e5bc969c4152bf7c2d83f92f5a504b92b0eb/mypy-1.18.2-cp314-cp314-win_amd64.whl", hash = "sha256:749b5f83198f1ca64345603118a6f01a4e99ad4bf9d103ddc5a3200cc4614adf", size = 9982959, upload-time = "2025-09-19T00:10:37.344Z" }, + { url = "https://files.pythonhosted.org/packages/87/e3/be76d87158ebafa0309946c4a73831974d4d6ab4f4ef40c3b53a385a66fd/mypy-1.18.2-py3-none-any.whl", hash = "sha256:22a1748707dd62b58d2ae53562ffc4d7f8bcc727e8ac7cbc69c053ddc874d47e", size = 2352367, upload-time = "2025-09-19T00:10:15.489Z" }, +] + [[package]] name = "mypy-boto3-dynamodb" version = "1.43.64" @@ -3472,6 +3510,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/bf/781db91d144e0f2a4c91649499722de9b57f6a3c251b34d339820f3f7f0f/mypy_boto3_glue-1.43.100-py3-none-any.whl", hash = "sha256:104b3d95133e10b4ad825432cfeeeb97ec56d5bb7ee70509ffc2a6a749b45a03", size = 154974, upload-time = "2026-09-22T20:27:52.045Z" }, ] +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + [[package]] name = "natsort" version = "8.4.0" @@ -4602,6 +4649,7 @@ dev = [ { name = "google-cloud-bigquery" }, { name = "ipykernel" }, { name = "moto", extra = ["server"] }, + { name = "mypy" }, { name = "mypy-boto3-dynamodb" }, { name = "mypy-boto3-glue" }, { name = "nbformat" }, @@ -4636,6 +4684,13 @@ docs = [ notebook = [ { name = "jupyterlab" }, ] +types = [ + { name = "ty" }, + { name = "types-cachetools" }, + { name = "types-pytest-lazy-fixture" }, + { name = "types-pytz" }, + { name = "types-requests" }, +] [package.metadata] requires-dist = [ @@ -4697,6 +4752,7 @@ dev = [ { name = "google-cloud-bigquery", specifier = ">=3.33.0,<4" }, { name = "ipykernel", specifier = ">=6.29.0" }, { name = "moto", extras = ["server"], specifier = ">=5.0.2" }, + { name = "mypy", specifier = "==1.18.2" }, { name = "mypy-boto3-dynamodb", specifier = ">=1.28.18" }, { name = "mypy-boto3-glue", specifier = ">=1.28.18" }, { name = "nbformat", specifier = ">=5.10.0" }, @@ -4729,6 +4785,13 @@ docs = [ { name = "mkdocstrings-python", specifier = "==2.0.9" }, ] notebook = [{ name = "jupyterlab", specifier = ">=4.0.0" }] +types = [ + { name = "ty", specifier = "==0.0.84" }, + { name = "types-cachetools", specifier = "==7.0.0.20260713" }, + { name = "types-pytest-lazy-fixture", specifier = "==0.6.3.20260518" }, + { name = "types-pytz", specifier = "==2026.5.0.20261006" }, + { name = "types-requests", specifier = "==2.33.0.20261006" }, +] [[package]] name = "pyiceberg-core" @@ -6042,6 +6105,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/1c/e6e91852b72566b1ab9aca504712857dfb89324c12092a5bd61d42d34427/ty-0.0.84-py3-none-win_arm64.whl", hash = "sha256:6c9d3718c4a0d318bc9c92af9b2aa85c3294e7dff54cf1d69ad79568f56516da", size = 13656453, upload-time = "2026-09-24T13:16:21.51Z" }, ] +[[package]] +name = "types-cachetools" +version = "7.0.0.20260713" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/64/66d7efdb36ecf6826aca5415e59fe2df96e97d24157147e53acfbe8dda11/types_cachetools-7.0.0.20260713.tar.gz", hash = "sha256:f1acf079e9c66a81e096a897ef0b261a82117cf856834e37b4bd0c9a116a076a", size = 10199, upload-time = "2026-07-13T05:22:21.845Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/c7/d3525c9dbdc1be7786bad46655ef051b6e7993f656d304719ec40079c91c/types_cachetools-7.0.0.20260713-py3-none-any.whl", hash = "sha256:6db9bcc7a3840d39e91c04117d85a9d0937eacc9d14d12a873e2b01a2d24a71d", size = 9615, upload-time = "2026-07-13T05:22:20.76Z" }, +] + +[[package]] +name = "types-pytest-lazy-fixture" +version = "0.6.3.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/46/a79ab84cdace98870440968006c86eec5cab22b64f805e1770d8b19bf8d8/types_pytest_lazy_fixture-0.6.3.20260518.tar.gz", hash = "sha256:5e4c259c3c0eddbbf7719d81109050f4e9dcf9f650291a6ff3de42b964734acd", size = 7472, upload-time = "2026-05-18T06:05:45.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/3e/bfcb69336e4a1c69b87f634a3ddebeb6c77f55932631407d191d8b394128/types_pytest_lazy_fixture-0.6.3.20260518-py3-none-any.whl", hash = "sha256:5fc2bc417c2804145ddfcff91bd3f472c8cfbe79b84a2a3abae661ef45408ec1", size = 7599, upload-time = "2026-05-18T06:05:44.471Z" }, +] + +[[package]] +name = "types-pytz" +version = "2026.5.0.20261006" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/47/b493f47f2dd0971db06459439ebdeb114cf95029188268099b331bbc4727/types_pytz-2026.5.0.20261006.tar.gz", hash = "sha256:1a522c2ec03aad8d4baaf97105958019ad51704b1e471c882d1c6ccea3e5e64b", size = 11073, upload-time = "2026-10-06T08:15:12.87Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/38/e375026fb4ff74fed6bdb84d7d336571d0603b6b9675751579cec84cfc9b/types_pytz-2026.5.0.20261006-py3-none-any.whl", hash = "sha256:9e4a893b362a8eed4e10a348c80603ade65bdb3819419e589364afafe2ea08b1", size = 10168, upload-time = "2026-10-06T08:15:11.985Z" }, +] + +[[package]] +name = "types-requests" +version = "2.33.0.20261006" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/15/9b7e2e2e7c87d01366185b198b3febc6bc0c973f2bf21a62da4ab7d3495e/types_requests-2.33.0.20261006.tar.gz", hash = "sha256:0652999e9306aea345f40732d58fa49a7f6cade6a0d74d92119c5c8d82eddaf0", size = 25316, upload-time = "2026-10-06T08:15:57.782Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/72/b82789207b3d360ce9f5a4372c790c52cc3dd928aeb7d4faeec652b651b2/types_requests-2.33.0.20261006-py3-none-any.whl", hash = "sha256:26cc8146505cab33cda9737991929e4144c559bebe05078ccc6998f27c4ca2c1", size = 21445, upload-time = "2026-10-06T08:15:56.658Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0"