diff --git a/pyathena/glue.py b/pyathena/glue.py index 756230b1..56147943 100644 --- a/pyathena/glue.py +++ b/pyathena/glue.py @@ -243,12 +243,14 @@ def list_databases(self, catalog_name: str | None) -> list[AthenaDatabase]: def table_metadata(table: Mapping[str, Any]) -> AthenaTableMetadata: """Build the metadata Athena reports for a Glue table. - Athena flattens the storage descriptor into the table parameters: the - location and formats are always present, the SerDe library whenever - the descriptor has SerDe information, and SerDe parameters with a - ``serde.param.`` prefix. The Glue description is not the table comment. - Glue keeps an Iceberg table's dropped and renamed columns, marked as - not current, which Athena leaves out. + Athena flattens the storage descriptor into the table parameters, + replacing parameters of the same name: the location and formats, the + SerDe library whenever the descriptor has SerDe information, and SerDe + parameters with a ``serde.param.`` prefix. A value the descriptor + leaves unset is left out rather than reported as None; an empty string + is kept. The Glue description is not the table comment. Glue keeps an + Iceberg table's dropped and renamed columns, marked as not current, + which Athena leaves out. Args: table: A ``Table`` from a Glue ``GetTable`` or ``GetTables`` response. @@ -257,16 +259,23 @@ def table_metadata(table: Mapping[str, Any]) -> AthenaTableMetadata: The table's metadata as Athena reports it. """ descriptor = table.get("StorageDescriptor") or {} - parameters = dict(table.get("Parameters") or {}) - parameters["location"] = descriptor.get("Location") - parameters["inputformat"] = descriptor.get("InputFormat") - parameters["outputformat"] = descriptor.get("OutputFormat") + serde = descriptor.get("SerdeInfo") or {} + storage = { + "location": descriptor.get("Location"), + "inputformat": descriptor.get("InputFormat"), + "outputformat": descriptor.get("OutputFormat"), + } if "SerdeInfo" in descriptor: - serde = descriptor["SerdeInfo"] - parameters["serde.serialization.lib"] = serde.get("SerializationLibrary") - parameters.update( - {f"serde.param.{k}": v for k, v in (serde.get("Parameters") or {}).items()} - ) + storage["serde.serialization.lib"] = serde.get("SerializationLibrary") + parameters = dict(table.get("Parameters") or {}) + for key, value in storage.items(): + if value is None: + parameters.pop(key, None) + else: + parameters[key] = value + parameters.update( + {f"serde.param.{k}": v for k, v in (serde.get("Parameters") or {}).items()} + ) def column(c: Mapping[str, Any]) -> dict[str, Any]: return {k: c[k] for k in ("Name", "Type", "Comment") if k in c} diff --git a/tests/pyathena/test_glue.py b/tests/pyathena/test_glue.py index 82d786fa..eecd3443 100644 --- a/tests/pyathena/test_glue.py +++ b/tests/pyathena/test_glue.py @@ -210,7 +210,8 @@ def column(name, current): "serde.param.field.delim": "\t", }, ), - # A view has empty SerDe information, which Athena still reports. + # A view has no formats and empty SerDe information; Athena reports + # only its empty location. ( { "Parameters": {"comment": "Presto View", "presto_view": "true"}, @@ -220,12 +221,9 @@ def column(name, current): "comment": "Presto View", "presto_view": "true", "location": "", - "inputformat": None, - "outputformat": None, - "serde.serialization.lib": None, }, ), - # An Iceberg table has none, and Athena reports no SerDe library. + # An Iceberg table has no formats and no SerDe information. ( { "Parameters": {"table_type": "ICEBERG", "metadata_location": "s3://m"}, @@ -235,16 +233,44 @@ def column(name, current): "table_type": "ICEBERG", "metadata_location": "s3://m", "location": "s3://bucket/iceberg", - "inputformat": None, - "outputformat": None, }, ), + # An S3 Tables table has empty formats, which Athena reports. + ( + { + "Parameters": {"table_type": "ICEBERG"}, + "StorageDescriptor": { + "Location": "s3://bucket--table-s3", + "InputFormat": "", + "OutputFormat": "", + }, + }, + { + "table_type": "ICEBERG", + "location": "s3://bucket--table-s3", + "inputformat": "", + "outputformat": "", + }, + ), + # The descriptor replaces table parameters of the same name, also + # when it leaves the value unset. + ( + { + "Parameters": { + "location": "s3://table-parameter", + "inputformat": "TableParameterInputFormat", + "serde.serialization.lib": "TableParameterSerDe", + }, + "StorageDescriptor": {"Location": "", "SerdeInfo": {}}, + }, + {"location": ""}, + ), ], - ids=["hive", "view", "iceberg"], + ids=["hive", "view", "iceberg", "s3_tables", "table_parameter_collision"], ) def test_table_metadata(self, table, expected_parameters): # Glue responses measured against GetTableMetadata for the same tables in - # #786; Athena flattens them this way. + # #786 and #887; Athena flattens them this way. table = { "Name": "t", "TableType": "EXTERNAL_TABLE",