From e86ec41c753ca9359e1de8e71d7cf29be64ac3b4 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Tue, 29 Sep 2026 11:29:23 +0900 Subject: [PATCH 1/2] Leave unset storage values out of the Glue table parameters Athena's GetTableMetadata and ListTableMetadata now leave the storage values a table does not have out of its parameters: a view reports no inputformat, outputformat, or serde.serialization.lib, and an Iceberg table in AwsDataCatalog no inputformat or outputformat. The Glue conversion still added them as None, so the Glue metadata no longer matched Athena's for views and Iceberg tables, and reflected table properties gained None values when the Glue fallback answered. Add each storage value only when Glue sets it. Empty strings, such as a view's location or an S3 Tables table's formats, are still reported. Closes #887 Co-Authored-By: Claude Opus 5.5 --- pyathena/glue.py | 30 ++++++++++++++++-------------- tests/pyathena/test_glue.py | 31 ++++++++++++++++++++++--------- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/pyathena/glue.py b/pyathena/glue.py index 756230b1..00e50652 100644 --- a/pyathena/glue.py +++ b/pyathena/glue.py @@ -244,11 +244,11 @@ def table_metadata(table: Mapping[str, Any]) -> AthenaTableMetadata: """Build the metadata Athena reports for a Glue table. Athena flattens the storage descriptor into the table parameters: the - location and formats are always present, the SerDe library whenever - the descriptor has SerDe information, and SerDe parameters with a - ``serde.param.`` prefix. The Glue description is not the table comment. - Glue keeps an Iceberg table's dropped and renamed columns, marked as - not current, which Athena leaves out. + location, formats, and SerDe library whenever they are set (an empty + string included), and SerDe parameters with a ``serde.param.`` prefix. + The Glue description is not the table comment. Glue keeps an Iceberg + table's dropped and renamed columns, marked as not current, which + Athena leaves out. Args: table: A ``Table`` from a Glue ``GetTable`` or ``GetTables`` response. @@ -257,16 +257,18 @@ def table_metadata(table: Mapping[str, Any]) -> AthenaTableMetadata: The table's metadata as Athena reports it. """ descriptor = table.get("StorageDescriptor") or {} + serde = descriptor.get("SerdeInfo") or {} + storage = { + "location": descriptor.get("Location"), + "inputformat": descriptor.get("InputFormat"), + "outputformat": descriptor.get("OutputFormat"), + "serde.serialization.lib": serde.get("SerializationLibrary"), + } parameters = dict(table.get("Parameters") or {}) - parameters["location"] = descriptor.get("Location") - parameters["inputformat"] = descriptor.get("InputFormat") - parameters["outputformat"] = descriptor.get("OutputFormat") - if "SerdeInfo" in descriptor: - serde = descriptor["SerdeInfo"] - parameters["serde.serialization.lib"] = serde.get("SerializationLibrary") - parameters.update( - {f"serde.param.{k}": v for k, v in (serde.get("Parameters") or {}).items()} - ) + parameters.update({k: v for k, v in storage.items() if v is not None}) + parameters.update( + {f"serde.param.{k}": v for k, v in (serde.get("Parameters") or {}).items()} + ) def column(c: Mapping[str, Any]) -> dict[str, Any]: return {k: c[k] for k in ("Name", "Type", "Comment") if k in c} diff --git a/tests/pyathena/test_glue.py b/tests/pyathena/test_glue.py index 82d786fa..cd2191f4 100644 --- a/tests/pyathena/test_glue.py +++ b/tests/pyathena/test_glue.py @@ -210,7 +210,8 @@ def column(name, current): "serde.param.field.delim": "\t", }, ), - # A view has empty SerDe information, which Athena still reports. + # A view has no formats and empty SerDe information; Athena reports + # only its empty location. ( { "Parameters": {"comment": "Presto View", "presto_view": "true"}, @@ -220,12 +221,9 @@ def column(name, current): "comment": "Presto View", "presto_view": "true", "location": "", - "inputformat": None, - "outputformat": None, - "serde.serialization.lib": None, }, ), - # An Iceberg table has none, and Athena reports no SerDe library. + # An Iceberg table has no formats and no SerDe information. ( { "Parameters": {"table_type": "ICEBERG", "metadata_location": "s3://m"}, @@ -235,16 +233,31 @@ def column(name, current): "table_type": "ICEBERG", "metadata_location": "s3://m", "location": "s3://bucket/iceberg", - "inputformat": None, - "outputformat": None, + }, + ), + # An S3 Tables table has empty formats, which Athena reports. + ( + { + "Parameters": {"table_type": "ICEBERG"}, + "StorageDescriptor": { + "Location": "s3://bucket--table-s3", + "InputFormat": "", + "OutputFormat": "", + }, + }, + { + "table_type": "ICEBERG", + "location": "s3://bucket--table-s3", + "inputformat": "", + "outputformat": "", }, ), ], - ids=["hive", "view", "iceberg"], + ids=["hive", "view", "iceberg", "s3_tables"], ) def test_table_metadata(self, table, expected_parameters): # Glue responses measured against GetTableMetadata for the same tables in - # #786; Athena flattens them this way. + # #786 and #887; Athena flattens them this way. table = { "Name": "t", "TableType": "EXTERNAL_TABLE", From 278e47cbb80bd73c93bd21a3e4d5221fa67e2341 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Tue, 29 Sep 2026 11:37:36 +0900 Subject: [PATCH 2/2] Keep the storage descriptor's precedence over same-named table parameters Leaving an unset storage value out must not expose a table parameter of the same name, which the descriptor replaced before: remove that parameter too, and replace the SerDe library only when the descriptor has SerDe information, as before. Co-Authored-By: Claude Opus 5.5 --- pyathena/glue.py | 23 +++++++++++++++-------- tests/pyathena/test_glue.py | 15 ++++++++++++++- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/pyathena/glue.py b/pyathena/glue.py index 00e50652..56147943 100644 --- a/pyathena/glue.py +++ b/pyathena/glue.py @@ -243,12 +243,14 @@ def list_databases(self, catalog_name: str | None) -> list[AthenaDatabase]: def table_metadata(table: Mapping[str, Any]) -> AthenaTableMetadata: """Build the metadata Athena reports for a Glue table. - Athena flattens the storage descriptor into the table parameters: the - location, formats, and SerDe library whenever they are set (an empty - string included), and SerDe parameters with a ``serde.param.`` prefix. - The Glue description is not the table comment. Glue keeps an Iceberg - table's dropped and renamed columns, marked as not current, which - Athena leaves out. + Athena flattens the storage descriptor into the table parameters, + replacing parameters of the same name: the location and formats, the + SerDe library whenever the descriptor has SerDe information, and SerDe + parameters with a ``serde.param.`` prefix. A value the descriptor + leaves unset is left out rather than reported as None; an empty string + is kept. The Glue description is not the table comment. Glue keeps an + Iceberg table's dropped and renamed columns, marked as not current, + which Athena leaves out. Args: table: A ``Table`` from a Glue ``GetTable`` or ``GetTables`` response. @@ -262,10 +264,15 @@ def table_metadata(table: Mapping[str, Any]) -> AthenaTableMetadata: "location": descriptor.get("Location"), "inputformat": descriptor.get("InputFormat"), "outputformat": descriptor.get("OutputFormat"), - "serde.serialization.lib": serde.get("SerializationLibrary"), } + if "SerdeInfo" in descriptor: + storage["serde.serialization.lib"] = serde.get("SerializationLibrary") parameters = dict(table.get("Parameters") or {}) - parameters.update({k: v for k, v in storage.items() if v is not None}) + for key, value in storage.items(): + if value is None: + parameters.pop(key, None) + else: + parameters[key] = value parameters.update( {f"serde.param.{k}": v for k, v in (serde.get("Parameters") or {}).items()} ) diff --git a/tests/pyathena/test_glue.py b/tests/pyathena/test_glue.py index cd2191f4..eecd3443 100644 --- a/tests/pyathena/test_glue.py +++ b/tests/pyathena/test_glue.py @@ -252,8 +252,21 @@ def column(name, current): "outputformat": "", }, ), + # The descriptor replaces table parameters of the same name, also + # when it leaves the value unset. + ( + { + "Parameters": { + "location": "s3://table-parameter", + "inputformat": "TableParameterInputFormat", + "serde.serialization.lib": "TableParameterSerDe", + }, + "StorageDescriptor": {"Location": "", "SerdeInfo": {}}, + }, + {"location": ""}, + ), ], - ids=["hive", "view", "iceberg", "s3_tables"], + ids=["hive", "view", "iceberg", "s3_tables", "table_parameter_collision"], ) def test_table_metadata(self, table, expected_parameters): # Glue responses measured against GetTableMetadata for the same tables in