From 949386333ddef9c0e6c38b5bb512029822fec8ef Mon Sep 17 00:00:00 2001 From: kokila-19 Date: Mon, 31 Aug 2026 22:51:11 +0530 Subject: [PATCH] test --- .../mr/hive/HiveIcebergStorageHandler.java | 8 +- ...berg_select_after_partition_value_change.q | 41 +++++ ..._select_after_partition_value_change.q.out | 159 ++++++++++++++++++ 3 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_select_after_partition_value_change.q create mode 100644 iceberg/iceberg-handler/src/test/results/positive/iceberg_select_after_partition_value_change.q.out diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index b1857a742536..a93bc704cf4a 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -422,7 +422,6 @@ public boolean supportsExternalViewCatalog() { @Override public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer deserializer, ExprNodeDesc exprNodeDesc) { DecomposedPredicate predicate = new DecomposedPredicate(); - predicate.residualPredicate = (ExprNodeGenericFuncDesc) exprNodeDesc; ExprNodeDesc pushedPredicate = exprNodeDesc.clone(); List subExprNodes = pushedPredicate.getChildren(); @@ -441,6 +440,13 @@ public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer dese } } predicate.pushedPredicate = (ExprNodeGenericFuncDesc) pushedPredicate; + // When the full filter is pushed to Iceberg, do not keep a duplicate residual filter. + // PCR would re-evaluate partition predicates against HMS and can incorrectly fold them to false. + if (pushedPredicate != null && pushedPredicate.getExprString().equals(exprNodeDesc.getExprString())) { + predicate.residualPredicate = null; + } else { + predicate.residualPredicate = (ExprNodeGenericFuncDesc) exprNodeDesc; + } if (pushedPredicate != null) { SessionStateUtil.setConflictDetectionFilter(conf, jobConf.get(Catalogs.NAME), pushedPredicate); diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_select_after_partition_value_change.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_select_after_partition_value_change.q new file mode 100644 index 000000000000..7f4cdc4c8947 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_select_after_partition_value_change.q @@ -0,0 +1,41 @@ +-- Repro for PPD/PCR bug: filtered SELECT returns 0 after COW UPDATE changes an identity +-- partition value. UPDATE uses a data-column predicate (not str_col IS NULL) so this +-- does not depend on HIVE-29819. + +drop table if exists iceberg_select_ppd_repro; + +create external table iceberg_select_ppd_repro ( + index int, + string_col string, + boolean_col boolean, + str_col string, + tinyint_col int +) partitioned by spec(str_col, tinyint_col) +stored by iceberg +tblproperties ('write.update.mode'='copy-on-write'); + +insert into iceberg_select_ppd_repro partition (str_col, tinyint_col) + values (1, 'a', true, null, 0), + (2, 'b', false, null, 1); + +-- Avoid "where str_col is null" so UPDATE works without HIVE-29819 +update iceberg_select_ppd_repro set str_col = 'UPDATEDNULLS' where index = 1; + +select * from iceberg_select_ppd_repro; + +explain select * from iceberg_select_ppd_repro where str_col = 'UPDATEDNULLS'; +select * from iceberg_select_ppd_repro where str_col = 'UPDATEDNULLS'; + +select count(*) from iceberg_select_ppd_repro where str_col = 'UPDATEDNULLS'; + +select count(*) from iceberg_select_ppd_repro where tinyint_col = 1; + + +update iceberg_select_ppd_repro set tinyint_col = 2 where index = 2; + +select * from iceberg_select_ppd_repro; + +explain select * from iceberg_select_ppd_repro where tinyint_col = 2; +select * from iceberg_select_ppd_repro where tinyint_col = 2; + +select count(*) from iceberg_select_ppd_repro where tinyint_col = 2; diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_select_after_partition_value_change.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_select_after_partition_value_change.q.out new file mode 100644 index 000000000000..aa74843a479b --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_select_after_partition_value_change.q.out @@ -0,0 +1,159 @@ +PREHOOK: query: drop table if exists iceberg_select_ppd_repro +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists iceberg_select_ppd_repro +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create external table iceberg_select_ppd_repro ( + index int, + string_col string, + boolean_col boolean, + str_col string, + tinyint_col int +) partitioned by spec(str_col, tinyint_col) +stored by iceberg +tblproperties ('write.update.mode'='copy-on-write') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@iceberg_select_ppd_repro +POSTHOOK: query: create external table iceberg_select_ppd_repro ( + index int, + string_col string, + boolean_col boolean, + str_col string, + tinyint_col int +) partitioned by spec(str_col, tinyint_col) +stored by iceberg +tblproperties ('write.update.mode'='copy-on-write') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@iceberg_select_ppd_repro +PREHOOK: query: insert into iceberg_select_ppd_repro partition (str_col, tinyint_col) + values (1, 'a', true, null, 0), + (2, 'b', false, null, 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@iceberg_select_ppd_repro +POSTHOOK: query: insert into iceberg_select_ppd_repro partition (str_col, tinyint_col) + values (1, 'a', true, null, 0), + (2, 'b', false, null, 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@iceberg_select_ppd_repro +PREHOOK: query: update iceberg_select_ppd_repro set str_col = 'UPDATEDNULLS' where index = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_select_ppd_repro +PREHOOK: Output: default@iceberg_select_ppd_repro +POSTHOOK: query: update iceberg_select_ppd_repro set str_col = 'UPDATEDNULLS' where index = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_select_ppd_repro +POSTHOOK: Output: default@iceberg_select_ppd_repro +PREHOOK: query: select * from iceberg_select_ppd_repro +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_select_ppd_repro +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from iceberg_select_ppd_repro +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_select_ppd_repro +POSTHOOK: Output: hdfs://### HDFS PATH ### +2 b false NULL 1 +1 a true UPDATEDNULLS 0 +PREHOOK: query: explain select * from iceberg_select_ppd_repro where str_col = 'UPDATEDNULLS' +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_select_ppd_repro +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select * from iceberg_select_ppd_repro where str_col = 'UPDATEDNULLS' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_select_ppd_repro +POSTHOOK: Output: hdfs://### HDFS PATH ### +Plan optimized by CBO. + +Stage-0 + Fetch Operator + limit:-1 + Select Operator [SEL_2] + Output:["_col0","_col1","_col2","_col3","_col4"] + TableScan [TS_0] + Output:["index","string_col","boolean_col","tinyint_col"] + +PREHOOK: query: select * from iceberg_select_ppd_repro where str_col = 'UPDATEDNULLS' +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_select_ppd_repro +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from iceberg_select_ppd_repro where str_col = 'UPDATEDNULLS' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_select_ppd_repro +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 a true UPDATEDNULLS 0 +PREHOOK: query: select count(*) from iceberg_select_ppd_repro where str_col = 'UPDATEDNULLS' +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_select_ppd_repro +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(*) from iceberg_select_ppd_repro where str_col = 'UPDATEDNULLS' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_select_ppd_repro +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +PREHOOK: query: select count(*) from iceberg_select_ppd_repro where tinyint_col = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_select_ppd_repro +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(*) from iceberg_select_ppd_repro where tinyint_col = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_select_ppd_repro +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +PREHOOK: query: update iceberg_select_ppd_repro set tinyint_col = 2 where index = 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_select_ppd_repro +PREHOOK: Output: default@iceberg_select_ppd_repro +POSTHOOK: query: update iceberg_select_ppd_repro set tinyint_col = 2 where index = 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_select_ppd_repro +POSTHOOK: Output: default@iceberg_select_ppd_repro +PREHOOK: query: select * from iceberg_select_ppd_repro +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_select_ppd_repro +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from iceberg_select_ppd_repro +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_select_ppd_repro +POSTHOOK: Output: hdfs://### HDFS PATH ### +2 b false NULL 2 +1 a true UPDATEDNULLS 0 +PREHOOK: query: explain select * from iceberg_select_ppd_repro where tinyint_col = 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_select_ppd_repro +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select * from iceberg_select_ppd_repro where tinyint_col = 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_select_ppd_repro +POSTHOOK: Output: hdfs://### HDFS PATH ### +Plan optimized by CBO. + +Stage-0 + Fetch Operator + limit:-1 + Select Operator [SEL_2] + Output:["_col0","_col1","_col2","_col3","_col4"] + TableScan [TS_0] + Output:["index","string_col","boolean_col","str_col"] + +PREHOOK: query: select * from iceberg_select_ppd_repro where tinyint_col = 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_select_ppd_repro +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from iceberg_select_ppd_repro where tinyint_col = 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_select_ppd_repro +POSTHOOK: Output: hdfs://### HDFS PATH ### +2 b false NULL 2 +PREHOOK: query: select count(*) from iceberg_select_ppd_repro where tinyint_col = 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_select_ppd_repro +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(*) from iceberg_select_ppd_repro where tinyint_col = 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_select_ppd_repro +POSTHOOK: Output: hdfs://### HDFS PATH ### +1