From 6490743963bab41798af5767910f5102502d9aca Mon Sep 17 00:00:00 2001 From: Rishabh Gupta Date: Sat, 17 Oct 2026 11:59:16 -0700 Subject: [PATCH 1/2] fix: avoid pruning timestamp-to-time casts --- .../core/tests/parquet/row_group_pruning.rs | 49 +++++++++++++++++- datafusion/pruning/src/pruning_predicate.rs | 50 +++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rs index d24424f444e47..b002d1023fd35 100644 --- a/datafusion/core/tests/parquet/row_group_pruning.rs +++ b/datafusion/core/tests/parquet/row_group_pruning.rs @@ -7,8 +7,8 @@ //! expected. use std::sync::Arc; -use arrow::array::{ArrayRef, Int32Array, RecordBatch}; -use arrow_schema::{DataType, Field, Schema}; +use arrow::array::{ArrayRef, Int32Array, RecordBatch, TimestampSecondArray}; +use arrow_schema::{DataType, Field, Schema, TimeUnit}; use datafusion::prelude::SessionConfig; use datafusion_common::{DataFusionError, ScalarValue}; use itertools::Itertools; @@ -293,6 +293,51 @@ async fn prune_timestamps_seconds() { .await; } +#[tokio::test] +async fn timestamp_to_time_cast_across_midnight_is_not_pruned() { + let schema = Arc::new(Schema::new(vec![Field::new( + "ts", + DataType::Timestamp(TimeUnit::Second, None), + false, + )])); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(TimestampSecondArray::from(vec![ + 82_800, 86_400, 90_000, + ]))], + ) + .unwrap(); + + for (time_type, op, literal, expected_rows) in [ + ("Time32(Second)", "=", "00:00:00", 1), + ("Time32(Second)", "<", "02:00:00", 2), + ("Time64(Nanosecond)", "=", "00:00:00", 1), + ("Time64(Nanosecond)", "<", "02:00:00", 2), + ] { + let query = format!( + "SELECT ts FROM t WHERE arrow_cast(ts, '{time_type}') {op} \ + arrow_cast('{literal}', '{time_type}')" + ); + + RowGroupPruningTest::new() + .with_scenario(Scenario::Timestamps) + .with_query(&query) + .with_expected_errors(Some(0)) + .with_matched_by_stats(Some(1)) + .with_fully_matched_by_stats(Some(0)) + .with_pruned_by_stats(Some(0)) + .with_pruned_files(Some(0)) + .with_limit_pruned_row_groups(Some(0)) + .with_expected_rows(expected_rows) + .test_row_group_prune_with_custom_data( + Arc::clone(&schema), + vec![batch.clone()], + 3, + ) + .await; + } +} + #[tokio::test] async fn prune_date32() { RowGroupPruningTest::new() diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs index c8508ed3b52b9..62a2117384934 100644 --- a/datafusion/pruning/src/pruning_predicate.rs +++ b/datafusion/pruning/src/pruning_predicate.rs @@ -1349,6 +1349,16 @@ fn verify_support_type_for_prune(from_type: &DataType, to_type: &DataType) -> Re } _ => to_type, }; + // Casting a timestamp to a time-of-day wraps at midnight and does not + // preserve ordering across day boundaries. Casting timestamp min/max + // statistics could therefore produce invalid bounds and prune matches. + if matches!(from_type, DataType::Timestamp(_, _)) + && matches!(to_type, DataType::Time32(_) | DataType::Time64(_)) + { + return plan_err!( + "Try Cast/Cast with from type {from_type} to type {to_type} is not supported" + ); + } // If both types are strings or both are not strings (number, timestamp, etc) // then we can compare them. // PruningPredicate does not support casting of strings to numbers and such. @@ -4971,6 +4981,46 @@ mod tests { Ok(()) } + #[test] + fn row_group_predicate_cast_timestamp_to_time() { + let schema = Schema::new(vec![Field::new( + "ts", + DataType::Timestamp(TimeUnit::Second, None), + false, + )]); + + for (time_type, literal) in [ + ( + DataType::Time32(TimeUnit::Second), + ScalarValue::Time32Second(Some(0)), + ), + ( + DataType::Time32(TimeUnit::Millisecond), + ScalarValue::Time32Millisecond(Some(0)), + ), + ( + DataType::Time64(TimeUnit::Microsecond), + ScalarValue::Time64Microsecond(Some(0)), + ), + ( + DataType::Time64(TimeUnit::Nanosecond), + ScalarValue::Time64Nanosecond(Some(0)), + ), + ] { + for expr in [ + cast(col("ts"), time_type.clone()).eq(lit(literal.clone())), + try_cast(col("ts"), time_type).eq(lit(literal)), + ] { + let predicate_expr = test_build_predicate_expression( + &expr, + &schema, + &mut RequiredColumns::new(), + ); + assert_eq!(predicate_expr.to_string(), "true"); + } + } + } + #[test] fn row_group_predicate_cast_string_string() -> Result<()> { let schema = Schema::new(vec![Field::new("c1", DataType::Utf8View, false)]); From 8574947e0a4490ef04fcf99c8524d9092b0b9c66 Mon Sep 17 00:00:00 2001 From: Rishabh Gupta Date: Thu, 22 Oct 2026 01:31:15 -0700 Subject: [PATCH 2/2] test: expand timestamp-to-time pruning coverage --- datafusion/pruning/src/pruning_predicate.rs | 18 +- .../test_files/push_down_filter_parquet.slt | 162 ++++++++++++++++++ 2 files changed, 176 insertions(+), 4 deletions(-) diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs index 62a2117384934..aa48b75ae7923 100644 --- a/datafusion/pruning/src/pruning_predicate.rs +++ b/datafusion/pruning/src/pruning_predicate.rs @@ -4983,11 +4983,21 @@ mod tests { #[test] fn row_group_predicate_cast_timestamp_to_time() { - let schema = Schema::new(vec![Field::new( - "ts", + // A timezone shifts where the day boundary falls but does not remove + // it, so a timezone-aware timestamp is just as unsafe to prune on. + for timestamp_type in [ DataType::Timestamp(TimeUnit::Second, None), - false, - )]); + DataType::Timestamp(TimeUnit::Second, Some("+05:00".into())), + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + ] { + assert_timestamp_to_time_cast_is_not_prunable(timestamp_type); + } + } + + /// Asserts that no comparison of `CAST`/`TRY_CAST` from `timestamp_type` to + /// a time-of-day produces a pruning expression. + fn assert_timestamp_to_time_cast_is_not_prunable(timestamp_type: DataType) { + let schema = Schema::new(vec![Field::new("ts", timestamp_type, false)]); for (time_type, literal) in [ ( diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt index bd1213ac9945d..c5536e37da35d 100644 --- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt +++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt @@ -1333,6 +1333,168 @@ RESET datafusion.execution.parquet.pushdown_filters; statement ok drop table t; +########## +# Casting a timestamp to a time-of-day wraps at midnight +# +# https://github.com/apache/datafusion/issues/26451 +########## + +# Pruning must never change a query's answer. A cast whose output does not +# preserve the ordering of its input can invert the bounds derived from +# statistics, which makes pruning discard row groups that do contain matching +# rows. Every predicate below is therefore run twice: once against an +# in-memory table, which is never pruned and so gives the ground truth, and +# once against the same rows stored as Parquet. The two must agree. + +# Three timestamps at 23:00, 00:00 and 01:00, written as a single row group so +# that the row group's min/max bracket midnight. Casting those bounds to +# time-of-day turns them into 23:00:00 and 01:00:00, which excludes every value +# that actually lies between them. +statement ok +CREATE TABLE source_data(ts TIMESTAMP) AS VALUES + (TIMESTAMP '2024-03-10 23:00:00'), + (TIMESTAMP '2024-03-11 00:00:00'), + (TIMESTAMP '2024-03-11 01:00:00'); + +statement ok +COPY (SELECT ts FROM source_data) +TO 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps.parquet' +STORED AS PARQUET +OPTIONS ('format.max_row_group_size' '3'); + +statement ok +CREATE EXTERNAL TABLE timestamps +STORED AS PARQUET +LOCATION 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps.parquet'; + +# Equality against the row at midnight. +query P +SELECT ts FROM source_data WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00 + +query P +SELECT ts FROM timestamps WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00 + +# Range predicate spanning the wrap point. +query P +SELECT ts FROM source_data WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00 +2024-03-11T01:00:00 + +query P +SELECT ts FROM timestamps WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00 +2024-03-11T01:00:00 + +# The same predicates spelled with arrow_cast, which also reaches Time32. +query P +SELECT ts FROM source_data +WHERE arrow_cast(ts, 'Time32(Second)') = arrow_cast('00:00:00', 'Time32(Second)') +ORDER BY ts; +---- +2024-03-11T00:00:00 + +query P +SELECT ts FROM timestamps +WHERE arrow_cast(ts, 'Time32(Second)') = arrow_cast('00:00:00', 'Time32(Second)') +ORDER BY ts; +---- +2024-03-11T00:00:00 + +query P +SELECT ts FROM source_data +WHERE arrow_cast(ts, 'Time32(Second)') < arrow_cast('02:00:00', 'Time32(Second)') +ORDER BY ts; +---- +2024-03-11T00:00:00 +2024-03-11T01:00:00 + +query P +SELECT ts FROM timestamps +WHERE arrow_cast(ts, 'Time32(Second)') < arrow_cast('02:00:00', 'Time32(Second)') +ORDER BY ts; +---- +2024-03-11T00:00:00 +2024-03-11T01:00:00 + +statement ok +DROP TABLE timestamps; + +########## +# The same wrap happens at the local midnight of a timezone-aware column +########## + +# The same three local wall-clock times, tagged +05:00, so the instants +# actually stored are 18:00, 19:00 and 20:00 UTC on 2024-03-10. Those are +# monotonic and fall inside a single UTC day, so the min/max bounds only +# invert if the cast honours the column's timezone. +statement ok +CREATE TABLE source_data_tz AS +SELECT arrow_cast(ts, 'Timestamp(Nanosecond, Some("+05:00"))') AS ts +FROM source_data; + +statement ok +DROP TABLE source_data; + +statement ok +COPY (SELECT ts FROM source_data_tz) +TO 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps_tz.parquet' +STORED AS PARQUET +OPTIONS ('format.max_row_group_size' '3'); + +statement ok +CREATE EXTERNAL TABLE timestamps_tz +STORED AS PARQUET +LOCATION 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps_tz.parquet'; + +# Pin the shape of the data this section depends on: the local times have to +# straddle a local midnight while the stored instants stay monotonic within a +# single UTC day. Without this, a change to how a timezone is attached would +# leave the queries below passing while no longer exercising the wrap at all. +query TIT +SELECT + ts::string AS local_time, + to_unixtime(ts) AS stored_instant, + CAST(ts AS TIME)::string AS time_of_day +FROM timestamps_tz ORDER BY ts; +---- +2024-03-10T23:00:00+05:00 1710093600 23:00:00 +2024-03-11T00:00:00+05:00 1710097200 00:00:00 +2024-03-11T01:00:00+05:00 1710100800 01:00:00 + +query P +SELECT ts FROM source_data_tz WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00+05:00 + +query P +SELECT ts FROM timestamps_tz WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00+05:00 + +query P +SELECT ts FROM source_data_tz WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00+05:00 +2024-03-11T01:00:00+05:00 + +query P +SELECT ts FROM timestamps_tz WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00+05:00 +2024-03-11T01:00:00+05:00 + +statement ok +DROP TABLE timestamps_tz; + +statement ok +DROP TABLE source_data_tz; + # A predicate that the Parquet scan keeps only for pruning # (pushdown_filters = false) must not make the scan claim equivalences. # Each file is sorted by (a, b). If the scan claims that `a` is constant,