feat: add ProgressiveEval and rule to convert from SortPreservingMerge

datafusion/common/src/config.rs

@@ -1076,6 +1076,17 @@ config_namespace! {
         ///
         /// Disabled by default, set to a number greater than 0 for enabling it.
         pub hash_join_buffering_capacity: usize, default = 0
+
+        /// Number of input streams to prefetch ahead-of-time for `ProgressiveEvalExec`.
+        /// Since `ProgressiveEvalExec` only polls one stream at a time in order,
+        /// we do not need to prefetch all streams at once, saving resources. However, if the
+        /// streams' IO time is much greater than their CPU/processing time, prefetching them will
+        /// help improve performance.
+        /// Default is 1 which means we will prefetch one extra stream before it is polled.
+        /// 0 means streams are only fetched immediately before they are required.
+        /// Increase this value if IO time to read a stream is often much more than CPU time to
+        /// process the previous one.
+        pub progressive_eval_num_prefetch_input_streams: usize, default = 1
     }
 }
 
@@ -1753,6 +1764,12 @@ config_namespace! {
         /// Default: true
         pub enable_sort_pushdown: bool, default = true
 
+        /// When set to true, the physical plan optimizer will replace
+        /// `SortPreservingMergeExec` with `ProgressiveEvalExec` when the input
+        /// partitions are non-overlapping ranges of the merge ordering,
+        /// avoiding a merge by emitting the partitions sequentially.
+        pub sequence_sorted_inputs: bool, default = false
+
         /// When set to true, the optimizer will extract leaf expressions
         /// (such as `get_field`) from filter/sort/join nodes into projections
         /// closer to the leaf table scans, and push those projections down

datafusion/physical-optimizer/src/lib.rs

@@ -45,6 +45,7 @@ pub use datafusion_pruning as pruning;
 pub mod hash_join_buffering;
 pub mod pushdown_sort;
 pub mod sanity_checker;
+pub mod sequence_sorted_inputs;
 pub mod topk_aggregation;
 pub mod topk_repartition;
 pub mod update_aggr_exprs;

datafusion/physical-optimizer/src/optimizer.rs

@@ -38,6 +38,7 @@ use crate::update_aggr_exprs::OptimizeAggregateOrder;
 use crate::hash_join_buffering::HashJoinBuffering;
 use crate::limit_pushdown_past_window::LimitPushPastWindows;
 use crate::pushdown_sort::PushdownSort;
+use crate::sequence_sorted_inputs::SequenceSortedInputs;
 use crate::window_topn::WindowTopN;
 use datafusion_common::config::ConfigOptions;
 
@@ -174,6 +175,10 @@ impl PhysicalOptimizer {
             Arc::new(ProjectionPushdown::new()),
             // PushdownSort: Detect sorts that can be pushed down to data sources.
             Arc::new(PushdownSort::new()),
+            // SequenceSortedInputs: Replace SortPreservingMergeExec with ProgressiveEvalExec
+            // for partitions that don't overlap in the sort columns.
+            // Runs after PushdownSort which might introduce a SortPreservingMergeExec.
+            Arc::new(SequenceSortedInputs::new()),
             Arc::new(EnsureCooperative::new()),
             // This FilterPushdown handles dynamic filters that may have references to the source ExecutionPlan.
             // Therefore, it should be run at the end of the optimization process since any changes to the plan may break the dynamic filter's references.