From 21a7a9e4da8d00e988aa42d2da71477f8cec0b64 Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Wed, 12 Aug 2026 08:15:44 -0400 Subject: [PATCH 1/2] Streamline the proxy reduce tactic The tactic now works directly from the presented windows: per key bracket it lifts the join partners by the seed meet and re-sorts the bracket to time-major order in place, with suffix meets in shared columns, replacing the per-key EditList/ValueHistory copies. Pending times flatten from BTreeMaps into sorted (key, time) columns consumed by one ascending cursor per retire. Novel times enter the one reached set, matching round_coverage's single novel witness set. Retire scratch pools across retires in the tactic, which names the backend's diff types and so re-parameterizes the tactic over (B1, B2, Bk). Co-Authored-By: Claude Fable 5 --- .../src/operators/int_proxy/history.rs | 5 - .../src/operators/int_proxy/join.rs | 3 +- .../src/operators/int_proxy/mod.rs | 1 - .../src/operators/int_proxy/reduce.rs | 631 ++++++++++++------ .../tests/int_proxy_bench.rs | 4 +- 5 files changed, 418 insertions(+), 226 deletions(-) delete mode 100644 differential-dataflow/src/operators/int_proxy/history.rs diff --git a/differential-dataflow/src/operators/int_proxy/history.rs b/differential-dataflow/src/operators/int_proxy/history.rs deleted file mode 100644 index 905614ac0..000000000 --- a/differential-dataflow/src/operators/int_proxy/history.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! Time-ordered replay of proxy update histories, with meet-advancement. - -/// A value history suitable for integer proxy values. -pub(in crate::operators) type IdHistory = crate::operators::ValueHistory; - diff --git a/differential-dataflow/src/operators/int_proxy/join.rs b/differential-dataflow/src/operators/int_proxy/join.rs index 401cad3fe..61e19567e 100644 --- a/differential-dataflow/src/operators/int_proxy/join.rs +++ b/differential-dataflow/src/operators/int_proxy/join.rs @@ -15,7 +15,8 @@ use super::ProxyBridge; use crate::operators::join::{Fresh, JoinTactic}; use crate::operators::ValueHistory; -use super::history::IdHistory; +/// A value history suitable for integer proxy values. +type IdHistory = ValueHistory; /// A type that can interpret and retire pairs of lists of batches, joined by key hashes. /// diff --git a/differential-dataflow/src/operators/int_proxy/mod.rs b/differential-dataflow/src/operators/int_proxy/mod.rs index e247fc44a..536987be8 100644 --- a/differential-dataflow/src/operators/int_proxy/mod.rs +++ b/differential-dataflow/src/operators/int_proxy/mod.rs @@ -42,7 +42,6 @@ //! Both are welcome to efficiently notice that there have been no collisions and optimize, //! or to ignore the risk entirely and live dangerously. -mod history; pub mod join; pub mod reduce; diff --git a/differential-dataflow/src/operators/int_proxy/reduce.rs b/differential-dataflow/src/operators/int_proxy/reduce.rs index 736fcf817..ecf03fbcc 100644 --- a/differential-dataflow/src/operators/int_proxy/reduce.rs +++ b/differential-dataflow/src/operators/int_proxy/reduce.rs @@ -3,8 +3,6 @@ //! A conventional differential reduce against `(u64, u64)`, where the backend supplies the //! implementation of the interpretation of the integers. -use std::collections::BTreeMap; - use timely::PartialOrder; use timely::progress::{Antichain, Timestamp}; use timely::progress::frontier::AntichainRef; @@ -14,7 +12,6 @@ use crate::lattice::Lattice; use crate::trace::{BatchReader, Description}; use super::ProxyBridge; use crate::operators::reduce::{sort_dedup, ReduceTactic}; -use crate::operators::ValueHistory; /// A unit of proxied reduce work, presented to the backend. pub struct ReduceInstance<'a, B1: BatchReader, B2: BatchReader