From 7ff18554271c354601d8f30d7ae32db98e11b02c Mon Sep 17 00:00:00 2001
From: lxy <38709059+lxy-9602@users.noreply.github.com>
Date: Wed, 19 Aug 2026 19:48:34 +0800
Subject: [PATCH 01/93] feat(read): support file index and predicate pushdown
 for data evolution (#215)

---
 .../operation/data_evolution_split_read.cpp   | 139 +++++++++++-
 .../operation/data_evolution_split_read.h     |  22 +-
 .../data_evolution_split_read_test.cpp        |  27 +++
 test/inte/data_evolution_table_test.cpp       | 214 +++++++++++++++---
 4 files changed, 358 insertions(+), 44 deletions(-)

diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp
index 52a1bb51c..9e71fe47d 100644
--- a/src/paimon/core/operation/data_evolution_split_read.cpp
+++ b/src/paimon/core/operation/data_evolution_split_read.cpp
@@ -22,6 +22,7 @@
 #include <cassert>
 #include <limits>
 #include <map>
+#include <set>
 #include <string>
 #include <string_view>
 #include <thread>
@@ -53,8 +54,13 @@
 #include "paimon/core/core_options.h"
 #include "paimon/core/deletionvectors/apply_deletion_vector_batch_reader.h"
 #include "paimon/core/global_index/indexed_split_impl.h"
+#include "paimon/core/io/file_index_evaluator.h"
 #include "paimon/core/utils/blob_view_lookup.h"
 #include "paimon/core/utils/data_evolution_utils.h"
+#include "paimon/core/utils/field_mapping.h"
+#include "paimon/file_index/bitmap_index_result.h"
+#include "paimon/file_index/file_index_result.h"
+#include "paimon/predicate/predicate_utils.h"
 
 namespace paimon {
 namespace {
@@ -389,12 +395,22 @@ Result<std::unique_ptr<BatchReader>> DataEvolutionSplitRead::InnerCreateReader(
         path_factory_->CreateDataFilePathFactory(split_impl->Partition(), split_impl->Bucket()));
     auto metas = split_impl->DataFiles();
     DeletionVector::Factory split_dv_factory = CreateSplitDvFactory(*split_impl);
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Predicate> push_down_predicate,
+                           CreatePushDownPredicate(context_->GetPredicate(), raw_read_schema_));
 
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::vector<std::shared_ptr<DataFileMeta>>> split_by_row_id,
                            MergeRangesAndSort(std::move(metas)));
 
     std::vector<std::unique_ptr<BatchReader>> sub_readers;
     for (const std::vector<std::shared_ptr<DataFileMeta>>& need_merge_files : split_by_row_id) {
+        if (need_merge_files.size() > 1) {
+            PAIMON_ASSIGN_OR_RAISE(
+                bool skip_group,
+                SkipByFileIndex(push_down_predicate, need_merge_files, data_file_path_factory));
+            if (skip_group) {
+                continue;
+            }
+        }
         PAIMON_ASSIGN_OR_RAISE(std::optional<GroupDeletionVector> group_dv,
                                ReadGroupDeletionVector(need_merge_files, split_dv_factory));
         PAIMON_ASSIGN_OR_RAISE(DeletionVector::Factory group_dv_factory,
@@ -404,10 +420,15 @@ Result<std::unique_ptr<BatchReader>> DataEvolutionSplitRead::InnerCreateReader(
             PAIMON_ASSIGN_OR_RAISE(
                 std::vector<std::unique_ptr<FileBatchReader>> raw_file_readers,
                 CreateRawFileReaders(split_impl->Partition(), need_merge_files, raw_read_schema_,
-                                     /*predicate=*/nullptr, group_dv_factory, row_ranges,
+                                     push_down_predicate, group_dv_factory, row_ranges,
                                      data_file_path_factory,
                                      /*extra_format_options=*/{}));
-            assert(raw_file_readers.size() == 1);
+            if (raw_file_readers.empty()) {
+                continue;
+            }
+            if (raw_file_readers.size() != 1) {
+                return Status::Invalid("Single-file data evolution group created multiple readers");
+            }
             sub_readers.push_back(std::move(raw_file_readers[0]));
         } else {
             PAIMON_ASSIGN_OR_RAISE(
@@ -424,17 +445,110 @@ Result<std::unique_ptr<BatchReader>> DataEvolutionSplitRead::InnerCreateReader(
     return std::make_unique<CompleteRowKindBatchReader>(std::move(batch_reader), pool_);
 }
 
+Result<std::shared_ptr<Predicate>> DataEvolutionSplitRead::CreatePushDownPredicate(
+    const std::shared_ptr<Predicate>& predicate,
+    const std::shared_ptr<arrow::Schema>& read_schema) {
+    std::map<std::string, int32_t> picked_field_name_to_idx;
+    for (int32_t i = 0; i < read_schema->num_fields(); ++i) {
+        const std::string& field_name = read_schema->field(i)->name();
+        if (!SpecialFields::IsSystemField(field_name)) {
+            picked_field_name_to_idx.emplace(field_name, i);
+        }
+    }
+    return PredicateUtils::CreatePickedFieldFilter(predicate, picked_field_name_to_idx);
+}
+
+Result<bool> DataEvolutionSplitRead::SkipByFileIndex(
+    const std::shared_ptr<Predicate>& predicate,
+    const std::vector<std::shared_ptr<DataFileMeta>>& files,
+    const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) const {
+    if (!options_.FileIndexReadEnabled() || !predicate) {
+        return false;
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<FieldMappingBuilder> field_mapping_builder,
+        FieldMappingBuilder::Create(raw_read_schema_, context_->GetPartitionKeys(), predicate));
+    std::set<int32_t> claimed_field_ids;
+    for (const auto& file : files) {
+        // Blob and vector-store files may cover only part of the row range, so their indexes
+        // cannot prove that the complete merged group misses the predicate.
+        if (!DataEvolutionUtils::IsNormalFile(file->file_name)) {
+            continue;
+        }
+
+        std::shared_ptr<TableSchema> data_schema = context_->GetTableSchema();
+        if (file->schema_id != data_schema->Id()) {
+            PAIMON_ASSIGN_OR_RAISE(data_schema, schema_manager_->ReadSchema(file->schema_id));
+        }
+        std::vector<DataField> written_fields;
+        if (file->write_cols) {
+            std::vector<std::string> data_write_cols;
+            data_write_cols.reserve(file->write_cols->size());
+            for (const auto& write_col : file->write_cols.value()) {
+                if (!SpecialFields::IsSystemField(write_col)) {
+                    data_write_cols.push_back(write_col);
+                }
+            }
+            PAIMON_ASSIGN_OR_RAISE(written_fields, data_schema->GetFields(data_write_cols));
+        } else {
+            written_fields = data_schema->Fields();
+        }
+
+        std::set<std::string> overwritten_field_names;
+        for (const auto& field : written_fields) {
+            if (!claimed_field_ids.insert(field.Id()).second) {
+                overwritten_field_names.insert(field.Name());
+            }
+        }
+
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FieldMapping> field_mapping,
+                               field_mapping_builder->CreateFieldMapping(written_fields));
+        std::shared_ptr<Predicate> data_predicate =
+            field_mapping->non_partition_info.non_partition_filter;
+        if (!overwritten_field_names.empty()) {
+            PAIMON_ASSIGN_OR_RAISE(data_predicate, PredicateUtils::ExcludePredicateWithFields(
+                                                       data_predicate, overwritten_field_names));
+        }
+        if (!data_predicate) {
+            continue;
+        }
+
+        auto written_schema = DataField::ConvertDataFieldsToArrowSchema(written_fields);
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr<FileIndexResult> index_result,
+            FileIndexEvaluator::Evaluate(written_schema, data_predicate, data_file_path_factory,
+                                         file, options_.GetFileSystem(), pool_));
+        PAIMON_ASSIGN_OR_RAISE(bool is_remain, index_result->IsRemain());
+        if (!is_remain) {
+            return true;
+        }
+    }
+    return false;
+}
+
 Result<std::unique_ptr<FileBatchReader>> DataEvolutionSplitRead::ApplyIndexAndDvReaderIfNeeded(
     std::unique_ptr<FileBatchReader>&& file_reader, const std::shared_ptr<DataFileMeta>& file,
     const std::shared_ptr<arrow::Schema>& data_schema,
     const std::shared_ptr<arrow::Schema>& read_schema, const std::shared_ptr<Predicate>& predicate,
     DeletionVector::Factory dv_factory, const std::optional<std::vector<Range>>& row_ranges,
     const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) const {
-    if (predicate) {
-        assert(false);
-        // as DataEvolutionSplitRead will skip predicate
-        return Status::Invalid("DataEvolutionSplitRead do not support predicate");
+    std::shared_ptr<FileIndexResult> file_index_result;
+    if (options_.FileIndexReadEnabled()) {
+        PAIMON_ASSIGN_OR_RAISE(
+            file_index_result,
+            FileIndexEvaluator::Evaluate(data_schema, predicate, data_file_path_factory, file,
+                                         options_.GetFileSystem(), pool_));
+        PAIMON_ASSIGN_OR_RAISE(bool is_remain, file_index_result->IsRemain());
+        if (!is_remain) {
+            return std::unique_ptr<FileBatchReader>();
+        }
+    }
+    const RoaringBitmap32* index_selection = nullptr;
+    if (auto* bitmap_index = dynamic_cast<BitmapIndexResult*>(file_index_result.get())) {
+        PAIMON_ASSIGN_OR_RAISE(index_selection, bitmap_index->GetBitmap());
     }
+
     // the factory is per row range group and already returns a view taking file-local positions.
     // Unlike RawFileSplitRead the vector is not folded into the format reader's selection: it is
     // no BitmapDeletionVector, and the blob fallback path's gap segments have no format reader.
@@ -444,10 +558,19 @@ Result<std::unique_ptr<FileBatchReader>> DataEvolutionSplitRead::ApplyIndexAndDv
     }
     PAIMON_ASSIGN_OR_RAISE(std::optional<RoaringBitmap32> selection_row_ids,
                            file->ToFileSelection(row_ranges));
+    if (index_selection) {
+        if (selection_row_ids) {
+            selection_row_ids.value() &= *index_selection;
+        } else {
+            selection_row_ids = *index_selection;
+        }
+    }
+    if (selection_row_ids && selection_row_ids->IsEmpty()) {
+        return std::unique_ptr<FileBatchReader>();
+    }
     ::ArrowSchema c_read_schema;
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, &c_read_schema));
-    PAIMON_RETURN_NOT_OK(
-        file_reader->SetReadSchema(&c_read_schema, /*predicate=*/nullptr, selection_row_ids));
+    PAIMON_RETURN_NOT_OK(file_reader->SetReadSchema(&c_read_schema, predicate, selection_row_ids));
 
     std::unique_ptr<FileBatchReader> reader;
     if (!file_reader->SupportPreciseBitmapSelection() && selection_row_ids) {
diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h
index 983ca29d7..94a59fa02 100644
--- a/src/paimon/core/operation/data_evolution_split_read.h
+++ b/src/paimon/core/operation/data_evolution_split_read.h
@@ -68,9 +68,10 @@ struct DeletionFile;
 /// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader
 ///
 ///
-/// A union `SplitRead` to read multiple inner files to merge columns, note that this class
-/// does not support filtering push down: a predicate would have to be evaluated consistently
-/// across the files being merged, which is not implemented here.
+/// A union `SplitRead` to read multiple inner files to merge columns. A single-file row range
+/// group gets both file-index and format-level predicate pushdown. A merged group only uses file
+/// indexes to skip the whole group: filtering its child readers independently would break their
+/// positional alignment.
 ///
 /// Deletion vectors are supported: a row range group's vector is maintained against the
 /// group's anchor file (DataEvolutionUtils::RetrieveAnchorFile), so its positions are
@@ -172,6 +173,21 @@ class DataEvolutionSplitRead : public AbstractSplitRead {
         const std::shared_ptr<DataSplit>& data_split,
         const std::optional<std::vector<Range>>& row_ranges) const;
 
+    /// Keeps top-level conjuncts whose fields all belong to `read_schema`, excluding conjuncts
+    /// over system fields. The returned predicate is for pushdown only; the original predicate is
+    /// still evaluated as a residual filter when requested by the read context.
+    static Result<std::shared_ptr<Predicate>> CreatePushDownPredicate(
+        const std::shared_ptr<Predicate>& predicate,
+        const std::shared_ptr<arrow::Schema>& read_schema);
+
+    /// Returns true when file indexes prove that no row in a merged row range group can match.
+    /// Only normal files are considered, and an older copy of a field is excluded after a newer
+    /// file has claimed the same field id.
+    Result<bool> SkipByFileIndex(
+        const std::shared_ptr<Predicate>& predicate,
+        const std::vector<std::shared_ptr<DataFileMeta>>& files,
+        const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) const;
+
     /// Builds the deletion vector factory over the split's deletion files, keyed by data file
     /// name. Only anchor files carry one. Returns a null factory when the split has none.
     DeletionVector::Factory CreateSplitDvFactory(const DataSplitImpl& split_impl) const;
diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp
index 809933e9f..03bc95b6f 100644
--- a/src/paimon/core/operation/data_evolution_split_read_test.cpp
+++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp
@@ -25,6 +25,7 @@
 
 #include "gtest/gtest.h"
 #include "paimon/common/data/binary_row.h"
+#include "paimon/common/table/special_fields.h"
 #include "paimon/core/deletionvectors/bitmap_deletion_vector.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/manifest/file_source.h"
@@ -37,6 +38,8 @@
 #include "paimon/executor.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate_builder.h"
 #include "paimon/read_context.h"
 #include "paimon/status.h"
 #include "paimon/testing/utils/testharness.h"
@@ -102,6 +105,30 @@ class DataEvolutionSplitReadTest : public ::testing::Test {
     std::shared_ptr<MemoryPool> pool_ = GetDefaultPool();
 };
 
+TEST_F(DataEvolutionSplitReadTest, TestCreatePushDownPredicate) {
+    auto f0_predicate =
+        PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::INT, Literal(1));
+    auto f1_predicate =
+        PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::INT, Literal(2));
+    auto row_id_predicate = PredicateBuilder::Equal(
+        /*field_index=*/2, SpecialFields::RowId().Name(), FieldType::BIGINT, Literal(3l));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> predicate,
+                         PredicateBuilder::And({f0_predicate, f1_predicate, row_id_predicate}));
+
+    auto read_schema = DataField::ConvertDataFieldsToArrowSchema(
+        {DataField(0, arrow::field("f0", arrow::int32())), SpecialFields::RowId()});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> push_down,
+                         DataEvolutionSplitRead::CreatePushDownPredicate(predicate, read_schema));
+    ASSERT_TRUE(push_down);
+    ASSERT_EQ(*push_down, *f0_predicate);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> or_predicate,
+                         PredicateBuilder::Or({f0_predicate, f1_predicate}));
+    ASSERT_OK_AND_ASSIGN(
+        push_down, DataEvolutionSplitRead::CreatePushDownPredicate(or_predicate, read_schema));
+    ASSERT_FALSE(push_down);
+}
+
 TEST_F(DataEvolutionSplitReadTest, TestAddSingleBlobEntry) {
     auto blob_entry =
         CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100,
diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp
index 6232c1e7e..13fede801 100644
--- a/test/inte/data_evolution_table_test.cpp
+++ b/test/inte/data_evolution_table_test.cpp
@@ -400,10 +400,11 @@ class DataEvolutionTableTest : public ::testing::Test,
                        const std::shared_ptr<arrow::StructArray>& expected_array,
                        const std::shared_ptr<Predicate>& predicate = nullptr,
                        const std::vector<Range>& row_ranges = {},
-                       bool check_scan_plan_when_empty_result = true) const {
+                       bool check_scan_plan_when_empty_result = true,
+                       bool apply_predicate_to_scan = true) const {
         // scan
         ScanContextBuilder scan_context_builder(table_path);
-        scan_context_builder.SetPredicate(predicate);
+        scan_context_builder.SetPredicate(apply_predicate_to_scan ? predicate : nullptr);
         if (!row_ranges.empty()) {
             auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges);
             scan_context_builder.SetGlobalIndexResult(global_index_result);
@@ -1880,7 +1881,7 @@ TEST_P(DataEvolutionTableTest, TestScanAndReadWithIndex) {
                               expected_array));
     }
     {
-        // first 4 records read with data evolution, ignore index
+        // The old file's f2 index does not contain 102, but the newer file owns f2.
         auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2",
                                                  FieldType::INT, Literal(102));
         auto expected_array = std::dynamic_pointer_cast<arrow::StructArray>(
@@ -1892,51 +1893,45 @@ TEST_P(DataEvolutionTableTest, TestScanAndReadWithIndex) {
     ])")
                 .ValueOrDie());
         ASSERT_OK(ScanAndRead(table_path, arrow::schema(arrow_data_type->fields())->field_names(),
-                              expected_array, predicate));
+                              expected_array, predicate,
+                              /*row_ranges=*/{},
+                              /*check_scan_plan_when_empty_result=*/true,
+                              /*apply_predicate_to_scan=*/false));
     }
     {
-        // f2 has bitmap index, but data evolution scan and read ignore index
+        // The bitmap proves that neither row range group contains f2 = 103.
         auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2",
                                                  FieldType::INT, Literal(103));
-        auto expected_array = std::dynamic_pointer_cast<arrow::StructArray>(
-            arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type, R"([
-        ["Lily", 2, 102, 2.1],
-        ["Alice", 4, 104, 3.1],
-        ["Bob", 6, 106, 4.1],
-        ["David", 8, 108, 5.1]
-    ])")
-                .ValueOrDie());
         ASSERT_OK(ScanAndRead(table_path, arrow::schema(arrow_data_type->fields())->field_names(),
-                              expected_array, predicate));
+                              /*expected_array=*/nullptr, predicate,
+                              /*row_ranges=*/{},
+                              /*check_scan_plan_when_empty_result=*/false,
+                              /*apply_predicate_to_scan=*/false));
     }
     {
-        // f2 has bitmap index, data evolution scan will ignore index => not empty plan
-        // data evolution split read will also ignore index => not empty read batch
+        // Scan planning keeps the split, but reader-side indexes skip both row range groups.
         auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2",
                                                  FieldType::INT, Literal(203));
-        auto expected_array = std::dynamic_pointer_cast<arrow::StructArray>(
-            arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type, R"([
-        [null, null, 202, 6.1],
-        [null, null, 204, 7.1]
-    ])")
-                .ValueOrDie());
         ASSERT_OK(ScanAndRead(table_path, arrow::schema(arrow_data_type->fields())->field_names(),
-                              expected_array, predicate,
+                              /*expected_array=*/nullptr, predicate,
                               /*row_ranges=*/{},
-                              /*check_scan_plan_when_empty_result=*/true));
+                              /*check_scan_plan_when_empty_result=*/false,
+                              /*apply_predicate_to_scan=*/false));
     }
     {
-        // f2 has bitmap index, data evolution split read will ignore index
+        // A single-file group applies the exact bitmap row selection.
         auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2",
                                                  FieldType::INT, Literal(202));
         auto expected_array = std::dynamic_pointer_cast<arrow::StructArray>(
             arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type, R"([
-        [null, null, 202, 6.1],
-        [null, null, 204, 7.1]
+        [null, null, 202, 6.1]
     ])")
                 .ValueOrDie());
         ASSERT_OK(ScanAndRead(table_path, arrow::schema(arrow_data_type->fields())->field_names(),
-                              expected_array, predicate));
+                              expected_array, predicate,
+                              /*row_ranges=*/{},
+                              /*check_scan_plan_when_empty_result=*/true,
+                              /*apply_predicate_to_scan=*/false));
     }
     {
         auto predicate =
@@ -1953,7 +1948,7 @@ TEST_P(DataEvolutionTableTest, TestScanAndReadWithIndex) {
     {
         // test row id with predicate
         std::vector<Range> row_ranges = {Range(0l, 2l)};
-        // row id = {0, 1, 2}, while data evolution split read will ignore index
+        // A merged group keeps all selected row ids to preserve column alignment.
         auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2",
                                                  FieldType::INT, Literal(106));
         CheckScanResult(table_path, /*predicate=*/predicate, /*row_ranges=*/row_ranges,
@@ -1967,26 +1962,179 @@ TEST_P(DataEvolutionTableTest, TestScanAndReadWithIndex) {
                 .ValueOrDie());
         ASSERT_OK(ScanAndRead(table_path, arrow::schema(arrow_data_type->fields())->field_names(),
                               expected_array, predicate,
-                              /*row_ranges=*/row_ranges));
+                              /*row_ranges=*/row_ranges,
+                              /*check_scan_plan_when_empty_result=*/true,
+                              /*apply_predicate_to_scan=*/false));
     }
     {
         // test row id with predicate
         std::vector<Range> row_ranges = {Range(4l, 5l)};
-        // row id = {4, 5}, data evolution split read will ignore bitmap index
+        // The single-file bitmap selection is intersected with the row-id selection.
         auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2",
                                                  FieldType::INT, Literal(204));
         CheckScanResult(table_path, /*predicate=*/predicate, /*row_ranges=*/row_ranges,
                         /*expected_first_row_ids=*/{4}, /*expected_row_counts=*/{2});
         auto expected_array = std::dynamic_pointer_cast<arrow::StructArray>(
             arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type, R"([
-        [null, null, 202, 6.1],
         [null, null, 204, 7.1]
     ])")
                 .ValueOrDie());
         ASSERT_OK(ScanAndRead(table_path, arrow::schema(arrow_data_type->fields())->field_names(),
                               expected_array, predicate,
-                              /*row_ranges=*/row_ranges));
+                              /*row_ranges=*/row_ranges,
+                              /*check_scan_plan_when_empty_result=*/true,
+                              /*apply_predicate_to_scan=*/false));
+    }
+}
+
+TEST_P(DataEvolutionTableTest, TestDataEvolutionPredicatePushDownBoundaries) {
+    auto file_format = FileFormat();
+    if (file_format == "avro") {
+        return;
+    }
+    std::string table_path = paimon::test::GetDataDir() + file_format +
+                             "/data_evolution_with_index.db/data_evolution_with_index";
+
+    {
+        // A file without f0 must not interpret the predicate as f0 = null.
+        auto predicate = PredicateBuilder::Equal(
+            /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING,
+            Literal(FieldType::STRING, "Lily", 4));
+        auto read_type =
+            arrow::struct_({arrow::field("f0", arrow::utf8()), arrow::field("f2", arrow::int32())});
+        auto expected_array = std::dynamic_pointer_cast<arrow::StructArray>(
+            arrow::ipc::internal::json::ArrayFromJSON(read_type, R"([
+        ["Lily", 102],
+        ["Alice", 104],
+        ["Bob", 106],
+        ["David", 108],
+        [null, 202],
+        [null, 204]
+    ])")
+                .ValueOrDie());
+        ASSERT_OK(ScanAndRead(table_path, {"f0", "f2"}, expected_array, predicate,
+                              /*row_ranges=*/{},
+                              /*check_scan_plan_when_empty_result=*/true,
+                              /*apply_predicate_to_scan=*/false));
     }
+    {
+        // System fields are completed after reading and cannot be pushed into data files.
+        auto predicate = PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"_ROW_ID",
+                                                 FieldType::BIGINT, Literal(99l));
+        auto read_type =
+            arrow::struct_({arrow::field("f2", arrow::int32()), SpecialFields::RowId().field_});
+        auto expected_array = std::dynamic_pointer_cast<arrow::StructArray>(
+            arrow::ipc::internal::json::ArrayFromJSON(read_type, R"([
+        [102, 0],
+        [104, 1],
+        [106, 2],
+        [108, 3],
+        [202, 4],
+        [204, 5]
+    ])")
+                .ValueOrDie());
+        ASSERT_OK(ScanAndRead(table_path, {"f2", "_ROW_ID"}, expected_array, predicate,
+                              /*row_ranges=*/{},
+                              /*check_scan_plan_when_empty_result=*/true,
+                              /*apply_predicate_to_scan=*/false));
+    }
+    {
+        // Dropping a system-field conjunct must not drop a pushable data conjunct.
+        auto data_predicate = PredicateBuilder::Equal(
+            /*field_index=*/0, /*field_name=*/"f2", FieldType::INT, Literal(103));
+        auto system_predicate = PredicateBuilder::Equal(
+            /*field_index=*/1, /*field_name=*/"_ROW_ID", FieldType::BIGINT, Literal(0l));
+        ASSERT_OK_AND_ASSIGN(auto predicate,
+                             PredicateBuilder::And({data_predicate, system_predicate}));
+        ASSERT_OK(ScanAndRead(table_path, {"f2", "_ROW_ID"}, /*expected_array=*/nullptr, predicate,
+                              /*row_ranges=*/{},
+                              /*check_scan_plan_when_empty_result=*/false,
+                              /*apply_predicate_to_scan=*/false));
+    }
+    {
+        // Bitmap positions compose with row ranges without changing the physical row id.
+        auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f2",
+                                                 FieldType::INT, Literal(204));
+        auto read_type =
+            arrow::struct_({arrow::field("f2", arrow::int32()), SpecialFields::RowId().field_});
+        auto expected_array = std::dynamic_pointer_cast<arrow::StructArray>(
+            arrow::ipc::internal::json::ArrayFromJSON(read_type, R"([
+        [204, 5]
+    ])")
+                .ValueOrDie());
+        ASSERT_OK(ScanAndRead(table_path, {"f2", "_ROW_ID"}, expected_array, predicate,
+                              /*row_ranges=*/{Range(4l, 5l)},
+                              /*check_scan_plan_when_empty_result=*/true,
+                              /*apply_predicate_to_scan=*/false));
+    }
+    {
+        // The bitmap selects row id 5 while the global-index selection keeps only row id 4.
+        auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f2",
+                                                 FieldType::INT, Literal(204));
+        ASSERT_OK(ScanAndRead(table_path, {"f2", "_ROW_ID"}, /*expected_array=*/nullptr, predicate,
+                              /*row_ranges=*/{Range(4l, 4l)},
+                              /*check_scan_plan_when_empty_result=*/false,
+                              /*apply_predicate_to_scan=*/false));
+    }
+    {
+        // The predicate keeps row ids {4, 5}; the global-index selection keeps {0, 1, 2, 3, 4}.
+        auto equal_202 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f2",
+                                                 FieldType::INT, Literal(202));
+        auto equal_204 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f2",
+                                                 FieldType::INT, Literal(204));
+        ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({equal_202, equal_204}));
+        auto read_type =
+            arrow::struct_({arrow::field("f2", arrow::int32()), SpecialFields::RowId().field_});
+        auto expected_array = std::dynamic_pointer_cast<arrow::StructArray>(
+            arrow::ipc::internal::json::ArrayFromJSON(read_type, R"([
+        [202, 4]
+    ])")
+                .ValueOrDie());
+        ASSERT_OK(ScanAndRead(table_path, {"f2", "_ROW_ID"}, expected_array, predicate,
+                              /*row_ranges=*/{Range(0l, 4l)},
+                              /*check_scan_plan_when_empty_result=*/true,
+                              /*apply_predicate_to_scan=*/false));
+    }
+}
+
+TEST_P(DataEvolutionTableTest, TestFormatPredicatePushDownWithoutFileIndex) {
+    if (FileFormat() == "avro") {
+        return;
+    }
+
+    CreateDataEvolutionTable(
+        /*deletion_vectors_enabled=*/false, {{Options::FILE_INDEX_READ_ENABLED, "false"},
+                                             {Options::WRITE_BATCH_SIZE, "1"},
+                                             {"parquet.page.size", "1"},
+                                             {"parquet.enable-dictionary", "false"},
+                                             {"parquet.write.enable-page-index", "true"},
+                                             {"parquet.read.enable-page-index-filter", "true"},
+                                             {"orc.stripe.size", "1"},
+                                             {"orc.row.index.stride", "1"}});
+    std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+
+    auto input = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([
+        [1, "a", "x"],
+        [2, "b", "y"],
+        [3, "c", "z"],
+        [4, "d", "w"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK_AND_ASSIGN(auto commit_messages, WriteArray(table_path, {"f0", "f1", "f2"}, input));
+    ASSERT_OK(Commit(table_path, commit_messages));
+
+    auto predicate =
+        PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::INT, Literal(3));
+    auto expected = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([
+        [3, "c", "z"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2"}, expected, predicate,
+                          /*row_ranges=*/{},
+                          /*check_scan_plan_when_empty_result=*/true,
+                          /*apply_predicate_to_scan=*/true));
 }
 
 TEST_P(DataEvolutionTableTest, TestPredicate) {

From ec8541515c0be97056271ac8cc272b6588dd94ea Mon Sep 17 00:00:00 2001
From: Yonghao Fang <yonghao.fyh@alibaba-inc.com>
Date: Wed, 19 Aug 2026 22:10:55 +0800
Subject: [PATCH 02/93] chore(build): reformat arrow.diff patch sections (#217)

---
 cmake_modules/arrow.diff | 1524 +++++++++++++++++++-------------------
 1 file changed, 766 insertions(+), 758 deletions(-)

diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff
index f86f36e8a..ce63af352 100644
--- a/cmake_modules/arrow.diff
+++ b/cmake_modules/arrow.diff
@@ -1,22 +1,197 @@
-diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc
-index ec3890a41f..943f69bb6c 100644
---- a/cpp/src/parquet/arrow/schema.cc
-+++ b/cpp/src/parquet/arrow/schema.cc
-@@ -178,7 +178,7 @@ static Status GetTimestampMetadata(const ::arrow::TimestampType& type,
+diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
+index e7523add27..e079a1ad41 100644
+--- a/cpp/cmake_modules/BuildUtils.cmake
++++ b/cpp/cmake_modules/BuildUtils.cmake
+@@ -112,7 +112,7 @@ function(arrow_create_merged_static_lib output_target)
+     execute_process(COMMAND ${LIBTOOL_MACOS} -V
+                     OUTPUT_VARIABLE LIBTOOL_V_OUTPUT
+                     OUTPUT_STRIP_TRAILING_WHITESPACE)
+-    if(NOT "${LIBTOOL_V_OUTPUT}" MATCHES ".*cctools-([0-9.]+).*")
++    if(NOT "${LIBTOOL_V_OUTPUT}" MATCHES ".*cctools(_ld)?-([0-9.]+).*")
+       message(FATAL_ERROR "libtool found appears to be the incompatible GNU libtool: ${LIBTOOL_MACOS}"
+       )
+     endif()
+diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+index 8cb3ec83f5..0765df8fa8 100644
+--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+@@ -983,6 +983,11 @@ if(CMAKE_TOOLCHAIN_FILE)
+   list(APPEND EP_COMMON_CMAKE_ARGS -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE})
+ endif()
 
-   // The user is explicitly asking for Impala int96 encoding, there is no
-   // logical type.
--  if (arrow_properties.support_deprecated_int96_timestamps()) {
-+  if (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO) {
-     *physical_type = ParquetType::INT96;
-     return Status::OK();
-   }
++# Compatibility with bundled dependencies that require old CMake versions.
++if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.30")
++  list(APPEND EP_COMMON_CMAKE_ARGS -DCMAKE_POLICY_VERSION_MINIMUM=3.5)
++endif()
++
+ # and crosscompiling emulator (for try_run() )
+ if(CMAKE_CROSSCOMPILING_EMULATOR)
+   string(REPLACE ";" ${EP_LIST_SEPARATOR} EP_CMAKE_CROSSCOMPILING_EMULATOR
+@@ -1716,6 +1721,7 @@ macro(build_thrift)
+       -DWITH_JAVASCRIPT=OFF
+       -DWITH_LIBEVENT=OFF
+       -DWITH_NODEJS=OFF
++      -DWITH_OPENSSL=OFF
+       -DWITH_PYTHON=OFF
+       -DWITH_QT5=OFF
+       -DWITH_ZLIB=OFF)
+diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
+index b36c38c6d4..f974a33073 100644
+--- a/cpp/src/arrow/io/interfaces.h
++++ b/cpp/src/arrow/io/interfaces.h
+@@ -210,7 +210,7 @@ class ARROW_EXPORT InputStream : virtual public FileInterface, virtual public Re
+   /// \brief Advance or skip stream indicated number of bytes
+   /// \param[in] nbytes the number to move forward
+   /// \return Status
+-  Status Advance(int64_t nbytes);
++  virtual Status Advance(int64_t nbytes);
 
+   /// \brief Return zero-copy string_view to upcoming bytes.
+   ///
 diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc
-index 285e2a5973..aa6f92f077 100644
+index 285e2a5973..db919d7ef8 100644
 --- a/cpp/src/parquet/arrow/reader.cc
 +++ b/cpp/src/parquet/arrow/reader.cc
-@@ -1013,25 +1013,32 @@ Status FileReaderImpl::GetRecordBatchReader(const std::vector<int>& row_groups,
+@@ -254,6 +254,11 @@ class FileReaderImpl : public FileReader {
+     return GetColumn(i, AllRowGroupsFactory(), out);
+   }
+
++  ::arrow::Status GetColumn(
++      int i, const std::vector<int>& column_indices,
++      FileColumnIteratorFactory iterator_factory,
++      std::unique_ptr<ColumnReader>* out) override;
++
+   Status GetSchema(std::shared_ptr<::arrow::Schema>* out) override {
+     return FromParquetSchema(reader_->metadata()->schema(), reader_properties_,
+                              reader_->metadata()->key_value_metadata(), out);
+@@ -493,10 +498,40 @@ class LeafReader : public ColumnReaderImpl {
+
+   ::arrow::Status BuildArray(int64_t length_upper_bound,
+                              std::shared_ptr<::arrow::ChunkedArray>* out) final {
++    if (!out_) {
++      BEGIN_PARQUET_CATCH_EXCEPTIONS
++      RETURN_NOT_OK(
++          TransferColumnData(record_reader_.get(), field_, descr_, ctx_->pool, &out_));
++      END_PARQUET_CATCH_EXCEPTIONS
++    }
+     *out = out_;
+     return Status::OK();
+   }
+
++  std::vector<int> LeafColumnIndices() const final {
++    return {input_->column_index()};
++  }
++
++  ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final {
++    if (col_idx != input_->column_index()) return Status::OK();
++    BEGIN_PARQUET_CATCH_EXCEPTIONS
++    out_ = nullptr;
++    record_reader_->Reset();
++    record_reader_->Reserve(reserve);
++    return Status::OK();
++    END_PARQUET_CATCH_EXCEPTIONS
++  }
++
++  int64_t SkipRecords(int col_idx, int64_t num_records) final {
++    if (col_idx != input_->column_index() || num_records <= 0) return 0;
++    return record_reader_->SkipRecords(num_records);
++  }
++
++  int64_t ReadRecords(int col_idx, int64_t num_records) final {
++    if (col_idx != input_->column_index() || num_records <= 0) return 0;
++    return record_reader_->ReadRecords(num_records);
++  }
++
+   const std::shared_ptr<Field> field() override { return field_; }
+
+  private:
+@@ -532,6 +567,22 @@ class ExtensionReader : public ColumnReaderImpl {
+     return storage_reader_->LoadBatch(number_of_records);
+   }
+
++  std::vector<int> LeafColumnIndices() const final {
++    return storage_reader_->LeafColumnIndices();
++  }
++
++  ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final {
++    return storage_reader_->ResetLeaf(col_idx, reserve);
++  }
++
++  int64_t SkipRecords(int col_idx, int64_t num_records) final {
++    return storage_reader_->SkipRecords(col_idx, num_records);
++  }
++
++  int64_t ReadRecords(int col_idx, int64_t num_records) final {
++    return storage_reader_->ReadRecords(col_idx, num_records);
++  }
++
+   Status BuildArray(int64_t length_upper_bound,
+                     std::shared_ptr<ChunkedArray>* out) override {
+     std::shared_ptr<ChunkedArray> storage;
+@@ -576,6 +627,22 @@ class ListReader : public ColumnReaderImpl {
+     return item_reader_->LoadBatch(number_of_records);
+   }
+
++  std::vector<int> LeafColumnIndices() const final {
++    return item_reader_->LeafColumnIndices();
++  }
++
++  ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final {
++    return item_reader_->ResetLeaf(col_idx, reserve);
++  }
++
++  int64_t SkipRecords(int col_idx, int64_t num_records) final {
++    return item_reader_->SkipRecords(col_idx, num_records);
++  }
++
++  int64_t ReadRecords(int col_idx, int64_t num_records) final {
++    return item_reader_->ReadRecords(col_idx, num_records);
++  }
++
+   virtual ::arrow::Result<std::shared_ptr<ChunkedArray>> AssembleArray(
+       std::shared_ptr<ArrayData> data) {
+     if (field_->type()->id() == ::arrow::Type::MAP) {
+@@ -709,6 +776,39 @@ class PARQUET_NO_EXPORT StructReader : public ColumnReaderImpl {
+     }
+     return Status::OK();
+   }
++
++  std::vector<int> LeafColumnIndices() const override {
++    std::vector<int> indices;
++    for (const std::unique_ptr<ColumnReaderImpl>& reader : children_) {
++      std::vector<int> child_indices = reader->LeafColumnIndices();
++      indices.insert(indices.end(), child_indices.begin(), child_indices.end());
++    }
++    return indices;
++  }
++
++  ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) override {
++    for (const std::unique_ptr<ColumnReaderImpl>& reader : children_) {
++      RETURN_NOT_OK(reader->ResetLeaf(col_idx, reserve));
++    }
++    return Status::OK();
++  }
++
++  int64_t SkipRecords(int col_idx, int64_t num_records) override {
++    int64_t skipped = 0;
++    for (const std::unique_ptr<ColumnReaderImpl>& reader : children_) {
++      skipped += reader->SkipRecords(col_idx, num_records);
++    }
++    return skipped;
++  }
++
++  int64_t ReadRecords(int col_idx, int64_t num_records) override {
++    int64_t read = 0;
++    for (const std::unique_ptr<ColumnReaderImpl>& reader : children_) {
++      read += reader->ReadRecords(col_idx, num_records);
++    }
++    return read;
++  }
++
+   Status BuildArray(int64_t length_upper_bound,
+                     std::shared_ptr<ChunkedArray>* out) override;
+   Status GetDefLevels(const int16_t** data, int64_t* length) override;
+@@ -1013,25 +1113,32 @@ Status FileReaderImpl::GetRecordBatchReader(const std::vector<int>& row_groups,
      return Status::OK();
    }
 
@@ -55,581 +230,49 @@ index 285e2a5973..aa6f92f077 100644
 
          RETURN_NOT_OK(::arrow::internal::OptionalParallelFor(
              reader_properties_.use_threads(), static_cast<int>(readers.size()),
-diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc
-index 4fd7ef1b47..87326a54f1 100644
---- a/cpp/src/parquet/arrow/writer.cc
-+++ b/cpp/src/parquet/arrow/writer.cc
-@@ -314,6 +314,14 @@ class FileWriterImpl : public FileWriter {
-     return Status::OK();
-   }
+@@ -1224,6 +1331,23 @@ Status FileReaderImpl::GetColumn(int i, FileColumnIteratorFactory iterator_facto
+   return Status::OK();
+ }
 
-+  int64_t GetBufferedSize() override {
-+    if (row_group_writer_ == nullptr) {
-+      return 0;
-+    }
-+    return row_group_writer_->total_compressed_bytes() +
-+      row_group_writer_->total_compressed_bytes_written();
-+  }
++::arrow::Status FileReaderImpl::GetColumn(
++    int i, const std::vector<int>& column_indices,
++    FileColumnIteratorFactory iterator_factory,
++    std::unique_ptr<ColumnReader>* out) {
++  RETURN_NOT_OK(BoundsCheckColumn(i));
++  auto ctx = std::make_shared<ReaderContext>();
++  ctx->reader = reader_.get();
++  ctx->pool = pool_;
++  ctx->iterator_factory = iterator_factory;
++  ctx->filter_leaves = true;
++  ctx->included_leaves = VectorToSharedSet(column_indices);
++  std::unique_ptr<ColumnReaderImpl> result;
++  RETURN_NOT_OK(GetReader(manifest_.schema_fields[i], ctx, &result));
++  *out = std::move(result);
++  return Status::OK();
++}
 +
-   Status Close() override {
-     if (!closed_) {
-       // Make idempotent
-@@ -418,10 +426,13 @@ class FileWriterImpl : public FileWriter {
+ Status FileReaderImpl::ReadRowGroups(const std::vector<int>& row_groups,
+                                      const std::vector<int>& column_indices,
+                                      std::shared_ptr<Table>* out) {
+diff --git a/cpp/src/parquet/arrow/reader.h b/cpp/src/parquet/arrow/reader.h
+index 6e46ca43f7..e86ff0ef52 100644
+--- a/cpp/src/parquet/arrow/reader.h
++++ b/cpp/src/parquet/arrow/reader.h
+@@ -21,6 +21,7 @@
+ // N.B. we don't include async_generator.h as it's relatively heavy
+ #include <functional>
+ #include <memory>
++#include <utility>
+ #include <vector>
 
-     // Max number of rows allowed in a row group.
-     const int64_t max_row_group_length = this->properties().max_row_group_length();
-+    const int64_t max_row_group_size = this->properties().max_row_group_size();
+ #include "parquet/file_reader.h"
+@@ -48,9 +49,13 @@ namespace arrow {
 
-     // Initialize a new buffered row group writer if necessary.
-     if (row_group_writer_ == nullptr || !row_group_writer_->buffered() ||
--        row_group_writer_->num_rows() >= max_row_group_length) {
-+        row_group_writer_->num_rows() >= max_row_group_length ||
-+        (row_group_writer_->total_compressed_bytes_written() +
-+         row_group_writer_->total_compressed_bytes() >= max_row_group_size)) {
-       RETURN_NOT_OK(NewBufferedRowGroup());
-     }
-
-diff --git a/cpp/src/parquet/arrow/writer.h b/cpp/src/parquet/arrow/writer.h
-index 4a1a033a7b..0f13d05e44 100644
---- a/cpp/src/parquet/arrow/writer.h
-+++ b/cpp/src/parquet/arrow/writer.h
-@@ -138,6 +138,9 @@ class PARQUET_EXPORT FileWriter {
-   /// option in this case.
-   virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch) = 0;
-
-+  /// \brief Return the buffered size in bytes.
-+  virtual int64_t GetBufferedSize() = 0;
-+
-   /// \brief Write the footer and close the file.
-   virtual ::arrow::Status Close() = 0;
-   virtual ~FileWriter();
-diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h
-index 4d3acb491e..3906ff3c59 100644
---- a/cpp/src/parquet/properties.h
-+++ b/cpp/src/parquet/properties.h
-@@ -139,6 +139,7 @@ static constexpr bool DEFAULT_IS_DICTIONARY_ENABLED = true;
- static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = kDefaultDataPageSize;
- static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024;
- static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024;
-+static constexpr int64_t DEFAULT_MAX_ROW_GROUP_SIZE = 128 * 1024 * 1024;
- static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true;
- static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096;
- static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN;
-@@ -232,6 +233,7 @@ class PARQUET_EXPORT WriterProperties {
-           dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT),
-           write_batch_size_(DEFAULT_WRITE_BATCH_SIZE),
-           max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH),
-+          max_row_group_size_(DEFAULT_MAX_ROW_GROUP_SIZE),
-           pagesize_(kDefaultDataPageSize),
-           version_(ParquetVersion::PARQUET_2_6),
-           data_page_version_(ParquetDataPageVersion::V1),
-@@ -244,6 +246,7 @@ class PARQUET_EXPORT WriterProperties {
-           dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()),
-           write_batch_size_(properties.write_batch_size()),
-           max_row_group_length_(properties.max_row_group_length()),
-+          max_row_group_size_(properties.max_row_group_size()),
-           pagesize_(properties.data_pagesize()),
-           version_(properties.version()),
-           data_page_version_(properties.data_page_version()),
-@@ -321,6 +324,13 @@ class PARQUET_EXPORT WriterProperties {
-       return this;
-     }
-
-+    /// Specify the max bytes size to put in a single row group.
-+    /// Default 128 M.
-+    Builder* max_row_group_size(int64_t max_row_group_size) {
-+      max_row_group_size_ = max_row_group_size;
-+      return this;
-+    }
-+
-     /// Specify the data page size.
-     /// Default 1MB.
-     Builder* data_pagesize(int64_t pg_size) {
-@@ -664,7 +674,7 @@ class PARQUET_EXPORT WriterProperties {
-
-       return std::shared_ptr<WriterProperties>(new WriterProperties(
-           pool_, dictionary_pagesize_limit_, write_batch_size_, max_row_group_length_,
--          pagesize_, version_, created_by_, page_checksum_enabled_,
-+          max_row_group_size_, pagesize_, version_, created_by_, page_checksum_enabled_,
-           std::move(file_encryption_properties_), default_column_properties_,
-           column_properties, data_page_version_, store_decimal_as_integer_,
-           std::move(sorting_columns_)));
-@@ -675,6 +685,7 @@ class PARQUET_EXPORT WriterProperties {
-     int64_t dictionary_pagesize_limit_;
-     int64_t write_batch_size_;
-     int64_t max_row_group_length_;
-+    int64_t max_row_group_size_;
-     int64_t pagesize_;
-     ParquetVersion::type version_;
-     ParquetDataPageVersion data_page_version_;
-@@ -705,6 +716,8 @@ class PARQUET_EXPORT WriterProperties {
-
-   inline int64_t max_row_group_length() const { return max_row_group_length_; }
-
-+  inline int64_t max_row_group_size() const { return max_row_group_size_; }
-+
-   inline int64_t data_pagesize() const { return pagesize_; }
-
-   inline ParquetDataPageVersion data_page_version() const {
-@@ -810,7 +823,7 @@ class PARQUET_EXPORT WriterProperties {
-  private:
-   explicit WriterProperties(
-       MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t write_batch_size,
--      int64_t max_row_group_length, int64_t pagesize, ParquetVersion::type version,
-+      int64_t max_row_group_length, int64_t max_row_group_size, int64_t pagesize, ParquetVersion::type version,
-       const std::string& created_by, bool page_write_checksum_enabled,
-       std::shared_ptr<FileEncryptionProperties> file_encryption_properties,
-       const ColumnProperties& default_column_properties,
-@@ -821,6 +834,7 @@ class PARQUET_EXPORT WriterProperties {
-         dictionary_pagesize_limit_(dictionary_pagesize_limit),
-         write_batch_size_(write_batch_size),
-         max_row_group_length_(max_row_group_length),
-+        max_row_group_size_(max_row_group_size),
-         pagesize_(pagesize),
-         parquet_data_page_version_(data_page_version),
-         parquet_version_(version),
-@@ -836,6 +850,7 @@ class PARQUET_EXPORT WriterProperties {
-   int64_t dictionary_pagesize_limit_;
-   int64_t write_batch_size_;
-   int64_t max_row_group_length_;
-+  int64_t max_row_group_size_;
-   int64_t pagesize_;
-   ParquetDataPageVersion parquet_data_page_version_;
-   ParquetVersion::type parquet_version_;
-
---- a/cpp/src/parquet/file_reader.h
-+++ b/cpp/src/parquet/file_reader.h
-@@ -210,6 +210,17 @@
-   ::arrow::Future<> WhenBuffered(const std::vector<int>& row_groups,
-                                  const std::vector<int>& column_indices) const;
-
-+  /// Pre-buffer arbitrary byte ranges (e.g., page-level ranges from OffsetIndex).
-+  /// Unlike PreBuffer(), this does NOT set the column bitmap, so
-+  /// GetColumnPageReader will use CachedInputStream (page-level cache path).
-+  void PreBufferRanges(const std::vector<::arrow::io::ReadRange>& ranges,
-+                       const ::arrow::io::IOContext& ctx,
-+                       const ::arrow::io::CacheOptions& options);
-+
-+  /// Wait for arbitrary byte ranges to be pre-buffered.
-+  ::arrow::Future<> WhenBufferedRanges(
-+      const std::vector<::arrow::io::ReadRange>& ranges) const;
-+
-  private:
-   // Holds a pointer to an instance of Contents implementation
-   std::unique_ptr<Contents> contents_;
-
---- a/cpp/src/parquet/file_reader.cc
-+++ b/cpp/src/parquet/file_reader.cc
-@@ -207,6 +207,117 @@
-   return {col_start, col_length};
- }
-
-+// CachedInputStream: InputStream adapter that reads through ReadRangeCache with
-+// zero-cost skip for non-cached pages. Used for page-level caching where only
-+// specific pages are pre-buffered.
-+//
-+// Key behavior:
-+// - Read(): On cache hit, returns cached data. On cache miss, returns zero-filled
-+//   buffer (zero I/O). This makes InputStream::Advance() (which calls Read() and
-+//   discards) effectively free for skipped pages.
-+// - Peek(): Always falls back to source on cache miss, because PageReader uses
-+//   Peek() to read Thrift page headers (~30 bytes) which must have real data.
-+class CachedInputStream : public ::arrow::io::InputStream {
-+ public:
-+  CachedInputStream(
-+      std::shared_ptr<::arrow::io::internal::ReadRangeCache> cache,
-+      std::shared_ptr<ArrowInputFile> source,
-+      int64_t offset, int64_t length)
-+      : cache_(std::move(cache)),
-+        source_(std::move(source)),
-+        base_offset_(offset),
-+        length_(length) {}
-+
-+  ::arrow::Status Close() override {
-+    closed_ = true;
-+    return ::arrow::Status::OK();
-+  }
-+
-+  bool closed() const override { return closed_; }
-+
-+  ::arrow::Result<int64_t> Tell() const override { return position_; }
-+
-+  ::arrow::Result<std::string_view> Peek(int64_t nbytes) override {
-+    int64_t to_read = std::min(nbytes, length_ - position_);
-+    if (to_read <= 0) {
-+      return std::string_view();
-+    }
-+    ::arrow::io::ReadRange range{base_offset_ + position_, to_read};
-+    auto result = cache_->Read(range);
-+    if (result.ok()) {
-+      peek_buffer_ = *result;
-+    } else {
-+      // Peek is used for Thrift page headers (~30 bytes) — must read real data
-+      ARROW_ASSIGN_OR_RAISE(peek_buffer_,
-+                            source_->ReadAt(range.offset, range.length));
-+    }
-+    return std::string_view(
-+        reinterpret_cast<const char*>(peek_buffer_->data()),
-+        static_cast<size_t>(peek_buffer_->size()));
-+  }
-+
-+  ::arrow::Result<int64_t> Read(int64_t nbytes, void* out) override {
-+    int64_t to_read = std::min(nbytes, length_ - position_);
-+    if (to_read <= 0) return 0;
-+    ::arrow::io::ReadRange range{base_offset_ + position_, to_read};
-+    auto result = cache_->Read(range);
-+    if (result.ok()) {
-+      auto& buf = *result;
-+      memcpy(out, buf->data(), static_cast<size_t>(buf->size()));
-+      position_ += buf->size();
-+      return buf->size();
-+    }
-+    // Cache miss: fall back to real I/O from source
-+    ARROW_ASSIGN_OR_RAISE(auto buf, source_->ReadAt(range.offset, range.length));
-+    memcpy(out, buf->data(), static_cast<size_t>(buf->size()));
-+    position_ += buf->size();
-+    return buf->size();
-+  }
-+
-+  ::arrow::Result<std::shared_ptr<::arrow::Buffer>> Read(int64_t nbytes) override {
-+    int64_t to_read = std::min(nbytes, length_ - position_);
-+    if (to_read <= 0) {
-+      return std::make_shared<::arrow::Buffer>(nullptr, 0);
-+    }
-+    ::arrow::io::ReadRange range{base_offset_ + position_, to_read};
-+    auto result = cache_->Read(range);
-+    if (result.ok()) {
-+      position_ += (*result)->size();
-+      return *result;
-+    }
-+    // Cache miss: fall back to real I/O from source
-+    ARROW_ASSIGN_OR_RAISE(auto buf, source_->ReadAt(range.offset, range.length));
-+    position_ += buf->size();
-+    return std::shared_ptr<::arrow::Buffer>(std::move(buf));
-+  }
-+
-+  // Override Advance to avoid real I/O for skipped pages.
-+  // The default InputStream::Advance() calls Read() and discards the result,
-+  // which would trigger source_->ReadAt() on cache miss — defeating page-level
-+  // I/O skipping via data_page_filter. Since Advance() is only used to skip
-+  // over data that will not be consumed, we can safely just move the position.
-+  ::arrow::Status Advance(int64_t nbytes) override {
-+    if (nbytes <= 0) {
-+      return ::arrow::Status::OK();
-+    }
-+    int64_t remaining = length_ - position_;
-+    if (remaining <= 0) {
-+      return ::arrow::Status::OK();
-+    }
-+    position_ += std::min(nbytes, remaining);
-+    return ::arrow::Status::OK();
-+  }
-+
-+ private:
-+  std::shared_ptr<::arrow::io::internal::ReadRangeCache> cache_;
-+  std::shared_ptr<ArrowInputFile> source_;
-+  int64_t base_offset_;
-+  int64_t length_;
-+  int64_t position_ = 0;
-+  bool closed_ = false;
-+  std::shared_ptr<::arrow::Buffer> peek_buffer_;
-+};
-+
- // RowGroupReader::Contents implementation for the Parquet file specification
- class SerializedRowGroup : public RowGroupReader::Contents {
-  public:
-@@ -242,6 +343,11 @@
-       // segments.
-       PARQUET_ASSIGN_OR_THROW(auto buffer, cached_source_->Read(col_range));
-       stream = std::make_shared<::arrow::io::BufferReader>(buffer);
-+    } else if (cached_source_) {
-+      // Page-level caching: read through cache with fallback to source.
-+      // Advance() is zero-cost for skipped pages via data_page_filter.
-+      stream = std::make_shared<CachedInputStream>(
-+          cached_source_, source_, col_range.offset, col_range.length);
-     } else {
-       stream = properties_.GetStream(source_, col_range.offset, col_range.length);
-     }
-@@ -417,6 +523,26 @@
-     return cached_source_->WaitFor(ranges);
-   }
-
-+  void PreBufferRanges(const std::vector<::arrow::io::ReadRange>& ranges,
-+                       const ::arrow::io::IOContext& ctx,
-+                       const ::arrow::io::CacheOptions& options) {
-+    cached_source_ =
-+        std::make_shared<::arrow::io::internal::ReadRangeCache>(source_, ctx, options);
-+    // Do NOT set prebuffered_column_chunks_ bitmap — GetColumnPageReader will
-+    // use CachedInputStream path instead of full-chunk BufferReader path.
-+    prebuffered_column_chunks_.clear();
-+    PARQUET_THROW_NOT_OK(cached_source_->Cache(ranges));
-+  }
-+
-+  ::arrow::Future<> WhenBufferedRanges(
-+      const std::vector<::arrow::io::ReadRange>& ranges) const {
-+    if (!cached_source_) {
-+      return ::arrow::Status::Invalid(
-+          "Must call PreBufferRanges before WhenBufferedRanges");
-+    }
-+    return cached_source_->WaitFor(ranges);
-+  }
-+
-   // Metadata/footer parsing. Divided up to separate sync/async paths, and to use
-   // exceptions for error handling (with the async path converting to Future/Status).
-
-@@ -911,6 +1037,22 @@
-   return file->WhenBuffered(row_groups, column_indices);
- }
-
-+void ParquetFileReader::PreBufferRanges(
-+    const std::vector<::arrow::io::ReadRange>& ranges,
-+    const ::arrow::io::IOContext& ctx,
-+    const ::arrow::io::CacheOptions& options) {
-+  SerializedFile* file =
-+      ::arrow::internal::checked_cast<SerializedFile*>(contents_.get());
-+  file->PreBufferRanges(ranges, ctx, options);
-+}
-+
-+::arrow::Future<> ParquetFileReader::WhenBufferedRanges(
-+    const std::vector<::arrow::io::ReadRange>& ranges) const {
-+  SerializedFile* file =
-+      ::arrow::internal::checked_cast<SerializedFile*>(contents_.get());
-+  return file->WhenBufferedRanges(ranges);
-+}
-+
- // ----------------------------------------------------------------------
- // File metadata helpers
-
-diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
---- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
-+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
-@@ -981,6 +981,11 @@ if(CMAKE_TOOLCHAIN_FILE)
-   list(APPEND EP_COMMON_CMAKE_ARGS -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE})
- endif()
-
-+# Compatibility with bundled dependencies that require old CMake versions.
-+if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.30")
-+  list(APPEND EP_COMMON_CMAKE_ARGS -DCMAKE_POLICY_VERSION_MINIMUM=3.5)
-+endif()
-+
- # and crosscompiling emulator (for try_run() )
- if(CMAKE_CROSSCOMPILING_EMULATOR)
-   string(REPLACE ";" ${EP_LIST_SEPARATOR} EP_CMAKE_CROSSCOMPILING_EMULATOR
-@@ -1720,6 +1725,7 @@ macro(build_thrift)
-       -DWITH_JAVASCRIPT=OFF
-       -DWITH_LIBEVENT=OFF
-       -DWITH_NODEJS=OFF
-+      -DWITH_OPENSSL=OFF
-       -DWITH_PYTHON=OFF
-       -DWITH_QT5=OFF
-       -DWITH_ZLIB=OFF)
-diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
---- a/cpp/cmake_modules/BuildUtils.cmake
-+++ b/cpp/cmake_modules/BuildUtils.cmake
-@@ -112,7 +112,7 @@ function(arrow_create_merged_static_lib output_target)
-     execute_process(COMMAND ${LIBTOOL_MACOS} -V
-                     OUTPUT_VARIABLE LIBTOOL_V_OUTPUT
-                     OUTPUT_STRIP_TRAILING_WHITESPACE)
--    if(NOT "${LIBTOOL_V_OUTPUT}" MATCHES ".*cctools-([0-9.]+).*")
-+    if(NOT "${LIBTOOL_V_OUTPUT}" MATCHES ".*cctools(_ld)?-([0-9.]+).*")
-       message(FATAL_ERROR "libtool found appears to be the incompatible GNU libtool: ${LIBTOOL_MACOS}"
-       )
-     endif()
-
-diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
---- a/cpp/src/arrow/io/interfaces.h
-+++ b/cpp/src/arrow/io/interfaces.h
-@@ -210,7 +210,7 @@
-   /// \brief Advance or skip stream indicated number of bytes
-   /// \param[in] nbytes the number to move forward
-   /// \return Status
--  Status Advance(int64_t nbytes);
-+  virtual Status Advance(int64_t nbytes);
-
-   /// \brief Return zero-copy string_view to upcoming bytes.
-   ///
---- a/cpp/src/parquet/arrow/reader.cc
-+++ b/cpp/src/parquet/arrow/reader.cc
-@@ -254,6 +254,11 @@
-     return GetColumn(i, AllRowGroupsFactory(), out);
-   }
-
-+  ::arrow::Status GetColumn(
-+      int i, const std::vector<int>& column_indices,
-+      FileColumnIteratorFactory iterator_factory,
-+      std::unique_ptr<ColumnReader>* out) override;
-+
-   Status GetSchema(std::shared_ptr<::arrow::Schema>* out) override {
-     return FromParquetSchema(reader_->metadata()->schema(), reader_properties_,
-                              reader_->metadata()->key_value_metadata(), out);
-@@ -493,10 +498,40 @@
-
-   ::arrow::Status BuildArray(int64_t length_upper_bound,
-                              std::shared_ptr<::arrow::ChunkedArray>* out) final {
-+    if (!out_) {
-+      BEGIN_PARQUET_CATCH_EXCEPTIONS
-+      RETURN_NOT_OK(
-+          TransferColumnData(record_reader_.get(), field_, descr_, ctx_->pool, &out_));
-+      END_PARQUET_CATCH_EXCEPTIONS
-+    }
-     *out = out_;
-     return Status::OK();
-   }
-
-+  std::vector<int> LeafColumnIndices() const final {
-+    return {input_->column_index()};
-+  }
-+
-+  ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final {
-+    if (col_idx != input_->column_index()) return Status::OK();
-+    BEGIN_PARQUET_CATCH_EXCEPTIONS
-+    out_ = nullptr;
-+    record_reader_->Reset();
-+    record_reader_->Reserve(reserve);
-+    return Status::OK();
-+    END_PARQUET_CATCH_EXCEPTIONS
-+  }
-+
-+  int64_t SkipRecords(int col_idx, int64_t num_records) final {
-+    if (col_idx != input_->column_index() || num_records <= 0) return 0;
-+    return record_reader_->SkipRecords(num_records);
-+  }
-+
-+  int64_t ReadRecords(int col_idx, int64_t num_records) final {
-+    if (col_idx != input_->column_index() || num_records <= 0) return 0;
-+    return record_reader_->ReadRecords(num_records);
-+  }
-+
-   const std::shared_ptr<Field> field() override { return field_; }
-
-  private:
-@@ -532,6 +567,22 @@
-     return storage_reader_->LoadBatch(number_of_records);
-   }
-
-+  std::vector<int> LeafColumnIndices() const final {
-+    return storage_reader_->LeafColumnIndices();
-+  }
-+
-+  ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final {
-+    return storage_reader_->ResetLeaf(col_idx, reserve);
-+  }
-+
-+  int64_t SkipRecords(int col_idx, int64_t num_records) final {
-+    return storage_reader_->SkipRecords(col_idx, num_records);
-+  }
-+
-+  int64_t ReadRecords(int col_idx, int64_t num_records) final {
-+    return storage_reader_->ReadRecords(col_idx, num_records);
-+  }
-+
-   Status BuildArray(int64_t length_upper_bound,
-                     std::shared_ptr<ChunkedArray>* out) override {
-     std::shared_ptr<ChunkedArray> storage;
-@@ -576,6 +627,22 @@
-     return item_reader_->LoadBatch(number_of_records);
-   }
-
-+  std::vector<int> LeafColumnIndices() const final {
-+    return item_reader_->LeafColumnIndices();
-+  }
-+
-+  ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final {
-+    return item_reader_->ResetLeaf(col_idx, reserve);
-+  }
-+
-+  int64_t SkipRecords(int col_idx, int64_t num_records) final {
-+    return item_reader_->SkipRecords(col_idx, num_records);
-+  }
-+
-+  int64_t ReadRecords(int col_idx, int64_t num_records) final {
-+    return item_reader_->ReadRecords(col_idx, num_records);
-+  }
-+
-   virtual ::arrow::Result<std::shared_ptr<ChunkedArray>> AssembleArray(
-       std::shared_ptr<ArrayData> data) {
-     if (field_->type()->id() == ::arrow::Type::MAP) {
-@@ -709,6 +776,39 @@
-     }
-     return Status::OK();
-   }
-+
-+  std::vector<int> LeafColumnIndices() const override {
-+    std::vector<int> indices;
-+    for (const std::unique_ptr<ColumnReaderImpl>& reader : children_) {
-+      std::vector<int> child_indices = reader->LeafColumnIndices();
-+      indices.insert(indices.end(), child_indices.begin(), child_indices.end());
-+    }
-+    return indices;
-+  }
-+
-+  ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) override {
-+    for (const std::unique_ptr<ColumnReaderImpl>& reader : children_) {
-+      RETURN_NOT_OK(reader->ResetLeaf(col_idx, reserve));
-+    }
-+    return Status::OK();
-+  }
-+
-+  int64_t SkipRecords(int col_idx, int64_t num_records) override {
-+    int64_t skipped = 0;
-+    for (const std::unique_ptr<ColumnReaderImpl>& reader : children_) {
-+      skipped += reader->SkipRecords(col_idx, num_records);
-+    }
-+    return skipped;
-+  }
-+
-+  int64_t ReadRecords(int col_idx, int64_t num_records) override {
-+    int64_t read = 0;
-+    for (const std::unique_ptr<ColumnReaderImpl>& reader : children_) {
-+      read += reader->ReadRecords(col_idx, num_records);
-+    }
-+    return read;
-+  }
-+
-   Status BuildArray(int64_t length_upper_bound,
-                     std::shared_ptr<ChunkedArray>* out) override;
-   Status GetDefLevels(const int16_t** data, int64_t* length) override;
-@@ -1228,6 +1328,23 @@
-   std::unique_ptr<ColumnReaderImpl> result;
-   RETURN_NOT_OK(GetReader(manifest_.schema_fields[i], ctx, &result));
-   *out = std::move(result);
-+  return Status::OK();
-+}
-+
-+::arrow::Status FileReaderImpl::GetColumn(
-+    int i, const std::vector<int>& column_indices,
-+    FileColumnIteratorFactory iterator_factory,
-+    std::unique_ptr<ColumnReader>* out) {
-+  RETURN_NOT_OK(BoundsCheckColumn(i));
-+  auto ctx = std::make_shared<ReaderContext>();
-+  ctx->reader = reader_.get();
-+  ctx->pool = pool_;
-+  ctx->iterator_factory = iterator_factory;
-+  ctx->filter_leaves = true;
-+  ctx->included_leaves = VectorToSharedSet(column_indices);
-+  std::unique_ptr<ColumnReaderImpl> result;
-+  RETURN_NOT_OK(GetReader(manifest_.schema_fields[i], ctx, &result));
-+  *out = std::move(result);
-   return Status::OK();
- }
-
---- a/cpp/src/parquet/arrow/reader.h
-+++ b/cpp/src/parquet/arrow/reader.h
-@@ -21,6 +21,7 @@
- // N.B. we don't include async_generator.h as it's relatively heavy
- #include <functional>
- #include <memory>
-+#include <utility>
- #include <vector>
-
- #include "parquet/file_reader.h"
-@@ -48,9 +49,13 @@
-
- class ColumnChunkReader;
- class ColumnReader;
-+class FileColumnIterator;
- struct SchemaManifest;
- class RowGroupReader;
+ class ColumnChunkReader;
+ class ColumnReader;
++class FileColumnIterator;
+ struct SchemaManifest;
+ class RowGroupReader;
 
 +using FileColumnIteratorFactory =
 +    std::function<FileColumnIterator*(int, ParquetFileReader*)>;
@@ -637,7 +280,7 @@ diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
  /// \brief Arrow read adapter class for deserializing Parquet files as Arrow row batches.
  ///
  /// This interfaces caters for different use cases and thus provides different
-@@ -136,6 +141,27 @@
+@@ -136,6 +141,27 @@ class PARQUET_EXPORT FileReader {
    // The indicated column index is relative to the schema
    virtual ::arrow::Status GetColumn(int i, std::unique_ptr<ColumnReader>* out) = 0;
 
@@ -665,7 +308,7 @@ diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
    /// \brief Return arrow schema for all the columns.
    virtual ::arrow::Status GetSchema(std::shared_ptr<::arrow::Schema>* out) = 0;
 
-@@ -316,6 +342,43 @@
+@@ -316,6 +342,43 @@ class PARQUET_EXPORT ColumnReader {
    // the data available in the file.
    virtual ::arrow::Status NextBatch(int64_t batch_size,
                                      std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
@@ -698,50 +341,269 @@ diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
 +  /// error; callers convert it to Status at the public boundary.
 +  virtual int64_t ReadRecords(int col_idx, int64_t num_records) { return 0; }
 +
-+  /// \brief Build the Arrow array from previously loaded data.
-+  /// For leaf readers, calls TransferColumnData if not already done.
-+  /// For nested readers, assembles the nested array from child arrays.
-+  virtual ::arrow::Status BuildArray(
-+      int64_t length_upper_bound,
-+      std::shared_ptr<::arrow::ChunkedArray>* out) {
-+    return ::arrow::Status::NotImplemented("BuildArray not implemented");
-+  }
- };
-
- /// \brief Experimental helper class for bindings (like Python) that struggle
---- a/cpp/src/parquet/arrow/reader_internal.h
-+++ b/cpp/src/parquet/arrow/reader_internal.h
-@@ -26,6 +26,7 @@
- #include <utility>
- #include <vector>
-
-+#include "parquet/arrow/reader.h"
- #include "parquet/arrow/schema.h"
- #include "parquet/column_reader.h"
- #include "parquet/file_reader.h"
-@@ -70,7 +71,10 @@
++  /// \brief Build the Arrow array from previously loaded data.
++  /// For leaf readers, calls TransferColumnData if not already done.
++  /// For nested readers, assembles the nested array from child arrays.
++  virtual ::arrow::Status BuildArray(
++      int64_t length_upper_bound,
++      std::shared_ptr<::arrow::ChunkedArray>* out) {
++    return ::arrow::Status::NotImplemented("BuildArray not implemented");
++  }
+ };
+
+ /// \brief Experimental helper class for bindings (like Python) that struggle
+diff --git a/cpp/src/parquet/arrow/reader_internal.h b/cpp/src/parquet/arrow/reader_internal.h
+index cf9dbb8657..9216f18289 100644
+--- a/cpp/src/parquet/arrow/reader_internal.h
++++ b/cpp/src/parquet/arrow/reader_internal.h
+@@ -26,6 +26,7 @@
+ #include <utility>
+ #include <vector>
+
++#include "parquet/arrow/reader.h"
+ #include "parquet/arrow/schema.h"
+ #include "parquet/column_reader.h"
+ #include "parquet/file_reader.h"
+@@ -70,7 +71,10 @@ class FileColumnIterator {
+
+   virtual ~FileColumnIterator() {}
+
+-  std::unique_ptr<::parquet::PageReader> NextChunk() {
++  /// \brief Fetch the PageReader for the next row group in this iterator's
++  /// range. Virtual so subclasses can decorate the returned PageReader, e.g.
++  /// to install a data_page_filter for I/O-level page skipping.
++  virtual std::unique_ptr<::parquet::PageReader> NextChunk() {
+     if (row_groups_.empty()) {
+       return nullptr;
+     }
+@@ -95,9 +99,6 @@ class FileColumnIterator {
+   std::deque<int> row_groups_;
+ };
+
+-using FileColumnIteratorFactory =
+-    std::function<FileColumnIterator*(int, ParquetFileReader*)>;
+-
+ Status TransferColumnData(::parquet::internal::RecordReader* reader,
+                           const std::shared_ptr<::arrow::Field>& value_field,
+                           const ColumnDescriptor* descr, ::arrow::MemoryPool* pool,
+diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc
+index ec3890a41f..943f69bb6c 100644
+--- a/cpp/src/parquet/arrow/schema.cc
++++ b/cpp/src/parquet/arrow/schema.cc
+@@ -178,7 +178,7 @@ static Status GetTimestampMetadata(const ::arrow::TimestampType& type,
+
+   // The user is explicitly asking for Impala int96 encoding, there is no
+   // logical type.
+-  if (arrow_properties.support_deprecated_int96_timestamps()) {
++  if (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO) {
+     *physical_type = ParquetType::INT96;
+     return Status::OK();
+   }
+diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc
+index 4fd7ef1b47..87326a54f1 100644
+--- a/cpp/src/parquet/arrow/writer.cc
++++ b/cpp/src/parquet/arrow/writer.cc
+@@ -314,6 +314,14 @@ class FileWriterImpl : public FileWriter {
+     return Status::OK();
+   }
+
++  int64_t GetBufferedSize() override {
++    if (row_group_writer_ == nullptr) {
++      return 0;
++    }
++    return row_group_writer_->total_compressed_bytes() +
++      row_group_writer_->total_compressed_bytes_written();
++  }
++
+   Status Close() override {
+     if (!closed_) {
+       // Make idempotent
+@@ -418,10 +426,13 @@ class FileWriterImpl : public FileWriter {
+
+     // Max number of rows allowed in a row group.
+     const int64_t max_row_group_length = this->properties().max_row_group_length();
++    const int64_t max_row_group_size = this->properties().max_row_group_size();
+
+     // Initialize a new buffered row group writer if necessary.
+     if (row_group_writer_ == nullptr || !row_group_writer_->buffered() ||
+-        row_group_writer_->num_rows() >= max_row_group_length) {
++        row_group_writer_->num_rows() >= max_row_group_length ||
++        (row_group_writer_->total_compressed_bytes_written() +
++         row_group_writer_->total_compressed_bytes() >= max_row_group_size)) {
+       RETURN_NOT_OK(NewBufferedRowGroup());
+     }
+
+diff --git a/cpp/src/parquet/arrow/writer.h b/cpp/src/parquet/arrow/writer.h
+index 4a1a033a7b..0f13d05e44 100644
+--- a/cpp/src/parquet/arrow/writer.h
++++ b/cpp/src/parquet/arrow/writer.h
+@@ -138,6 +138,9 @@ class PARQUET_EXPORT FileWriter {
+   /// option in this case.
+   virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch) = 0;
+
++  /// \brief Return the buffered size in bytes.
++  virtual int64_t GetBufferedSize() = 0;
++
+   /// \brief Write the footer and close the file.
+   virtual ::arrow::Status Close() = 0;
+   virtual ~FileWriter();
+diff --git a/cpp/src/parquet/column_reader.cc b/cpp/src/parquet/column_reader.cc
+index ebf9515f27..0abc7d2320 100644
+--- a/cpp/src/parquet/column_reader.cc
++++ b/cpp/src/parquet/column_reader.cc
+@@ -208,6 +208,39 @@ ReaderProperties default_reader_properties() {
+   return default_reader_properties;
+ }
+
++void PageReader::set_data_page_read_plan(
++    int64_t first_data_page_offset,
++    std::vector<DataPageReadPlanEntry> data_pages) {
++  if (data_page_filter_) {
++    throw ParquetException(
++        "data_page_filter and data_page_read_plan cannot be enabled together");
++  }
++  if (first_data_page_offset < 0) {
++    throw ParquetException("Invalid negative first data page offset");
++  }
++
++  int64_t previous_end = first_data_page_offset;
++  int32_t previous_ordinal = -1;
++  for (const auto& page : data_pages) {
++    int64_t page_end;
++    if (page.page_ordinal < 0 || page.offset < first_data_page_offset ||
++        page.compressed_page_size <= 0 ||
++        AddWithOverflow(page.offset, page.compressed_page_size, &page_end)) {
++      throw ParquetException("Invalid data page read plan entry");
++    }
++    if (page.offset < previous_end || page.page_ordinal <= previous_ordinal) {
++      throw ParquetException("Data page read plan entries must be ordered");
++    }
++    previous_end = page_end;
++    previous_ordinal = page.page_ordinal;
++  }
++
++  data_page_read_plan_enabled_ = true;
++  first_data_page_offset_ = first_data_page_offset;
++  data_page_read_plan_ = std::move(data_pages);
++  next_data_page_ = 0;
++}
++
+ namespace {
+
+ // Extracts encoded statistics from V1 and V2 data page headers
+@@ -430,9 +463,43 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
+
+   // Loop here because there may be unhandled page types that we skip until
+   // finding a page that we do know what to do with
+-  while (seen_num_values_ < total_num_values_) {
++  while (data_page_read_plan_enabled_ || seen_num_values_ < total_num_values_) {
++    const DataPageReadPlanEntry* planned_data_page = nullptr;
++    uint32_t page_header_limit = max_page_header_size_;
++
++    if (data_page_read_plan_enabled_) {
++      if (next_data_page_ >= data_page_read_plan_.size()) {
++        return nullptr;
++      }
++
++      PARQUET_ASSIGN_OR_THROW(int64_t current_position, stream_->Tell());
++      if (current_position < first_data_page_offset_) {
++        page_header_limit = static_cast<uint32_t>(std::min<int64_t>(
++            page_header_limit, first_data_page_offset_ - current_position));
++      } else {
++        planned_data_page = &data_page_read_plan_[next_data_page_];
++        if (current_position > planned_data_page->offset) {
++          throw ParquetException("Data page read plan points behind stream position");
++        }
++        PARQUET_THROW_NOT_OK(
++            stream_->Advance(planned_data_page->offset - current_position));
++        PARQUET_ASSIGN_OR_THROW(int64_t target_position, stream_->Tell());
++        if (target_position != planned_data_page->offset) {
++          throw ParquetException("Failed to seek to planned data page");
++        }
++        page_ordinal_ = planned_data_page->page_ordinal;
++        page_header_limit = static_cast<uint32_t>(std::min<int64_t>(
++            page_header_limit, planned_data_page->compressed_page_size));
++      }
++    }
++
++    if (page_header_limit == 0) {
++      throw ParquetException("No bytes available for page header");
++    }
++
+     uint32_t header_size = 0;
+-    uint32_t allowed_page_size = kDefaultPageHeaderSize;
++    uint32_t allowed_page_size =
++        std::min<uint32_t>(kDefaultPageHeaderSize, page_header_limit);
 
-   virtual ~FileColumnIterator() {}
+     // Page headers can be very large because of page statistics
+     // We try to deserialize a larger buffer progressively
+@@ -458,11 +525,12 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
+         // Failed to deserialize. Double the allowed page header size and try again
+         std::stringstream ss;
+         ss << e.what();
+-        allowed_page_size *= 2;
+-        if (allowed_page_size > max_page_header_size_) {
++        if (allowed_page_size >= page_header_limit) {
+           ss << "Deserializing page header failed.\n";
+           throw ParquetException(ss.str());
+         }
++        allowed_page_size =
++            std::min<uint32_t>(allowed_page_size * 2, page_header_limit);
+       }
+     }
+     // Advance the stream offset
+@@ -474,6 +542,20 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
+       throw ParquetException("Invalid page header");
+     }
 
--  std::unique_ptr<::parquet::PageReader> NextChunk() {
-+  /// \brief Fetch the PageReader for the next row group in this iterator's
-+  /// range. Virtual so subclasses can decorate the returned PageReader, e.g.
-+  /// to install a data_page_filter for I/O-level page skipping.
-+  virtual std::unique_ptr<::parquet::PageReader> NextChunk() {
-     if (row_groups_.empty()) {
-       return nullptr;
++    const PageType::type page_type = LoadEnumSafe(&current_page_header_.type);
++    if (planned_data_page != nullptr) {
++      if (page_type != PageType::DATA_PAGE && page_type != PageType::DATA_PAGE_V2) {
++        throw ParquetException("Data page read plan points to a non-data page");
++      }
++      int64_t total_compressed_size;
++      if (AddWithOverflow(static_cast<int64_t>(header_size),
++                          static_cast<int64_t>(compressed_len),
++                          &total_compressed_size) ||
++          total_compressed_size != planned_data_page->compressed_page_size) {
++        throw ParquetException("Planned data page size does not match page header");
++      }
++    }
++
+     EncodedStatistics data_page_statistics;
+     if (ShouldSkipPage(&data_page_statistics)) {
+       PARQUET_THROW_NOT_OK(stream_->Advance(compressed_len));
+@@ -494,8 +576,6 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
+       ParquetException::EofException(ss.str());
      }
-@@ -95,9 +99,6 @@
-   std::deque<int> row_groups_;
- };
 
--using FileColumnIteratorFactory =
--    std::function<FileColumnIterator*(int, ParquetFileReader*)>;
+-    const PageType::type page_type = LoadEnumSafe(&current_page_header_.type);
 -
- Status TransferColumnData(::parquet::internal::RecordReader* reader,
-                           const std::shared_ptr<::arrow::Field>& value_field,
-                           const ColumnDescriptor* descr, ::arrow::MemoryPool* pool,
+     if (properties_.page_checksum_verification() && current_page_header_.__isset.crc &&
+         PageCanUseChecksum(page_type)) {
+       // verify crc
+@@ -534,6 +614,9 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
+                                               LoadEnumSafe(&dict_header.encoding),
+                                               is_sorted);
+     } else if (page_type == PageType::DATA_PAGE) {
++      if (planned_data_page != nullptr) {
++        ++next_data_page_;
++      }
+       ++page_ordinal_;
+       const format::DataPageHeader& header = current_page_header_.data_page_header;
+       page_buffer =
+@@ -545,6 +628,9 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
+           LoadEnumSafe(&header.repetition_level_encoding), uncompressed_len,
+           std::move(data_page_statistics));
+     } else if (page_type == PageType::DATA_PAGE_V2) {
++      if (planned_data_page != nullptr) {
++        ++next_data_page_;
++      }
+       ++page_ordinal_;
+       const format::DataPageHeaderV2& header = current_page_header_.data_page_header_v2;
+
 diff --git a/cpp/src/parquet/column_reader.h b/cpp/src/parquet/column_reader.h
+index 29e1b2a25e..386e574644 100644
 --- a/cpp/src/parquet/column_reader.h
 +++ b/cpp/src/parquet/column_reader.h
 @@ -76,6 +76,18 @@ struct PARQUET_EXPORT DataPageStats {
@@ -797,156 +659,302 @@ diff --git a/cpp/src/parquet/column_reader.h b/cpp/src/parquet/column_reader.h
  };
 
  class PARQUET_EXPORT ColumnReader {
-diff --git a/cpp/src/parquet/column_reader.cc b/cpp/src/parquet/column_reader.cc
---- a/cpp/src/parquet/column_reader.cc
-+++ b/cpp/src/parquet/column_reader.cc
-@@ -207,6 +207,39 @@ ReaderProperties default_reader_properties() {
-   return default_reader_properties;
+diff --git a/cpp/src/parquet/file_reader.cc b/cpp/src/parquet/file_reader.cc
+index 3e9eeea6c6..671ebe4644 100644
+--- a/cpp/src/parquet/file_reader.cc
++++ b/cpp/src/parquet/file_reader.cc
+@@ -207,6 +207,117 @@ const RowGroupMetaData* RowGroupReader::metadata() const { return contents_->met
+   return {col_start, col_length};
  }
 
-+void PageReader::set_data_page_read_plan(
-+    int64_t first_data_page_offset,
-+    std::vector<DataPageReadPlanEntry> data_pages) {
-+  if (data_page_filter_) {
-+    throw ParquetException(
-+        "data_page_filter and data_page_read_plan cannot be enabled together");
++// CachedInputStream: InputStream adapter that reads through ReadRangeCache with
++// zero-cost skip for non-cached pages. Used for page-level caching where only
++// specific pages are pre-buffered.
++//
++// Key behavior:
++// - Read(): On cache hit, returns cached data. On cache miss, returns zero-filled
++//   buffer (zero I/O). This makes InputStream::Advance() (which calls Read() and
++//   discards) effectively free for skipped pages.
++// - Peek(): Always falls back to source on cache miss, because PageReader uses
++//   Peek() to read Thrift page headers (~30 bytes) which must have real data.
++class CachedInputStream : public ::arrow::io::InputStream {
++ public:
++  CachedInputStream(
++      std::shared_ptr<::arrow::io::internal::ReadRangeCache> cache,
++      std::shared_ptr<ArrowInputFile> source,
++      int64_t offset, int64_t length)
++      : cache_(std::move(cache)),
++        source_(std::move(source)),
++        base_offset_(offset),
++        length_(length) {}
++
++  ::arrow::Status Close() override {
++    closed_ = true;
++    return ::arrow::Status::OK();
 +  }
-+  if (first_data_page_offset < 0) {
-+    throw ParquetException("Invalid negative first data page offset");
++
++  bool closed() const override { return closed_; }
++
++  ::arrow::Result<int64_t> Tell() const override { return position_; }
++
++  ::arrow::Result<std::string_view> Peek(int64_t nbytes) override {
++    int64_t to_read = std::min(nbytes, length_ - position_);
++    if (to_read <= 0) {
++      return std::string_view();
++    }
++    ::arrow::io::ReadRange range{base_offset_ + position_, to_read};
++    auto result = cache_->Read(range);
++    if (result.ok()) {
++      peek_buffer_ = *result;
++    } else {
++      // Peek is used for Thrift page headers (~30 bytes) — must read real data
++      ARROW_ASSIGN_OR_RAISE(peek_buffer_,
++                            source_->ReadAt(range.offset, range.length));
++    }
++    return std::string_view(
++        reinterpret_cast<const char*>(peek_buffer_->data()),
++        static_cast<size_t>(peek_buffer_->size()));
++  }
++
++  ::arrow::Result<int64_t> Read(int64_t nbytes, void* out) override {
++    int64_t to_read = std::min(nbytes, length_ - position_);
++    if (to_read <= 0) return 0;
++    ::arrow::io::ReadRange range{base_offset_ + position_, to_read};
++    auto result = cache_->Read(range);
++    if (result.ok()) {
++      auto& buf = *result;
++      memcpy(out, buf->data(), static_cast<size_t>(buf->size()));
++      position_ += buf->size();
++      return buf->size();
++    }
++    // Cache miss: fall back to real I/O from source
++    ARROW_ASSIGN_OR_RAISE(auto buf, source_->ReadAt(range.offset, range.length));
++    memcpy(out, buf->data(), static_cast<size_t>(buf->size()));
++    position_ += buf->size();
++    return buf->size();
++  }
++
++  ::arrow::Result<std::shared_ptr<::arrow::Buffer>> Read(int64_t nbytes) override {
++    int64_t to_read = std::min(nbytes, length_ - position_);
++    if (to_read <= 0) {
++      return std::make_shared<::arrow::Buffer>(nullptr, 0);
++    }
++    ::arrow::io::ReadRange range{base_offset_ + position_, to_read};
++    auto result = cache_->Read(range);
++    if (result.ok()) {
++      position_ += (*result)->size();
++      return *result;
++    }
++    // Cache miss: fall back to real I/O from source
++    ARROW_ASSIGN_OR_RAISE(auto buf, source_->ReadAt(range.offset, range.length));
++    position_ += buf->size();
++    return std::shared_ptr<::arrow::Buffer>(std::move(buf));
++  }
++
++  // Override Advance to avoid real I/O for skipped pages.
++  // The default InputStream::Advance() calls Read() and discards the result,
++  // which would trigger source_->ReadAt() on cache miss — defeating page-level
++  // I/O skipping via data_page_filter. Since Advance() is only used to skip
++  // over data that will not be consumed, we can safely just move the position.
++  ::arrow::Status Advance(int64_t nbytes) override {
++    if (nbytes <= 0) {
++      return ::arrow::Status::OK();
++    }
++    int64_t remaining = length_ - position_;
++    if (remaining <= 0) {
++      return ::arrow::Status::OK();
++    }
++    position_ += std::min(nbytes, remaining);
++    return ::arrow::Status::OK();
++  }
++
++ private:
++  std::shared_ptr<::arrow::io::internal::ReadRangeCache> cache_;
++  std::shared_ptr<ArrowInputFile> source_;
++  int64_t base_offset_;
++  int64_t length_;
++  int64_t position_ = 0;
++  bool closed_ = false;
++  std::shared_ptr<::arrow::Buffer> peek_buffer_;
++};
++
+ // RowGroupReader::Contents implementation for the Parquet file specification
+ class SerializedRowGroup : public RowGroupReader::Contents {
+  public:
+@@ -242,6 +353,11 @@ class SerializedRowGroup : public RowGroupReader::Contents {
+       // segments.
+       PARQUET_ASSIGN_OR_THROW(auto buffer, cached_source_->Read(col_range));
+       stream = std::make_shared<::arrow::io::BufferReader>(buffer);
++    } else if (cached_source_) {
++      // Page-level caching: read through cache with fallback to source.
++      // Advance() is zero-cost for skipped pages via data_page_filter.
++      stream = std::make_shared<CachedInputStream>(
++          cached_source_, source_, col_range.offset, col_range.length);
+     } else {
+       stream = properties_.GetStream(source_, col_range.offset, col_range.length);
+     }
+@@ -417,6 +533,26 @@ class SerializedFile : public ParquetFileReader::Contents {
+     return cached_source_->WaitFor(ranges);
+   }
+
++  void PreBufferRanges(const std::vector<::arrow::io::ReadRange>& ranges,
++                       const ::arrow::io::IOContext& ctx,
++                       const ::arrow::io::CacheOptions& options) {
++    cached_source_ =
++        std::make_shared<::arrow::io::internal::ReadRangeCache>(source_, ctx, options);
++    // Do NOT set prebuffered_column_chunks_ bitmap — GetColumnPageReader will
++    // use CachedInputStream path instead of full-chunk BufferReader path.
++    prebuffered_column_chunks_.clear();
++    PARQUET_THROW_NOT_OK(cached_source_->Cache(ranges));
 +  }
 +
-+  int64_t previous_end = first_data_page_offset;
-+  int32_t previous_ordinal = -1;
-+  for (const auto& page : data_pages) {
-+    int64_t page_end;
-+    if (page.page_ordinal < 0 || page.offset < first_data_page_offset ||
-+        page.compressed_page_size <= 0 ||
-+        AddWithOverflow(page.offset, page.compressed_page_size, &page_end)) {
-+      throw ParquetException("Invalid data page read plan entry");
-+    }
-+    if (page.offset < previous_end || page.page_ordinal <= previous_ordinal) {
-+      throw ParquetException("Data page read plan entries must be ordered");
++  ::arrow::Future<> WhenBufferedRanges(
++      const std::vector<::arrow::io::ReadRange>& ranges) const {
++    if (!cached_source_) {
++      return ::arrow::Status::Invalid(
++          "Must call PreBufferRanges before WhenBufferedRanges");
 +    }
-+    previous_end = page_end;
-+    previous_ordinal = page.page_ordinal;
++    return cached_source_->WaitFor(ranges);
 +  }
 +
-+  data_page_read_plan_enabled_ = true;
-+  first_data_page_offset_ = first_data_page_offset;
-+  data_page_read_plan_ = std::move(data_pages);
-+  next_data_page_ = 0;
+   // Metadata/footer parsing. Divided up to separate sync/async paths, and to use
+   // exceptions for error handling (with the async path converting to Future/Status).
+
+@@ -911,6 +1047,22 @@ void ParquetFileReader::PreBuffer(const std::vector<int>& row_groups,
+   return file->WhenBuffered(row_groups, column_indices);
+ }
+
++void ParquetFileReader::PreBufferRanges(
++    const std::vector<::arrow::io::ReadRange>& ranges,
++    const ::arrow::io::IOContext& ctx,
++    const ::arrow::io::CacheOptions& options) {
++  SerializedFile* file =
++      ::arrow::internal::checked_cast<SerializedFile*>(contents_.get());
++  file->PreBufferRanges(ranges, ctx, options);
 +}
 +
- namespace {
++::arrow::Future<> ParquetFileReader::WhenBufferedRanges(
++    const std::vector<::arrow::io::ReadRange>& ranges) const {
++  SerializedFile* file =
++      ::arrow::internal::checked_cast<SerializedFile*>(contents_.get());
++  return file->WhenBufferedRanges(ranges);
++}
++
+ // ----------------------------------------------------------------------
+ // File metadata helpers
 
- // Extracts encoded statistics from V1 and V2 data page headers
-@@ -430,9 +463,43 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
+diff --git a/cpp/src/parquet/file_reader.h b/cpp/src/parquet/file_reader.h
+index b59b59f95c..657a438a3a 100644
+--- a/cpp/src/parquet/file_reader.h
++++ b/cpp/src/parquet/file_reader.h
+@@ -210,6 +210,17 @@ class PARQUET_EXPORT ParquetFileReader {
+   ::arrow::Future<> WhenBuffered(const std::vector<int>& row_groups,
+                                  const std::vector<int>& column_indices) const;
 
-   // Loop here because there may be unhandled page types that we skip until
-   // finding a page that we do know what to do with
--  while (seen_num_values_ < total_num_values_) {
-+  while (data_page_read_plan_enabled_ || seen_num_values_ < total_num_values_) {
-+    const DataPageReadPlanEntry* planned_data_page = nullptr;
-+    uint32_t page_header_limit = max_page_header_size_;
-+
-+    if (data_page_read_plan_enabled_) {
-+      if (next_data_page_ >= data_page_read_plan_.size()) {
-+        return nullptr;
-+      }
++  /// Pre-buffer arbitrary byte ranges (e.g., page-level ranges from OffsetIndex).
++  /// Unlike PreBuffer(), this does NOT set the column bitmap, so
++  /// GetColumnPageReader will use CachedInputStream (page-level cache path).
++  void PreBufferRanges(const std::vector<::arrow::io::ReadRange>& ranges,
++                       const ::arrow::io::IOContext& ctx,
++                       const ::arrow::io::CacheOptions& options);
 +
-+      PARQUET_ASSIGN_OR_THROW(int64_t current_position, stream_->Tell());
-+      if (current_position < first_data_page_offset_) {
-+        page_header_limit = static_cast<uint32_t>(std::min<int64_t>(
-+            page_header_limit, first_data_page_offset_ - current_position));
-+      } else {
-+        planned_data_page = &data_page_read_plan_[next_data_page_];
-+        if (current_position > planned_data_page->offset) {
-+          throw ParquetException("Data page read plan points behind stream position");
-+        }
-+        PARQUET_THROW_NOT_OK(
-+            stream_->Advance(planned_data_page->offset - current_position));
-+        PARQUET_ASSIGN_OR_THROW(int64_t target_position, stream_->Tell());
-+        if (target_position != planned_data_page->offset) {
-+          throw ParquetException("Failed to seek to planned data page");
-+        }
-+        page_ordinal_ = planned_data_page->page_ordinal;
-+        page_header_limit = static_cast<uint32_t>(std::min<int64_t>(
-+            page_header_limit, planned_data_page->compressed_page_size));
-+      }
-+    }
++  /// Wait for arbitrary byte ranges to be pre-buffered.
++  ::arrow::Future<> WhenBufferedRanges(
++      const std::vector<::arrow::io::ReadRange>& ranges) const;
 +
-+    if (page_header_limit == 0) {
-+      throw ParquetException("No bytes available for page header");
+  private:
+   // Holds a pointer to an instance of Contents implementation
+   std::unique_ptr<Contents> contents_;
+diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h
+index 4d3acb491e..3906ff3c59 100644
+--- a/cpp/src/parquet/properties.h
++++ b/cpp/src/parquet/properties.h
+@@ -139,6 +139,7 @@ static constexpr bool DEFAULT_IS_DICTIONARY_ENABLED = true;
+ static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = kDefaultDataPageSize;
+ static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024;
+ static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024;
++static constexpr int64_t DEFAULT_MAX_ROW_GROUP_SIZE = 128 * 1024 * 1024;
+ static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true;
+ static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096;
+ static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN;
+@@ -232,6 +233,7 @@ class PARQUET_EXPORT WriterProperties {
+           dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT),
+           write_batch_size_(DEFAULT_WRITE_BATCH_SIZE),
+           max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH),
++          max_row_group_size_(DEFAULT_MAX_ROW_GROUP_SIZE),
+           pagesize_(kDefaultDataPageSize),
+           version_(ParquetVersion::PARQUET_2_6),
+           data_page_version_(ParquetDataPageVersion::V1),
+@@ -244,6 +246,7 @@ class PARQUET_EXPORT WriterProperties {
+           dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()),
+           write_batch_size_(properties.write_batch_size()),
+           max_row_group_length_(properties.max_row_group_length()),
++          max_row_group_size_(properties.max_row_group_size()),
+           pagesize_(properties.data_pagesize()),
+           version_(properties.version()),
+           data_page_version_(properties.data_page_version()),
+@@ -321,6 +324,13 @@ class PARQUET_EXPORT WriterProperties {
+       return this;
+     }
+
++    /// Specify the max bytes size to put in a single row group.
++    /// Default 128 M.
++    Builder* max_row_group_size(int64_t max_row_group_size) {
++      max_row_group_size_ = max_row_group_size;
++      return this;
 +    }
 +
-     uint32_t header_size = 0;
--    uint32_t allowed_page_size = kDefaultPageHeaderSize;
-+    uint32_t allowed_page_size =
-+        std::min<uint32_t>(kDefaultPageHeaderSize, page_header_limit);
+     /// Specify the data page size.
+     /// Default 1MB.
+     Builder* data_pagesize(int64_t pg_size) {
+@@ -664,7 +674,7 @@ class PARQUET_EXPORT WriterProperties {
 
-     // Page headers can be very large because of page statistics
-     // We try to deserialize a larger buffer progressively
-@@ -458,11 +525,12 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
-         // Failed to deserialize. Double the allowed page header size and try again
-         std::stringstream ss;
-         ss << e.what();
--        allowed_page_size *= 2;
--        if (allowed_page_size > max_page_header_size_) {
-+        if (allowed_page_size >= page_header_limit) {
-           ss << "Deserializing page header failed.\n";
-           throw ParquetException(ss.str());
-         }
-+        allowed_page_size =
-+            std::min<uint32_t>(allowed_page_size * 2, page_header_limit);
-       }
-     }
-     // Advance the stream offset
-@@ -474,6 +542,20 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
-       throw ParquetException("Invalid page header");
-     }
+       return std::shared_ptr<WriterProperties>(new WriterProperties(
+           pool_, dictionary_pagesize_limit_, write_batch_size_, max_row_group_length_,
+-          pagesize_, version_, created_by_, page_checksum_enabled_,
++          max_row_group_size_, pagesize_, version_, created_by_, page_checksum_enabled_,
+           std::move(file_encryption_properties_), default_column_properties_,
+           column_properties, data_page_version_, store_decimal_as_integer_,
+           std::move(sorting_columns_)));
+@@ -675,6 +685,7 @@ class PARQUET_EXPORT WriterProperties {
+     int64_t dictionary_pagesize_limit_;
+     int64_t write_batch_size_;
+     int64_t max_row_group_length_;
++    int64_t max_row_group_size_;
+     int64_t pagesize_;
+     ParquetVersion::type version_;
+     ParquetDataPageVersion data_page_version_;
+@@ -705,6 +716,8 @@ class PARQUET_EXPORT WriterProperties {
 
-+    const PageType::type page_type = LoadEnumSafe(&current_page_header_.type);
-+    if (planned_data_page != nullptr) {
-+      if (page_type != PageType::DATA_PAGE && page_type != PageType::DATA_PAGE_V2) {
-+        throw ParquetException("Data page read plan points to a non-data page");
-+      }
-+      int64_t total_compressed_size;
-+      if (AddWithOverflow(static_cast<int64_t>(header_size),
-+                          static_cast<int64_t>(compressed_len),
-+                          &total_compressed_size) ||
-+          total_compressed_size != planned_data_page->compressed_page_size) {
-+        throw ParquetException("Planned data page size does not match page header");
-+      }
-+    }
+   inline int64_t max_row_group_length() const { return max_row_group_length_; }
+
++  inline int64_t max_row_group_size() const { return max_row_group_size_; }
 +
-     EncodedStatistics data_page_statistics;
-     if (ShouldSkipPage(&data_page_statistics)) {
-       PARQUET_THROW_NOT_OK(stream_->Advance(compressed_len));
-@@ -494,8 +576,6 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
-       ParquetException::EofException(ss.str());
-     }
+   inline int64_t data_pagesize() const { return pagesize_; }
 
--    const PageType::type page_type = LoadEnumSafe(&current_page_header_.type);
--
-     if (properties_.page_checksum_verification() && current_page_header_.__isset.crc &&
-         PageCanUseChecksum(page_type)) {
-       // verify crc
-@@ -534,6 +614,9 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
-                                               LoadEnumSafe(&dict_header.encoding),
-                                               is_sorted);
-     } else if (page_type == PageType::DATA_PAGE) {
-+      if (planned_data_page != nullptr) {
-+        ++next_data_page_;
-+      }
-       ++page_ordinal_;
-       const format::DataPageHeader& header = current_page_header_.data_page_header;
-       page_buffer =
-@@ -545,6 +628,9 @@ std::shared_ptr<Page> SerializedPageReader::NextPage() {
-           LoadEnumSafe(&header.repetition_level_encoding), uncompressed_len,
-           std::move(data_page_statistics));
-     } else if (page_type == PageType::DATA_PAGE_V2) {
-+      if (planned_data_page != nullptr) {
-+        ++next_data_page_;
-+      }
-       ++page_ordinal_;
-       const format::DataPageHeaderV2& header = current_page_header_.data_page_header_v2;
+   inline ParquetDataPageVersion data_page_version() const {
+@@ -810,7 +823,7 @@ class PARQUET_EXPORT WriterProperties {
+  private:
+   explicit WriterProperties(
+       MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t write_batch_size,
+-      int64_t max_row_group_length, int64_t pagesize, ParquetVersion::type version,
++      int64_t max_row_group_length, int64_t max_row_group_size, int64_t pagesize, ParquetVersion::type version,
+       const std::string& created_by, bool page_write_checksum_enabled,
+       std::shared_ptr<FileEncryptionProperties> file_encryption_properties,
+       const ColumnProperties& default_column_properties,
+@@ -821,6 +834,7 @@ class PARQUET_EXPORT WriterProperties {
+         dictionary_pagesize_limit_(dictionary_pagesize_limit),
+         write_batch_size_(write_batch_size),
+         max_row_group_length_(max_row_group_length),
++        max_row_group_size_(max_row_group_size),
+         pagesize_(pagesize),
+         parquet_data_page_version_(data_page_version),
+         parquet_version_(version),
+@@ -836,6 +850,7 @@ class PARQUET_EXPORT WriterProperties {
+   int64_t dictionary_pagesize_limit_;
+   int64_t write_batch_size_;
+   int64_t max_row_group_length_;
++  int64_t max_row_group_size_;
+   int64_t pagesize_;
+   ParquetDataPageVersion parquet_data_page_version_;
+   ParquetVersion::type parquet_version_;

From 223e566b8dfb794fe9b9f7bb4c096e95b648c3a5 Mon Sep 17 00:00:00 2001
From: Yonghao Fang <yonghao.fyh@alibaba-inc.com>
Date: Wed, 19 Aug 2026 23:08:01 +0800
Subject: [PATCH 03/93] feat(prefetch): support read-ahead cache for Parquet
 reads (#209)

---
 include/paimon/format/read_hints.h            |  35 ++
 include/paimon/format/reader_builder.h        |  11 +
 include/paimon/read_context.h                 |  21 +-
 include/paimon/utils/prefetch_cache_config.h  |  76 +++
 .../apply_bitmap_index_batch_reader_test.cpp  |   4 +-
 src/paimon/common/io/cache_input_stream.h     |  20 +-
 .../common/io/cache_input_stream_test.cpp     |   6 +-
 .../prefetch_file_batch_reader_impl.cpp       |  70 +--
 .../reader/prefetch_file_batch_reader_impl.h  |  11 +-
 .../prefetch_file_batch_reader_impl_test.cpp  | 348 ++++++------
 .../common/utils/byte_range_combiner.cpp      |  15 +
 src/paimon/common/utils/byte_range_combiner.h |   2 +-
 .../common/utils/byte_range_combiner_test.cpp |  26 +-
 src/paimon/common/utils/read_ahead_cache.cpp  | 312 ++++++++---
 .../paimon/common}/utils/read_ahead_cache.h   | 136 ++---
 .../common/utils/read_ahead_cache_test.cpp    | 509 +++++++++++++++---
 ...pply_deletion_vector_batch_reader_test.cpp |   4 +-
 .../core/operation/abstract_split_read.cpp    |   9 +-
 .../core/operation/internal_read_context.h    |   4 +-
 src/paimon/core/operation/read_context.cpp    |  14 +-
 .../core/operation/read_context_test.cpp      |  11 +-
 src/paimon/core/table/bucket_mode.cpp         |  10 +-
 src/paimon/core/table/bucket_mode.h           |   6 +
 src/paimon/core/table/bucket_mode_test.cpp    |   8 +-
 .../table/system/audit_log_system_table.cpp   |   2 +-
 .../system/read_optimized_system_table.cpp    |   2 +-
 .../format/parquet/file_reader_wrapper.cpp    | 137 +++--
 .../format/parquet/file_reader_wrapper.h      |  33 +-
 .../parquet/file_reader_wrapper_test.cpp      | 331 +++++++++++-
 .../page_filtered_row_group_reader_test.cpp   |   9 +-
 .../parquet/parquet_file_batch_reader.cpp     |  36 +-
 .../parquet/parquet_file_batch_reader.h       |  10 +-
 .../parquet_file_batch_reader_test.cpp        | 278 +++++++++-
 .../format/parquet/parquet_reader_builder.h   |  16 +-
 .../parquet/predicate_pushdown_test.cpp       |   3 +-
 .../format/parquet/variant_parquet_test.cpp   |  14 +-
 test/inte/read_inte_test.cpp                  | 154 ++++--
 37 files changed, 2054 insertions(+), 639 deletions(-)
 create mode 100644 include/paimon/format/read_hints.h
 create mode 100644 include/paimon/utils/prefetch_cache_config.h
 rename {include/paimon => src/paimon/common}/utils/read_ahead_cache.h (50%)

diff --git a/include/paimon/format/read_hints.h b/include/paimon/format/read_hints.h
new file mode 100644
index 000000000..d60b41320
--- /dev/null
+++ b/include/paimon/format/read_hints.h
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "paimon/visibility.h"
+
+namespace paimon {
+
+/// Runtime state of the framework read path, passed to format layers via
+/// `ReaderBuilder::WithReadHints` so each format can adapt its internal behavior
+/// (e.g. whether parquet enables its own pre-buffering).
+struct PAIMON_EXPORT ReadHints {
+    /// Whether framework-level prefetch is enabled for this read.
+    bool prefetch_enabled = false;
+    /// Whether the shared read-ahead cache is enabled for this read.
+    bool read_ahead_cache_enabled = false;
+};
+
+}  // namespace paimon
diff --git a/include/paimon/format/reader_builder.h b/include/paimon/format/reader_builder.h
index b0837a262..5a28077a8 100644
--- a/include/paimon/format/reader_builder.h
+++ b/include/paimon/format/reader_builder.h
@@ -19,7 +19,9 @@
 #pragma once
 
 #include <memory>
+#include <optional>
 
+#include "paimon/format/read_hints.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/reader/file_batch_reader.h"
 #include "paimon/type_fwd.h"
@@ -41,6 +43,15 @@ class PAIMON_EXPORT ReaderBuilder {
         return this;
     }
 
+    /// Inject runtime read state from the framework layer, so the format can adapt
+    /// its internal behavior accordingly. When present, the hints describe the
+    /// authoritative runtime state of this read; when absent, the format should fall
+    /// back to its own options.
+    virtual ReaderBuilder* WithReadHints(const std::optional<ReadHints>& hints) {
+        (void)hints;
+        return this;
+    }
+
     /// Build a file batch reader based on the created `InputStream`.
     virtual Result<std::unique_ptr<FileBatchReader>> Build(
         const std::shared_ptr<InputStream>& path) const = 0;
diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h
index 9bed54024..3e58b1c45 100644
--- a/include/paimon/read_context.h
+++ b/include/paimon/read_context.h
@@ -30,7 +30,7 @@
 #include "paimon/predicate/predicate.h"
 #include "paimon/result.h"
 #include "paimon/type_fwd.h"
-#include "paimon/utils/read_ahead_cache.h"
+#include "paimon/utils/prefetch_cache_config.h"
 #include "paimon/visibility.h"
 
 namespace paimon {
@@ -59,9 +59,8 @@ class PAIMON_EXPORT ReadContext {
                 const std::shared_ptr<FileSystem>& specific_file_system,
                 const std::map<std::string, std::string>& fs_scheme_to_identifier_map,
                 const std::shared_ptr<RealtimeContext>& realtime_context,
-                const std::map<std::string, std::string>& options,
-                PrefetchCacheMode prefetch_cache_mode, const CacheConfig& cache_config,
-                const std::shared_ptr<Cache>& cache);
+                const std::map<std::string, std::string>& options, bool read_ahead_cache_enabled,
+                const CacheConfig& cache_config, const std::shared_ptr<Cache>& cache);
     ~ReadContext();
 
     const std::string& GetPath() const {
@@ -128,8 +127,8 @@ class PAIMON_EXPORT ReadContext {
         return realtime_context_;
     }
 
-    PrefetchCacheMode GetPrefetchCacheMode() const {
-        return prefetch_cache_mode_;
+    bool ReadAheadCacheEnabled() const {
+        return read_ahead_cache_enabled_;
     }
 
     const CacheConfig& GetCacheConfig() const {
@@ -175,7 +174,7 @@ class PAIMON_EXPORT ReadContext {
     std::map<std::string, std::string> fs_scheme_to_identifier_map_;
     std::shared_ptr<RealtimeContext> realtime_context_;
     std::map<std::string, std::string> options_;
-    PrefetchCacheMode prefetch_cache_mode_;
+    bool read_ahead_cache_enabled_;
     CacheConfig cache_config_;
     std::shared_ptr<Cache> cache_;
     // Owns schema resources and releases ArrowSchema::release in destructor.
@@ -307,13 +306,13 @@ class PAIMON_EXPORT ReadContextBuilder {
     /// @return Reference to this builder for method chaining.
     ReadContextBuilder& EnablePrefetch(bool enabled);
 
-    /// Set prefetch cache mode for read operations.
+    /// Enable or disable the read-ahead cache for read operations.
     ///
-    /// A prefetch cache is used to prebuffer data ranges before they are needed,
+    /// A read-ahead cache is used to prebuffer data ranges before they are needed,
     /// which can improve read performance by reducing redundant I/O operations.
-    /// @param mode (default: PrefetchCacheMode::ALWAYS)
+    /// @param enabled Whether to enable the read-ahead cache (default: true)
     /// @return Reference to this builder for method chaining.
-    ReadContextBuilder& SetPrefetchCacheMode(PrefetchCacheMode mode);
+    ReadContextBuilder& SetReadAheadCacheEnabled(bool enabled);
 
     /// Set the cache configuration for prefetch read operations.
     ///
diff --git a/include/paimon/utils/prefetch_cache_config.h b/include/paimon/utils/prefetch_cache_config.h
new file mode 100644
index 000000000..4bbf1ecd3
--- /dev/null
+++ b/include/paimon/utils/prefetch_cache_config.h
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Adapted from Apache ORC
+// https://github.com/apache/orc/blob/main/c%2B%2B/src/io/Cache.hh
+
+#pragma once
+
+#include <cstdint>
+
+#include "paimon/visibility.h"
+
+namespace paimon {
+
+/// Configuration parameters for the read-ahead cache behavior.
+///
+/// This struct controls various limits and prefetching strategies used by
+/// ReadAheadCache to balance memory usage, I/O efficiency, and latency hiding.
+class PAIMON_EXPORT CacheConfig {
+ public:
+    CacheConfig();
+    CacheConfig(uint64_t range_size_limit, uint64_t hole_size_limit, uint64_t pre_buffer_limit);
+
+    /// Returns the maximum allowed size (in bytes) for a single cached range.
+    uint64_t GetRangeSizeLimit() const {
+        return range_size_limit_;
+    }
+
+    /// Sets the maximum allowed size (in bytes) for a single cached range.
+    void SetRangeSizeLimit(uint64_t range_size_limit) {
+        range_size_limit_ = range_size_limit;
+    }
+
+    /// Returns the maximum gap size (in bytes) considered mergeable between adjacent ranges.
+    uint64_t GetHoleSizeLimit() const {
+        return hole_size_limit_;
+    }
+
+    /// Sets the maximum gap size (in bytes) considered mergeable between adjacent ranges.
+    void SetHoleSizeLimit(uint64_t hole_size_limit) {
+        hole_size_limit_ = hole_size_limit;
+    }
+
+    /// Returns the maximum size to pre-buffer ahead of the current read position.
+    uint64_t GetPreBufferLimit() const {
+        return pre_buffer_limit_;
+    }
+
+    /// Sets the maximum size to pre-buffer ahead of the current read position.
+    void SetPreBufferLimit(uint64_t pre_buffer_limit) {
+        pre_buffer_limit_ = pre_buffer_limit;
+    }
+
+ private:
+    uint64_t range_size_limit_;
+    uint64_t hole_size_limit_;
+    uint64_t pre_buffer_limit_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp b/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp
index d068d3c2b..1082e6695 100644
--- a/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp
+++ b/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp
@@ -30,6 +30,7 @@
 #include "gtest/gtest.h"
 #include "paimon/common/reader/prefetch_file_batch_reader_impl.h"
 #include "paimon/common/utils/date_time_utils.h"
+#include "paimon/common/utils/read_ahead_cache.h"
 #include "paimon/executor.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/testing/mock/mock_file_batch_reader.h"
@@ -37,7 +38,6 @@
 #include "paimon/testing/mock/mock_format_reader_builder.h"
 #include "paimon/testing/utils/read_result_collector.h"
 #include "paimon/testing/utils/testharness.h"
-#include "paimon/utils/read_ahead_cache.h"
 
 namespace arrow {
 class Array;
@@ -97,7 +97,7 @@ class ApplyBitmapIndexBatchReaderTest : public ::testing::Test,
                         prefetch_batch_count, batch_size, prefetch_batch_count * 2,
                         /*enable_adaptive_prefetch_strategy=*/false, executor_,
                         /*initialize_read_ranges=*/true,
-                        /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), pool_));
+                        /*read_ahead_cache_enabled=*/true, CacheConfig(), pool_));
             } else {
                 file_batch_reader =
                     std::make_unique<MockFileBatchReader>(data, target_type_, batch_size);
diff --git a/src/paimon/common/io/cache_input_stream.h b/src/paimon/common/io/cache_input_stream.h
index 9ccbf2608..015b082c3 100644
--- a/src/paimon/common/io/cache_input_stream.h
+++ b/src/paimon/common/io/cache_input_stream.h
@@ -18,13 +18,12 @@
 
 #pragma once
 
-#include <cstring>
 #include <memory>
 #include <string>
 
 #include "paimon/common/utils/math.h"
+#include "paimon/common/utils/read_ahead_cache.h"
 #include "paimon/fs/file_system.h"
-#include "paimon/utils/read_ahead_cache.h"
 
 namespace paimon {
 
@@ -48,10 +47,9 @@ class CacheInputStream : public InputStream {
             PAIMON_RETURN_NOT_OK(ValidateValueInRange<uint64_t>(offset, "read offset"));
             PAIMON_RETURN_NOT_OK(ValidateValueInRange<uint64_t>(size, "read size"));
             ByteRange range{static_cast<uint64_t>(offset), static_cast<uint64_t>(size)};
-            PAIMON_ASSIGN_OR_RAISE(ByteSlice slice, cache_->Read(range));
-            if (slice.buffer) {
-                std::memcpy(buffer, slice.buffer->data() + slice.offset, slice.length);
-                return slice.length;
+            PAIMON_ASSIGN_OR_RAISE(bool hit, cache_->Read(range, buffer));
+            if (hit) {
+                return size;
             }
         }
         return input_stream_->Read(buffer, size, offset);
@@ -70,14 +68,12 @@ class CacheInputStream : public InputStream {
                 return;
             }
             ByteRange range{static_cast<uint64_t>(offset), static_cast<uint64_t>(size)};
-            Result<ByteSlice> slice = cache_->Read(range);
-            if (!slice.ok()) {
-                callback(slice.status());
+            Result<bool> hit = cache_->Read(range, buffer);
+            if (!hit.ok()) {
+                callback(hit.status());
                 return;
             }
-            if (slice.value().buffer) {
-                std::memcpy(buffer, slice.value().buffer->data() + slice.value().offset,
-                            slice.value().length);
+            if (hit.value()) {
                 callback(Status::OK());
                 return;
             }
diff --git a/src/paimon/common/io/cache_input_stream_test.cpp b/src/paimon/common/io/cache_input_stream_test.cpp
index d4a61854f..0b14dc33b 100644
--- a/src/paimon/common/io/cache_input_stream_test.cpp
+++ b/src/paimon/common/io/cache_input_stream_test.cpp
@@ -26,6 +26,7 @@
 
 #include "gtest/gtest.h"
 #include "paimon/common/factories/io_hook.h"
+#include "paimon/common/utils/read_ahead_cache.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/fs/file_system_factory.h"
@@ -33,7 +34,6 @@
 #include "paimon/memory/memory_pool.h"
 #include "paimon/testing/utils/io_exception_helper.h"
 #include "paimon/testing/utils/testharness.h"
-#include "paimon/utils/read_ahead_cache.h"
 
 namespace paimon::test {
 
@@ -60,7 +60,7 @@ class CacheInputStreamTest : public ::testing::Test {
 
     std::shared_ptr<ReadAheadCache> CreateCache(std::vector<ByteRange> ranges) {
         auto stream = OpenFile();
-        CacheConfig config(/*buffer_size_limit=*/1024 * 1024, /*range_size_limit=*/1024,
+        CacheConfig config(/*range_size_limit=*/1024,
                            /*hole_size_limit=*/0, /*pre_buffer_limit=*/1024 * 1024);
         auto cache = std::make_shared<ReadAheadCache>(std::move(stream), config, pool_);
         EXPECT_OK(cache->Init(std::move(ranges)));
@@ -204,7 +204,7 @@ TEST_F(CacheInputStreamTest, TestReadAsyncCacheReadError) {
         ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFactory::Get("local", file_path_, {}));
         ASSERT_OK_AND_ASSIGN(auto cache_stream, fs->Open(file_path_));
         ASSERT_OK_AND_ASSIGN(auto underlying, fs->Open(file_path_));
-        CacheConfig config(/*buffer_size_limit=*/1024 * 1024, /*range_size_limit=*/1024,
+        CacheConfig config(/*range_size_limit=*/1024,
                            /*hole_size_limit=*/0, /*pre_buffer_limit=*/1024 * 1024);
         auto cache = std::make_shared<ReadAheadCache>(std::move(cache_stream), config, pool_);
         ASSERT_OK(cache->Init(std::vector<ByteRange>{{0, 10}}));
diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp
index c44651790..12e38966d 100644
--- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp
+++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp
@@ -31,10 +31,10 @@
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/reader/reader_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/read_ahead_cache.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/format/reader_builder.h"
 #include "paimon/fs/file_system.h"
-#include "paimon/utils/read_ahead_cache.h"
 
 namespace arrow {
 class Schema;
@@ -60,7 +60,7 @@ Result<std::unique_ptr<PrefetchFileBatchReaderImpl>> PrefetchFileBatchReaderImpl
     const std::shared_ptr<FileSystem>& fs, uint32_t prefetch_max_parallel_num, int32_t batch_size,
     uint32_t prefetch_batch_count, bool enable_adaptive_prefetch_strategy,
     const std::shared_ptr<Executor>& executor, bool initialize_read_ranges,
-    PrefetchCacheMode prefetch_cache_mode, const CacheConfig& cache_config,
+    bool read_ahead_cache_enabled, const CacheConfig& cache_config,
     const std::shared_ptr<MemoryPool>& pool) {
     if (prefetch_max_parallel_num == 0) {
         return Status::Invalid("prefetch max parallel num should be greater than 0.");
@@ -82,7 +82,7 @@ Result<std::unique_ptr<PrefetchFileBatchReaderImpl>> PrefetchFileBatchReaderImpl
     }
 
     std::shared_ptr<ReadAheadCache> cache;
-    if (prefetch_cache_mode != PrefetchCacheMode::NEVER) {
+    if (read_ahead_cache_enabled) {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> input_stream,
                                fs->Open(FileStatus(data_file_path, data_file_size)));
         cache = std::make_shared<ReadAheadCache>(input_stream, cache_config, pool);
@@ -119,9 +119,9 @@ Result<std::unique_ptr<PrefetchFileBatchReaderImpl>> PrefetchFileBatchReaderImpl
     }
     uint32_t prefetch_queue_capacity = prefetch_batch_count / readers.size();
 
-    auto reader = std::unique_ptr<PrefetchFileBatchReaderImpl>(new PrefetchFileBatchReaderImpl(
-        readers, batch_size, prefetch_queue_capacity, enable_adaptive_prefetch_strategy, executor,
-        cache, prefetch_cache_mode));
+    auto reader = std::unique_ptr<PrefetchFileBatchReaderImpl>(
+        new PrefetchFileBatchReaderImpl(readers, batch_size, prefetch_queue_capacity,
+                                        enable_adaptive_prefetch_strategy, executor, cache));
     if (initialize_read_ranges) {
         // normally initialize read ranges should be false, as set read schema will refresh read
         // ranges, and set read schema will always be called before read.
@@ -133,13 +133,11 @@ Result<std::unique_ptr<PrefetchFileBatchReaderImpl>> PrefetchFileBatchReaderImpl
 PrefetchFileBatchReaderImpl::PrefetchFileBatchReaderImpl(
     const std::vector<std::shared_ptr<PrefetchFileBatchReader>>& readers, int32_t batch_size,
     uint32_t prefetch_queue_capacity, bool enable_adaptive_prefetch_strategy,
-    const std::shared_ptr<Executor>& executor, const std::shared_ptr<ReadAheadCache>& cache,
-    PrefetchCacheMode cache_mode)
+    const std::shared_ptr<Executor>& executor, const std::shared_ptr<ReadAheadCache>& cache)
     : readers_(std::move(readers)),
       batch_size_(batch_size),
       executor_(executor),
       cache_(cache),
-      cache_mode_(cache_mode),
       prefetch_queue_capacity_(prefetch_queue_capacity),
       enable_adaptive_prefetch_strategy_(enable_adaptive_prefetch_strategy) {
     for (size_t i = 0; i < readers_.size(); i++) {
@@ -158,6 +156,9 @@ Status PrefetchFileBatchReaderImpl::SetReadSchema(
     ::ArrowSchema* read_schema, const std::shared_ptr<Predicate>& predicate,
     const std::optional<RoaringBitmap32>& selection_bitmap) {
     PAIMON_RETURN_NOT_OK(CleanUp());
+    if (cache_) {
+        cache_->Reset();
+    }
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> schema,
                                       arrow::ImportSchema(read_schema));
     for (const auto& reader : readers_) {
@@ -172,6 +173,9 @@ Status PrefetchFileBatchReaderImpl::SetReadSchema(
 
 Status PrefetchFileBatchReaderImpl::RefreshReadRanges() {
     PAIMON_RETURN_NOT_OK(CleanUp());
+    if (cache_) {
+        cache_->Reset();
+    }
     return RefreshReadRangesAfterCleanUp();
 }
 
@@ -294,35 +298,14 @@ Status PrefetchFileBatchReaderImpl::CleanUp() {
         reader_is_working_[i] = false;
     }
     is_shutdown_ = false;
-    if (cache_) {
-        cache_->Reset();
-    }
     SetReadStatus(Status::OK());
     return Status::OK();
 }
 
-bool PrefetchFileBatchReaderImpl::NeedInitCache() const {
-    switch (cache_mode_) {
-        case PrefetchCacheMode::NEVER:
-            return false;
-        case PrefetchCacheMode::EXCLUDE_PREDICATE:
-            return predicate_ == nullptr;
-        case PrefetchCacheMode::EXCLUDE_BITMAP:
-            return selection_bitmap_ == std::nullopt;
-        case PrefetchCacheMode::EXCLUDE_BITMAP_OR_PREDICATE:
-            return predicate_ == nullptr && selection_bitmap_ == std::nullopt;
-        case PrefetchCacheMode::ALWAYS:
-            return true;
-        default:
-            assert(false);
-            return true;
-    }
-}
-
 void PrefetchFileBatchReaderImpl::Workloop() {
     std::vector<std::future<void>> futures;
     futures.resize(readers_.size());
-    if (cache_ && NeedInitCache()) {
+    if (cache_) {
         auto read_ranges = readers_[0]->PreBufferRange();
         if (read_ranges.ok()) {
             std::vector<ByteRange> ranges;
@@ -332,6 +315,11 @@ void PrefetchFileBatchReaderImpl::Workloop() {
             auto s = cache_->Init(std::move(ranges));
             if (!s.ok()) {
                 SetReadStatus(s);
+            } else {
+                // Init() only registers the ranges, so without this the first
+                // cache fetch races the readers' first reads instead of running
+                // ahead of them.
+                cache_->Warmup();
             }
         } else {
             SetReadStatus(read_ranges.status());
@@ -622,7 +610,15 @@ Status PrefetchFileBatchReaderImpl::SeekToRow(uint64_t row_number) {
 }
 
 std::shared_ptr<Metrics> PrefetchFileBatchReaderImpl::GetReaderMetrics() const {
-    return MetricsImpl::CollectReadMetrics(readers_);
+    auto res_metrics = MetricsImpl::CollectReadMetrics(readers_);
+    if (cache_) {
+        // The shared read-ahead cache serves reads of all sub-readers, so its
+        // hit/miss counters are file-level and merge into the reader metrics.
+        std::shared_ptr<Metrics> cache_metrics = std::make_shared<MetricsImpl>();
+        cache_->CollectMetrics(&cache_metrics);
+        res_metrics->Merge(cache_metrics);
+    }
+    return res_metrics;
 }
 
 Result<std::unique_ptr<::ArrowSchema>> PrefetchFileBatchReaderImpl::GetFileSchema() const {
@@ -676,7 +672,17 @@ Result<std::pair<uint64_t, uint64_t>> PrefetchFileBatchReaderImpl::EofRange() co
 }
 
 void PrefetchFileBatchReaderImpl::Close() {
+    // CleanUp() no longer resets the read-ahead cache: ConcatBatchReader closes file readers as
+    // soon as they reach EOF, and the cache hit/miss counters must remain readable through
+    // GetReaderMetrics() after that. The cache is reset only when the reader is reused via
+    // SetReadSchema()/RefreshReadRanges().
     (void)CleanUp();
+    if (cache_) {
+        // Free the prefetched buffers of this file right away (ConcatBatchReader keeps
+        // closed file readers alive until the whole scan finishes), but keep the
+        // counters for GetReaderMetrics().
+        cache_->ReleaseBuffers();
+    }
     for (const auto& reader : readers_) {
         reader->Close();
     }
diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h
index 78cfbb5f9..c21856d09 100644
--- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h
+++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h
@@ -35,12 +35,12 @@
 #include <vector>
 
 #include "arrow/c/abi.h"
+#include "paimon/common/utils/read_ahead_cache.h"
 #include "paimon/common/utils/threadsafe_queue.h"
 #include "paimon/reader/batch_reader.h"
 #include "paimon/reader/prefetch_file_batch_reader.h"
 #include "paimon/result.h"
 #include "paimon/status.h"
-#include "paimon/utils/read_ahead_cache.h"
 #include "paimon/utils/roaring_bitmap32.h"
 
 struct ArrowSchema;
@@ -60,8 +60,8 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader {
         const ReaderBuilder* reader_builder, const std::shared_ptr<FileSystem>& fs,
         uint32_t prefetch_max_parallel_num, int32_t batch_size, uint32_t prefetch_batch_count,
         bool enable_adaptive_prefetch_strategy, const std::shared_ptr<Executor>& executor,
-        bool initialize_read_ranges, PrefetchCacheMode prefetch_cache_mode,
-        const CacheConfig& cache_config, const std::shared_ptr<MemoryPool>& pool);
+        bool initialize_read_ranges, bool read_ahead_cache_enabled, const CacheConfig& cache_config,
+        const std::shared_ptr<MemoryPool>& pool);
 
     ~PrefetchFileBatchReaderImpl() override;
 
@@ -113,8 +113,7 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader {
     PrefetchFileBatchReaderImpl(
         const std::vector<std::shared_ptr<PrefetchFileBatchReader>>& readers, int32_t batch_size,
         uint32_t prefetch_queue_capacity, bool enable_adaptive_prefetch_strategy,
-        const std::shared_ptr<Executor>& executor, const std::shared_ptr<ReadAheadCache>& cache,
-        PrefetchCacheMode cache_mode);
+        const std::shared_ptr<Executor>& executor, const std::shared_ptr<ReadAheadCache>& cache);
 
     Status CleanUp();
     void Workloop();
@@ -143,7 +142,6 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader {
                                 const std::pair<uint64_t, uint64_t>& read_range) const;
     Status HandleReadResult(size_t reader_idx, const std::pair<uint64_t, uint64_t>& read_range,
                             FileBatchReader::ReadBatchWithBitmap&& read_batch_with_bitmap);
-    bool NeedInitCache() const;
 
  private:
     std::vector<std::shared_ptr<PrefetchFileBatchReader>> readers_;
@@ -162,7 +160,6 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader {
     std::condition_variable cv_;
     std::shared_ptr<Executor> executor_;
     std::shared_ptr<ReadAheadCache> cache_;
-    PrefetchCacheMode cache_mode_;
 
     mutable std::shared_mutex rw_mutex_;
     std::unique_ptr<std::thread> background_thread_;
diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp
index b3828f7e5..192028ac4 100644
--- a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp
+++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp
@@ -27,6 +27,7 @@
 #include "gtest/gtest.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/read_ahead_cache.h"
 #include "paimon/executor.h"
 #include "paimon/format/file_format.h"
 #include "paimon/format/file_format_factory.h"
@@ -39,7 +40,6 @@
 #include "paimon/testing/mock/mock_format_reader_builder.h"
 #include "paimon/testing/utils/read_result_collector.h"
 #include "paimon/testing/utils/testharness.h"
-#include "paimon/utils/read_ahead_cache.h"
 
 namespace paimon::test {
 
@@ -115,7 +115,7 @@ class ControlledMockFormatReaderBuilder : public ReaderBuilder {
 
 struct TestParam {
     std::string file_format;
-    PrefetchCacheMode cache_mode;
+    bool read_ahead_cache_enabled;
 };
 
 class PrefetchFileBatchReaderImplTest : public ::testing::Test,
@@ -194,7 +194,7 @@ class PrefetchFileBatchReaderImplTest : public ::testing::Test,
         const std::string& file_format_str, const arrow::Schema* read_schema,
         const std::shared_ptr<Predicate>& predicate,
         const std::optional<RoaringBitmap32>& selection_bitmap, int32_t batch_size,
-        int32_t prefetch_max_parallel_num, PrefetchCacheMode cache_mode) const {
+        int32_t prefetch_max_parallel_num, bool read_ahead_cache_enabled) const {
         EXPECT_OK_AND_ASSIGN(std::unique_ptr<FileFormat> file_format,
                              FileFormatFactory::Get(file_format_str, {}));
         EXPECT_OK_AND_ASSIGN(auto reader_builder, file_format->CreateReaderBuilder(batch_size));
@@ -209,7 +209,8 @@ class PrefetchFileBatchReaderImplTest : public ::testing::Test,
                 data_file_path, data_file_status.GetLen(), reader_builder.get(), local_fs_,
                 prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                 /*enable_adaptive_prefetch_strategy=*/false, executor,
-                /*initialize_read_ranges=*/false, cache_mode, CacheConfig(), GetDefaultPool()));
+                /*initialize_read_ranges=*/false, read_ahead_cache_enabled, CacheConfig(),
+                GetDefaultPool()));
         std::unique_ptr<ArrowSchema> c_schema = std::make_unique<ArrowSchema>();
         auto arrow_status = arrow::ExportSchema(*read_schema, c_schema.get());
         EXPECT_TRUE(arrow_status.ok());
@@ -275,18 +276,11 @@ CollectResultAndRowIds(FileBatchReader* reader) {
 }
 
 std::vector<TestParam> PrepareTestParam() {
-    std::vector<TestParam> values = {
-        TestParam{"parquet", PrefetchCacheMode::ALWAYS},
-        TestParam{"parquet", PrefetchCacheMode::EXCLUDE_BITMAP},
-        TestParam{"parquet", PrefetchCacheMode::EXCLUDE_PREDICATE},
-        TestParam{"parquet", PrefetchCacheMode::EXCLUDE_BITMAP_OR_PREDICATE},
-        TestParam{"parquet", PrefetchCacheMode::NEVER}};
+    std::vector<TestParam> values = {TestParam{"parquet", /*read_ahead_cache_enabled=*/true},
+                                     TestParam{"parquet", /*read_ahead_cache_enabled=*/false}};
 #ifdef PAIMON_ENABLE_ORC
-    values.emplace_back(TestParam{"orc", PrefetchCacheMode::ALWAYS});
-    values.emplace_back(TestParam{"orc", PrefetchCacheMode::EXCLUDE_BITMAP});
-    values.emplace_back(TestParam{"orc", PrefetchCacheMode::EXCLUDE_PREDICATE});
-    values.emplace_back(TestParam{"orc", PrefetchCacheMode::EXCLUDE_BITMAP_OR_PREDICATE});
-    values.emplace_back(TestParam{"orc", PrefetchCacheMode::NEVER});
+    values.emplace_back(TestParam{"orc", /*read_ahead_cache_enabled=*/true});
+    values.emplace_back(TestParam{"orc", /*read_ahead_cache_enabled=*/false});
 #endif
     return values;
 }
@@ -300,13 +294,12 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestSimple) {
     for (auto prefetch_max_parallel_num : {1, 2, 3, 5, 8, 10}) {
         MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
         ASSERT_OK_AND_ASSIGN(
-            auto reader,
-            PrefetchFileBatchReaderImpl::Create(
-                /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-                prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-                /*enable_adaptive_prefetch_strategy=*/false, executor_,
-                /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-                CacheConfig(), GetDefaultPool()));
+            auto reader, PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
         ASSERT_NOK(reader->GetPreviousBatchFileRowId(0));
         ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get()));
 
@@ -323,14 +316,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestReadWithLimits) {
     int32_t prefetch_max_parallel_num = 12;
 
     MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
     // simulate read limits, only read 8 batches
     for (int32_t i = 0; i < 8; i++) {
         ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch_with_bitmap,
@@ -353,14 +345,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestReadWithoutInitializeReadRanges) {
     int32_t prefetch_max_parallel_num = 12;
 
     MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/false, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
     // simulate read limits, only read 8 batches
     ASSERT_NOK_WITH_MSG(reader->NextBatchWithBitmap(),
                         "prefetch reader read ranges are not initialized");
@@ -430,14 +421,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, RefreshReadRanges) {
     int32_t batch_size = 30;
     int32_t prefetch_max_parallel_num = 3;
     MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/false, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     ASSERT_OK(prefetch_reader->RefreshReadRanges());
     std::vector<std::pair<uint64_t, uint64_t>> read_ranges_0 = {{0, 30}, {90, 101}};
@@ -460,15 +450,14 @@ TEST_F(PrefetchFileBatchReaderImplTest, RefreshReadRangesDisablePrefetchByAdapti
                                                      /*need_prefetch=*/true,
                                                      /*set_read_ranges_statuses=*/{});
 
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size,
-            /*prefetch_batch_count=*/2,
-            /*enable_adaptive_prefetch_strategy=*/true, executor_,
-            /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size,
+                             /*prefetch_batch_count=*/2,
+                             /*enable_adaptive_prefetch_strategy=*/true, executor_,
+                             /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
 
     ASSERT_FALSE(reader->NeedPrefetch());
 }
@@ -478,14 +467,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, SetReadRanges) {
     int32_t batch_size = 30;
     int32_t prefetch_max_parallel_num = 3;
     MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/false, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     ASSERT_FALSE(prefetch_reader->need_prefetch_);
     prefetch_reader->need_prefetch_ = true;
@@ -522,14 +510,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, SetReadRangesReturnErrorWhenPushDownFail
         /*set_read_ranges_statuses=*/
         {Status::IOError("set read ranges failed"), Status::IOError("set read ranges failed")});
 
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/false, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
 
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     prefetch_reader->need_prefetch_ = true;
@@ -539,43 +526,23 @@ TEST_F(PrefetchFileBatchReaderImplTest, SetReadRangesReturnErrorWhenPushDownFail
     ASSERT_TRUE(status.IsIOError());
 }
 
-TEST_F(PrefetchFileBatchReaderImplTest, NeedInitCacheNeverMode) {
-    auto data_array = PrepareArray(10);
-    int32_t batch_size = 5;
-    int32_t prefetch_max_parallel_num = 1;
-    MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/false, /*prefetch_cache_mode=*/PrefetchCacheMode::NEVER,
-            CacheConfig(), GetDefaultPool()));
-
-    auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
-    ASSERT_FALSE(prefetch_reader->NeedInitCache());
-}
-
 TEST_F(PrefetchFileBatchReaderImplTest, WorkloopSetReadStatusWhenCacheInitFailed) {
     auto data_array = PrepareArray(10);
     int32_t batch_size = 5;
     int32_t prefetch_max_parallel_num = 1;
     MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
     CacheConfig invalid_cache_config(
-        /*buffer_size_limit=*/512 * 1024,
         /*range_size_limit=*/4 * 1024,
         /*hole_size_limit=*/8 * 1024,
         /*pre_buffer_limit=*/128 * 1024);
 
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/false, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            invalid_cache_config, GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true,
+                             invalid_cache_config, GetDefaultPool()));
 
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     prefetch_reader->Workloop();
@@ -589,14 +556,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, DoReadBatchReturnOkWhenShutdown) {
     int32_t batch_size = 5;
     int32_t prefetch_max_parallel_num = 1;
     MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/false, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
 
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     prefetch_reader->is_shutdown_ = true;
@@ -608,14 +574,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, DoReadBatchReturnOkWhenNoCurrentReadRang
     int32_t batch_size = 5;
     int32_t prefetch_max_parallel_num = 1;
     MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/false, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
 
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     prefetch_reader->read_ranges_in_group_ = {{}};
@@ -627,14 +592,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestReadWithLargeBatchSize) {
     int32_t batch_size = 150;
     int32_t prefetch_max_parallel_num = 3;
     MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
     ASSERT_NOK(reader->GetPreviousBatchFileRowId(0));
     ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get()));
     auto row_ids = array_and_row_ids.second;
@@ -648,14 +612,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPartialReaderSuccessRead) {
     int32_t batch_size = 10;
     int32_t prefetch_max_parallel_num = 3;
     MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     for (int32_t i = 0; i < prefetch_max_parallel_num; i++) {
         dynamic_cast<MockFileBatchReader*>(prefetch_reader->readers_[i].get())
@@ -694,14 +657,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestAllReaderFailedWithIOError) {
     int32_t batch_size = 10;
     int32_t prefetch_max_parallel_num = 3;
     MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
 
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     for (int32_t i = 0; i < prefetch_max_parallel_num; i++) {
@@ -730,14 +692,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPrefetchWithEmptyData) {
     int32_t batch_size = 10;
     int32_t prefetch_max_parallel_num = 3;
     MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
     ASSERT_NOK(reader->GetPreviousBatchFileRowId(0));
     ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get()));
     auto row_ids = array_and_row_ids.second;
@@ -750,14 +711,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestCallNextBatchAfterReadingEof) {
     int32_t batch_size = 10;
     int32_t prefetch_max_parallel_num = 6;
     MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
     ASSERT_NOK(reader->GetPreviousBatchFileRowId(0));
     ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get()));
     auto row_ids = array_and_row_ids.second;
@@ -776,14 +736,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestCreateReaderWithoutNextBatch) {
     int32_t batch_size = 10;
     int32_t prefetch_max_parallel_num = 3;
     MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size);
-    ASSERT_OK_AND_ASSIGN(
-        auto reader,
-        PrefetchFileBatchReaderImpl::Create(
-            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-            prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-            /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
 }
 
 TEST_F(PrefetchFileBatchReaderImplTest, TestInvalidCase) {
@@ -797,16 +756,16 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestInvalidCase) {
             data_file_path, /*data_file_size=*/0, &reader_builder, mock_fs_,
             /*prefetch_max_parallel_num=*/0, batch_size, 2,
             /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+            /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true, CacheConfig(),
+            GetDefaultPool()));
     }
     {
         ASSERT_NOK(PrefetchFileBatchReaderImpl::Create(
             data_file_path, /*data_file_size=*/0, &reader_builder, mock_fs_,
             prefetch_max_parallel_num, /*batch_size=*/-1, prefetch_max_parallel_num * 2,
             /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+            /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true, CacheConfig(),
+            GetDefaultPool()));
     }
     {
         ASSERT_NOK(PrefetchFileBatchReaderImpl::Create(
@@ -814,33 +773,32 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestInvalidCase) {
             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
             /*enable_adaptive_prefetch_strategy=*/false,
             /*executor=*/nullptr, /*initialize_read_ranges=*/true,
-            /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), GetDefaultPool()));
+            /*read_ahead_cache_enabled=*/true, CacheConfig(), GetDefaultPool()));
     }
     {
         ASSERT_NOK(PrefetchFileBatchReaderImpl::Create(
             data_file_path, /*data_file_size=*/0, /*reader_builder=*/nullptr, mock_fs_,
             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
             /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+            /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true, CacheConfig(),
+            GetDefaultPool()));
     }
     {
         ASSERT_NOK(PrefetchFileBatchReaderImpl::Create(
             data_file_path, /*data_file_size=*/0, &reader_builder,
             /*fs=*/nullptr, prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
             /*enable_adaptive_prefetch_strategy=*/false, executor_,
-            /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-            CacheConfig(), GetDefaultPool()));
+            /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true, CacheConfig(),
+            GetDefaultPool()));
     }
     {
         ASSERT_OK_AND_ASSIGN(
-            auto reader,
-            PrefetchFileBatchReaderImpl::Create(
-                data_file_path, /*data_file_size=*/0, &reader_builder, mock_fs_,
-                prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
-                /*enable_adaptive_prefetch_strategy=*/false, executor_,
-                /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-                CacheConfig(), GetDefaultPool()));
+            auto reader, PrefetchFileBatchReaderImpl::Create(
+                             data_file_path, /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
+                             CacheConfig(), GetDefaultPool()));
         ASSERT_NOK_WITH_MSG(reader->SeekToRow(/*row_number=*/101),
                             "not support seek to row for prefetch reader");
     }
@@ -850,7 +808,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestInvalidCase) {
 /// [30,60) will be filtered out.
 /// The read range is [0,30), [30,60), [60,90). So, expected results is [0,30), [60,90)
 TEST_P(PrefetchFileBatchReaderImplTest, TestPrefetchWithPredicatePushdownWithCompleteFiltering) {
-    auto [file_format, cache_mode] = GetParam();
+    auto [file_format, read_ahead_cache_enabled] = GetParam();
     auto data_array = PrepareArray(90);
     int32_t batch_size = 10;
     PrepareTestData(file_format, data_array, /*stripe_row_count=*/30, /*row_index_stride=*/30);
@@ -866,7 +824,7 @@ TEST_P(PrefetchFileBatchReaderImplTest, TestPrefetchWithPredicatePushdownWithCom
     auto reader = PreparePrefetchReader(file_format, schema.get(), predicate,
                                         /*selection_bitmap=*/std::nullopt,
                                         /*batch_size=*/batch_size, /*prefetch_max_parallel_num=*/3,
-                                        cache_mode);
+                                        read_ahead_cache_enabled);
     ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get()));
 
     arrow::ArrayVector expected_array_vector;
@@ -883,7 +841,7 @@ TEST_P(PrefetchFileBatchReaderImplTest, TestPrefetchWithPredicatePushdownWithCom
 /// The read range is [0,30), [30,60), [60,90).
 TEST_P(PrefetchFileBatchReaderImplTest,
        TestPrefetchWithOrcPredicatePushdownWithRowGroupGranularity) {
-    auto [file_format, cache_mode] = GetParam();
+    auto [file_format, read_ahead_cache_enabled] = GetParam();
     auto data_array = PrepareArray(90);
     int32_t batch_size = 10;
     PrepareTestData(file_format, data_array, /*stripe_row_count=*/30, /*row_index_stride=*/10);
@@ -900,7 +858,7 @@ TEST_P(PrefetchFileBatchReaderImplTest,
     auto reader = PreparePrefetchReader(file_format, schema.get(), predicate,
                                         /*selection_bitmap=*/std::nullopt,
                                         /*batch_size=*/batch_size, /*prefetch_max_parallel_num=*/3,
-                                        cache_mode);
+                                        read_ahead_cache_enabled);
     ASSERT_OK(reader->RefreshReadRanges());
     ASSERT_NOK(reader->GetPreviousBatchFileRowId(0));
     ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get()));
@@ -925,14 +883,14 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPrefetchWithBitmap) {
     MockFormatReaderBuilder reader_builder(data_array, data_type_, bitmap,
                                            /*read_batch_size=*/100);
     int32_t prefetch_max_parallel_num = 3;
-    ASSERT_OK_AND_ASSIGN(auto reader, PrefetchFileBatchReaderImpl::Create(
-                                          /*data_file_path=*/"", /*data_file_size=*/0,
-                                          &reader_builder, mock_fs_, prefetch_max_parallel_num,
-                                          /*batch_size=*/100, prefetch_max_parallel_num * 2,
-                                          /*enable_adaptive_prefetch_strategy=*/false, executor_,
-                                          /*initialize_read_ranges=*/true,
-                                          /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS,
-                                          CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         PrefetchFileBatchReaderImpl::Create(
+                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
+                             prefetch_max_parallel_num,
+                             /*batch_size=*/100, prefetch_max_parallel_num * 2,
+                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                             /*initialize_read_ranges=*/true,
+                             /*read_ahead_cache_enabled=*/true, CacheConfig(), GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(auto result_chunk_array, ReadResultCollector::CollectResult(reader.get()));
 
     ASSERT_OK_AND_ASSIGN(auto data_batch, ReadResultCollector::GetReadBatch(data_array));
@@ -946,7 +904,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPrefetchWithBitmap) {
 }
 
 TEST_P(PrefetchFileBatchReaderImplTest, TestRowMapping) {
-    auto [file_format, cache_mode] = GetParam();
+    auto [file_format, read_ahead_cache_enabled] = GetParam();
     auto data_array = PrepareArray(90);
     PrepareTestData(file_format, data_array, /*stripe_row_count=*/30, /*row_index_stride=*/10);
     auto schema = arrow::schema(fields_);
@@ -959,10 +917,10 @@ TEST_P(PrefetchFileBatchReaderImplTest, TestRowMapping) {
                                       Literal(70l), Literal(79l)),
         }));
 
-    auto reader =
-        PreparePrefetchReader(file_format, schema.get(), predicate,
-                              /*selection_bitmap=*/std::nullopt,
-                              /*batch_size=*/10, /*prefetch_max_parallel_num=*/3, cache_mode);
+    auto reader = PreparePrefetchReader(file_format, schema.get(), predicate,
+                                        /*selection_bitmap=*/std::nullopt,
+                                        /*batch_size=*/10, /*prefetch_max_parallel_num=*/3,
+                                        read_ahead_cache_enabled);
     ASSERT_NOK(reader->GetPreviousBatchFileRowId(0));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> batch,
                          paimon::test::ReadResultCollector::CollectResultOneBatch(reader.get()));
diff --git a/src/paimon/common/utils/byte_range_combiner.cpp b/src/paimon/common/utils/byte_range_combiner.cpp
index 428770037..c306c100f 100644
--- a/src/paimon/common/utils/byte_range_combiner.cpp
+++ b/src/paimon/common/utils/byte_range_combiner.cpp
@@ -23,6 +23,7 @@
 
 #include <algorithm>
 #include <cassert>
+#include <limits>
 
 #include "fmt/format.h"
 
@@ -39,6 +40,20 @@ Result<std::vector<ByteRange>> ByteRangeCombiner::CoalesceByteRanges(
         return ranges;
     }
 
+    // Reject ranges that exceed the int64 bound before any offset + length arithmetic
+    // below. Such ranges can originate from corrupt file metadata (e.g. negative signed
+    // values cast to uint64_t) and would otherwise wrap around or make the splitting
+    // loop run until memory is exhausted.
+    constexpr auto kMaxRangeValue = static_cast<uint64_t>(std::numeric_limits<int64_t>::max());
+    for (const auto& range : ranges) {
+        if (range.offset > kMaxRangeValue || range.length > kMaxRangeValue ||
+            range.offset + range.length > kMaxRangeValue) {
+            return Status::Invalid(
+                fmt::format("byte range (offset={}, length={}) exceeds the int64 bound",
+                            range.offset, range.length));
+        }
+    }
+
     std::vector<ByteRange> adjusted_ranges;
     for (const auto& range : ranges) {
         uint64_t range_start = range.offset;
diff --git a/src/paimon/common/utils/byte_range_combiner.h b/src/paimon/common/utils/byte_range_combiner.h
index 599ca59cf..90942719b 100644
--- a/src/paimon/common/utils/byte_range_combiner.h
+++ b/src/paimon/common/utils/byte_range_combiner.h
@@ -23,8 +23,8 @@
 
 #include <vector>
 
+#include "paimon/common/utils/read_ahead_cache.h"
 #include "paimon/result.h"
-#include "paimon/utils/read_ahead_cache.h"
 
 namespace paimon {
 
diff --git a/src/paimon/common/utils/byte_range_combiner_test.cpp b/src/paimon/common/utils/byte_range_combiner_test.cpp
index 19d739a41..5a3cc1c6e 100644
--- a/src/paimon/common/utils/byte_range_combiner_test.cpp
+++ b/src/paimon/common/utils/byte_range_combiner_test.cpp
@@ -21,9 +21,11 @@
 
 #include "paimon/common/utils/byte_range_combiner.h"
 
+#include <limits>
+
 #include "gtest/gtest.h"
+#include "paimon/common/utils/read_ahead_cache.h"
 #include "paimon/testing/utils/testharness.h"
-#include "paimon/utils/read_ahead_cache.h"
 
 namespace paimon::test {
 
@@ -75,4 +77,26 @@ TEST(ByteRangeCombinerTest, TestBasics) {
     check({{20, 5}, {20, 5}, {21, 2}}, {{20, 5}});
 }
 
+// Ranges beyond the int64 bound (e.g. negative signed metadata values cast to uint64_t)
+// must be rejected before the unchecked offset + length arithmetic, which would otherwise
+// wrap around or spin the splitting loop until memory is exhausted.
+TEST(ByteRangeCombinerTest, TestRejectsRangesBeyondInt64Bound) {
+    constexpr auto kInt64Max = static_cast<uint64_t>(std::numeric_limits<int64_t>::max());
+    auto check_invalid = [](std::vector<ByteRange> ranges) -> void {
+        ASSERT_NOK_WITH_MSG(
+            ByteRangeCombiner::CoalesceByteRanges(std::move(ranges), /*hole_size_limit=*/9,
+                                                  /*range_size_limit=*/99),
+            "exceeds the int64 bound");
+    };
+
+    // Offset beyond int64 (negative int64 cast to uint64_t lands here).
+    check_invalid({{kInt64Max + 1, 1}});
+    // Length beyond int64 (e.g. -1 cast to uint64_t), which would explode the split loop.
+    check_invalid({{0, std::numeric_limits<uint64_t>::max()}});
+    // Both in range individually, but the end position overflows the int64 bound.
+    check_invalid({{kInt64Max - 10, 20}});
+    // One bad range among valid ones still fails the whole batch.
+    check_invalid({{100, 10}, {0, std::numeric_limits<uint64_t>::max()}});
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/common/utils/read_ahead_cache.cpp b/src/paimon/common/utils/read_ahead_cache.cpp
index b0001189b..a74b35202 100644
--- a/src/paimon/common/utils/read_ahead_cache.cpp
+++ b/src/paimon/common/utils/read_ahead_cache.cpp
@@ -20,15 +20,19 @@
 // Adapted from Apache ORC
 // https://github.com/apache/orc/blob/main/c%2B%2B/src/io/Cache.cc
 
-#include "paimon/utils/read_ahead_cache.h"
+#include "paimon/common/utils/read_ahead_cache.h"
 
 #include <algorithm>
+#include <atomic>
 #include <cassert>
+#include <cstring>
 #include <future>
 #include <shared_mutex>
 
 #include "paimon/common/utils/byte_range_combiner.h"
 #include "paimon/common/utils/math.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/metrics.h"
 
 namespace paimon {
 
@@ -47,18 +51,53 @@ struct RangeCacheEntry {
     }
 };
 
-CacheConfig::CacheConfig(uint64_t buffer_size_limit, uint64_t range_size_limit,
-                         uint64_t hole_size_limit, uint64_t pre_buffer_limit)
-    : buffer_size_limit_(buffer_size_limit),
-      range_size_limit_(range_size_limit),
+// Everything needed to dispatch the prefetch IO of an entry AFTER the entry
+// has been published into entries_: the promise resolves the entry's future
+// and the buffer capture keeps the destination alive for the async IO.
+struct PendingFetch {
+    ByteRange range;
+    std::shared_ptr<Bytes> buffer;
+    std::shared_ptr<std::promise<Status>> promise;
+};
+
+namespace {
+
+// Copy the requested window out of the covering entries into dest. The
+// entries must fully cover the range and their futures must be resolved.
+void CopyRangeFromEntries(const std::vector<RangeCacheEntry>& covering, const ByteRange& range,
+                          char* dest) {
+    size_t pos = 0;
+    for (const auto& entry : covering) {
+        const uint64_t entry_end = entry.range.offset + entry.range.length;
+        const uint64_t copy_begin = std::max(range.offset, entry.range.offset);
+        const uint64_t copy_end = std::min(range.offset + range.length, entry_end);
+        const auto copy_len = static_cast<size_t>(copy_end - copy_begin);
+        std::memcpy(dest + pos, entry.buffer->data() + (copy_begin - entry.range.offset), copy_len);
+        pos += copy_len;
+    }
+}
+
+}  // namespace
+
+CacheConfig::CacheConfig(uint64_t range_size_limit, uint64_t hole_size_limit,
+                         uint64_t pre_buffer_limit)
+    : range_size_limit_(range_size_limit),
       hole_size_limit_(hole_size_limit),
       pre_buffer_limit_(pre_buffer_limit) {}
 
 CacheConfig::CacheConfig()
-    : CacheConfig(/*buffer_size_limit=*/512 * 1024 * 1024,
-                  /*range_size_limit=*/16 * 1024 * 1024,
+    // Aligned with the reader's request granularity and with realistic data
+    // file sizes:
+    // - range_size_limit matches the parquet reader's 32 MiB request blocks
+    //   (Arrow ReadRangeCache's own range limit); a smaller limit cuts entries
+    //   below the request size, so a request can never be served from one piece.
+    // - pre_buffer_limit must exceed the LARGEST single read a reader issues
+    //   (coalesced column-chunk reads of ~128 MiB were observed): fetches are
+    //   only dispatched up to this window, so a request reaching past it can
+    //   never be served and falls back to a second fetch of the same bytes.
+    : CacheConfig(/*range_size_limit=*/32 * 1024 * 1024,
                   /*hole_size_limit=*/8 * 1024,
-                  /*pre_buffer_limit=*/128 * 1024 * 1024) {}
+                  /*pre_buffer_limit=*/256 * 1024 * 1024) {}
 
 class ReadAheadCache::Impl {
  public:
@@ -67,18 +106,37 @@ class ReadAheadCache::Impl {
     ~Impl();
 
     Status Init(std::vector<ByteRange>&& ranges);
-    Result<ByteSlice> Read(const ByteRange& range);
+    Result<bool> Read(const ByteRange& range, char* dest);
     void Reset();
+    void ReleaseBuffers();
+    void Warmup();
+    void CollectMetrics(std::shared_ptr<Metrics>* metrics) const;
 
  private:
-    std::vector<RangeCacheEntry> MakeCacheEntries(const std::vector<ByteRange>& ranges) const;
+    /// Dispatch the prefetch IOs for entries that have already been published
+    /// into entries_.
+    void DispatchFetches(const std::vector<PendingFetch>& fetches);
+    /// Find the entries fully covering the given range under the read lock.
+    /// Returns an empty vector on miss. Entries are copied (shared buffers)
+    /// so the caller may use them after releasing the lock.
+    std::vector<RangeCacheEntry> FindCoveringEntries(const ByteRange& range);
     void PreBuffer(uint64_t offset);
+    void CountHit(uint64_t size) {
+        hits_.fetch_add(1, std::memory_order_relaxed);
+        hit_bytes_.fetch_add(size, std::memory_order_relaxed);
+    }
+    void CountMiss(uint64_t size) {
+        misses_.fetch_add(1, std::memory_order_relaxed);
+        miss_bytes_.fetch_add(size, std::memory_order_relaxed);
+    }
 
-    /// Cache the given ranges in the background.
+    /// Mark, publish and fetch the pending ranges at the given indices.
     ///
-    /// The caller must ensure that the ranges do not overlap with each other,
-    /// nor with previously cached ranges.  Otherwise, behaviour will be undefined.
-    void Cache(std::vector<ByteRange> ranges);
+    /// Marking is_cached_ and publishing the promise-backed entries happen
+    /// atomically under the write lock, before any IO is dispatched, so a
+    /// reader racing the prefetch waits on the in-flight entries instead of
+    /// re-fetching the same bytes.
+    void Cache(std::vector<size_t> pending_indices);
 
     std::shared_ptr<InputStream> stream_;
     CacheConfig config_;
@@ -89,38 +147,52 @@ class ReadAheadCache::Impl {
     std::vector<std::atomic<bool>> is_cached_;
     std::vector<ByteRange> pending_ranges_;
     bool is_initialized_ = false;
+    // Statistics of the Read() requests issued to the cache, aggregated over
+    // all streams sharing this cache.
+    std::atomic<uint64_t> read_count_{0};
+    std::atomic<uint64_t> read_bytes_{0};
+    std::atomic<uint64_t> hits_{0};
+    std::atomic<uint64_t> hit_bytes_{0};
+    std::atomic<uint64_t> misses_{0};
+    std::atomic<uint64_t> miss_bytes_{0};
+    // Prefetch IO statistics: how many requests and bytes were actually issued
+    // to the underlying stream.
+    std::atomic<uint64_t> io_count_{0};
+    std::atomic<uint64_t> io_bytes_{0};
 };
 
-void ReadAheadCache::Impl::Cache(std::vector<ByteRange> ranges) {
-    std::sort(ranges.begin(), ranges.end(),
-              [](const ByteRange& a, const ByteRange& b) { return a.offset < b.offset; });
-    std::vector<RangeCacheEntry> new_entries = MakeCacheEntries(ranges);
-    // Add new entries, themselves ordered by offset
-    std::unique_lock<std::shared_mutex> lock(rw_mutex_);
-    if (entries_.size() > 0) {
-        size_t new_entries_size = 0;
-        for (const auto& e : new_entries) {
-            new_entries_size += e.range.length;
-        }
-
-        size_t total_size = 0;
-        for (const auto& e : entries_) {
-            total_size += e.range.length;
+void ReadAheadCache::Impl::Cache(std::vector<size_t> pending_indices) {
+    std::vector<RangeCacheEntry> new_entries;
+    std::vector<PendingFetch> fetches;
+    // Mark is_cached_, publish the promise-backed entries and only then
+    // dispatch the IOs. The mark and the publication happen atomically under
+    // the write lock: a reader racing the prefetch observes is_cached_=true
+    // only once the covering entries are already visible, so it waits on
+    // their futures instead of issuing a duplicate underlying read.
+    {
+        std::unique_lock<std::shared_mutex> lock(rw_mutex_);
+        for (size_t idx : pending_indices) {
+            if (is_cached_[idx].exchange(true)) {
+                continue;
+            }
+            const ByteRange& range = pending_ranges_[idx];
+            auto promise = std::make_shared<std::promise<Status>>();
+            auto future = promise->get_future();
+            auto buffer = std::make_shared<Bytes>(range.length, memory_pool_.get());
+            fetches.push_back({range, buffer, promise});
+            new_entries.emplace_back(range, std::move(buffer), std::move(future));
         }
-        size_t limit = config_.GetBufferSizeLimit();
-        while (!entries_.empty() && total_size + new_entries_size > limit) {
-            auto iter = entries_.begin();
-            total_size -= entries_.front().range.length;
-            entries_.erase(iter);
+        if (!new_entries.empty()) {
+            // Entries are never evicted: the cache holds every published
+            // range until ReleaseBuffers()/Reset(), so an in-flight fetch
+            // always keeps its entry and thus its future reachable.
+            std::vector<RangeCacheEntry> merged(entries_.size() + new_entries.size());
+            std::merge(entries_.begin(), entries_.end(), new_entries.begin(), new_entries.end(),
+                       merged.begin());
+            entries_ = std::move(merged);
         }
-
-        std::vector<RangeCacheEntry> merged(entries_.size() + new_entries.size());
-        std::merge(entries_.begin(), entries_.end(), new_entries.begin(), new_entries.end(),
-                   merged.begin());
-        entries_ = std::move(merged);
-    } else {
-        entries_ = std::move(new_entries);
     }
+    DispatchFetches(fetches);
 }
 
 Status ReadAheadCache::Impl::Init(std::vector<ByteRange>&& ranges) {
@@ -154,22 +226,18 @@ void ReadAheadCache::Impl::PreBuffer(uint64_t offset) {
     }
 
     size_t start_idx = std::distance(pending_ranges_.begin(), it);
-    std::vector<ByteRange> ranges;
+    std::vector<size_t> pending_indices;
     size_t total_bytes = 0;
     for (size_t i = start_idx; i < pending_ranges_.size(); ++i) {
-        size_t range_size = pending_ranges_[i].length;
-        total_bytes += range_size;
+        total_bytes += pending_ranges_[i].length;
         if (total_bytes > config_.GetPreBufferLimit()) {
             break;
         }
-        if (is_cached_[i].exchange(true)) {
-            continue;
-        }
-        ranges.emplace_back(pending_ranges_[i]);
+        pending_indices.push_back(i);
     }
 
-    if (!ranges.empty()) {
-        Cache(std::move(ranges));
+    if (!pending_indices.empty()) {
+        Cache(std::move(pending_indices));
     }
 }
 
@@ -185,7 +253,22 @@ ReadAheadCache::Impl::~Impl() {
 }
 
 void ReadAheadCache::Impl::Reset() {
+    ReleaseBuffers();
+    read_count_.store(0, std::memory_order_relaxed);
+    read_bytes_.store(0, std::memory_order_relaxed);
+    hits_.store(0, std::memory_order_relaxed);
+    hit_bytes_.store(0, std::memory_order_relaxed);
+    misses_.store(0, std::memory_order_relaxed);
+    miss_bytes_.store(0, std::memory_order_relaxed);
+    io_count_.store(0, std::memory_order_relaxed);
+    io_bytes_.store(0, std::memory_order_relaxed);
+}
+
+void ReadAheadCache::Impl::ReleaseBuffers() {
     std::unique_lock<std::shared_mutex> lock(rw_mutex_);
+    // Entries are never evicted, so waiting on entries_ covers every
+    // dispatched fetch: no async callback can outlive the stream or the
+    // memory pool its buffer belongs to.
     for (auto& entry : entries_) {
         entry.future.wait();
     }
@@ -193,45 +276,106 @@ void ReadAheadCache::Impl::Reset() {
     is_cached_.clear();
     pending_ranges_.clear();
     is_initialized_ = false;
+    // The read/io counters are deliberately kept: a reader closed at EOF must
+    // still be able to report them through CollectMetrics().
 }
 
-Result<ByteSlice> ReadAheadCache::Impl::Read(const ByteRange& range) {
+void ReadAheadCache::Impl::CollectMetrics(std::shared_ptr<Metrics>* metrics) const {
+    if (metrics == nullptr || !*metrics) {
+        return;
+    }
+    auto& m = *metrics;
+    m->SetCounter(ReadAheadCacheMetrics::READ_COUNT, read_count_.load(std::memory_order_relaxed));
+    m->SetCounter(ReadAheadCacheMetrics::READ_BYTES, read_bytes_.load(std::memory_order_relaxed));
+    m->SetCounter(ReadAheadCacheMetrics::READ_HITS, hits_.load(std::memory_order_relaxed));
+    m->SetCounter(ReadAheadCacheMetrics::READ_HIT_BYTES,
+                  hit_bytes_.load(std::memory_order_relaxed));
+    m->SetCounter(ReadAheadCacheMetrics::READ_MISSES, misses_.load(std::memory_order_relaxed));
+    m->SetCounter(ReadAheadCacheMetrics::READ_MISS_BYTES,
+                  miss_bytes_.load(std::memory_order_relaxed));
+    m->SetCounter(ReadAheadCacheMetrics::IO_COUNT, io_count_.load(std::memory_order_relaxed));
+    m->SetCounter(ReadAheadCacheMetrics::IO_BYTES, io_bytes_.load(std::memory_order_relaxed));
+}
+
+void ReadAheadCache::Impl::Warmup() {
+    // Init() only registers the pending ranges; without this the first fetch
+    // starts when the first Read() arrives, racing the reader's own miss fetch.
+    if (!pending_ranges_.empty()) {
+        PreBuffer(pending_ranges_.front().offset);
+    }
+}
+
+std::vector<RangeCacheEntry> ReadAheadCache::Impl::FindCoveringEntries(const ByteRange& range) {
+    std::vector<RangeCacheEntry> covering;
+    std::shared_lock<std::shared_mutex> lock(rw_mutex_);
+    // Find the entry holding the start of the range: the first entry whose
+    // end is beyond range.offset (entries are disjoint and sorted by offset).
+    auto it = std::lower_bound(entries_.begin(), entries_.end(), range.offset,
+                               [](const RangeCacheEntry& e, uint64_t offset) {
+                                   return e.range.offset + e.range.length <= offset;
+                               });
+    if (it == entries_.end() || it->range.offset > range.offset) {
+        return covering;
+    }
+    if (it->range.Contains(range)) {
+        covering.push_back(*it);
+        return covering;
+    }
+    // The request spans several adjacent entries (a column chunk larger than
+    // one coalesced range): collect the contiguous run and check it covers
+    // the whole request. Entries are published before their fetch is
+    // dispatched, so a reader racing the prefetch waits for the in-flight
+    // fetch instead of issuing a second one for the same bytes.
+    uint64_t covered_end = it->range.offset + it->range.length;
+    covering.push_back(*it);
+    auto next = std::next(it);
+    while (covered_end < range.offset + range.length && next != entries_.end() &&
+           next->range.offset == covered_end) {
+        covered_end = next->range.offset + next->range.length;
+        covering.push_back(*next);
+        ++next;
+    }
+    if (covered_end < range.offset + range.length) {
+        covering.clear();
+    }
+    return covering;
+}
+
+Result<bool> ReadAheadCache::Impl::Read(const ByteRange& range, char* dest) {
     if (range.length == 0) {
-        return ByteSlice{std::make_shared<Bytes>(0, memory_pool_.get()), 0, 0};
+        return true;
     }
+    read_count_.fetch_add(1, std::memory_order_relaxed);
+    read_bytes_.fetch_add(range.length, std::memory_order_relaxed);
     PreBuffer(range.offset);
-    ByteSlice result{};
-    {
-        std::shared_lock<std::shared_mutex> lock(rw_mutex_);
-        auto it = std::lower_bound(entries_.begin(), entries_.end(), range.offset,
-                                   [](const RangeCacheEntry& e, uint64_t offset) {
-                                       return e.range.offset + e.range.length <= offset;
-                                   });
-        if (it != entries_.end() && it->range.Contains(range)) {
-            PAIMON_RETURN_NOT_OK(it->future.get());
-            result = ByteSlice{it->buffer, range.offset - it->range.offset, range.length};
-            return result;
-        }
+    std::vector<RangeCacheEntry> covering = FindCoveringEntries(range);
+    if (covering.empty()) {
+        CountMiss(range.length);
+        return false;
+    }
+    // Wait OUTSIDE the lock: the futures resolve when the prefetch stream's
+    // async reads complete, and holding rw_mutex_ would block Cache().
+    for (const auto& entry : covering) {
+        PAIMON_RETURN_NOT_OK(entry.future.get());
     }
-    return result;
+    // The data copy runs OUTSIDE the lock for the same reason.
+    CopyRangeFromEntries(covering, range, dest);
+    CountHit(range.length);
+    return true;
 }
 
-std::vector<RangeCacheEntry> ReadAheadCache::Impl::MakeCacheEntries(
-    const std::vector<ByteRange>& ranges) const {
-    std::vector<RangeCacheEntry> new_entries;
-    new_entries.reserve(ranges.size());
-    for (const auto& range : ranges) {
-        auto promise = std::make_shared<std::promise<Status>>();
-        auto future = promise->get_future();
-        auto buffer = std::make_shared<Bytes>(range.length, memory_pool_.get());
+void ReadAheadCache::Impl::DispatchFetches(const std::vector<PendingFetch>& fetches) {
+    for (const auto& fetch : fetches) {
+        auto promise = fetch.promise;
+        auto buffer = fetch.buffer;
         auto read_size = static_cast<int64_t>(buffer->size());
-        auto read_offset = static_cast<int64_t>(range.offset);
+        auto read_offset = static_cast<int64_t>(fetch.range.offset);
         stream_->ReadAsync(
             buffer->data(), read_size, read_offset,
             [promise, buffer](Status status) mutable { promise->set_value(status); });
-        new_entries.emplace_back(range, std::move(buffer), std::move(future));
+        io_count_.fetch_add(1, std::memory_order_relaxed);
+        io_bytes_.fetch_add(fetch.range.length, std::memory_order_relaxed);
     }
-    return new_entries;
 }
 
 ReadAheadCache::ReadAheadCache(const std::shared_ptr<InputStream>& stream,
@@ -245,12 +389,24 @@ Status ReadAheadCache::Init(std::vector<ByteRange>&& ranges) {
     return impl_->Init(std::move(ranges));
 }
 
-Result<ByteSlice> ReadAheadCache::Read(const ByteRange& range) {
-    return impl_->Read(range);
+Result<bool> ReadAheadCache::Read(const ByteRange& range, char* dest) {
+    return impl_->Read(range, dest);
 }
 
 void ReadAheadCache::Reset() {
     return impl_->Reset();
 }
 
+void ReadAheadCache::ReleaseBuffers() {
+    return impl_->ReleaseBuffers();
+}
+
+void ReadAheadCache::Warmup() {
+    impl_->Warmup();
+}
+
+void ReadAheadCache::CollectMetrics(std::shared_ptr<Metrics>* metrics) const {
+    impl_->CollectMetrics(metrics);
+}
+
 }  // namespace paimon
diff --git a/include/paimon/utils/read_ahead_cache.h b/src/paimon/common/utils/read_ahead_cache.h
similarity index 50%
rename from include/paimon/utils/read_ahead_cache.h
rename to src/paimon/common/utils/read_ahead_cache.h
index 196045b7e..f039c22e8 100644
--- a/include/paimon/utils/read_ahead_cache.h
+++ b/src/paimon/common/utils/read_ahead_cache.h
@@ -27,87 +27,31 @@
 #include <vector>
 
 #include "paimon/fs/file_system.h"
-#include "paimon/memory/bytes.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/result.h"
 #include "paimon/status.h"
+#include "paimon/utils/prefetch_cache_config.h"
 #include "paimon/visibility.h"
 
 namespace paimon {
 
-/// PrefetchCacheMode
-/// Cache prefetch switch modes.
-/// Controls whether to enable cache prefetching under different circumstances, such as queries with
-/// predicates or bitmap indexes.
-///
-/// - ALWAYS: Enable cache in all scenarios.
-/// - EXCLUDE_PREDICATE: Disable cache when query has predicates.
-/// - EXCLUDE_BITMAP: Disable cache when using bitmap index.
-/// - EXCLUDE_BITMAP_OR_PREDICATE: Disable cache if query has predicates or bitmap index.
-/// - NEVER: Always disable cache.
-enum class PAIMON_EXPORT PrefetchCacheMode {
-    ALWAYS = 1,
-    EXCLUDE_PREDICATE = 2,
-    EXCLUDE_BITMAP = 3,
-    EXCLUDE_BITMAP_OR_PREDICATE = 4,
-    NEVER = 5
-};
+class Metrics;
 
-/// Configuration parameters for the read-ahead cache behavior.
-///
-/// This struct controls various limits and prefetching strategies used by
-/// ReadAheadCache to balance memory usage, I/O efficiency, and latency hiding.
-class PAIMON_EXPORT CacheConfig {
+/// Metric names for the read-ahead cache.
+class PAIMON_EXPORT ReadAheadCacheMetrics {
  public:
-    CacheConfig();
-    CacheConfig(uint64_t buffer_size_limit, uint64_t range_size_limit, uint64_t hole_size_limit,
-                uint64_t pre_buffer_limit);
-
-    /// Returns the maximum total size (in bytes) of cached data.
-    uint64_t GetBufferSizeLimit() const {
-        return buffer_size_limit_;
-    }
-
-    /// Sets the maximum total size (in bytes) of cached data.
-    void SetBufferSizeLimit(uint64_t buffer_size_limit) {
-        buffer_size_limit_ = buffer_size_limit;
-    }
-
-    /// Returns the maximum allowed size (in bytes) for a single cached range.
-    uint64_t GetRangeSizeLimit() const {
-        return range_size_limit_;
-    }
-
-    /// Sets the maximum allowed size (in bytes) for a single cached range.
-    void SetRangeSizeLimit(uint64_t range_size_limit) {
-        range_size_limit_ = range_size_limit;
-    }
-
-    /// Returns the maximum gap size (in bytes) considered mergeable between adjacent ranges.
-    uint64_t GetHoleSizeLimit() const {
-        return hole_size_limit_;
-    }
-
-    /// Sets the maximum gap size (in bytes) considered mergeable between adjacent ranges.
-    void SetHoleSizeLimit(uint64_t hole_size_limit) {
-        hole_size_limit_ = hole_size_limit;
-    }
-
-    /// Returns the maximum size to pre-buffer ahead of the current read position.
-    uint64_t GetPreBufferLimit() const {
-        return pre_buffer_limit_;
-    }
-
-    /// Sets the maximum size to pre-buffer ahead of the current read position.
-    void SetPreBufferLimit(uint64_t pre_buffer_limit) {
-        pre_buffer_limit_ = pre_buffer_limit;
-    }
-
- private:
-    uint64_t buffer_size_limit_;
-    uint64_t range_size_limit_;
-    uint64_t hole_size_limit_;
-    uint64_t pre_buffer_limit_;
+    /// Number of non-zero-sized Read() requests issued to the cache.
+    static inline const char READ_COUNT[] = "read-ahead-cache.read.count";
+    /// Total bytes requested by the Read() requests issued to the cache.
+    static inline const char READ_BYTES[] = "read-ahead-cache.read.bytes";
+    static inline const char READ_HITS[] = "read-ahead-cache.read.hits";
+    static inline const char READ_HIT_BYTES[] = "read-ahead-cache.read.hit-bytes";
+    static inline const char READ_MISSES[] = "read-ahead-cache.read.misses";
+    static inline const char READ_MISS_BYTES[] = "read-ahead-cache.read.miss-bytes";
+    /// Number of prefetch IO requests actually issued to the underlying stream.
+    static inline const char IO_COUNT[] = "read-ahead-cache.io.count";
+    /// Total bytes requested by the prefetch IOs issued to the underlying stream.
+    static inline const char IO_BYTES[] = "read-ahead-cache.io.bytes";
 };
 
 /// A byte range with offset and length.
@@ -132,22 +76,15 @@ struct PAIMON_EXPORT ByteRange {
     }
 };
 
-/// A byte slice with buffer, offset and length.
-struct PAIMON_EXPORT ByteSlice {
-    std::shared_ptr<Bytes> buffer = nullptr;
-    uint64_t offset = 0;
-    uint64_t length = 0;
-};
-
 /// A read cache designed to hide IO latencies when reading.
 /// Prefetching strategy: When a range is read, the cache will prefetch up to
 /// `pre_buffer_range_count` additional adjacent ranges ahead of the requested offset. This helps
 /// hide I/O latency for sequential access. Example: If you read range [0, 100), and
 /// pre_buffer_range_count=2, the next two configured ranges will also be prefetched.
 ///
-/// Eviction policy: The cache uses a simple FIFO eviction policy based on total cached byte size.
-/// When adding new ranges would exceed `buffer_size_limit`, the oldest cached ranges are evicted
-/// first until there is enough space for the new data.
+/// The cache never evicts: every published range stays cached until
+/// ReleaseBuffers() or Reset(). It is meant to hold the prefetched ranges of
+/// a single data file, whose size is bounded by the reader's scan scope.
 class PAIMON_EXPORT ReadAheadCache {
  public:
     /// Construct a read cache with given options
@@ -162,11 +99,30 @@ class PAIMON_EXPORT ReadAheadCache {
     /// on the cache configuration.
     Status Init(std::vector<ByteRange>&& ranges);
 
-    /// Read a range previously provided to Init().
+    /// Read a range previously provided to Init(), copying the cached data
+    /// directly into the given destination buffer.
+    ///
+    /// Multi-segment hits are copied into `dest` segment by segment, without
+    /// an intermediate assembled buffer.
     /// @param range The byte range to read.
-    /// @return The byte slice containing the requested data. If the data is not yet cached
-    /// (cache miss), the returned `ByteSlice` will have a null buffer (`buffer == nullptr`)
-    Result<ByteSlice> Read(const ByteRange& range);
+    /// @param dest Destination buffer with at least `range.length` bytes.
+    /// @return true if the range was served from the cache and `dest` was
+    /// filled; false on cache miss (`dest` is left untouched).
+    Result<bool> Read(const ByteRange& range, char* dest);
+
+    /// Start fetching the first batch of pending ranges immediately.
+    /// Init() only registers the ranges; without Warmup() the first fetch starts
+    /// when the first Read() arrives, racing the caller's own miss fetch.
+    void Warmup();
+
+    /// Collect hit/miss counters of Read() calls and the prefetch IO
+    /// counters into the given metrics as counters named after
+    /// `ReadAheadCacheMetrics`. Only reads issued through Read() are counted
+    /// as hits/misses; prefetch fetches dispatched by the cache itself are
+    /// counted in the fetch counters instead.
+    /// @param metrics The metrics to write the counters into. A null
+    /// pointer or a null shared pointer is a no-op.
+    void CollectMetrics(std::shared_ptr<Metrics>* metrics) const;
 
     /// Reset the cache to its initial state, clearing all cached data and configuration.
     ///
@@ -175,6 +131,14 @@ class PAIMON_EXPORT ReadAheadCache {
     /// After calling Reset, the cache can be safely re-initialized with new ranges.
     void Reset();
 
+    /// Release all cached buffers and pending ranges while keeping the hit/miss
+    /// counters intact.
+    ///
+    /// Unlike Reset(), the counters recorded by Read() remain readable through
+    /// CollectMetrics() afterwards, so this is safe to call when the owning reader
+    /// is closed while its metrics are still being aggregated.
+    void ReleaseBuffers();
+
  private:
     class Impl;
     std::unique_ptr<Impl> impl_;
diff --git a/src/paimon/common/utils/read_ahead_cache_test.cpp b/src/paimon/common/utils/read_ahead_cache_test.cpp
index 676254220..ab1d0ed3d 100644
--- a/src/paimon/common/utils/read_ahead_cache_test.cpp
+++ b/src/paimon/common/utils/read_ahead_cache_test.cpp
@@ -17,12 +17,18 @@
  * under the License.
  */
 
-#include "paimon/utils/read_ahead_cache.h"
+#include "paimon/common/utils/read_ahead_cache.h"
 
+#include <chrono>
 #include <fstream>
+#include <mutex>
+#include <thread>
 #include <vector>
 
 #include "gtest/gtest.h"
+#include "paimon/common/factories/io_hook.h"
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/utils/scope_guard.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/fs/file_system_factory.h"
 #include "paimon/testing/utils/testharness.h"
@@ -61,8 +67,94 @@ TestCacheEnv CreateTestFileAndCache(const std::string& filename, const std::stri
     return {path, cache, pool};
 }
 
+// Assert that reading the range is a cache hit filling the destination with
+// the expected content.
+void AssertReadEquals(const ByteRange& range, const std::string& expected, ReadAheadCache* cache) {
+    std::string dest(std::max<size_t>(range.length, 1), 'X');
+    bool hit = false;
+    ASSERT_OK_AND_ASSIGN(hit, cache->Read(range, dest.data()));
+    ASSERT_TRUE(hit) << expected;
+    EXPECT_EQ(expected, std::string_view(dest.data(), range.length));
+}
+
+// Assert that reading the range misses and leaves the destination untouched.
+void AssertReadMiss(const ByteRange& range, ReadAheadCache* cache) {
+    std::string dest(std::max<size_t>(range.length, 1), 'X');
+    bool hit = true;
+    ASSERT_OK_AND_ASSIGN(hit, cache->Read(range, dest.data()));
+    ASSERT_FALSE(hit);
+    EXPECT_EQ(std::string(dest.size(), 'X'), dest);
+}
+
+// An InputStream wrapper that holds ReadAsync callbacks until ReleaseAll() is
+// called, letting tests observe the cache while prefetch IOs are in flight.
+class GatedAsyncInputStream : public InputStream {
+ public:
+    explicit GatedAsyncInputStream(std::shared_ptr<InputStream> inner) : inner_(std::move(inner)) {}
+
+    Status Close() override {
+        return inner_->Close();
+    }
+    Status Seek(int64_t offset, SeekOrigin origin) override {
+        return inner_->Seek(offset, origin);
+    }
+    Result<int64_t> GetPos() const override {
+        return inner_->GetPos();
+    }
+    Result<int64_t> Read(char* buffer, int64_t size) override {
+        return inner_->Read(buffer, size);
+    }
+    Result<int64_t> Read(char* buffer, int64_t size, int64_t offset) override {
+        return inner_->Read(buffer, size, offset);
+    }
+    void ReadAsync(char* buffer, int64_t size, int64_t offset,
+                   std::function<void(Status)>&& callback) override {
+        std::lock_guard<std::mutex> lock(mutex_);
+        async_read_count_++;
+        pending_.push_back({buffer, size, offset, std::move(callback)});
+    }
+    Result<std::string> GetUri() const override {
+        return inner_->GetUri();
+    }
+    Result<int64_t> Length() const override {
+        return inner_->Length();
+    }
+
+    int AsyncReadCount() {
+        std::lock_guard<std::mutex> lock(mutex_);
+        return async_read_count_;
+    }
+
+    /// Complete all held fetches against the underlying stream.
+    void ReleaseAll() {
+        std::vector<PendingRead> taken;
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            taken = std::move(pending_);
+            pending_.clear();
+        }
+        for (auto& read : taken) {
+            Result<int64_t> res = inner_->Read(read.buffer, read.size, read.offset);
+            read.callback(res.ok() ? Status::OK() : res.status());
+        }
+    }
+
+ private:
+    struct PendingRead {
+        char* buffer;
+        int64_t size;
+        int64_t offset;
+        std::function<void(Status)> callback;
+    };
+
+    std::shared_ptr<InputStream> inner_;
+    std::mutex mutex_;
+    std::vector<PendingRead> pending_;
+    int async_read_count_ = 0;
+};
+
 TEST(TestReadAheadCache, TestBasics) {
-    CacheConfig config(/*buffer_size_limit=*/256 * 1024 * 1024, /*range_size_limit=*/10,
+    CacheConfig config(/*range_size_limit=*/10,
                        /*hole_size_limit=*/2, /*pre_buffer_limit=*/128 * 1024 * 1024);
     std::string content = "abcdefghijklmnopqrstuvwxyz";
     auto env = CreateTestFileAndCache(
@@ -70,90 +162,379 @@ TEST(TestReadAheadCache, TestBasics) {
         {{1, 2}, {3, 2}, {8, 2}, {10, 4}, {14, 0}, {15, 4}, {20, 2}, {25, 0}});
     auto& cache = *env.cache;
 
-    auto assert_slice_equal = [](const ByteSlice& slice, const std::string& expected) {
-        ASSERT_TRUE(slice.buffer) << expected;
-        EXPECT_EQ(expected, std::string_view(slice.buffer->data() + slice.offset, slice.length));
-    };
-
-    ByteSlice slice;
-
-    ASSERT_OK_AND_ASSIGN(slice, cache.Read({20, 2}));
-    assert_slice_equal(slice, "uv");
+    AssertReadEquals({20, 2}, "uv", &cache);
+    AssertReadEquals({1, 2}, "bc", &cache);
+    AssertReadEquals({3, 2}, "de", &cache);
+    AssertReadEquals({8, 2}, "ij", &cache);
+    AssertReadEquals({10, 4}, "klmn", &cache);
+    AssertReadEquals({15, 4}, "pqrs", &cache);
+    AssertReadEquals({19, 3}, "tuv", &cache);
 
-    ASSERT_OK_AND_ASSIGN(slice, cache.Read({1, 2}));
-    assert_slice_equal(slice, "bc");
+    // Zero-sized reads are immediate hits touching nothing.
+    AssertReadEquals({14, 0}, "", &cache);
+    AssertReadEquals({25, 0}, "", &cache);
 
-    ASSERT_OK_AND_ASSIGN(slice, cache.Read({3, 2}));
-    assert_slice_equal(slice, "de");
-
-    ASSERT_OK_AND_ASSIGN(slice, cache.Read({8, 2}));
-    assert_slice_equal(slice, "ij");
-
-    ASSERT_OK_AND_ASSIGN(slice, cache.Read({10, 4}));
-    assert_slice_equal(slice, "klmn");
+    // Non-cached ranges miss and leave the destination untouched.
+    AssertReadMiss({20, 3}, &cache);
+    AssertReadMiss({0, 3}, &cache);
+    AssertReadMiss({25, 2}, &cache);
+}
 
-    ASSERT_OK_AND_ASSIGN(slice, cache.Read({15, 4}));
-    assert_slice_equal(slice, "pqrs");
+// Test that a read spanning several adjacent cache entries is served from the
+// contiguous run of entries and counted as a single hit.
+TEST(TestReadAheadCache, TestMultiSegmentContiguousHit) {
+    CacheConfig config(/*range_size_limit=*/10,
+                       /*hole_size_limit=*/2, /*pre_buffer_limit=*/1024);
+    std::string content = "abcdefghijklmnopqrstuvwxyz";
+    // A single 25-byte range exceeds range_size_limit, so Init() coalesces it
+    // into three adjacent entries: {0,10}, {10,10} and {20,5}.
+    auto env = CreateTestFileAndCache("data_file", content, config, {{0, 25}});
+    auto& cache = *env.cache;
 
-    ASSERT_OK_AND_ASSIGN(slice, cache.Read({19, 3}));
-    assert_slice_equal(slice, "tuv");
+    // Spans all three entries.
+    AssertReadEquals({5, 20}, "fghijklmnopqrstuvwxy", &cache);
 
-    // Zero-sized
-    ASSERT_OK_AND_ASSIGN(slice, cache.Read({14, 0}));
-    assert_slice_equal(slice, "");
-    ASSERT_OK_AND_ASSIGN(slice, cache.Read({25, 0}));
-    assert_slice_equal(slice, "");
+    // Spans the first two entries only, trimming both ends of the run.
+    AssertReadEquals({5, 10}, "fghijklmno", &cache);
 
-    // Non-cached ranges
+    // Runs past the end of the last entry: no contiguous cover, a miss.
+    AssertReadMiss({5, 21}, &cache);
 
-    ASSERT_FALSE(cache.Read({20, 3}).value().buffer);
-    ASSERT_FALSE(cache.Read({0, 3}).value().buffer);
-    ASSERT_FALSE(cache.Read({25, 2}).value().buffer);
+    // A multi-segment hit counts once with the full requested length.
+    std::shared_ptr<Metrics> metrics = std::make_shared<MetricsImpl>();
+    cache.CollectMetrics(&metrics);
+    ASSERT_OK_AND_ASSIGN(uint64_t hits, metrics->GetCounter(ReadAheadCacheMetrics::READ_HITS));
+    ASSERT_EQ(hits, 2u);
+    ASSERT_OK_AND_ASSIGN(uint64_t hit_bytes,
+                         metrics->GetCounter(ReadAheadCacheMetrics::READ_HIT_BYTES));
+    ASSERT_EQ(hit_bytes, 30u);
+    ASSERT_OK_AND_ASSIGN(uint64_t misses, metrics->GetCounter(ReadAheadCacheMetrics::READ_MISSES));
+    ASSERT_EQ(misses, 1u);
+    ASSERT_OK_AND_ASSIGN(uint64_t miss_bytes,
+                         metrics->GetCounter(ReadAheadCacheMetrics::READ_MISS_BYTES));
+    ASSERT_EQ(miss_bytes, 21u);
 }
 
 // Test repeated reads to the same range to ensure cache reuse.
 TEST(TestReadAheadCache, TestRepeatedReadCacheReuse) {
-    CacheConfig config(/*buffer_size_limit=*/64, /*range_size_limit=*/10,
+    CacheConfig config(/*range_size_limit=*/10,
                        /*hole_size_limit=*/2, /*pre_buffer_limit=*/64);
     std::string content = "abcdefghijklmnopqrstuvwxyz";
     auto env = CreateTestFileAndCache("data_file", content, config, {{0, 5}, {7, 5}});
     auto& cache = *env.cache;
 
-    ByteSlice slice;
-    ASSERT_OK_AND_ASSIGN(slice, cache.Read({0, 5}));
-    ASSERT_TRUE(slice.buffer);
-    std::string first_read(slice.buffer->data() + slice.offset, slice.length);
-    ASSERT_EQ(first_read, "abcde");
-
-    ASSERT_OK_AND_ASSIGN(slice, cache.Read({0, 5}));
-    ASSERT_TRUE(slice.buffer);
-    std::string second_read(slice.buffer->data() + slice.offset, slice.length);
-    ASSERT_EQ(second_read, "abcde");
+    AssertReadEquals({0, 5}, "abcde", &cache);
+    AssertReadEquals({0, 5}, "abcde", &cache);
 }
 
-// Test cache eviction when buffer size is limited.
-TEST(TestReadAheadCache, TestCacheEviction) {
-    CacheConfig config(/*buffer_size_limit=*/10, /*range_size_limit=*/5,
-                       /*hole_size_limit=*/2, /*pre_buffer_limit=*/10);
+// The cache never evicts: every prefetched range stays cached until
+// ReleaseBuffers()/Reset(), regardless of how much data accumulates.
+TEST(TestReadAheadCache, TestNoEvictionKeepsAllRanges) {
+    CacheConfig config(/*range_size_limit=*/5, /*hole_size_limit=*/2,
+                       /*pre_buffer_limit=*/10);
     std::string content = "abcdefghijklmnopqrstuvwxyz";
     auto env = CreateTestFileAndCache("data_file", content, config, {{0, 5}, {8, 5}, {16, 5}});
     auto& cache = *env.cache;
 
-    ByteSlice slice;
-    ASSERT_OK_AND_ASSIGN(slice, cache.Read({0, 5}));
-    ASSERT_TRUE(slice.buffer);
-    std::string first_read(slice.buffer->data() + slice.offset, slice.length);
-    ASSERT_EQ(first_read, "abcde");
-
-    // Reading another range should evict the first one due to buffer size limit
-    ASSERT_OK_AND_ASSIGN(slice, cache.Read({8, 5}));
-    ASSERT_TRUE(slice.buffer);
-    std::string second_read(slice.buffer->data() + slice.offset, slice.length);
-    ASSERT_EQ(second_read, "ijklm");
-
-    // The first range should now be a cache miss (buffer is nullptr)
-    auto miss = cache.Read({0, 5});
-    ASSERT_FALSE(miss.value().buffer);
+    AssertReadEquals({0, 5}, "abcde", &cache);
+
+    // Reading further ranges keeps the earlier ones cached.
+    AssertReadEquals({8, 5}, "ijklm", &cache);
+    AssertReadEquals({16, 5}, "qrstu", &cache);
+    AssertReadEquals({0, 5}, "abcde", &cache);
+}
+
+// Test that Read() hits and misses are recorded in the cache metrics.
+TEST(TestReadAheadCache, TestMetrics) {
+    CacheConfig config(/*range_size_limit=*/10,
+                       /*hole_size_limit=*/2, /*pre_buffer_limit=*/128 * 1024 * 1024);
+    std::string content = "abcdefghijklmnopqrstuvwxyz";
+    auto env = CreateTestFileAndCache("data_file", content, config, {{0, 5}, {8, 5}});
+    auto& cache = *env.cache;
+
+    AssertReadEquals({0, 5}, "abcde", &cache);
+    // Out of any cached range: a miss.
+    AssertReadMiss({20, 3}, &cache);
+
+    std::shared_ptr<Metrics> metrics = std::make_shared<MetricsImpl>();
+    cache.CollectMetrics(&metrics);
+    // Both Read() requests are counted, regardless of hit or miss.
+    ASSERT_OK_AND_ASSIGN(uint64_t read_count,
+                         metrics->GetCounter(ReadAheadCacheMetrics::READ_COUNT));
+    ASSERT_EQ(read_count, 2u);
+    ASSERT_OK_AND_ASSIGN(uint64_t read_bytes,
+                         metrics->GetCounter(ReadAheadCacheMetrics::READ_BYTES));
+    ASSERT_EQ(read_bytes, 8u);
+    ASSERT_OK_AND_ASSIGN(uint64_t hits, metrics->GetCounter(ReadAheadCacheMetrics::READ_HITS));
+    ASSERT_EQ(hits, 1u);
+    ASSERT_OK_AND_ASSIGN(uint64_t hit_bytes,
+                         metrics->GetCounter(ReadAheadCacheMetrics::READ_HIT_BYTES));
+    ASSERT_EQ(hit_bytes, 5u);
+    ASSERT_OK_AND_ASSIGN(uint64_t misses, metrics->GetCounter(ReadAheadCacheMetrics::READ_MISSES));
+    ASSERT_EQ(misses, 1u);
+    ASSERT_OK_AND_ASSIGN(uint64_t miss_bytes,
+                         metrics->GetCounter(ReadAheadCacheMetrics::READ_MISS_BYTES));
+    ASSERT_EQ(miss_bytes, 3u);
+    // The hit prefetches both pending ranges in one window: two IO requests
+    // for 10 bytes in total; the miss issues no further fetch.
+    ASSERT_OK_AND_ASSIGN(uint64_t io_count, metrics->GetCounter(ReadAheadCacheMetrics::IO_COUNT));
+    ASSERT_EQ(io_count, 2u);
+    ASSERT_OK_AND_ASSIGN(uint64_t io_bytes, metrics->GetCounter(ReadAheadCacheMetrics::IO_BYTES));
+    ASSERT_EQ(io_bytes, 10u);
+}
+
+// Test that ReleaseBuffers() drops the cached data but keeps the hit/miss counters
+// readable, while Reset() zeroes them as well.
+TEST(TestReadAheadCache, TestReleaseBuffersKeepsMetrics) {
+    CacheConfig config(/*range_size_limit=*/10,
+                       /*hole_size_limit=*/2, /*pre_buffer_limit=*/128 * 1024 * 1024);
+    std::string content = "abcdefghijklmnopqrstuvwxyz";
+    auto env = CreateTestFileAndCache("data_file", content, config, {{0, 5}});
+    auto& cache = *env.cache;
+
+    AssertReadEquals({0, 5}, "abcde", &cache);
+
+    cache.ReleaseBuffers();
+
+    // The previously cached range is gone: the read now misses.
+    AssertReadMiss({0, 5}, &cache);
+
+    std::shared_ptr<Metrics> metrics = std::make_shared<MetricsImpl>();
+    cache.CollectMetrics(&metrics);
+    // The read counters survive ReleaseBuffers() as well.
+    ASSERT_OK_AND_ASSIGN(uint64_t read_count,
+                         metrics->GetCounter(ReadAheadCacheMetrics::READ_COUNT));
+    ASSERT_EQ(read_count, 2u);
+    ASSERT_OK_AND_ASSIGN(uint64_t read_bytes,
+                         metrics->GetCounter(ReadAheadCacheMetrics::READ_BYTES));
+    ASSERT_EQ(read_bytes, 10u);
+    ASSERT_OK_AND_ASSIGN(uint64_t hits, metrics->GetCounter(ReadAheadCacheMetrics::READ_HITS));
+    ASSERT_EQ(hits, 1u);
+    ASSERT_OK_AND_ASSIGN(uint64_t hit_bytes,
+                         metrics->GetCounter(ReadAheadCacheMetrics::READ_HIT_BYTES));
+    ASSERT_EQ(hit_bytes, 5u);
+    ASSERT_OK_AND_ASSIGN(uint64_t misses, metrics->GetCounter(ReadAheadCacheMetrics::READ_MISSES));
+    ASSERT_EQ(misses, 1u);
+    // The io counters survive ReleaseBuffers() as well.
+    ASSERT_OK_AND_ASSIGN(uint64_t io_count, metrics->GetCounter(ReadAheadCacheMetrics::IO_COUNT));
+    ASSERT_EQ(io_count, 1u);
+    ASSERT_OK_AND_ASSIGN(uint64_t io_bytes, metrics->GetCounter(ReadAheadCacheMetrics::IO_BYTES));
+    ASSERT_EQ(io_bytes, 5u);
+
+    // Reset() clears the counters too.
+    cache.Reset();
+    std::shared_ptr<Metrics> reset_metrics = std::make_shared<MetricsImpl>();
+    cache.CollectMetrics(&reset_metrics);
+    ASSERT_OK_AND_ASSIGN(read_count, reset_metrics->GetCounter(ReadAheadCacheMetrics::READ_COUNT));
+    ASSERT_EQ(read_count, 0u);
+    ASSERT_OK_AND_ASSIGN(hits, reset_metrics->GetCounter(ReadAheadCacheMetrics::READ_HITS));
+    ASSERT_EQ(hits, 0u);
+    ASSERT_OK_AND_ASSIGN(misses, reset_metrics->GetCounter(ReadAheadCacheMetrics::READ_MISSES));
+    ASSERT_EQ(misses, 0u);
+    ASSERT_OK_AND_ASSIGN(io_count, reset_metrics->GetCounter(ReadAheadCacheMetrics::IO_COUNT));
+    ASSERT_EQ(io_count, 0u);
+    ASSERT_OK_AND_ASSIGN(io_bytes, reset_metrics->GetCounter(ReadAheadCacheMetrics::IO_BYTES));
+    ASSERT_EQ(io_bytes, 0u);
+}
+
+// Test that a failed prefetch surfaces as an error Status from Read(), not as
+// a miss: the entry exists from the moment its fetch is submitted and its
+// future carries the IO error.
+TEST(TestReadAheadCache, TestPrefetchIOErrorPropagation) {
+    CacheConfig config(/*range_size_limit=*/10,
+                       /*hole_size_limit=*/2, /*pre_buffer_limit=*/1024);
+    std::string content = "abcdefghijklmnopqrstuvwxyz";
+    auto io_hook = paimon::IOHook::GetInstance();
+
+    // Single entry: the prefetch is the first IO after the hook is armed.
+    {
+        auto env = CreateTestFileAndCache("data_file", content, config, {{0, 10}});
+        paimon::ScopeGuard guard([&io_hook]() { io_hook->Clear(); });
+        io_hook->Reset(0, paimon::IOHook::Mode::RETURN_ERROR);
+        std::string dest(5, 'X');
+        ASSERT_NOK_WITH_MSG(env.cache->Read({0, 5}, dest.data()),
+                            "io hook triggered io error at position");
+    }
+
+    // Several adjacent entries: the error of any segment aborts the read.
+    {
+        auto env = CreateTestFileAndCache("data_file", content, config, {{0, 25}});
+        paimon::ScopeGuard guard([&io_hook]() { io_hook->Clear(); });
+        io_hook->Reset(1, paimon::IOHook::Mode::RETURN_ERROR);
+        std::string dest(20, 'X');
+        ASSERT_NOK_WITH_MSG(env.cache->Read({0, 20}, dest.data()),
+                            "io hook triggered io error at position");
+    }
+}
+
+// Test that Warmup() fetches the pending ranges up front so the first Read()
+// issues no further IO, while without Warmup() the first Read() triggers the
+// prefetch itself.
+TEST(TestReadAheadCache, TestWarmupPrefetchesBeforeFirstRead) {
+    CacheConfig config(/*range_size_limit=*/10,
+                       /*hole_size_limit=*/2, /*pre_buffer_limit=*/1024);
+    std::string content = "abcdefghijklmnopqrstuvwxyz";
+    auto env1 = CreateTestFileAndCache("data_file", content, config, {{0, 5}, {8, 5}});
+    env1.cache->Warmup();
+    auto env2 = CreateTestFileAndCache("data_file", content, config, {{0, 5}});
+
+    auto io_hook = paimon::IOHook::GetInstance();
+    paimon::ScopeGuard guard([&io_hook]() { io_hook->Clear(); });
+    // Any new IO fails: the warmed-up reads must be served without fetching.
+    io_hook->Reset(0, paimon::IOHook::Mode::RETURN_ERROR);
+
+    AssertReadEquals({0, 5}, "abcde", env1.cache.get());
+    AssertReadEquals({8, 5}, "ijklm", env1.cache.get());
+
+    // Without Warmup() the first Read() starts the prefetch and sees the error.
+    std::string dest(5, 'X');
+    ASSERT_NOK(env2.cache->Read({0, 5}, dest.data()));
+}
+
+// Warmup() without any pending ranges is a safe no-op.
+TEST(TestReadAheadCache, TestWarmupWithEmptyRanges) {
+    CacheConfig config(/*range_size_limit=*/10,
+                       /*hole_size_limit=*/2, /*pre_buffer_limit=*/1024);
+    std::string content = "abcdefghijklmnopqrstuvwxyz";
+    auto env = CreateTestFileAndCache("data_file", content, config, {});
+    env.cache->Warmup();
+    AssertReadMiss({0, 5}, env.cache.get());
+}
+
+// A reader racing an in-flight prefetch must find the published entry and wait
+// on its future instead of missing and re-fetching the same bytes: entries are
+// published under the lock before their fetch is dispatched.
+TEST(TestReadAheadCache, TestInFlightEntryServesRacingReader) {
+    CacheConfig config(/*range_size_limit=*/10,
+                       /*hole_size_limit=*/2, /*pre_buffer_limit=*/1024);
+    std::string content = "abcdefghijklmnopqrstuvwxyz";
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    std::string path = dir->Str() + "/data_file";
+    std::ofstream file(path, std::ios::binary);
+    ASSERT_TRUE(file.is_open());
+    file.write(content.data(), content.size());
+    ASSERT_FALSE(file.fail());
+    file.close();
+    ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFactory::Get("local", path, {}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs->Open(path));
+    auto gated = std::make_shared<GatedAsyncInputStream>(std::move(in));
+
+    ReadAheadCache cache(gated, config, GetDefaultPool());
+    ASSERT_OK(cache.Init({{0, 5}}));
+    cache.Warmup();
+
+    // The prefetch entry is published, but its fetch is still held.
+    ASSERT_EQ(gated->AsyncReadCount(), 1);
+
+    // A racing reader blocks on the in-flight entry's future and is served
+    // from it once the fetch completes, without triggering a second fetch.
+    std::thread reader([&cache, &gated]() {
+        std::string dest(5, 'X');
+        Result<bool> res = cache.Read({0, 5}, dest.data());
+        EXPECT_TRUE(res.ok());
+        if (res.ok()) {
+            EXPECT_TRUE(res.value());
+        }
+        EXPECT_EQ("abcde", std::string_view(dest.data(), 5));
+        EXPECT_EQ(gated->AsyncReadCount(), 1);
+    });
+    // Give the reader time to block on the in-flight entry's future before
+    // completing the fetch.
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    gated->ReleaseAll();
+    reader.join();
+
+    std::shared_ptr<Metrics> metrics = std::make_shared<MetricsImpl>();
+    cache.CollectMetrics(&metrics);
+    ASSERT_OK_AND_ASSIGN(uint64_t misses, metrics->GetCounter(ReadAheadCacheMetrics::READ_MISSES));
+    ASSERT_EQ(misses, 0u);
+    ASSERT_OK_AND_ASSIGN(uint64_t io_count, metrics->GetCounter(ReadAheadCacheMetrics::IO_COUNT));
+    ASSERT_EQ(io_count, 1u);
+}
+
+// Test that pre_buffer_limit truncates the prefetch window: only ranges within
+// the window are fetched at once, later reads fetch the remaining batches.
+TEST(TestReadAheadCache, TestPreBufferWindowLimit) {
+    CacheConfig config(/*range_size_limit=*/10,
+                       /*hole_size_limit=*/0, /*pre_buffer_limit=*/10);
+    std::string content = "abcdefghijklmnopqrstuvwxyz";
+    auto env = CreateTestFileAndCache("data_file", content, config, {{0, 10}, {16, 10}});
+    auto& cache = *env.cache;
+
+    auto io_hook = paimon::IOHook::GetInstance();
+    paimon::ScopeGuard guard([&io_hook]() { io_hook->Clear(); });
+    io_hook->Clear();
+
+    AssertReadEquals({0, 10}, "abcdefghij", &cache);
+    // The second range did not fit into the window: only one prefetch IO.
+    ASSERT_EQ(io_hook->IOCount(), 1);
+
+    AssertReadEquals({16, 10}, "qrstuvwxyz", &cache);
+    // The second read triggered exactly one more prefetch IO.
+    ASSERT_EQ(io_hook->IOCount(), 2);
+
+    // The range is cached now: re-reading it issues no IO at all.
+    io_hook->Clear();
+    AssertReadEquals({16, 10}, "qrstuvwxyz", &cache);
+    ASSERT_EQ(io_hook->IOCount(), 0);
+}
+
+// Test that Init() rejects a second call until the cache is reset.
+TEST(TestReadAheadCache, TestDoubleInit) {
+    CacheConfig config(/*range_size_limit=*/10,
+                       /*hole_size_limit=*/2, /*pre_buffer_limit=*/1024);
+    std::string content = "abcdefghijklmnopqrstuvwxyz";
+    auto env = CreateTestFileAndCache("data_file", content, config, {{0, 5}});
+    auto& cache = *env.cache;
+
+    Status status = cache.Init({{8, 5}});
+    ASSERT_FALSE(status.ok());
+
+    // The original ranges still work.
+    AssertReadEquals({0, 5}, "abcde", &cache);
+}
+
+// Test that the cache can be re-initialized after Reset() and serves the new ranges.
+TEST(TestReadAheadCache, TestReinitAfterReset) {
+    CacheConfig config(/*range_size_limit=*/10,
+                       /*hole_size_limit=*/2, /*pre_buffer_limit=*/1024);
+    std::string content = "abcdefghijklmnopqrstuvwxyz";
+    auto env = CreateTestFileAndCache("data_file", content, config, {{0, 5}});
+    auto& cache = *env.cache;
+
+    AssertReadEquals({0, 5}, "abcde", &cache);
+
+    cache.Reset();
+    ASSERT_OK(cache.Init({{3, 4}}));
+    AssertReadEquals({3, 4}, "defg", &cache);
+
+    // The old ranges are gone.
+    AssertReadMiss({20, 2}, &cache);
+}
+
+// Test that Init() merges ranges separated by a small hole, so a read
+// spanning the hole is served by the single coalesced entry.
+TEST(TestReadAheadCache, TestInitCoalescesSmallHoles) {
+    CacheConfig config(/*range_size_limit=*/1024,
+                       /*hole_size_limit=*/2, /*pre_buffer_limit=*/1024);
+    std::string content = "abcdefghijklmnopqrstuvwxyz";
+    // Byte 5 sits in a 1-byte hole, within hole_size_limit: one entry {0,11}.
+    auto env = CreateTestFileAndCache("data_file", content, config, {{0, 5}, {6, 5}});
+    auto& cache = *env.cache;
+
+    AssertReadEquals({4, 3}, "efg", &cache);
+}
+
+// CollectMetrics() with a null metrics output is a safe no-op.
+TEST(TestReadAheadCache, TestCollectMetricsWithNullMetrics) {
+    CacheConfig config(/*range_size_limit=*/10,
+                       /*hole_size_limit=*/2, /*pre_buffer_limit=*/1024);
+    std::string content = "abcdefghijklmnopqrstuvwxyz";
+    auto env = CreateTestFileAndCache("data_file", content, config, {{0, 5}});
+    env.cache->CollectMetrics(/*metrics=*/nullptr);
+    std::shared_ptr<Metrics> null_metrics;
+    env.cache->CollectMetrics(&null_metrics);
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp b/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp
index 1fca5f374..5451c9d29 100644
--- a/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp
+++ b/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp
@@ -26,13 +26,13 @@
 #include "arrow/ipc/json_simple.h"
 #include "gtest/gtest.h"
 #include "paimon/common/reader/prefetch_file_batch_reader_impl.h"
+#include "paimon/common/utils/read_ahead_cache.h"
 #include "paimon/executor.h"
 #include "paimon/testing/mock/mock_file_batch_reader.h"
 #include "paimon/testing/mock/mock_file_system.h"
 #include "paimon/testing/mock/mock_format_reader_builder.h"
 #include "paimon/testing/utils/read_result_collector.h"
 #include "paimon/testing/utils/testharness.h"
-#include "paimon/utils/read_ahead_cache.h"
 
 namespace arrow {
 class Array;
@@ -87,7 +87,7 @@ class ApplyDeletionVectorBatchReaderTest : public ::testing::Test,
                         prefetch_batch_count, batch_size, prefetch_batch_count * 2,
                         /*enable_adaptive_prefetch_strategy=*/false, executor_,
                         /*initialize_read_ranges=*/true,
-                        /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), pool_));
+                        /*read_ahead_cache_enabled=*/true, CacheConfig(), pool_));
             } else {
                 file_batch_reader =
                     std::make_unique<MockFileBatchReader>(data, target_type_, batch_size);
diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp
index 2a3d9e10d..40fb6e5be 100644
--- a/src/paimon/core/operation/abstract_split_read.cpp
+++ b/src/paimon/core/operation/abstract_split_read.cpp
@@ -139,6 +139,13 @@ Result<std::unique_ptr<ReaderBuilder>> AbstractSplitRead::PrepareReaderBuilder(
                            file_format->CreateReaderBuilder(options_.GetReadBatchSize()));
     reader_builder->WithMemoryPool(pool_);
     reader_builder->WithCache(options_.GetCache());
+    // Propagate the framework runtime read state so each format can adapt its own
+    // behavior (e.g. parquet disabling its pre-buffer when the shared read-ahead cache
+    // takes over prefetching), instead of mutating format options here.
+    ReadHints read_hints;
+    read_hints.prefetch_enabled = context_->EnablePrefetch();
+    read_hints.read_ahead_cache_enabled = context_->ReadAheadCacheEnabled();
+    reader_builder->WithReadHints(read_hints);
     return reader_builder;
 }
 
@@ -154,7 +161,7 @@ Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::CreateFileBatchReade
                 context_->GetPrefetchMaxParallelNum(), options_.GetReadBatchSize(),
                 context_->GetPrefetchBatchCount(), options_.EnableAdaptivePrefetchStrategy(),
                 executor_,
-                /*initialize_read_ranges=*/false, context_->GetPrefetchCacheMode(),
+                /*initialize_read_ranges=*/false, context_->ReadAheadCacheEnabled(),
                 context_->GetCacheConfig(), pool_));
         return std::make_unique<DelegatingPrefetchReader>(std::move(prefetch_reader));
     } else {
diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h
index f33b7a359..8ef9f2d26 100644
--- a/src/paimon/core/operation/internal_read_context.h
+++ b/src/paimon/core/operation/internal_read_context.h
@@ -96,8 +96,8 @@ class InternalReadContext {
         return read_context_->GetRealtimeContext();
     }
 
-    PrefetchCacheMode GetPrefetchCacheMode() const {
-        return read_context_->GetPrefetchCacheMode();
+    bool ReadAheadCacheEnabled() const {
+        return read_context_->ReadAheadCacheEnabled();
     }
 
     const CacheConfig& GetCacheConfig() const {
diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp
index eb3d8826d..08a854d84 100644
--- a/src/paimon/core/operation/read_context.cpp
+++ b/src/paimon/core/operation/read_context.cpp
@@ -42,7 +42,7 @@ ReadContext::ReadContext(
     const std::shared_ptr<FileSystem>& specific_file_system,
     const std::map<std::string, std::string>& fs_scheme_to_identifier_map,
     const std::shared_ptr<RealtimeContext>& realtime_context,
-    const std::map<std::string, std::string>& options, PrefetchCacheMode prefetch_cache_mode,
+    const std::map<std::string, std::string>& options, bool read_ahead_cache_enabled,
     const CacheConfig& cache_config, const std::shared_ptr<Cache>& cache)
     : path_(path),
       branch_(branch),
@@ -62,7 +62,7 @@ ReadContext::ReadContext(
       fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map),
       realtime_context_(realtime_context),
       options_(options),
-      prefetch_cache_mode_(prefetch_cache_mode),
+      read_ahead_cache_enabled_(read_ahead_cache_enabled),
       cache_config_(cache_config),
       cache_(cache) {}
 
@@ -97,7 +97,7 @@ class ReadContextBuilder::Impl {
         predicate_.reset();
         enable_predicate_filter_ = false;
         enable_prefetch_ = false;
-        prefetch_cache_mode_ = PrefetchCacheMode::ALWAYS;
+        read_ahead_cache_enabled_ = true;
         prefetch_batch_count_ = 600;
         prefetch_max_parallel_num_ = 3;
         enable_multi_thread_row_to_batch_ = false;
@@ -131,7 +131,7 @@ class ReadContextBuilder::Impl {
     std::shared_ptr<Executor> executor_;
     std::shared_ptr<FileSystem> specific_file_system_;
     std::shared_ptr<RealtimeContext> realtime_context_;
-    PrefetchCacheMode prefetch_cache_mode_ = PrefetchCacheMode::ALWAYS;
+    bool read_ahead_cache_enabled_ = true;
     CacheConfig cache_config_;
     std::shared_ptr<Cache> cache_;
 };
@@ -250,8 +250,8 @@ ReadContextBuilder& ReadContextBuilder::WithFileSystem(
     return *this;
 }
 
-ReadContextBuilder& ReadContextBuilder::SetPrefetchCacheMode(PrefetchCacheMode mode) {
-    impl_->prefetch_cache_mode_ = mode;
+ReadContextBuilder& ReadContextBuilder::SetReadAheadCacheEnabled(bool enabled) {
+    impl_->read_ahead_cache_enabled_ = enabled;
     return *this;
 }
 
@@ -301,7 +301,7 @@ Result<std::unique_ptr<ReadContext>> ReadContextBuilder::Finish() {
         impl_->enable_multi_thread_row_to_batch_, impl_->row_to_batch_thread_number_,
         impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_,
         impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_,
-        impl_->prefetch_cache_mode_, impl_->cache_config_, impl_->cache_);
+        impl_->read_ahead_cache_enabled_, impl_->cache_config_, impl_->cache_);
     if (impl_->read_schema_ && impl_->read_schema_->release) {
         ctx->SetReadSchema(std::move(impl_->read_schema_));
     }
diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp
index 8d5a78e14..c686cccb5 100644
--- a/src/paimon/core/operation/read_context_test.cpp
+++ b/src/paimon/core/operation/read_context_test.cpp
@@ -45,7 +45,7 @@ TEST(ReadContextTest, TestDefaultValue) {
     ASSERT_FALSE(ctx->GetPredicate());
     ASSERT_FALSE(ctx->EnablePredicateFilter());
     ASSERT_FALSE(ctx->EnablePrefetch());
-    ASSERT_EQ(PrefetchCacheMode::ALWAYS, ctx->GetPrefetchCacheMode());
+    ASSERT_TRUE(ctx->ReadAheadCacheEnabled());
     ASSERT_EQ(600, ctx->GetPrefetchBatchCount());
     ASSERT_EQ(3, ctx->GetPrefetchMaxParallelNum());
     ASSERT_FALSE(ctx->EnableMultiThreadRowToBatch());
@@ -59,8 +59,8 @@ TEST(ReadContextTest, TestSetContent) {
     ReadContextBuilder builder("table_root_path");
     std::shared_ptr<MemoryPool> memory_pool = GetDefaultPool();
     std::shared_ptr<Executor> executor = CreateDefaultExecutor();
-    CacheConfig cache_config(/*buffer_size_limit=*/1024, /*range_size_limit=*/512,
-                             /*hole_size_limit=*/128, /*pre_buffer_limit=*/2048);
+    CacheConfig cache_config(/*range_size_limit=*/512, /*hole_size_limit=*/128,
+                             /*pre_buffer_limit=*/2048);
 
     builder.AddOption("key", "value");
     builder.SetReadFieldNames({"f1", "f2"});
@@ -70,7 +70,7 @@ TEST(ReadContextTest, TestSetContent) {
     builder.SetPredicate(predicate);
     builder.EnablePredicateFilter(true);
     builder.EnablePrefetch(true);
-    builder.SetPrefetchCacheMode(PrefetchCacheMode::NEVER);
+    builder.SetReadAheadCacheEnabled(false);
     builder.SetPrefetchBatchCount(1200);
     builder.SetPrefetchMaxParallelNum(6);
     builder.EnableMultiThreadRowToBatch(true);
@@ -95,7 +95,7 @@ TEST(ReadContextTest, TestSetContent) {
     ASSERT_EQ(*predicate, *(ctx->GetPredicate()));
     ASSERT_TRUE(ctx->EnablePredicateFilter());
     ASSERT_TRUE(ctx->EnablePrefetch());
-    ASSERT_EQ(PrefetchCacheMode::NEVER, ctx->GetPrefetchCacheMode());
+    ASSERT_FALSE(ctx->ReadAheadCacheEnabled());
     ASSERT_EQ(1200, ctx->GetPrefetchBatchCount());
     ASSERT_EQ(6, ctx->GetPrefetchMaxParallelNum());
     ASSERT_TRUE(ctx->EnableMultiThreadRowToBatch());
@@ -105,7 +105,6 @@ TEST(ReadContextTest, TestSetContent) {
     ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value());
     ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value());
     ASSERT_EQ("rt", ctx->GetBranch());
-    ASSERT_EQ(1024U, ctx->GetCacheConfig().GetBufferSizeLimit());
     ASSERT_EQ(512U, ctx->GetCacheConfig().GetRangeSizeLimit());
     ASSERT_EQ(128U, ctx->GetCacheConfig().GetHoleSizeLimit());
     ASSERT_EQ(2048U, ctx->GetCacheConfig().GetPreBufferLimit());
diff --git a/src/paimon/core/table/bucket_mode.cpp b/src/paimon/core/table/bucket_mode.cpp
index d46739f89..429a787d2 100644
--- a/src/paimon/core/table/bucket_mode.cpp
+++ b/src/paimon/core/table/bucket_mode.cpp
@@ -24,15 +24,13 @@
 namespace paimon {
 
 BucketMode ResolveBucketMode(int32_t bucket, const std::shared_ptr<TableSchema>& table_schema) {
-    if (bucket == BucketModeDefine::POSTPONE_BUCKET) {
+    bool has_primary_keys = !table_schema->PrimaryKeys().empty();
+    // Postpone bucket is only valid for primary key tables.
+    if (has_primary_keys && bucket == BucketModeDefine::POSTPONE_BUCKET) {
         return BucketMode::POSTPONE_MODE;
     }
     if (bucket == -1) {
-        return table_schema->PrimaryKeys().empty() ? BucketMode::BUCKET_UNAWARE
-                                                   : BucketMode::HASH_DYNAMIC;
-    }
-    if (bucket == BucketModeDefine::UNAWARE_BUCKET) {
-        return BucketMode::BUCKET_UNAWARE;
+        return has_primary_keys ? BucketMode::HASH_DYNAMIC : BucketMode::BUCKET_UNAWARE;
     }
     return BucketMode::HASH_FIXED;
 }
diff --git a/src/paimon/core/table/bucket_mode.h b/src/paimon/core/table/bucket_mode.h
index f87ab3467..e6e6491c6 100644
--- a/src/paimon/core/table/bucket_mode.h
+++ b/src/paimon/core/table/bucket_mode.h
@@ -63,10 +63,16 @@ enum class BucketMode {
 
 class BucketModeDefine {
  public:
+    /// The bucket id that all data of a `BucketMode::BUCKET_UNAWARE` table is written to. Note that
+    /// this is a bucket id, not a valid value of the 'bucket' option.
     static constexpr int32_t UNAWARE_BUCKET = 0;
+    /// The value of the 'bucket' option which enables `BucketMode::POSTPONE_MODE`, it is also used
+    /// as the bucket id of the data waiting to be assigned to a real bucket.
     static constexpr int32_t POSTPONE_BUCKET = -2;
 };
 
+/// Resolves the bucket mode from the 'bucket' option and the table schema. Note that an invalid
+/// 'bucket' value is rejected by `SchemaValidation::ValidateBucket` instead of here.
 BucketMode ResolveBucketMode(int32_t bucket, const std::shared_ptr<TableSchema>& table_schema);
 
 }  // namespace paimon
diff --git a/src/paimon/core/table/bucket_mode_test.cpp b/src/paimon/core/table/bucket_mode_test.cpp
index 2a77e5ea9..8feeb6e8c 100644
--- a/src/paimon/core/table/bucket_mode_test.cpp
+++ b/src/paimon/core/table/bucket_mode_test.cpp
@@ -50,13 +50,17 @@ TEST(BucketModeTest, TestResolveBucketMode) {
     std::shared_ptr<TableSchema> append_schema = CreateTableSchema(/*primary_keys=*/{});
     std::shared_ptr<TableSchema> pk_schema = CreateTableSchema(/*primary_keys=*/{"f0"});
 
+    // Postpone bucket only applies to primary key tables.
     EXPECT_EQ(BucketMode::POSTPONE_MODE,
+              ResolveBucketMode(BucketModeDefine::POSTPONE_BUCKET, pk_schema));
+    EXPECT_EQ(BucketMode::HASH_FIXED,
               ResolveBucketMode(BucketModeDefine::POSTPONE_BUCKET, append_schema));
+
     EXPECT_EQ(BucketMode::BUCKET_UNAWARE, ResolveBucketMode(-1, append_schema));
     EXPECT_EQ(BucketMode::HASH_DYNAMIC, ResolveBucketMode(-1, pk_schema));
-    EXPECT_EQ(BucketMode::BUCKET_UNAWARE,
-              ResolveBucketMode(BucketModeDefine::UNAWARE_BUCKET, pk_schema));
+
     EXPECT_EQ(BucketMode::HASH_FIXED, ResolveBucketMode(4, append_schema));
+    EXPECT_EQ(BucketMode::HASH_FIXED, ResolveBucketMode(4, pk_schema));
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp
index e384146ed..7572f906e 100644
--- a/src/paimon/core/table/system/audit_log_system_table.cpp
+++ b/src/paimon/core/table/system/audit_log_system_table.cpp
@@ -440,7 +440,7 @@ Result<std::unique_ptr<TableRead>> AuditLogSystemTable::NewChangelogRead(
         .SetPrefetchMaxParallelNum(context->GetPrefetchMaxParallelNum())
         .EnableMultiThreadRowToBatch(context->EnableMultiThreadRowToBatch())
         .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber())
-        .SetPrefetchCacheMode(context->GetPrefetchCacheMode())
+        .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled())
         .WithCacheConfig(context->GetCacheConfig())
         .WithCache(context->GetCache());
 
diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp
index 516861cb5..6abec946b 100644
--- a/src/paimon/core/table/system/read_optimized_system_table.cpp
+++ b/src/paimon/core/table/system/read_optimized_system_table.cpp
@@ -109,7 +109,7 @@ Result<std::unique_ptr<TableRead>> ReadOptimizedSystemTable::NewRead(
         .WithExecutor(context->GetExecutor())
         .WithFileSystem(context->GetSpecificFileSystem())
         .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap())
-        .SetPrefetchCacheMode(context->GetPrefetchCacheMode())
+        .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled())
         .WithCacheConfig(context->GetCacheConfig())
         .WithCache(context->GetCache())
         .SetReadFieldNames(context->GetReadFieldNames())
diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp
index 48a4430ad..fb5b65741 100644
--- a/src/paimon/format/parquet/file_reader_wrapper.cpp
+++ b/src/paimon/format/parquet/file_reader_wrapper.cpp
@@ -21,12 +21,14 @@
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
+#include <limits>
 
 #include "arrow/io/interfaces.h"
 #include "arrow/record_batch.h"
 #include "arrow/util/range.h"
 #include "fmt/format.h"
 #include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/format/parquet/column_index_filter.h"
 #include "paimon/format/parquet/page_filtered_row_group_reader.h"
 #include "paimon/format/parquet/parquet_format_defs.h"
@@ -202,6 +204,17 @@ Status FileReaderWrapper::SeekToRow(uint64_t row_number) {
                 current_row_group_idx_ = i;
                 next_row_to_read_ = rg_start;
 
+                if (!reader_initialized_) {
+                    // PrepareForReading (first Next()) will build batch_reader_, so just
+                    // record the seeked start for it. Building batch_reader_ here would be
+                    // discarded by PrepareForReading, and the arrow GetRecordBatchReader
+                    // eagerly reads every column chunk, so building twice doubles the
+                    // requested bytes.
+                    pending_start_idx_ = i;
+                    batch_reader_.reset();
+                    return Status::OK();
+                }
+
                 // Rebuild batch_reader_ for non-page-filtered RGs at/after seek position.
                 std::vector<int32_t> fully_matched_indices;
                 for (uint64_t j = i; j < target_row_groups_.size(); j++) {
@@ -221,6 +234,12 @@ Status FileReaderWrapper::SeekToRow(uint64_t row_number) {
         }
         next_row_to_read_ = num_rows_;
         current_row_group_idx_ = target_row_groups_.size();
+        if (!reader_initialized_) {
+            // Seek past the last row group before initialization: the deferred
+            // PrepareForReading must start at EOF as well.
+            pending_start_idx_ = target_row_groups_.size();
+            batch_reader_.reset();
+        }
         return Status::OK();
     }
     PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::SeekToRow")
@@ -376,39 +395,77 @@ Status FileReaderWrapper::PrepareForReadingLazy(
     target_row_groups_ = target_row_groups;
     target_column_indices_ = column_indices;
     reader_initialized_ = false;
+    pending_start_idx_.reset();
     return Status::OK();
 }
 
-std::vector<::arrow::io::ReadRange> FileReaderWrapper::CollectPreBufferRanges(
-    const std::vector<int32_t>& column_indices) {
-    std::vector<::arrow::io::ReadRange> ranges;
-    auto file_metadata = file_reader_->parquet_reader()->metadata();
-
-    for (const auto& trg : target_row_groups_) {
-        if (trg.IsExcludedByReadRange()) continue;
-
-        if (trg.IsPartiallyMatched()) {
-            // Page-filtered RGs: only matching page byte ranges.
-            auto row_group_page_index_reader = GetRowGroupPageIndexReader(trg.GetRowGroupIndex());
-            auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges(
-                trg, column_indices, row_group_page_index_reader, file_reader_->parquet_reader());
-            ranges.insert(ranges.end(), std::make_move_iterator(page_ranges.begin()),
-                          std::make_move_iterator(page_ranges.end()));
-        } else {
-            // Fully-matched RGs: entire column chunk ranges.
-            auto rg_metadata = file_metadata->RowGroup(trg.GetRowGroupIndex());
-            for (int32_t col_idx : column_indices) {
-                auto col_chunk = rg_metadata->ColumnChunk(col_idx);
-                int64_t offset = col_chunk->data_page_offset();
-                if (col_chunk->has_dictionary_page() && col_chunk->dictionary_page_offset() > 0 &&
-                    offset > col_chunk->dictionary_page_offset()) {
-                    offset = col_chunk->dictionary_page_offset();
+Result<std::vector<::arrow::io::ReadRange>> FileReaderWrapper::CollectPreBufferRanges(
+    const std::vector<int32_t>& column_indices, uint64_t start_idx) {
+    return DoCollectPreBufferRanges(column_indices, /*skip_read_range_excluded=*/true, start_idx);
+}
+
+Result<std::vector<::arrow::io::ReadRange>> FileReaderWrapper::DoCollectPreBufferRanges(
+    const std::vector<int32_t>& column_indices, bool skip_read_range_excluded, uint64_t start_idx) {
+    try {
+        std::vector<::arrow::io::ReadRange> ranges;
+        auto file_metadata = file_reader_->parquet_reader()->metadata();
+
+        for (uint64_t idx = start_idx; idx < target_row_groups_.size(); idx++) {
+            const auto& trg = target_row_groups_[idx];
+            if (skip_read_range_excluded && trg.IsExcludedByReadRange()) {
+                continue;
+            }
+
+            if (trg.IsPartiallyMatched()) {
+                // Page-filtered RGs: only matching page byte ranges.
+                auto row_group_page_index_reader =
+                    GetRowGroupPageIndexReader(trg.GetRowGroupIndex());
+                auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges(
+                    trg, column_indices, row_group_page_index_reader,
+                    file_reader_->parquet_reader());
+                ranges.insert(ranges.end(), std::make_move_iterator(page_ranges.begin()),
+                              std::make_move_iterator(page_ranges.end()));
+            } else {
+                // Fully-matched RGs: entire column chunk ranges.
+                auto rg_metadata = file_metadata->RowGroup(trg.GetRowGroupIndex());
+                for (int32_t col_idx : column_indices) {
+                    auto col_chunk = rg_metadata->ColumnChunk(col_idx);
+                    int64_t offset = col_chunk->data_page_offset();
+                    if (col_chunk->has_dictionary_page() &&
+                        col_chunk->dictionary_page_offset() > 0 &&
+                        offset > col_chunk->dictionary_page_offset()) {
+                        offset = col_chunk->dictionary_page_offset();
+                    }
+                    ranges.push_back({offset, col_chunk->total_compressed_size()});
                 }
-                ranges.push_back({offset, col_chunk->total_compressed_size()});
             }
         }
+        return ranges;
+    }
+    PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::DoCollectPreBufferRanges")
+}
+
+Result<std::vector<std::pair<uint64_t, uint64_t>>> FileReaderWrapper::GetPreBufferRanges() {
+    PAIMON_ASSIGN_OR_RAISE(std::vector<::arrow::io::ReadRange> ranges,
+                           DoCollectPreBufferRanges(target_column_indices_,
+                                                    /*skip_read_range_excluded=*/false,
+                                                    /*start_idx=*/0));
+    std::vector<std::pair<uint64_t, uint64_t>> pre_buffer_ranges;
+    pre_buffer_ranges.reserve(ranges.size());
+    for (const auto& range : ranges) {
+        // Ranges come from signed parquet metadata; a corrupt footer may hold negative or
+        // overflowing values. Validate before converting to uint64_t, since downstream
+        // range coalescing does unchecked offset + length arithmetic on them.
+        PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(range.offset, "pre-buffer range offset"));
+        PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(range.length, "pre-buffer range length"));
+        if (range.offset > std::numeric_limits<int64_t>::max() - range.length) {
+            return Status::Invalid(fmt::format("pre-buffer range overflows: offset={}, length={}",
+                                               range.offset, range.length));
+        }
+        pre_buffer_ranges.emplace_back(static_cast<uint64_t>(range.offset),
+                                       static_cast<uint64_t>(range.length));
     }
-    return ranges;
+    return pre_buffer_ranges;
 }
 
 void FileReaderWrapper::DispatchPreBuffer(std::vector<::arrow::io::ReadRange> ranges) {
@@ -429,10 +486,24 @@ Status FileReaderWrapper::PrepareForReading(const std::vector<TargetRowGroup>& t
         target_row_groups_ = target_row_groups;
         target_column_indices_ = column_indices;
 
+        // Find the first row group to read: skip read-range-excluded ones, and honor a
+        // seek issued while the reader was still uninitialized (SeekToRow defers reader
+        // construction to here).
+        uint64_t first_active_idx = 0;
+        while (first_active_idx < target_row_groups_.size() &&
+               target_row_groups_[first_active_idx].IsExcludedByReadRange()) {
+            first_active_idx++;
+        }
+        if (pending_start_idx_.has_value()) {
+            first_active_idx = std::max(first_active_idx, pending_start_idx_.value());
+            pending_start_idx_.reset();
+        }
+
         // Partition into fully-matched and page-filtered row groups, skipping excluded ones.
         std::vector<int32_t> fully_matched_row_groups;
         uint64_t active_count = 0;
-        for (const auto& trg : target_row_groups_) {
+        for (uint64_t i = first_active_idx; i < target_row_groups_.size(); i++) {
+            const auto& trg = target_row_groups_[i];
             if (trg.IsExcludedByReadRange()) {
                 continue;
             }
@@ -462,16 +533,12 @@ Status FileReaderWrapper::PrepareForReading(const std::vector<TargetRowGroup>& t
         // When page-filtered RGs exist, issue a single PreBuffer covering both kinds.
         // Otherwise GetRecordBatchReader already issued PreBuffer internally.
         if (has_partially_matched) {
-            auto all_ranges = CollectPreBufferRanges(column_indices);
+            PAIMON_ASSIGN_OR_RAISE(std::vector<::arrow::io::ReadRange> all_ranges,
+                                   CollectPreBufferRanges(column_indices, first_active_idx));
             DispatchPreBuffer(std::move(all_ranges));
         }
 
-        // Reset read state. Find the first non-excluded row group.
-        uint64_t first_active_idx = 0;
-        while (first_active_idx < target_row_groups_.size() &&
-               target_row_groups_[first_active_idx].IsExcludedByReadRange()) {
-            first_active_idx++;
-        }
+        // Reset read state to the first row group that will be read.
         if (first_active_idx >= target_row_groups_.size()) {
             next_row_to_read_ = num_rows_;
         } else {
@@ -489,6 +556,8 @@ Status FileReaderWrapper::PrepareForReading(const std::vector<TargetRowGroup>& t
 
 Status FileReaderWrapper::ApplyReadRanges(
     const std::vector<std::pair<uint64_t, uint64_t>>& read_ranges) {
+    // A read-range change invalidates any seek recorded before initialization.
+    pending_start_idx_.reset();
     if (read_ranges.empty()) {
         for (auto& trg : target_row_groups_) {
             trg.SetExcludedByReadRange(true);
diff --git a/src/paimon/format/parquet/file_reader_wrapper.h b/src/paimon/format/parquet/file_reader_wrapper.h
index 78a07c794..02e6ae5b1 100644
--- a/src/paimon/format/parquet/file_reader_wrapper.h
+++ b/src/paimon/format/parquet/file_reader_wrapper.h
@@ -22,6 +22,7 @@
 #include <limits>
 #include <map>
 #include <memory>
+#include <optional>
 #include <set>
 #include <utility>
 #include <vector>
@@ -63,6 +64,9 @@ class FileReaderWrapper {
 
     /// Seek to the specified row number.
     /// @param row_number The row to seek to (must be at a row group boundary).
+    /// When the reader is not yet initialized (before the first Next()), the reader
+    /// construction is deferred to PrepareForReading to avoid building a batch reader
+    /// that would be immediately discarded.
     Status SeekToRow(uint64_t row_number);
 
     /// Read the next batch of rows.
@@ -147,6 +151,13 @@ class FileReaderWrapper {
     std::shared_ptr<::parquet::RowGroupPageIndexReader> GetRowGroupPageIndexReader(
         int32_t row_group_index);
 
+    /// Compute the (offset, length) byte ranges required by the current target row groups
+    /// and columns. Unlike the arrow-internal PreBuffer path, this covers row groups that
+    /// are excluded by read-range dispatch as well, because the shared prefetch cache must
+    /// serve data consumed by all sub-readers. Relies only on file metadata, so it is safe
+    /// to call before the lazy reader initialization.
+    Result<std::vector<std::pair<uint64_t, uint64_t>>> GetPreBufferRanges();
+
  private:
     FileReaderWrapper(std::unique_ptr<::parquet::arrow::FileReader>&& file_reader,
                       const std::vector<std::pair<uint64_t, uint64_t>>& all_row_group_ranges,
@@ -165,9 +176,21 @@ class FileReaderWrapper {
     /// Read next batch from the fully-matched batch_reader_. Returns nullptr when exhausted.
     Result<std::shared_ptr<arrow::RecordBatch>> NextFullyMatched();
 
-    /// Collect all byte ranges that need pre-buffering (page-filtered + fully-matched).
-    std::vector<::arrow::io::ReadRange> CollectPreBufferRanges(
-        const std::vector<int32_t>& column_indices);
+    /// Collect all byte ranges that need pre-buffering (page-filtered + fully-matched),
+    /// skipping row groups excluded by ApplyReadRanges and row groups before start_idx
+    /// (already skipped by a deferred seek).
+    Result<std::vector<::arrow::io::ReadRange>> CollectPreBufferRanges(
+        const std::vector<int32_t>& column_indices, uint64_t start_idx);
+
+    /// Core byte-range collection shared by CollectPreBufferRanges and GetPreBufferRanges.
+    /// When skip_read_range_excluded is true, row groups excluded by ApplyReadRanges are
+    /// skipped (arrow-internal PreBuffer for this reader); when false, they are included
+    /// (shared prefetch cache covering all sub-readers). Ranges before start_idx are
+    /// never collected. Metadata and page index lookups throw on malformed files or IO
+    /// failures, so the exceptions are converted to a Status here.
+    Result<std::vector<::arrow::io::ReadRange>> DoCollectPreBufferRanges(
+        const std::vector<int32_t>& column_indices, bool skip_read_range_excluded,
+        uint64_t start_idx);
 
     /// Dispatch a single PreBufferRanges call with merged ranges.
     void DispatchPreBuffer(std::vector<::arrow::io::ReadRange> ranges);
@@ -186,6 +209,10 @@ class FileReaderWrapper {
     uint64_t previous_first_row_ = std::numeric_limits<uint64_t>::max();
     uint64_t current_row_group_idx_ = 0;
     bool reader_initialized_ = false;
+    // Target index recorded by SeekToRow when the reader was still uninitialized;
+    // consumed by PrepareForReading so the deferred initialization starts at the seeked
+    // position instead of rebuilding readers twice.
+    std::optional<uint64_t> pending_start_idx_;
 
     // Streaming reader for the currently-active page-filtered row group. Created lazily
     // on the first Next() call into a page-filtered RG, drained batch-by-batch, then reset
diff --git a/src/paimon/format/parquet/file_reader_wrapper_test.cpp b/src/paimon/format/parquet/file_reader_wrapper_test.cpp
index aaef711e0..41dcde193 100644
--- a/src/paimon/format/parquet/file_reader_wrapper_test.cpp
+++ b/src/paimon/format/parquet/file_reader_wrapper_test.cpp
@@ -18,8 +18,13 @@
 
 #include "paimon/format/parquet/file_reader_wrapper.h"
 
+#include <fstream>
+#include <functional>
+#include <iterator>
 #include <map>
+#include <mutex>
 #include <string>
+#include <vector>
 
 #include "arrow/api.h"
 #include "arrow/array/builder_binary.h"
@@ -52,6 +57,64 @@ class Array;
 
 namespace paimon::parquet::test {
 
+// Tracks positional reads (Read at offset / ReadAsync) issued through the stream.
+class ReadTrackingInputStream : public InputStream {
+ public:
+    explicit ReadTrackingInputStream(std::shared_ptr<InputStream> input)
+        : input_(std::move(input)) {}
+
+    Status Seek(int64_t offset, SeekOrigin origin) override {
+        return input_->Seek(offset, origin);
+    }
+
+    Result<int64_t> GetPos() const override {
+        return input_->GetPos();
+    }
+
+    Result<int64_t> Read(char* buffer, int64_t size) override {
+        return input_->Read(buffer, size);
+    }
+
+    Result<int64_t> Read(char* buffer, int64_t size, int64_t offset) override {
+        RecordPositionalRead(offset, size);
+        return input_->Read(buffer, size, offset);
+    }
+
+    void ReadAsync(char* buffer, int64_t size, int64_t offset,
+                   std::function<void(Status)>&& callback) override {
+        RecordPositionalRead(offset, size);
+        input_->ReadAsync(buffer, size, offset, std::move(callback));
+    }
+
+    Result<std::string> GetUri() const override {
+        return input_->GetUri();
+    }
+
+    Result<int64_t> Length() const override {
+        return input_->Length();
+    }
+
+    Status Close() override {
+        return input_->Close();
+    }
+
+    int64_t GetPositionalReadBytes() const {
+        std::lock_guard<std::mutex> lock(mutex_);
+        return positional_read_bytes_;
+    }
+
+ private:
+    void RecordPositionalRead(int64_t offset, int64_t size) {
+        (void)offset;
+        std::lock_guard<std::mutex> lock(mutex_);
+        positional_read_bytes_ += size;
+    }
+
+    std::shared_ptr<InputStream> input_;
+    mutable std::mutex mutex_;
+    int64_t positional_read_bytes_ = 0;
+};
+
 class FileReaderWrapperTest : public ::testing::Test {
  public:
     void SetUp() override {
@@ -121,8 +184,14 @@ class FileReaderWrapperTest : public ::testing::Test {
     Result<std::unique_ptr<FileReaderWrapper>> PrepareReaderWrapper(
         const std::string& file_path, int64_t wrapper_batch_size = 0) {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> in, fs_->Open(file_path));
+        return PrepareReaderWrapperOnStream(std::move(in), wrapper_batch_size);
+    }
+
+    Result<std::unique_ptr<FileReaderWrapper>> PrepareReaderWrapperOnStream(
+        std::shared_ptr<InputStream> in, int64_t wrapper_batch_size = 0) {
         PAIMON_ASSIGN_OR_RAISE(int64_t file_length, in->Length());
-        auto input_stream = std::make_unique<ArrowInputStreamAdapter>(in, file_length, arrow_pool_);
+        auto input_stream =
+            std::make_unique<ArrowInputStreamAdapter>(std::move(in), file_length, arrow_pool_);
         ::parquet::arrow::FileReaderBuilder file_reader_builder;
         ::parquet::ReaderProperties reader_properties;
         reader_properties.enable_buffered_stream();
@@ -250,6 +319,52 @@ TEST_F(FileReaderWrapperTest, Simple) {
     ASSERT_EQ(5500, reader_wrapper->GetPreviousBatchFirstRowNumber().value());
 }
 
+/// The prefetch framework always issues SeekToRow right after SetReadRanges, while the
+/// wrapper is still uninitialized (before the first Next()). That seek must not build
+/// the arrow batch reader eagerly: building it once in SeekToRow and again in
+/// PrepareForReading makes the arrow reader request every column chunk twice (2x read
+/// amplification). The deferred construction must also honor the seeked start position.
+TEST_F(FileReaderWrapperTest, SeekBeforeInitIssuesNoReadsAndStartsAtSeekPosition) {
+    std::string file_path = PathUtil::JoinPath(dir_->Str(), "seek_before_init.parquet");
+    PrepareParquetFile(file_path, /*row_count=*/5500);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_path));
+    auto tracking_stream = std::make_shared<ReadTrackingInputStream>(std::move(in));
+    auto* tracking = tracking_stream.get();
+    ASSERT_OK_AND_ASSIGN(auto reader_wrapper,
+                         PrepareReaderWrapperOnStream(std::move(tracking_stream)));
+    ASSERT_EQ(6, reader_wrapper->GetNumberOfRowGroups());
+
+    // Baseline: only metadata reads happened so far (footer etc. during Open/Build).
+    int64_t baseline_read_bytes = tracking->GetPositionalReadBytes();
+
+    // Seek to the start of RG2 while still uninitialized. This must only record the
+    // position, not build a batch reader that would eagerly read column chunks.
+    ASSERT_OK(reader_wrapper->SeekToRow(2000));
+    ASSERT_EQ(2000, reader_wrapper->GetNextRowToRead());
+    ASSERT_EQ(baseline_read_bytes, tracking->GetPositionalReadBytes())
+        << "SeekToRow before initialization issued eager column chunk reads; the deferred "
+           "PrepareForReader would build a second reader and read everything twice";
+
+    // The first Next() performs the single deferred initialization at the seeked position.
+    int64_t total_rows = 0;
+    bool checked_first_batch = false;
+    while (true) {
+        ASSERT_OK_AND_ASSIGN(auto batch, reader_wrapper->Next());
+        if (!batch) {
+            break;
+        }
+        if (!checked_first_batch) {
+            ASSERT_EQ(2000, reader_wrapper->GetPreviousBatchFirstRowNumber().value());
+            checked_first_batch = true;
+        }
+        total_rows += batch->num_rows();
+    }
+    // RG2..RG5 cover rows [2000, 5500).
+    ASSERT_EQ(3500, total_rows);
+    ASSERT_EQ(5500, reader_wrapper->GetNextRowToRead());
+}
+
 /// Regression: when batch_size_ is 0 (the default) and a row group is consumed via
 /// the page-filtered streaming path, we must not pass 0 to TableBatchReader::set_chunksize
 /// — that would make ReadNext spin forever on zero-row batches. The wrapper now
@@ -569,4 +684,218 @@ TEST_F(FileReaderWrapperTest, PrepareForReading) {
               reader_wrapper->GetPreviousBatchFirstRowNumber().value());
 }
 
+namespace {
+
+// A minimal Thrift compact-protocol walker, just enough to locate and corrupt one
+// i64 field inside a Parquet footer. Only the field types that may appear in an
+// unencrypted footer are supported; anything else makes the walk fail.
+class CompactThriftFooter {
+ public:
+    CompactThriftFooter(std::string* data, size_t pos) : data_(data), pos_(pos) {}
+
+    size_t pos() const {
+        return pos_;
+    }
+
+    // Advance to the struct field with the given id, checking it has the expected
+    // type, and leave the cursor at the start of its value.
+    bool SeekField(int32_t target_id, uint8_t target_type) {
+        int32_t field_id = 0;
+        while (true) {
+            uint8_t type = 0;
+            if (!NextField(&field_id, &type)) return false;
+            if (field_id == 0) return false;  // STOP without finding the field
+            if (field_id == target_id) return type == target_type;
+            if (!SkipValue(type)) return false;
+        }
+    }
+
+    // Enter the body of the first element of the list at the cursor; the element
+    // must be a struct.
+    bool EnterFirstListElement() {
+        uint64_t header = 0;
+        if (!ReadByte(&header)) return false;
+        uint64_t size = (header >> 4) & 0x0F;
+        if ((header & 0x0F) != kStruct) return false;
+        if (size == 15 && !ReadVarint(&size)) return false;
+        return size > 0;  // Cursor is now at the first element's struct body.
+    }
+
+    static constexpr uint8_t kI64 = 6;
+    static constexpr uint8_t kList = 9;
+    static constexpr uint8_t kStruct = 12;
+
+ private:
+    static constexpr uint8_t kBoolTrue = 1;
+    static constexpr uint8_t kBoolFalse = 2;
+    static constexpr uint8_t kByte = 3;
+    static constexpr uint8_t kI16 = 4;
+    static constexpr uint8_t kI32 = 5;
+    static constexpr uint8_t kDouble = 7;
+    static constexpr uint8_t kBinary = 8;
+    static constexpr uint8_t kSet = 10;
+    static constexpr uint8_t kMap = 11;
+
+    bool ReadByte(uint64_t* out) {
+        if (pos_ >= data_->size()) return false;
+        *out = static_cast<uint8_t>((*data_)[pos_++]);
+        return true;
+    }
+
+    bool ReadVarint(uint64_t* out) {
+        *out = 0;
+        for (int shift = 0; shift < 64; shift += 7) {
+            uint64_t byte = 0;
+            if (!ReadByte(&byte)) return false;
+            *out |= (byte & 0x7F) << shift;
+            if ((byte & 0x80) == 0) return true;
+        }
+        return false;  // Varints longer than 10 bytes are malformed.
+    }
+
+    bool NextField(int32_t* field_id, uint8_t* type) {
+        uint64_t header = 0;
+        if (!ReadByte(&header)) return false;
+        if (header == 0) {
+            *field_id = 0;  // STOP
+            return true;
+        }
+        *type = header & 0x0F;
+        uint64_t delta = (header >> 4) & 0x0F;
+        if (delta != 0) {
+            *field_id += static_cast<int32_t>(delta);
+            return true;
+        }
+        uint64_t zigzag = 0;
+        if (!ReadVarint(&zigzag)) return false;
+        *field_id = static_cast<int32_t>((zigzag >> 1) ^ -(zigzag & 1));
+        return true;
+    }
+
+    bool SkipValue(uint8_t type) {
+        switch (type) {
+            case kBoolTrue:
+            case kBoolFalse:
+                return true;  // The value is encoded in the field header itself.
+            case kByte: {
+                uint64_t unused = 0;
+                return ReadByte(&unused);
+            }
+            case kI16:
+            case kI32:
+            case kI64: {
+                uint64_t unused = 0;
+                return ReadVarint(&unused);
+            }
+            case kDouble:
+                if (pos_ + 8 > data_->size()) return false;
+                pos_ += 8;
+                return true;
+            case kBinary: {
+                uint64_t length = 0;
+                if (!ReadVarint(&length)) return false;
+                if (pos_ + length > data_->size()) return false;
+                pos_ += length;
+                return true;
+            }
+            case kList:
+            case kSet:
+                return SkipCollection();
+            case kMap: {
+                uint64_t size = 0;
+                if (!ReadVarint(&size)) return false;
+                if (size == 0) return true;
+                uint64_t kv_types = 0;
+                if (!ReadByte(&kv_types)) return false;
+                for (uint64_t i = 0; i < size; ++i) {
+                    if (!SkipValue((kv_types >> 4) & 0x0F)) return false;
+                    if (!SkipValue(kv_types & 0x0F)) return false;
+                }
+                return true;
+            }
+            case kStruct: {
+                int32_t field_id = 0;
+                while (true) {
+                    uint8_t field_type = 0;
+                    if (!NextField(&field_id, &field_type)) return false;
+                    if (field_id == 0) return true;  // STOP
+                    if (!SkipValue(field_type)) return false;
+                }
+            }
+            default:
+                return false;
+        }
+    }
+
+    bool SkipCollection() {
+        uint64_t header = 0;
+        if (!ReadByte(&header)) return false;
+        uint64_t size = (header >> 4) & 0x0F;
+        uint8_t elem_type = header & 0x0F;
+        if (size == 15 && !ReadVarint(&size)) return false;
+        for (uint64_t i = 0; i < size; ++i) {
+            if (elem_type == kBoolTrue || elem_type == kBoolFalse) {
+                uint64_t unused = 0;
+                if (!ReadByte(&unused)) return false;
+                continue;
+            }
+            if (!SkipValue(elem_type)) return false;
+        }
+        return true;
+    }
+
+    std::string* data_;
+    size_t pos_;
+};
+
+}  // namespace
+
+// A corrupt footer may carry negative column chunk offsets. GetPreBufferRanges must
+// reject them instead of casting them into huge uint64_t ranges that would blow up
+// the downstream range coalescing.
+TEST_F(FileReaderWrapperTest, GetPreBufferRangesRejectsNegativeMetadataOffset) {
+    std::string file_path = PathUtil::JoinPath(dir_->Str(), "test.parquet");
+    PrepareParquetFile(file_path, /*row_count=*/100);
+
+    std::ifstream file(file_path, std::ios::binary);
+    std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
+    file.close();
+    ASSERT_GT(content.size(), size_t{12});
+
+    // Footer layout: [thrift FileMetaData][footer length (4-byte LE)]["PAR1"].
+    size_t tail = content.size();
+    ASSERT_EQ("PAR1", content.substr(tail - 4));
+    uint32_t footer_length = static_cast<uint8_t>(content[tail - 8]) |
+                             (static_cast<uint8_t>(content[tail - 7]) << 8) |
+                             (static_cast<uint8_t>(content[tail - 6]) << 16) |
+                             (static_cast<uint8_t>(content[tail - 5]) << 24);
+    ASSERT_LT(static_cast<uint64_t>(footer_length) + 8, content.size());
+    size_t footer_start = tail - 8 - footer_length;
+
+    // Walk to FileMetaData.row_groups[0].columns[0].meta_data.data_page_offset and
+    // flip the sign of its zigzag varint (positive -> negative, byte length kept).
+    CompactThriftFooter footer(&content, footer_start);
+    ASSERT_TRUE(footer.SeekField(/*FileMetaData.row_groups=*/4, CompactThriftFooter::kList));
+    ASSERT_TRUE(footer.EnterFirstListElement());
+    ASSERT_TRUE(footer.SeekField(/*RowGroup.columns=*/1, CompactThriftFooter::kList));
+    ASSERT_TRUE(footer.EnterFirstListElement());
+    ASSERT_TRUE(footer.SeekField(/*ColumnChunk.meta_data=*/3, CompactThriftFooter::kStruct));
+    ASSERT_TRUE(footer.SeekField(/*ColumnMetaData.data_page_offset=*/9, CompactThriftFooter::kI64));
+    size_t offset_pos = footer.pos();
+    ASSERT_EQ(0, content[offset_pos] & 0x01);  // Positive value: even zigzag encoding.
+    content[offset_pos] |= 0x01;               // Now decodes to a negative offset.
+
+    std::string corrupt_path = PathUtil::JoinPath(dir_->Str(), "corrupt.parquet");
+    std::ofstream corrupt_file(corrupt_path, std::ios::binary);
+    corrupt_file.write(content.data(), static_cast<std::streamsize>(content.size()));
+    corrupt_file.close();
+
+    ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(corrupt_path));
+    ASSERT_OK(reader_wrapper->PrepareForReadingLazy(
+        {TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false,
+                        /*ranges=*/RowRanges())},
+        /*column_indices=*/{0, 1, 2}));
+    ASSERT_NOK_WITH_MSG(reader_wrapper->GetPreBufferRanges(), "pre-buffer range offset");
+}
+
 }  // namespace paimon::parquet::test
diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp
index 67b24ac50..daacaceee 100644
--- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp
+++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp
@@ -207,7 +207,8 @@ class PageFilteredRowGroupReaderTest : public ::testing::Test {
         ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create(
                                                     std::move(in_stream), options, batch_size,
                                                     /*file_metadata=*/nullptr,
-                                                    /*storage_read_bytes=*/nullptr, arrow_pool_));
+                                                    /*storage_read_bytes=*/nullptr, arrow_pool_,
+                                                    /*hints=*/std::nullopt));
         auto c_schema = std::make_unique<ArrowSchema>();
         ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok());
         ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), predicate,
@@ -235,7 +236,8 @@ class PageFilteredRowGroupReaderTest : public ::testing::Test {
         ASSERT_OK_AND_ASSIGN(
             auto batch_reader,
             ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size, nullptr,
-                                           /*storage_read_bytes=*/nullptr, arrow_pool_));
+                                           /*storage_read_bytes=*/nullptr, arrow_pool_,
+                                           /*hints=*/std::nullopt));
         auto c_schema = std::make_unique<ArrowSchema>();
         ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok());
         ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), predicate, bitmap));
@@ -2120,7 +2122,8 @@ TEST_F(PageFilteredRowGroupReaderTest, BitmapInvalidStrategyTest) {
 
     ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create(
                                                 std::move(in_stream), options, 1024, nullptr,
-                                                /*storage_read_bytes=*/nullptr, arrow_pool_));
+                                                /*storage_read_bytes=*/nullptr, arrow_pool_,
+                                                /*hints=*/std::nullopt));
 
     auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
     auto c_schema = std::make_unique<ArrowSchema>();
diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp
index c0cd40e19..20e5e0ea7 100644
--- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp
+++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp
@@ -20,6 +20,7 @@
 
 #include <algorithm>
 #include <cstddef>
+#include <optional>
 #include <unordered_map>
 
 #include "arrow/acero/options.h"
@@ -123,6 +124,18 @@ bool HasSameNestedProjectionShape(const std::shared_ptr<arrow::DataType>& read_t
             return false;
     }
 }
+
+// Resolve whether parquet-level pre-buffering should be enabled. When the framework
+// provides runtime hints, they describe the authoritative state of this read: once the
+// shared read-ahead cache takes over prefetching, disable parquet's own pre-buffering so
+// the same byte ranges are not buffered twice. Without hints, fall back to the option.
+Result<bool> ResolvePreBufferEnabled(const std::map<std::string, std::string>& options,
+                                     const std::optional<ReadHints>& hints) {
+    if (hints.has_value() && hints->prefetch_enabled && hints->read_ahead_cache_enabled) {
+        return false;
+    }
+    return OptionsUtils::GetValueFromMap<bool>(options, PARQUET_READ_ENABLE_PRE_BUFFER, true);
+}
 }  // namespace
 
 ParquetFileBatchReader::ParquetFileBatchReader(
@@ -143,14 +156,14 @@ Result<std::unique_ptr<ParquetFileBatchReader>> ParquetFileBatchReader::Create(
     const std::map<std::string, std::string>& options, int32_t batch_size,
     std::shared_ptr<::parquet::FileMetaData> file_metadata,
     std::shared_ptr<std::atomic<uint64_t>> storage_read_bytes,
-    const std::shared_ptr<arrow::MemoryPool>& pool) {
+    const std::shared_ptr<arrow::MemoryPool>& pool, const std::optional<ReadHints>& hints) {
     try {
         assert(input_stream);
         PAIMON_ASSIGN_OR_RAISE(::parquet::ReaderProperties reader_properties,
-                               CreateReaderProperties(pool, options));
+                               CreateReaderProperties(pool, options, hints));
 
         PAIMON_ASSIGN_OR_RAISE(::parquet::ArrowReaderProperties arrow_reader_properties,
-                               CreateArrowReaderProperties(pool, options, batch_size));
+                               CreateArrowReaderProperties(pool, options, batch_size, hints));
 
         ::parquet::arrow::FileReaderBuilder file_reader_builder;
         PAIMON_RETURN_NOT_OK_FROM_ARROW(
@@ -635,14 +648,16 @@ Result<std::vector<std::pair<uint64_t, uint64_t>>> ParquetFileBatchReader::GenRe
     PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::GenReadRanges")
 }
 
+Result<std::vector<std::pair<uint64_t, uint64_t>>> ParquetFileBatchReader::PreBufferRange() {
+    return reader_->GetPreBufferRanges();
+}
+
 Result<::parquet::ReaderProperties> ParquetFileBatchReader::CreateReaderProperties(
     const std::shared_ptr<arrow::MemoryPool>& pool,
-    const std::map<std::string, std::string>& options) {
+    const std::map<std::string, std::string>& options, const std::optional<ReadHints>& hints) {
     ::parquet::ReaderProperties reader_properties;
     // TODO(jinli.zjw): set more ReaderProperties (compare with java)
-    PAIMON_ASSIGN_OR_RAISE(
-        bool enable_pre_buffer,
-        OptionsUtils::GetValueFromMap<bool>(options, PARQUET_READ_ENABLE_PRE_BUFFER, true));
+    PAIMON_ASSIGN_OR_RAISE(bool enable_pre_buffer, ResolvePreBufferEnabled(options, hints));
     if (enable_pre_buffer) {
         reader_properties.enable_buffered_stream();
     } else {
@@ -653,7 +668,8 @@ Result<::parquet::ReaderProperties> ParquetFileBatchReader::CreateReaderProperti
 
 Result<::parquet::ArrowReaderProperties> ParquetFileBatchReader::CreateArrowReaderProperties(
     const std::shared_ptr<arrow::MemoryPool>& pool,
-    const std::map<std::string, std::string>& options, int32_t batch_size) {
+    const std::map<std::string, std::string>& options, int32_t batch_size,
+    const std::optional<ReadHints>& hints) {
     PAIMON_ASSIGN_OR_RAISE(
         uint32_t executor_thread_count,
         OptionsUtils::GetValueFromMap<uint32_t>(options, PARQUET_READ_EXECUTOR_THREAD_COUNT,
@@ -661,9 +677,7 @@ Result<::parquet::ArrowReaderProperties> ParquetFileBatchReader::CreateArrowRead
 
     ::parquet::ArrowReaderProperties arrow_reader_props;
     // TODO(jinli.zjw): set more ArrowReaderProperties (compare with java)
-    PAIMON_ASSIGN_OR_RAISE(
-        bool enable_pre_buffer,
-        OptionsUtils::GetValueFromMap<bool>(options, PARQUET_READ_ENABLE_PRE_BUFFER, true));
+    PAIMON_ASSIGN_OR_RAISE(bool enable_pre_buffer, ResolvePreBufferEnabled(options, hints));
     arrow_reader_props.set_pre_buffer(enable_pre_buffer);
     arrow_reader_props.set_batch_size(static_cast<int64_t>(batch_size));
     if (executor_thread_count != 0) {
diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h
index 7e5e9afab..daa18f040 100644
--- a/src/paimon/format/parquet/parquet_file_batch_reader.h
+++ b/src/paimon/format/parquet/parquet_file_batch_reader.h
@@ -44,6 +44,7 @@
 #include "paimon/format/parquet/parquet_format_defs.h"
 #include "paimon/format/parquet/row_ranges.h"
 #include "paimon/format/parquet/target_row_group.h"
+#include "paimon/format/read_hints.h"
 #include "paimon/logging.h"
 #include "paimon/reader/prefetch_file_batch_reader.h"
 #include "paimon/result.h"
@@ -76,11 +77,11 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader {
         const std::map<std::string, std::string>& options, int32_t batch_size,
         std::shared_ptr<::parquet::FileMetaData> file_metadata,
         std::shared_ptr<std::atomic<uint64_t>> storage_read_bytes,
-        const std::shared_ptr<arrow::MemoryPool>& pool);
+        const std::shared_ptr<arrow::MemoryPool>& pool, const std::optional<ReadHints>& hints);
 
     static Result<::parquet::ReaderProperties> CreateReaderProperties(
         const std::shared_ptr<arrow::MemoryPool>& pool,
-        const std::map<std::string, std::string>& options);
+        const std::map<std::string, std::string>& options, const std::optional<ReadHints>& hints);
 
     // For timestamp type, we return the schema stored in file, e.g., second in parquet file will
     // store as milli.
@@ -102,6 +103,8 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader {
     Result<std::vector<std::pair<uint64_t, uint64_t>>> GenReadRanges(
         bool* need_prefetch) const override;
 
+    Result<std::vector<std::pair<uint64_t, uint64_t>>> PreBufferRange() override;
+
     Result<uint64_t> GetPreviousBatchFileRowId(uint64_t batch_row_id) const override {
         if (row_mapping_.empty()) {
             PAIMON_ASSIGN_OR_RAISE(uint64_t previous_first_row,
@@ -162,7 +165,8 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader {
 
     static Result<::parquet::ArrowReaderProperties> CreateArrowReaderProperties(
         const std::shared_ptr<arrow::MemoryPool>& pool,
-        const std::map<std::string, std::string>& options, int32_t batch_size);
+        const std::map<std::string, std::string>& options, int32_t batch_size,
+        const std::optional<ReadHints>& hints);
 
     static void FlattenSchema(const std::shared_ptr<arrow::DataType>& type, int32_t* index,
                               std::vector<int32_t>* index_vector) {
diff --git a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp
index bcd4af210..1409f24bf 100644
--- a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp
+++ b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp
@@ -18,12 +18,15 @@
 
 #include "paimon/format/parquet/parquet_file_batch_reader.h"
 
+#include <algorithm>
 #include <atomic>
 #include <functional>
 #include <iostream>
 #include <limits>
 #include <memory>
 #include <string>
+#include <utility>
+#include <vector>
 
 #include "arrow/api.h"
 #include "arrow/array/array_base.h"
@@ -37,6 +40,8 @@
 #include "arrow/ipc/api.h"
 #include "arrow/ipc/json_simple.h"
 #include "gtest/gtest.h"
+#include "paimon/common/io/cache_input_stream.h"
+#include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h"
 #include "paimon/common/utils/arrow/arrow_utils.h"
@@ -44,11 +49,13 @@
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/read_ahead_cache.h"
 #include "paimon/defs.h"
 #include "paimon/format/parquet/parquet_field_id_converter.h"
 #include "paimon/format/parquet/parquet_format_defs.h"
 #include "paimon/format/parquet/parquet_format_writer.h"
 #include "paimon/format/parquet/parquet_reader_builder.h"
+#include "paimon/format/read_hints.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/memory/memory_pool.h"
@@ -60,6 +67,7 @@
 #include "paimon/testing/utils/testharness.h"
 #include "paimon/testing/utils/timezone_guard.h"
 #include "paimon/utils/roaring_bitmap32.h"
+#include "parquet/file_reader.h"
 #include "parquet/properties.h"
 
 namespace paimon {
@@ -225,7 +233,8 @@ class ParquetFileBatchReaderTest : public ::testing::Test,
         EXPECT_OK_AND_ASSIGN(auto parquet_batch_reader,
                              ParquetFileBatchReader::Create(
                                  std::move(in_stream), options, batch_size,
-                                 /*file_metadata=*/nullptr, std::move(storage_read_bytes), pool_));
+                                 /*file_metadata=*/nullptr, std::move(storage_read_bytes), pool_,
+                                 /*hints=*/std::nullopt));
         std::unique_ptr<ArrowSchema> c_schema = std::make_unique<ArrowSchema>();
         auto arrow_status = arrow::ExportSchema(*read_schema, c_schema.get());
         EXPECT_TRUE(arrow_status.ok());
@@ -392,7 +401,8 @@ TEST_F(ParquetFileBatchReaderTest, TestSetReadSchema) {
     ASSERT_OK_AND_ASSIGN(auto parquet_batch_reader,
                          ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size_,
                                                         /*file_metadata=*/nullptr,
-                                                        /*storage_read_bytes=*/nullptr, pool_));
+                                                        /*storage_read_bytes=*/nullptr, pool_,
+                                                        /*hints=*/std::nullopt));
     // test GetFileSchema()
     ASSERT_OK_AND_ASSIGN(auto c_file_schema, parquet_batch_reader->GetFileSchema());
     auto arrow_file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie();
@@ -838,8 +848,9 @@ TEST_F(ParquetFileBatchReaderTest, TestCreateReaderProperties) {
     {
         // test default options
         std::map<std::string, std::string> options;
-        ASSERT_OK_AND_ASSIGN(auto reader_properties,
-                             ParquetFileBatchReader::CreateReaderProperties(pool_, options));
+        ASSERT_OK_AND_ASSIGN(auto reader_properties, ParquetFileBatchReader::CreateReaderProperties(
+                                                         pool_, options,
+                                                         /*hints=*/std::nullopt));
         ASSERT_EQ(reader_properties.is_buffered_stream_enabled(), true);
     }
 }
@@ -851,7 +862,8 @@ TEST_F(ParquetFileBatchReaderTest, TestCreateArrowReaderProperties) {
         int32_t batch_size = 1024;
         ASSERT_OK_AND_ASSIGN(
             auto arrow_reader_properties,
-            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, batch_size));
+            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, batch_size,
+                                                                /*hints=*/std::nullopt));
         ASSERT_EQ(arrow_reader_properties.pre_buffer(), true);
         ASSERT_EQ(arrow_reader_properties.batch_size(), 1024);
         ASSERT_EQ(arrow_reader_properties.use_threads(), true);
@@ -865,7 +877,8 @@ TEST_F(ParquetFileBatchReaderTest, TestCreateArrowReaderProperties) {
         int32_t batch_size = 1024;
         ASSERT_OK_AND_ASSIGN(
             auto arrow_reader_properties,
-            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, batch_size));
+            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, batch_size,
+                                                                /*hints=*/std::nullopt));
         ASSERT_EQ(arrow_reader_properties.use_threads(), false);
     }
     {
@@ -873,7 +886,8 @@ TEST_F(ParquetFileBatchReaderTest, TestCreateArrowReaderProperties) {
         int32_t batch_size = 1024;
         ASSERT_OK_AND_ASSIGN(
             auto arrow_reader_properties,
-            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, batch_size));
+            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, batch_size,
+                                                                /*hints=*/std::nullopt));
         ASSERT_EQ(arrow_reader_properties.use_threads(), true);
         ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), 6);
     }
@@ -886,7 +900,8 @@ TEST_F(ParquetFileBatchReaderTest, TestCreateArrowReaderProperties) {
         };
         ASSERT_OK_AND_ASSIGN(
             auto arrow_reader_properties,
-            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, 1024));
+            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, 1024,
+                                                                /*hints=*/std::nullopt));
         const auto& cache_options = arrow_reader_properties.cache_options();
         ASSERT_TRUE(cache_options.lazy);
         ASSERT_EQ(cache_options.prefetch_limit, 2);
@@ -898,7 +913,8 @@ TEST_F(ParquetFileBatchReaderTest, TestCreateArrowReaderProperties) {
             {PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT, "-1"},
         };
         ASSERT_NOK_WITH_MSG(
-            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, 1024),
+            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, 1024,
+                                                                /*hints=*/std::nullopt),
             "parquet.read.cache-option.hole-size-limit must be non-negative");
     }
     {
@@ -907,12 +923,79 @@ TEST_F(ParquetFileBatchReaderTest, TestCreateArrowReaderProperties) {
             {PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT, "1048576"},
         };
         ASSERT_NOK_WITH_MSG(
-            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, 1024),
+            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, 1024,
+                                                                /*hints=*/std::nullopt),
             "parquet.read.cache-option.range-size-limit must be greater than "
             "parquet.read.cache-option.hole-size-limit");
     }
 }
 
+TEST_F(ParquetFileBatchReaderTest, TestPreBufferReadHints) {
+    const int32_t batch_size = 1024;
+    // When both framework prefetch and the shared read-ahead cache are active, parquet's own
+    // pre-buffering must be disabled even if the option explicitly enables it (runtime state
+    // takes precedence over the option).
+    {
+        std::map<std::string, std::string> options = {{PARQUET_READ_ENABLE_PRE_BUFFER, "true"}};
+        ReadHints hints;
+        hints.prefetch_enabled = true;
+        hints.read_ahead_cache_enabled = true;
+        ASSERT_OK_AND_ASSIGN(auto arrow_props, ParquetFileBatchReader::CreateArrowReaderProperties(
+                                                   pool_, options, batch_size, hints));
+        ASSERT_EQ(arrow_props.pre_buffer(), false);
+        ASSERT_OK_AND_ASSIGN(auto reader_props,
+                             ParquetFileBatchReader::CreateReaderProperties(pool_, options, hints));
+        ASSERT_EQ(reader_props.is_buffered_stream_enabled(), false);
+    }
+    // Only prefetch enabled (cache disabled): fall back to the option, which defaults to true.
+    {
+        std::map<std::string, std::string> options;
+        ReadHints hints;
+        hints.prefetch_enabled = true;
+        hints.read_ahead_cache_enabled = false;
+        ASSERT_OK_AND_ASSIGN(auto arrow_props, ParquetFileBatchReader::CreateArrowReaderProperties(
+                                                   pool_, options, batch_size, hints));
+        ASSERT_EQ(arrow_props.pre_buffer(), true);
+    }
+    // Only cache enabled (prefetch disabled): fall back to the option, which defaults to true.
+    {
+        std::map<std::string, std::string> options;
+        ReadHints hints;
+        hints.prefetch_enabled = false;
+        hints.read_ahead_cache_enabled = true;
+        ASSERT_OK_AND_ASSIGN(auto arrow_props, ParquetFileBatchReader::CreateArrowReaderProperties(
+                                                   pool_, options, batch_size, hints));
+        ASSERT_EQ(arrow_props.pre_buffer(), true);
+    }
+    // Neither active and the option explicitly disabled: honor the option.
+    {
+        std::map<std::string, std::string> options = {{PARQUET_READ_ENABLE_PRE_BUFFER, "false"}};
+        ReadHints hints;
+        ASSERT_OK_AND_ASSIGN(auto arrow_props, ParquetFileBatchReader::CreateArrowReaderProperties(
+                                                   pool_, options, batch_size, hints));
+        ASSERT_EQ(arrow_props.pre_buffer(), false);
+        ASSERT_OK_AND_ASSIGN(auto reader_props,
+                             ParquetFileBatchReader::CreateReaderProperties(pool_, options, hints));
+        ASSERT_EQ(reader_props.is_buffered_stream_enabled(), false);
+    }
+    // Neither active and no option: default to enabled.
+    {
+        std::map<std::string, std::string> options;
+        ReadHints hints;
+        ASSERT_OK_AND_ASSIGN(auto arrow_props, ParquetFileBatchReader::CreateArrowReaderProperties(
+                                                   pool_, options, batch_size, hints));
+        ASSERT_EQ(arrow_props.pre_buffer(), true);
+    }
+    // No hints provided at all (builder used without WithReadHints): fall back to the option.
+    {
+        std::map<std::string, std::string> options = {{PARQUET_READ_ENABLE_PRE_BUFFER, "false"}};
+        ASSERT_OK_AND_ASSIGN(auto arrow_props, ParquetFileBatchReader::CreateArrowReaderProperties(
+                                                   pool_, options, batch_size,
+                                                   /*hints=*/std::nullopt));
+        ASSERT_EQ(arrow_props.pre_buffer(), false);
+    }
+}
+
 TEST_F(ParquetFileBatchReaderTest, TestBitmapRowGroupPushDownWithMultiRowGroups) {
     arrow::FieldVector fields = {arrow::field("f0", arrow::int32())};
     auto arrow_type = arrow::struct_(fields);
@@ -1472,4 +1555,179 @@ TEST_F(ParquetFileBatchReaderTest, TestRowMappingSetReadSchemaTwice) {
     ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(2).value(), 5);
 }
 
+// The shared prefetch cache is initialized from a single sub-reader's PreBufferRange(),
+// so the returned byte ranges must cover the column chunks of all target row groups,
+// including those excluded by read-range dispatch.
+TEST_F(ParquetFileBatchReaderTest, TestPreBufferRangeCoversDispatchExcludedRowGroups) {
+    arrow::FieldVector fields = {arrow::field("c0", arrow::int32()),
+                                 arrow::field("c1", arrow::int32())};
+    arrow::Int32Builder c0_builder;
+    arrow::Int32Builder c1_builder;
+    ASSERT_TRUE(c0_builder.Reserve(20).ok());
+    ASSERT_TRUE(c1_builder.Reserve(20).ok());
+    for (int32_t i = 0; i < 20; ++i) {
+        c0_builder.UnsafeAppend(i);
+        c1_builder.UnsafeAppend(i % 4);
+    }
+    auto c0_array = c0_builder.Finish().ValueOrDie();
+    auto c1_array = c1_builder.Finish().ValueOrDie();
+    auto src_array = arrow::StructArray::Make({c0_array, c1_array}, fields).ValueOrDie();
+    auto arrow_schema = arrow::schema(fields);
+    WriteArray(file_path_, src_array, arrow_schema, /*write_batch_size=*/10,
+               /*enable_dictionary=*/true, /*max_row_group_length=*/10);
+
+    auto parquet_batch_reader = PrepareParquetFileBatchReader(
+        file_path_, arrow_schema, /*predicate=*/nullptr, std::nullopt, batch_size_);
+
+    bool need_prefetch = false;
+    ASSERT_OK_AND_ASSIGN(auto row_group_ranges,
+                         parquet_batch_reader->GenReadRanges(&need_prefetch));
+    ASSERT_EQ(2u, row_group_ranges.size());
+
+    // Simulate prefetch dispatch: this sub-reader owns only the first row group.
+    ASSERT_OK(parquet_batch_reader->SetReadRanges({row_group_ranges[0]}));
+
+    ASSERT_OK_AND_ASSIGN(auto pre_buffer_ranges, parquet_batch_reader->PreBufferRange());
+
+    // Expected: column chunk ranges of every row group, including the dispatch-excluded one.
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_path_));
+    ASSERT_OK_AND_ASSIGN(int64_t file_length, in->Length());
+    auto adapter = std::make_shared<ArrowInputStreamAdapter>(std::move(in), file_length, pool_);
+    auto parquet_reader = ::parquet::ParquetFileReader::Open(adapter);
+    ASSERT_TRUE(parquet_reader);
+    auto file_metadata = parquet_reader->metadata();
+    ASSERT_EQ(2, file_metadata->num_row_groups());
+
+    std::vector<std::pair<uint64_t, uint64_t>> expected_ranges;
+    for (int32_t rg = 0; rg < file_metadata->num_row_groups(); ++rg) {
+        auto rg_metadata = file_metadata->RowGroup(rg);
+        for (int32_t col = 0; col < rg_metadata->num_columns(); ++col) {
+            auto col_chunk = rg_metadata->ColumnChunk(col);
+            int64_t offset = col_chunk->data_page_offset();
+            if (col_chunk->has_dictionary_page() && col_chunk->dictionary_page_offset() > 0 &&
+                offset > col_chunk->dictionary_page_offset()) {
+                offset = col_chunk->dictionary_page_offset();
+            }
+            expected_ranges.emplace_back(static_cast<uint64_t>(offset),
+                                         static_cast<uint64_t>(col_chunk->total_compressed_size()));
+        }
+    }
+    ASSERT_EQ(4u, expected_ranges.size());
+
+    std::vector<std::pair<uint64_t, uint64_t>> actual_ranges = pre_buffer_ranges;
+    std::sort(expected_ranges.begin(), expected_ranges.end());
+    std::sort(actual_ranges.begin(), actual_ranges.end());
+    ASSERT_EQ(expected_ranges, actual_ranges);
+}
+
+// A page-index partially-matched row group should contribute page-level byte ranges
+// that are strictly smaller than its full column chunk.
+TEST_F(ParquetFileBatchReaderTest, TestPreBufferRangeWithPageFilteredRowGroup) {
+    arrow::FieldVector fields = {arrow::field("f0", arrow::int32())};
+    auto src_array = MakeSequentialIntData(12);
+    auto arrow_schema = arrow::schema(fields);
+    // One row per page, three row groups of four rows each.
+    WriteArray(file_path_, src_array, arrow_schema, /*write_batch_size=*/1,
+               /*enable_dictionary=*/false, /*max_row_group_length=*/4, /*max_page_size=*/1);
+
+    // Only rows 10 and 11 of RowGroup 2 match, making it partially matched; RowGroups 0
+    // and 1 are excluded by the predicate entirely.
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::GreaterOrEqual(
+        /*field_index=*/0, /*field_name=*/"f0", FieldType::INT, Literal(10));
+    auto parquet_batch_reader = PrepareParquetFileBatchReader(file_path_, arrow_schema, predicate,
+                                                              std::nullopt, batch_size_,
+                                                              /*enable_page_level_filter=*/true);
+
+    ASSERT_OK_AND_ASSIGN(auto pre_buffer_ranges, parquet_batch_reader->PreBufferRange());
+    ASSERT_FALSE(pre_buffer_ranges.empty());
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_path_));
+    ASSERT_OK_AND_ASSIGN(int64_t file_length, in->Length());
+    auto adapter = std::make_shared<ArrowInputStreamAdapter>(std::move(in), file_length, pool_);
+    auto parquet_reader = ::parquet::ParquetFileReader::Open(adapter);
+    ASSERT_TRUE(parquet_reader);
+    auto col_chunk = parquet_reader->metadata()->RowGroup(2)->ColumnChunk(0);
+    auto chunk_offset = static_cast<uint64_t>(col_chunk->data_page_offset());
+    uint64_t chunk_end = chunk_offset + static_cast<uint64_t>(col_chunk->total_compressed_size());
+
+    uint64_t filtered_total = 0;
+    for (const auto& range : pre_buffer_ranges) {
+        ASSERT_GE(range.first, chunk_offset);
+        ASSERT_LE(range.first + range.second, chunk_end);
+        filtered_total += range.second;
+    }
+    ASSERT_LT(filtered_total, chunk_end - chunk_offset);
+}
+
+// End-to-end: PreBufferRange() feeds the shared ReadAheadCache through CacheInputStream,
+// and data reads are served from the cache.
+TEST_F(ParquetFileBatchReaderTest, TestPreBufferRangeFeedsReadAheadCache) {
+    arrow::FieldVector fields = {arrow::field("f0", arrow::int32())};
+    auto src_array = MakeSequentialIntData(20);
+    auto arrow_schema = arrow::schema(fields);
+    WriteArray(file_path_, src_array, arrow_schema, /*write_batch_size=*/10,
+               /*enable_dictionary=*/true, /*max_row_group_length=*/10);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> cache_stream, fs_->Open(file_path_));
+    auto cache = std::make_shared<ReadAheadCache>(cache_stream, CacheConfig(), GetDefaultPool());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<InputStream> reader_stream, fs_->Open(file_path_));
+    auto cache_input_stream = std::make_shared<CacheInputStream>(std::move(reader_stream), cache);
+
+    std::map<std::string, std::string> options;
+    ParquetReaderBuilder builder(options, batch_size_);
+    builder.WithMemoryPool(GetDefaultPool());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileBatchReader> base_reader,
+                         builder.Build(cache_input_stream));
+    auto parquet_batch_reader = dynamic_cast<ParquetFileBatchReader*>(base_reader.get());
+    ASSERT_TRUE(parquet_batch_reader);
+    std::unique_ptr<ArrowSchema> c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*arrow_schema, c_schema.get()).ok());
+    ASSERT_OK(
+        parquet_batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(auto pre_buffer_ranges, parquet_batch_reader->PreBufferRange());
+    ASSERT_FALSE(pre_buffer_ranges.empty());
+    std::vector<ByteRange> byte_ranges;
+    byte_ranges.reserve(pre_buffer_ranges.size());
+    for (const auto& range : pre_buffer_ranges) {
+        byte_ranges.emplace_back(range.first, range.second);
+    }
+    ASSERT_OK(cache->Init(std::move(byte_ranges)));
+    // Dispatch the prefetch immediately so every pre-buffered range is covered
+    // before the reads below consume them.
+    cache->Warmup();
+
+    // Baseline before draining: the Build() phase may have issued reads that no
+    // prefetch range can cover (e.g. footer parsing when no metadata cache is
+    // configured). Only the data-consumption reads below must be miss-free.
+    std::shared_ptr<Metrics> baseline_metrics = std::make_shared<MetricsImpl>();
+    cache->CollectMetrics(&baseline_metrics);
+    ASSERT_OK_AND_ASSIGN(uint64_t baseline_misses,
+                         baseline_metrics->GetCounter(ReadAheadCacheMetrics::READ_MISSES));
+    ASSERT_OK_AND_ASSIGN(uint64_t baseline_miss_bytes,
+                         baseline_metrics->GetCounter(ReadAheadCacheMetrics::READ_MISS_BYTES));
+
+    // Drain the file through the cache-backed stream.
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result,
+                         paimon::test::ReadResultCollector::CollectResult(parquet_batch_reader));
+    ASSERT_EQ(20, result->length());
+
+    // The cache metrics must show that the data reads were served by the cache:
+    // at least one hit, and no additional miss falling back to the wrapped stream.
+    std::shared_ptr<Metrics> cache_metrics = std::make_shared<MetricsImpl>();
+    cache->CollectMetrics(&cache_metrics);
+    ASSERT_OK_AND_ASSIGN(uint64_t hits,
+                         cache_metrics->GetCounter(ReadAheadCacheMetrics::READ_HITS));
+    ASSERT_GT(hits, 0u);
+    ASSERT_OK_AND_ASSIGN(uint64_t hit_bytes,
+                         cache_metrics->GetCounter(ReadAheadCacheMetrics::READ_HIT_BYTES));
+    ASSERT_GT(hit_bytes, 0u);
+    ASSERT_OK_AND_ASSIGN(uint64_t misses,
+                         cache_metrics->GetCounter(ReadAheadCacheMetrics::READ_MISSES));
+    ASSERT_EQ(misses, baseline_misses);
+    ASSERT_OK_AND_ASSIGN(uint64_t miss_bytes,
+                         cache_metrics->GetCounter(ReadAheadCacheMetrics::READ_MISS_BYTES));
+    ASSERT_EQ(miss_bytes, baseline_miss_bytes);
+}
+
 }  // namespace paimon::parquet::test
diff --git a/src/paimon/format/parquet/parquet_reader_builder.h b/src/paimon/format/parquet/parquet_reader_builder.h
index 519763724..112167bf3 100644
--- a/src/paimon/format/parquet/parquet_reader_builder.h
+++ b/src/paimon/format/parquet/parquet_reader_builder.h
@@ -23,6 +23,7 @@
 #include <limits>
 #include <map>
 #include <memory>
+#include <optional>
 #include <string>
 #include <utility>
 
@@ -34,6 +35,7 @@
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/format/parquet/parquet_file_batch_reader.h"
 #include "paimon/format/parquet/parquet_format_defs.h"
+#include "paimon/format/read_hints.h"
 #include "paimon/format/reader_builder.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/memory/memory_segment.h"
@@ -59,6 +61,11 @@ class ParquetReaderBuilder : public ReaderBuilder {
         return this;
     }
 
+    ReaderBuilder* WithReadHints(const std::optional<ReadHints>& hints) override {
+        hints_ = hints;
+        return this;
+    }
+
     Result<std::unique_ptr<FileBatchReader>> Build(
         const std::shared_ptr<InputStream>& path) const override {
         try {
@@ -78,9 +85,9 @@ class ParquetReaderBuilder : public ReaderBuilder {
                 std::move(unique_input_stream));
             PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<::parquet::FileMetaData> file_metadata,
                                    GetCachedParquetMetadata(input_stream, file_uri, arrow_pool));
-            return ParquetFileBatchReader::Create(std::move(input_stream), options_, batch_size_,
-                                                  std::move(file_metadata),
-                                                  std::move(storage_read_bytes), arrow_pool);
+            return ParquetFileBatchReader::Create(
+                std::move(input_stream), options_, batch_size_, std::move(file_metadata),
+                std::move(storage_read_bytes), arrow_pool, hints_);
         }
         PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetReaderBuilder::Build")
     }
@@ -137,7 +144,7 @@ class ParquetReaderBuilder : public ReaderBuilder {
         }
         PAIMON_ASSIGN_OR_RAISE(
             ::parquet::ReaderProperties reader_properties,
-            ParquetFileBatchReader::CreateReaderProperties(arrow_pool, options_));
+            ParquetFileBatchReader::CreateReaderProperties(arrow_pool, options_, hints_));
 
         auto cache_key = CacheKey::ForKind(file_uri, /*position=*/-1, /*length=*/-1,
                                            CacheKind::DATA_FILE_FOOTER);
@@ -162,6 +169,7 @@ class ParquetReaderBuilder : public ReaderBuilder {
     std::shared_ptr<MemoryPool> pool_;
     std::map<std::string, std::string> options_;
     std::shared_ptr<Cache> cache_;
+    std::optional<ReadHints> hints_;
 };
 
 }  // namespace paimon::parquet
diff --git a/src/paimon/format/parquet/predicate_pushdown_test.cpp b/src/paimon/format/parquet/predicate_pushdown_test.cpp
index 9723794eb..26175e1df 100644
--- a/src/paimon/format/parquet/predicate_pushdown_test.cpp
+++ b/src/paimon/format/parquet/predicate_pushdown_test.cpp
@@ -130,7 +130,8 @@ class PredicatePushdownTest : public ::testing::Test {
         ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create(
                                                     std::move(in_stream), options, batch_size_,
                                                     /*file_metadata=*/nullptr,
-                                                    /*storage_read_bytes=*/nullptr, arrow_pool_));
+                                                    /*storage_read_bytes=*/nullptr, arrow_pool_,
+                                                    /*hints=*/std::nullopt));
         std::unique_ptr<ArrowSchema> c_schema = std::make_unique<ArrowSchema>();
         auto arrow_status = arrow::ExportSchema(*read_schema, c_schema.get());
         ASSERT_TRUE(arrow_status.ok());
diff --git a/src/paimon/format/parquet/variant_parquet_test.cpp b/src/paimon/format/parquet/variant_parquet_test.cpp
index fd60e7b2f..404b0521d 100644
--- a/src/paimon/format/parquet/variant_parquet_test.cpp
+++ b/src/paimon/format/parquet/variant_parquet_test.cpp
@@ -386,7 +386,8 @@ class VariantParquetTest : public ::testing::Test {
                                                       std::move(in_stream), options,
                                                       /*batch_size=*/1024,
                                                       /*file_metadata=*/nullptr,
-                                                      /*storage_read_bytes=*/nullptr, arrow_pool_));
+                                                      /*storage_read_bytes=*/nullptr, arrow_pool_,
+                                                      /*hints=*/std::nullopt));
         *file_reader = std::move(parquet_reader);
         ASSERT_OK_AND_ASSIGN(std::unique_ptr<::ArrowSchema> c_file_schema,
                              (*file_reader)->GetFileSchema());
@@ -569,11 +570,12 @@ TEST_F(VariantParquetTest, WriteAndReadRoundTrip) {
     auto in_stream =
         std::make_unique<ArrowInputStreamAdapter>(std::move(input_stream), length, arrow_pool_);
     std::map<std::string, std::string> options = {};
-    ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create(
-                                                std::move(in_stream), options,
-                                                /*batch_size=*/1024,
-                                                /*file_metadata=*/nullptr,
-                                                /*storage_read_bytes=*/nullptr, arrow_pool_));
+    ASSERT_OK_AND_ASSIGN(auto batch_reader,
+                         ParquetFileBatchReader::Create(std::move(in_stream), options,
+                                                        /*batch_size=*/1024,
+                                                        /*file_metadata=*/nullptr,
+                                                        /*storage_read_bytes=*/nullptr, arrow_pool_,
+                                                        /*hints=*/std::nullopt));
     auto c_schema = std::make_unique<ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*paimon_schema_, c_schema.get()).ok());
     ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr,
diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp
index 6d7e55623..1e0952e08 100644
--- a/test/inte/read_inte_test.cpp
+++ b/test/inte/read_inte_test.cpp
@@ -47,6 +47,7 @@
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/read_ahead_cache.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/manifest/file_source.h"
@@ -91,7 +92,7 @@ struct TestParam {
     bool enable_prefetch;
     std::string enable_adaptive_prefetch_strategy;
     std::string file_format;
-    PrefetchCacheMode cache_mode;
+    bool read_ahead_cache_enabled;
 };
 
 // read_inte_test.cpp test mainly for raw file split read (pk+dv & append only)
@@ -354,19 +355,16 @@ Result<int64_t> CountDataFiles(const std::vector<std::shared_ptr<Split>>& splits
 
 std::vector<TestParam> PrepareTestParam() {
     std::vector<TestParam> values = {
-        TestParam{false, "false", "parquet", PrefetchCacheMode::ALWAYS},
-        TestParam{true, "true", "parquet", PrefetchCacheMode::ALWAYS},
-        TestParam{true, "false", "parquet", PrefetchCacheMode::ALWAYS},
-        TestParam{true, "false", "parquet", PrefetchCacheMode::NEVER},
-        TestParam{true, "false", "parquet", PrefetchCacheMode::EXCLUDE_BITMAP_OR_PREDICATE}};
+        TestParam{false, "false", "parquet", /*read_ahead_cache_enabled=*/true},
+        TestParam{true, "true", "parquet", /*read_ahead_cache_enabled=*/true},
+        TestParam{true, "false", "parquet", /*read_ahead_cache_enabled=*/true},
+        TestParam{true, "false", "parquet", /*read_ahead_cache_enabled=*/false}};
 
 #ifdef PAIMON_ENABLE_ORC
-    values.push_back(TestParam{false, "false", "orc", PrefetchCacheMode::ALWAYS});
-    values.push_back(TestParam{true, "true", "orc", PrefetchCacheMode::ALWAYS});
-    values.push_back(TestParam{true, "false", "orc", PrefetchCacheMode::ALWAYS});
-    values.push_back(TestParam{true, "false", "orc", PrefetchCacheMode::NEVER});
-    values.push_back(
-        TestParam{true, "false", "orc", PrefetchCacheMode::EXCLUDE_BITMAP_OR_PREDICATE});
+    values.push_back(TestParam{false, "false", "orc", /*read_ahead_cache_enabled=*/true});
+    values.push_back(TestParam{true, "true", "orc", /*read_ahead_cache_enabled=*/true});
+    values.push_back(TestParam{true, "false", "orc", /*read_ahead_cache_enabled=*/true});
+    values.push_back(TestParam{true, "false", "orc", /*read_ahead_cache_enabled=*/false});
 #endif
     return values;
 }
@@ -390,7 +388,7 @@ TEST_P(ReadInteTest, TestAppendSimple) {
         context_builder.EnablePrefetch(param.enable_prefetch)
             .AddOption("test.enable-adaptive-prefetch-strategy", "false")
             .AddOption("orc.read.enable-metrics", "true");
-        context_builder.SetPrefetchCacheMode(param.cache_mode);
+        context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
 
         if (specific_table_schema) {
             context_builder.SetTableSchema(specific_table_schema.value());
@@ -496,7 +494,7 @@ TEST_P(ReadInteTest, TestReadWithLimits) {
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption(Options::READ_BATCH_SIZE, "1");
     context_builder.EnablePrefetch(param.enable_prefetch)
-        .SetPrefetchCacheMode(param.cache_mode)
+        .SetReadAheadCacheEnabled(param.read_ahead_cache_enabled)
         .AddOption("test.enable-adaptive-prefetch-strategy",
                    param.enable_adaptive_prefetch_strategy)
         .AddOption("orc.read.enable-metrics", "true")
@@ -546,6 +544,76 @@ TEST_P(ReadInteTest, TestReadWithLimits) {
     }
 }
 
+TEST_P(ReadInteTest, TestReadAheadCacheMetrics) {
+    auto param = GetParam();
+    std::string path =
+        paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09";
+    ReadContextBuilder context_builder(path);
+    context_builder.AddOption(Options::FILE_FORMAT, param.file_format);
+    context_builder.EnablePrefetch(param.enable_prefetch)
+        .AddOption("test.enable-adaptive-prefetch-strategy", "false")
+        .SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
+
+    ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context)));
+
+    std::vector<std::string> file_list;
+    if (param.file_format == "orc") {
+        file_list = {"data-db2b44c0-0d73-449d-82a0-4075bd2cb6e3-0.orc",
+                     "data-b913a160-a4d1-4084-af2a-18333c35668e-0.orc"};
+    } else if (param.file_format == "parquet") {
+        file_list = {"data-b446f78a-2cfb-4b3b-add8-31295d24a277-0.parquet",
+                     "data-fd72a479-53ae-42f7-aec0-e982ee555928-0.parquet"};
+    }
+
+    DataSplitsSimple input_data_splits = {{paimon::test::GetDataDir() + "/" + param.file_format +
+                                               "/append_09.db/append_09/f1=20/"
+                                               "bucket-0",
+                                           BinaryRowGenerator::GenerateRow({20}, pool_.get()),
+                                           file_list}};
+
+    auto data_splits = CreateDataSplits(input_data_splits, /*snapshot_id=*/3);
+    ASSERT_EQ(data_splits.size(), 1);
+    ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits));
+    ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(batch_reader.get()));
+    ASSERT_TRUE(result_array);
+    ASSERT_EQ(result_array->length(), 2);
+
+    // Verify the read-ahead cache metrics are surfaced through the reader chain. The prefetch
+    // reader merges the cache counters into its reader metrics only when a cache is created,
+    // so the counters must be present and effective exactly in that case.
+    auto read_metrics = batch_reader->GetReaderMetrics();
+    ASSERT_TRUE(read_metrics);
+    if (param.enable_prefetch && param.read_ahead_cache_enabled) {
+        ASSERT_OK_AND_ASSIGN(uint64_t read_count,
+                             read_metrics->GetCounter(ReadAheadCacheMetrics::READ_COUNT));
+        ASSERT_GT(read_count, 0u);
+        ASSERT_OK_AND_ASSIGN(uint64_t read_bytes,
+                             read_metrics->GetCounter(ReadAheadCacheMetrics::READ_BYTES));
+        ASSERT_GT(read_bytes, 0u);
+        ASSERT_OK_AND_ASSIGN(uint64_t hits,
+                             read_metrics->GetCounter(ReadAheadCacheMetrics::READ_HITS));
+        ASSERT_GT(hits, 0u);
+        ASSERT_OK_AND_ASSIGN(uint64_t hit_bytes,
+                             read_metrics->GetCounter(ReadAheadCacheMetrics::READ_HIT_BYTES));
+        ASSERT_GT(hit_bytes, 0u);
+        ASSERT_OK(read_metrics->GetCounter(ReadAheadCacheMetrics::READ_MISSES));
+        ASSERT_OK(read_metrics->GetCounter(ReadAheadCacheMetrics::READ_MISS_BYTES));
+        // Serving hits requires prefetch IOs issued to the underlying stream.
+        ASSERT_OK_AND_ASSIGN(uint64_t io_count,
+                             read_metrics->GetCounter(ReadAheadCacheMetrics::IO_COUNT));
+        ASSERT_GT(io_count, 0u);
+        ASSERT_OK_AND_ASSIGN(uint64_t io_bytes,
+                             read_metrics->GetCounter(ReadAheadCacheMetrics::IO_BYTES));
+        ASSERT_GT(io_bytes, 0u);
+    } else {
+        ASSERT_NOK(read_metrics->GetCounter(ReadAheadCacheMetrics::READ_COUNT));
+        ASSERT_NOK(read_metrics->GetCounter(ReadAheadCacheMetrics::READ_HITS));
+        ASSERT_NOK(read_metrics->GetCounter(ReadAheadCacheMetrics::READ_MISSES));
+        ASSERT_NOK(read_metrics->GetCounter(ReadAheadCacheMetrics::IO_COUNT));
+    }
+}
+
 TEST_P(ReadInteTest, TestReadOnlyPartitionField) {
     auto param = GetParam();
     std::string path = paimon::test::GetDataDir() + "/" + param.file_format +
@@ -557,7 +625,7 @@ TEST_P(ReadInteTest, TestReadOnlyPartitionField) {
     ReadContextBuilder context_builder(path);
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format);
     context_builder.SetReadFieldNames({"dt"});
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.EnablePrefetch(param.enable_prefetch)
         .AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("test.enable-adaptive-prefetch-strategy",
@@ -1879,7 +1947,7 @@ TEST_P(ReadInteTest, TestAppendReadWithMultipleBuckets) {
         paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09";
     ReadContextBuilder context_builder(path);
     context_builder.SetReadFieldNames({"f3", "f0", "f1"});
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2")
         .AddOption("test.enable-adaptive-prefetch-strategy",
@@ -1959,7 +2027,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicate) {
 
     ReadContextBuilder context_builder(path);
     context_builder.SetReadFieldNames({"f3", "f0", "f1"});
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .SetPredicate(predicate)
         .EnablePredicateFilter(true)
@@ -2061,7 +2129,7 @@ TEST_P(ReadInteTest, TestAppendReadWithComplexTypePredicate) {
     std::string path = paimon::test::GetDataDir() + "/" + param.file_format +
                        "/append_complex_data.db/append_complex_data";
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.SetReadFieldNames({"f6", "f2", "f4", "f3", "f5"});
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2");
@@ -2134,7 +2202,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateOnlyPushdown) {
         paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09";
 
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.SetReadFieldNames({"f3", "f0", "f1"});
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2")
@@ -2210,7 +2278,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateAllFiltered) {
         paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09";
 
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.SetReadFieldNames({"f3", "f0", "f1"});
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2")
@@ -2297,7 +2365,7 @@ TEST_P(ReadInteTest, TestAppendReadIOException) {
         ReadContextBuilder context_builder(paimon::test::GetDataDir() + "/" + param.file_format +
                                            "/append_09.db/append_09/");
         context_builder.SetReadFieldNames({"f3", "f0", "f1"});
-        context_builder.SetPrefetchCacheMode(param.cache_mode);
+        context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
         context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
             .AddOption("read.batch-size", "2")
             .EnablePrefetch(param.enable_prefetch)
@@ -2342,7 +2410,7 @@ TEST_P(ReadInteTest, TestPkTableWithDeletionVectorSimple) {
     ReadContextBuilder context_builder(path);
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2");
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.EnablePrefetch(param.enable_prefetch)
         .AddOption("test.enable-adaptive-prefetch-strategy",
                    param.enable_adaptive_prefetch_strategy);
@@ -2388,7 +2456,7 @@ TEST_P(ReadInteTest, TestPkTableWithDeletionVector) {
 
     std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09";
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2")
         .EnablePrefetch(param.enable_prefetch)
@@ -2454,7 +2522,7 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot6) {
                                                    FieldType::DOUBLE, Literal(15.0));
     std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09";
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2");
     context_builder.SetPredicate(predicate);
@@ -2539,7 +2607,7 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot8) {
 
     std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09";
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.SetReadFieldNames({"f0", "f3", "f1"});
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2");
@@ -2617,7 +2685,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolution) {
                                               DataField(8, arrow::field("e", arrow::int32()))};
 
         ReadContextBuilder context_builder(path);
-        context_builder.SetPrefetchCacheMode(param.cache_mode);
+        context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
         context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
             .AddOption("read.batch-size", "2");
         context_builder.EnablePrefetch(param.enable_prefetch)
@@ -2713,13 +2781,13 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateFilter) {
     std::string path = paimon::test::GetDataDir() + "/" + param.file_format +
                        "/append_table_with_alter_table.db/append_table_with_alter_table/";
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.SetReadFieldNames({"a", "k", "key1", "d", "key0", "c"});
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2");
     context_builder.SetPredicate(predicate);
     context_builder.EnablePredicateFilter(true);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.EnablePrefetch(param.enable_prefetch)
         .AddOption("test.enable-adaptive-prefetch-strategy",
                    param.enable_adaptive_prefetch_strategy);
@@ -2792,7 +2860,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateOnlyPushDown)
                        "/append_table_with_alter_table.db/"
                        "append_table_with_alter_table/";
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.SetReadFieldNames({"a", "k", "key1", "d", "key0", "c"});
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2");
@@ -2865,7 +2933,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot5WithSchemaEvolution) {
     std::string path = paimon::test::GetDataDir() + "/" + param.file_format +
                        "/pk_table_with_alter_table.db/pk_table_with_alter_table/";
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"});
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2");
@@ -2950,7 +3018,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolution) {
     std::string path = paimon::test::GetDataDir() + "/" + param.file_format +
                        "/pk_table_with_alter_table.db/pk_table_with_alter_table/";
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"});
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2");
@@ -3038,7 +3106,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateOnlyPush
     context_builder.SetReadFieldNames({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}});
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2");
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.SetPredicate(predicate);
     context_builder.EnablePrefetch(param.enable_prefetch)
         .AddOption("test.enable-adaptive-prefetch-strategy",
@@ -3117,7 +3185,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateFilter)
     ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than}));
 
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"});
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2");
@@ -3209,7 +3277,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithBuildInFieldId) {
     }
 
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.SetReadFieldNames({"key0", "key1", "k", "c", "d", "a", "e"});
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2");
@@ -3272,7 +3340,7 @@ TEST_P(ReadInteTest, TestAppendReadNestedType) {
     std::string path = paimon::test::GetDataDir() + "/" + param.file_format +
                        "/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/";
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2");
     context_builder.EnablePrefetch(param.enable_prefetch)
@@ -3327,7 +3395,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCast) {
                        "/append_table_alter_table_with_cast.db/"
                        "append_table_alter_table_with_cast/";
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"});
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2");
@@ -3410,7 +3478,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCastWithPredicatePushD
                        "append_table_alter_table_with_cast/";
     ReadContextBuilder context_builder(path);
     context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"});
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
         .AddOption("read.batch-size", "2");
     context_builder.SetPredicate(predicate);
@@ -3488,7 +3556,7 @@ TEST_P(ReadInteTest, TestReadWithPKFallBackBranch) {
         };
 
         ReadContextBuilder context_builder(path);
-        context_builder.SetPrefetchCacheMode(param.cache_mode);
+        context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
         context_builder.EnablePrefetch(param.enable_prefetch)
             .AddOption(Options::FILE_FORMAT, param.file_format)
             .AddOption("test.enable-adaptive-prefetch-strategy",
@@ -3545,7 +3613,7 @@ TEST_P(ReadInteTest, TestReadWithAppendFallBackBranch) {
 
     ReadContextBuilder context_builder(path);
     context_builder.EnablePrefetch(param.enable_prefetch);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
     ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context)));
     ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits));
@@ -3587,7 +3655,7 @@ TEST_P(ReadInteTest, TestFallBackBranchStreamRead) {
                                           DataField(1, arrow::field("name", arrow::utf8())),
                                           DataField(2, arrow::field("amount", arrow::int32()))};
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.EnablePrefetch(param.enable_prefetch)
         .AddOption("test.enable-adaptive-prefetch-strategy",
                    param.enable_adaptive_prefetch_strategy);
@@ -3632,7 +3700,7 @@ TEST_P(ReadInteTest, TestReadWithPKRtBranch) {
         };
 
         ReadContextBuilder context_builder(path);
-        context_builder.SetPrefetchCacheMode(param.cache_mode);
+        context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
         context_builder.EnablePrefetch(param.enable_prefetch)
             .AddOption("test.enable-adaptive-prefetch-strategy",
                        param.enable_adaptive_prefetch_strategy)
@@ -3689,7 +3757,7 @@ TEST_P(ReadInteTest, TestReadWithAppendPtBranch) {
         };
 
         ReadContextBuilder context_builder(path);
-        context_builder.SetPrefetchCacheMode(param.cache_mode);
+        context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
         context_builder.EnablePrefetch(param.enable_prefetch)
             .AddOption("test.enable-adaptive-prefetch-strategy",
                        param.enable_adaptive_prefetch_strategy)
@@ -3820,7 +3888,7 @@ TEST_P(ReadInteTest, TestSpecificFs) {
     auto countable_fs =
         std::make_shared<CountableFileSystem>(std::make_shared<LocalFileSystem>(), &io_count);
     ReadContextBuilder context_builder(path);
-    context_builder.SetPrefetchCacheMode(param.cache_mode);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.AddOption(Options::FILE_FORMAT, param.file_format);
     context_builder.EnablePrefetch(param.enable_prefetch)
         .AddOption("test.enable-adaptive-prefetch-strategy", "false")

From c7f23c69c7c1f4abe2f68bcc7078a8cf002c6bbd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B0=8F=E6=98=8E=E5=90=8C=E5=AD=A6?=
 <72908278+ChaomingZhangCN@users.noreply.github.com>
Date: Thu, 20 Aug 2026 10:21:54 +0800
Subject: [PATCH 04/93] feat(parquet): support vector type storage (#198)

---
 docs/source/user_guide/data_types.rst         |  21 +
 include/paimon/defs.h                         |   2 +
 include/paimon/format/column_stats.h          |  12 +-
 src/paimon/CMakeLists.txt                     |   4 +
 .../common/predicate/literal_converter.cpp    |   1 +
 src/paimon/common/types/data_type.cpp         |   3 +
 .../common/types/data_type_json_parser.cpp    |  55 +++
 .../common/types/data_type_json_parser.h      |   2 +
 .../types/data_type_json_parser_test.cpp      |  50 ++
 src/paimon/common/types/data_type_test.cpp    |  14 +
 src/paimon/common/types/vector_type.h         |  76 +++
 src/paimon/common/utils/arrow/arrow_utils.cpp |  43 ++
 .../common/utils/arrow/arrow_utils_test.cpp   |  47 ++
 .../common/utils/arrow/vector_utils.cpp       | 128 +++++
 src/paimon/common/utils/arrow/vector_utils.h  |  54 +++
 .../common/utils/arrow/vector_utils_test.cpp  | 100 ++++
 src/paimon/common/utils/field_type_utils.h    |   4 +
 .../common/utils/field_type_utils_test.cpp    |   5 +
 .../core/io/vector_file_batch_reader.cpp      | 280 +++++++++++
 src/paimon/core/io/vector_file_batch_reader.h |  85 ++++
 .../core/io/vector_file_batch_reader_test.cpp | 232 ++++++++++
 .../core/operation/abstract_split_read.cpp    |   4 +
 .../operation/data_evolution_split_read.h     |   2 +-
 .../core/operation/raw_file_split_read.h      |   4 +-
 .../core/schema/arrow_schema_validator.cpp    |  22 +
 .../schema/arrow_schema_validator_test.cpp    |  23 +-
 src/paimon/core/schema/schema_validation.cpp  |  40 ++
 src/paimon/core/schema/schema_validation.h    |   2 +
 .../core/schema/schema_validation_test.cpp    |  63 +++
 src/paimon/core/schema/table_schema.cpp       |   8 +
 src/paimon/core/utils/field_mapping.cpp       |   2 +-
 src/paimon/format/parquet/CMakeLists.txt      |   3 +
 .../parquet/parquet_field_id_converter.cpp    |   6 +
 .../parquet_field_id_converter_test.cpp       |  11 +-
 .../parquet/parquet_file_batch_reader.cpp     |  20 +-
 .../format/parquet/parquet_format_writer.cpp  |  30 +-
 .../format/parquet/parquet_format_writer.h    |   2 +
 .../parquet/parquet_stats_extractor.cpp       |   4 +-
 .../parquet/parquet_stats_extractor_test.cpp  |  16 +-
 .../parquet/parquet_vector_converter.cpp      | 174 +++++++
 .../format/parquet/parquet_vector_converter.h |  46 ++
 .../parquet/parquet_vector_converter_test.cpp |  95 ++++
 .../format/parquet/parquet_vector_io_test.cpp | 437 ++++++++++++++++++
 test/inte/write_and_read_inte_test.cpp        | 177 +++++++
 .../parquet/vector_compatibility/README.md    |  38 ++
 .../vector_compatibility/java_vector.parquet  | Bin 0 -> 1303 bytes
 .../java_vector_nullable.parquet              | Bin 0 -> 765 bytes
 .../vector_compatibility/rust_vector.parquet  | Bin 0 -> 949 bytes
 .../rust_vector_nullable.parquet              | Bin 0 -> 932 bytes
 49 files changed, 2422 insertions(+), 25 deletions(-)
 create mode 100644 src/paimon/common/types/vector_type.h
 create mode 100644 src/paimon/common/utils/arrow/vector_utils.cpp
 create mode 100644 src/paimon/common/utils/arrow/vector_utils.h
 create mode 100644 src/paimon/common/utils/arrow/vector_utils_test.cpp
 create mode 100644 src/paimon/core/io/vector_file_batch_reader.cpp
 create mode 100644 src/paimon/core/io/vector_file_batch_reader.h
 create mode 100644 src/paimon/core/io/vector_file_batch_reader_test.cpp
 create mode 100644 src/paimon/format/parquet/parquet_vector_converter.cpp
 create mode 100644 src/paimon/format/parquet/parquet_vector_converter.h
 create mode 100644 src/paimon/format/parquet/parquet_vector_converter_test.cpp
 create mode 100644 src/paimon/format/parquet/parquet_vector_io_test.cpp
 create mode 100644 test/test_data/parquet/vector_compatibility/README.md
 create mode 100644 test/test_data/parquet/vector_compatibility/java_vector.parquet
 create mode 100644 test/test_data/parquet/vector_compatibility/java_vector_nullable.parquet
 create mode 100644 test/test_data/parquet/vector_compatibility/rust_vector.parquet
 create mode 100644 test/test_data/parquet/vector_compatibility/rust_vector_nullable.parquet

diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst
index 3d529332b..9fdecf6e5 100644
--- a/docs/source/user_guide/data_types.rst
+++ b/docs/source/user_guide/data_types.rst
@@ -186,6 +186,27 @@ and `Arrow DataTypes <https://arrow.apache.org/docs/format/Columnar.html#data-ty
 
        The type can be declared using ``ARRAY<t>`` where t is the data type of the contained elements.
 
+   * - ``VECTOR<t, n>``
+     - FixedSizeList
+     - Data type of a dense vector containing exactly ``n`` elements of type ``t``.
+
+       ``n`` must be positive. ``t`` can be ``BOOLEAN``, ``TINYINT``,
+       ``SMALLINT``, ``INT``, ``BIGINT``, ``FLOAT``, or ``DOUBLE``. A VECTOR
+       value may be NULL, but its elements cannot be NULL.
+
+       Paimon C++ currently supports VECTOR columns only in append-only tables
+       backed by Parquet data files. They use the standard Parquet LIST
+       representation on disk and are restored as Arrow ``FixedSizeList``
+       values on read. Primary-key tables and data-evolution tables containing
+       VECTOR fields are rejected. VECTOR columns also cannot be partition or
+       bucket keys. Dedicated vector storage is not included yet.
+
+       **Note:** A data file written by another engine that records the column as
+       Arrow ``FixedSizeList`` instead of ``LIST``, such as Paimon Rust or Python,
+       can only be read while it holds no NULL vector. Parquet stores a NULL list
+       slot with no values, which the Arrow 17 Parquet reader rejects for a
+       ``FixedSizeList`` column.
+
    * - ``MAP<kt, vt>``
      - Map
      - Data type of an associative array that maps keys (including NULL) to values (including NULL). A map cannot contain duplicate keys; each key can map to at most one value.
diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index 9fcf8e34d..d1ebf507a 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -50,6 +50,8 @@ enum class FieldType {
     STRUCT = 15,
     BLOB = 16,
     VARIANT = 17,
+    /// Fixed-length dense vector represented by Arrow FixedSizeList.
+    VECTOR = 18,
     UNKNOWN = 128,
 };
 
diff --git a/include/paimon/format/column_stats.h b/include/paimon/format/column_stats.h
index f16cb2547..a4e3de484 100644
--- a/include/paimon/format/column_stats.h
+++ b/include/paimon/format/column_stats.h
@@ -33,8 +33,8 @@ namespace paimon {
 /// ColumnStats is an abstract base class that represents statistical information for data columns
 /// in Paimon tables. It provides min/max values and null count statistics
 ///
-/// Only primitive data types support min/max statistics. Nested types (arrays, maps, structs) only
-/// track null counts through `NestedColumnStats`.
+/// Only primitive data types support min/max statistics. Nested types (arrays, vectors, maps,
+/// structs) only track null counts through `NestedColumnStats`.
 ///
 /// @note This is an abstract base class. Use the static factory methods `CreateXXXColumnStats()` to
 /// create concrete instances for specific data types.
@@ -52,7 +52,7 @@ class PAIMON_EXPORT ColumnStats {
     /// @name CreateXXXColumnStats()
     /// %Factory methods `CreateXXXColumnStats()` to create column statistics.
     /// - min/max/null_count for primitive data types
-    /// - null_count for nested data types (arrays, maps, structs)
+    /// - null_count for nested data types (arrays, vectors, maps, structs)
     ///
     /// @{
     static std::unique_ptr<ColumnStats> CreateBooleanColumnStats(std::optional<bool> min,
@@ -88,8 +88,8 @@ class PAIMON_EXPORT ColumnStats {
     static std::unique_ptr<ColumnStats> CreateDateColumnStats(std::optional<int32_t> min,
                                                               std::optional<int32_t> max,
                                                               std::optional<int64_t> null_count);
-    /// Creates column statistics for nested data types (arrays, maps, structs), which only track
-    /// null counts.
+    /// Creates column statistics for nested data types (arrays, vectors, maps, structs), which only
+    /// track null counts.
     static std::unique_ptr<ColumnStats> CreateNestedColumnStats(const FieldType& nested_type,
                                                                 std::optional<int64_t> null_count);
     /// @}
@@ -180,7 +180,7 @@ class PAIMON_EXPORT NestedColumnStats : public ColumnStats {
     NestedColumnStats(const FieldType& nested_type, std::optional<int64_t> null_count)
         : nested_type_(nested_type), null_count_(null_count) {
         assert(nested_type == FieldType::ARRAY || nested_type == FieldType::MAP ||
-               nested_type == FieldType::STRUCT);
+               nested_type == FieldType::STRUCT || nested_type == FieldType::VECTOR);
     }
 
     std::optional<int64_t> NullCount() const override {
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index b0fe91b08..9b0807b64 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -157,6 +157,7 @@ set(PAIMON_COMMON_SRCS
     common/utils/arrow/arrow_output_stream_adapter.cpp
     common/utils/arrow/arrow_utils.cpp
     common/utils/arrow/mem_utils.cpp
+    common/utils/arrow/vector_utils.cpp
     common/utils/binary_row_partition_computer.cpp
     common/utils/bit_set.cpp
     common/utils/bloom_filter.cpp
@@ -275,6 +276,7 @@ set(PAIMON_CORE_SRCS
     core/io/data_file_writer.cpp
     core/io/field_mapping_reader.cpp
     core/io/complete_row_tracking_fields_reader.cpp
+    core/io/vector_file_batch_reader.cpp
     core/io/file_index_evaluator.cpp
     core/io/key_value_data_file_record_reader.cpp
     core/io/key_value_data_file_writer_factory.cpp
@@ -611,6 +613,7 @@ if(PAIMON_BUILD_TESTS)
                     common/utils/row_range_index_test.cpp
                     common/utils/var_length_int_utils_test.cpp
                     common/utils/arrow/arrow_utils_test.cpp
+                    common/utils/arrow/vector_utils_test.cpp
                     common/utils/arrow/arrow_stream_adapter_test.cpp
                     common/utils/arrow/mem_utils_test.cpp
                     common/utils/arrow/status_utils_test.cpp
@@ -747,6 +750,7 @@ if(PAIMON_BUILD_TESTS)
                     core/io/key_value_in_memory_record_reader_test.cpp
                     core/io/merged_key_value_record_reader_test.cpp
                     core/io/complete_row_tracking_fields_reader_test.cpp
+                    core/io/vector_file_batch_reader_test.cpp
                     core/io/data_file_meta_test.cpp
                     core/io/file_index_evaluator_test.cpp
                     core/io/single_file_writer_test.cpp
diff --git a/src/paimon/common/predicate/literal_converter.cpp b/src/paimon/common/predicate/literal_converter.cpp
index 194107687..102348d70 100644
--- a/src/paimon/common/predicate/literal_converter.cpp
+++ b/src/paimon/common/predicate/literal_converter.cpp
@@ -166,6 +166,7 @@ Result<Literal> LiteralConverter::ConvertLiteralsFromRow(
         case FieldType::DATE:
             return Literal(FieldType::DATE, row.GetInt(field_idx));
         case FieldType::ARRAY:
+        case FieldType::VECTOR:
         case FieldType::MAP:
         case FieldType::STRUCT:
         default:
diff --git a/src/paimon/common/types/data_type.cpp b/src/paimon/common/types/data_type.cpp
index 9b6d3c903..2bf5d73cb 100644
--- a/src/paimon/common/types/data_type.cpp
+++ b/src/paimon/common/types/data_type.cpp
@@ -29,6 +29,7 @@
 #include "paimon/common/types/array_type.h"
 #include "paimon/common/types/map_type.h"
 #include "paimon/common/types/row_type.h"
+#include "paimon/common/types/vector_type.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/common/utils/decimal_utils.h"
@@ -52,6 +53,8 @@ std::unique_ptr<DataType> DataType::Create(
             return std::make_unique<MapType>(type, nullable, metadata);
         case arrow::Type::type::LIST:
             return std::make_unique<ArrayType>(type, nullable, metadata);
+        case arrow::Type::type::FIXED_SIZE_LIST:
+            return std::make_unique<VectorType>(type, nullable, metadata);
         case arrow::Type::type::STRUCT:
             if (VariantTypeUtils::IsVariantMetadata(metadata)) {
                 // A variant field is physically a struct<value, metadata> but is a scalar
diff --git a/src/paimon/common/types/data_type_json_parser.cpp b/src/paimon/common/types/data_type_json_parser.cpp
index 33308ab69..e95582a17 100644
--- a/src/paimon/common/types/data_type_json_parser.cpp
+++ b/src/paimon/common/types/data_type_json_parser.cpp
@@ -25,6 +25,7 @@
 #include <cstdint>
 #include <limits>
 #include <map>
+#include <optional>
 #include <sstream>
 #include <utility>
 #include <vector>
@@ -33,6 +34,7 @@
 #include "paimon/common/data/blob_utils.h"
 #include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/common/types/vector_type.h"
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/common/utils/rapidjson_util.h"
 #include "paimon/common/utils/string_utils.h"
@@ -148,6 +150,7 @@ enum class Keyword : int32_t {
     ROW,
     BLOB,
     VARIANT,
+    VECTOR,
     // NULL is keyword in c++
     NULL_,
     RAW,
@@ -197,6 +200,7 @@ const std::map<std::string, Keyword>& Keywords() {
         {"ROW", Keyword::ROW},
         {"BLOB", Keyword::BLOB},
         {"VARIANT", Keyword::VARIANT},
+        {"VECTOR", Keyword::VECTOR},
         {"NULL", Keyword::NULL_},
         {"RAW", Keyword::RAW},
         {"LEGACY", Keyword::LEGACY},
@@ -249,6 +253,7 @@ class TokenParser {
     Result<std::shared_ptr<arrow::DataType>> ParseDoubleType();
     Result<std::shared_ptr<arrow::DataType>> ParseTimestampType();
     Result<std::shared_ptr<arrow::DataType>> ParseTimestampLtzType();
+    Result<std::shared_ptr<arrow::DataType>> ParseVectorType();
     Result<int32_t> ParseOptionalPrecision(int32_t default_precision);
 
  private:
@@ -526,6 +531,8 @@ Result<std::shared_ptr<arrow::DataType>> TokenParser::ParseTypeByKeyword(
             return ParseTimestampType();
         case Keyword::TIMESTAMP_LTZ:
             return ParseTimestampLtzType();
+        case Keyword::VECTOR:
+            return ParseVectorType();
         default:
             return Status::Invalid(fmt::format("Unsupported type: {}", GetToken().value));
     }
@@ -607,6 +614,31 @@ Result<std::shared_ptr<arrow::DataType>> TokenParser::ParseTimestampLtzType() {
     return ts_type;
 }
 
+Result<std::shared_ptr<arrow::DataType>> TokenParser::ParseVectorType() {
+    PAIMON_RETURN_NOT_OK(NextToken(TokenType::BEGIN_SUBTYPE));
+    bool element_nullable = true;
+    AtomicTypeAttributes element_attributes;
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::DataType> element_type,
+                           ParseTypeWithNullability(&element_nullable, &element_attributes));
+    if (element_attributes.is_blob || element_attributes.is_variant ||
+        !VectorType::IsValidElementType(element_type)) {
+        return Status::Invalid(
+            fmt::format("Invalid element type for vector: {}", element_type->ToString()));
+    }
+    PAIMON_RETURN_NOT_OK(NextToken(TokenType::LIST_SEPARATOR));
+    PAIMON_RETURN_NOT_OK(NextToken(TokenType::LITERAL_INT));
+    const std::string& length_token = GetToken().value;
+    std::optional<int32_t> length = StringUtils::StringToValue<int32_t>(length_token);
+    if (!length || length.value() < 1) {
+        return Status::Invalid(
+            fmt::format("Vector length must be between 1 and {} (both inclusive), but was {}",
+                        std::numeric_limits<int32_t>::max(), length_token));
+    }
+    PAIMON_RETURN_NOT_OK(NextToken(TokenType::END_SUBTYPE));
+    return arrow::fixed_size_list(arrow::field("item", element_type, element_nullable),
+                                  length.value());
+}
+
 Result<int32_t> TokenParser::ParseOptionalPrecision(int32_t default_precision) {
     auto precision = default_precision;
     if (HasNextToken({TokenType::BEGIN_PARAMETER})) {
@@ -659,6 +691,8 @@ Result<std::shared_ptr<arrow::Field>> DataTypeJsonParser::ParseComplexTypeField(
 
     if (StringUtils::StartsWith(type_str, "ARRAY")) {
         return ParseArrayType(name, type_json_value, nullable);
+    } else if (StringUtils::StartsWith(type_str, "VECTOR")) {
+        return ParseVectorType(name, type_json_value, nullable);
     } else if (StringUtils::StartsWith(type_str, "MAP")) {
         return ParseMapType(name, type_json_value, nullable);
     } else if (StringUtils::StartsWith(type_str, "ROW")) {
@@ -681,6 +715,27 @@ Result<std::shared_ptr<arrow::Field>> DataTypeJsonParser::ParseArrayType(
     return arrow::field(name, arrow::list(element_field), nullable);
 }
 
+Result<std::shared_ptr<arrow::Field>> DataTypeJsonParser::ParseVectorType(
+    const std::string& name, const rapidjson::Value& type_json_value, bool nullable) {
+    if (!type_json_value.HasMember("element") || !type_json_value.HasMember("length")) {
+        return Status::Invalid("vector data type must have element and length");
+    }
+    if (!type_json_value["length"].IsInt()) {
+        return Status::Invalid("vector length must be an integer");
+    }
+    int32_t length = type_json_value["length"].GetInt();
+    if (length < 1) {
+        return Status::Invalid("Vector length must be between 1 and 2147483647 (both inclusive)");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Field> element_field,
+                           ParseType("item", type_json_value["element"]));
+    if (!VectorType::IsValidElementType(element_field->type())) {
+        return Status::Invalid(
+            fmt::format("Invalid element type for vector: {}", element_field->type()->ToString()));
+    }
+    return arrow::field(name, arrow::fixed_size_list(element_field, length), nullable);
+}
+
 Result<std::shared_ptr<arrow::Field>> DataTypeJsonParser::ParseMapType(
     const std::string& name, const rapidjson::Value& type_json_value, bool nullable) {
     if (!type_json_value.HasMember("key") || !type_json_value.HasMember("value")) {
diff --git a/src/paimon/common/types/data_type_json_parser.h b/src/paimon/common/types/data_type_json_parser.h
index 92cb5d55f..2134236a7 100644
--- a/src/paimon/common/types/data_type_json_parser.h
+++ b/src/paimon/common/types/data_type_json_parser.h
@@ -50,6 +50,8 @@ class DataTypeJsonParser {
 
     static Result<std::shared_ptr<arrow::Field>> ParseArrayType(
         const std::string& name, const rapidjson::Value& type_json_value, bool nullable);
+    static Result<std::shared_ptr<arrow::Field>> ParseVectorType(
+        const std::string& name, const rapidjson::Value& type_json_value, bool nullable);
     static Result<std::shared_ptr<arrow::Field>> ParseMapType(
         const std::string& name, const rapidjson::Value& type_json_value, bool nullable);
     static Result<std::shared_ptr<arrow::Field>> ParseRowType(
diff --git a/src/paimon/common/types/data_type_json_parser_test.cpp b/src/paimon/common/types/data_type_json_parser_test.cpp
index 5026db020..e5dfbc21e 100644
--- a/src/paimon/common/types/data_type_json_parser_test.cpp
+++ b/src/paimon/common/types/data_type_json_parser_test.cpp
@@ -48,6 +48,56 @@ TEST(DataTypeJsonParserTest, ParseTypeArrayTypeSuccess) {
     ASSERT_NE(field, nullptr);
 }
 
+TEST(DataTypeJsonParserTest, ParseVectorTypeSuccess) {
+    const char* json = R"({
+        "type": "VECTOR NOT NULL",
+        "element": "FLOAT",
+        "length": 3
+    })";
+    rapidjson::Document doc;
+    doc.Parse(json);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Field> field,
+                         DataTypeJsonParser::ParseType("embedding", doc));
+    ASSERT_FALSE(field->nullable());
+    ASSERT_EQ(field->type()->id(), arrow::Type::FIXED_SIZE_LIST);
+    auto vector_type = checked_pointer_cast<arrow::FixedSizeListType>(field->type());
+    ASSERT_EQ(vector_type->list_size(), 3);
+    ASSERT_TRUE(vector_type->value_type()->Equals(arrow::float32()));
+
+    rapidjson::Document sql_doc;
+    rapidjson::Value sql_value("VECTOR<BIGINT NOT NULL, 5>", sql_doc.GetAllocator());
+    ASSERT_OK_AND_ASSIGN(field, DataTypeJsonParser::ParseType("embedding", sql_value));
+    vector_type = checked_pointer_cast<arrow::FixedSizeListType>(field->type());
+    ASSERT_TRUE(field->nullable());
+    ASSERT_EQ(vector_type->list_size(), 5);
+    ASSERT_FALSE(vector_type->value_field()->nullable());
+    ASSERT_TRUE(vector_type->value_type()->Equals(arrow::int64()));
+}
+
+TEST(DataTypeJsonParserTest, ParseVectorTypeFailure) {
+    for (const char* json : {
+             R"({"type":"VECTOR","element":"FLOAT","length":0})",
+             R"({"type":"VECTOR","element":"STRING","length":3})",
+             R"({"type":"VECTOR","element":"FLOAT"})",
+             R"({"type":"VECTOR","element":"FLOAT","length":"3"})",
+         }) {
+        rapidjson::Document doc;
+        doc.Parse(json);
+        ASSERT_NOK(DataTypeJsonParser::ParseType("embedding", doc));
+    }
+
+    rapidjson::Document sql_doc;
+    rapidjson::Value sql_value("VECTOR<STRING, 3>", sql_doc.GetAllocator());
+    ASSERT_NOK_WITH_MSG(DataTypeJsonParser::ParseType("embedding", sql_value),
+                        "Invalid element type for vector");
+    sql_value.SetString("VECTOR<DOUBLE, 3>", sql_doc.GetAllocator());
+    ASSERT_OK(DataTypeJsonParser::ParseType("embedding", sql_value));
+    sql_value.SetString("VECTOR<FLOAT, 999999999999999999999999>", sql_doc.GetAllocator());
+    ASSERT_NOK_WITH_MSG(DataTypeJsonParser::ParseType("embedding", sql_value),
+                        "Vector length must be between 1 and 2147483647");
+}
+
 TEST(DataTypeJsonParserTest, ParseTypeMapTypeSuccess) {
     const std::string name = "map_field";
     const char* json = R"({
diff --git a/src/paimon/common/types/data_type_test.cpp b/src/paimon/common/types/data_type_test.cpp
index 9568c3ff0..d6eacdc1f 100644
--- a/src/paimon/common/types/data_type_test.cpp
+++ b/src/paimon/common/types/data_type_test.cpp
@@ -148,4 +148,18 @@ TEST(DataTypeTest, NestedTypeSerializationUsesChildMetadata) {
               R"({"type":"ARRAY","element":"INT"})");
 }
 
+TEST(DataTypeTest, VectorTypeSerialization) {
+    auto vector_field = arrow::field(
+        "embedding", arrow::fixed_size_list(arrow::field("item", arrow::float32()), 3), false);
+    auto data_type =
+        DataType::Create(vector_field->type(), vector_field->nullable(), vector_field->metadata());
+    rapidjson::Document doc;
+    auto value = data_type->ToJson(&doc.GetAllocator());
+    rapidjson::StringBuffer buffer;
+    rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
+    value.Accept(writer);
+    ASSERT_EQ(std::string(buffer.GetString()),
+              R"({"type":"VECTOR NOT NULL","element":"FLOAT","length":3})");
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/common/types/vector_type.h b/src/paimon/common/types/vector_type.h
new file mode 100644
index 000000000..9c55c165b
--- /dev/null
+++ b/src/paimon/common/types/vector_type.h
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "arrow/api.h"
+#include "paimon/common/types/data_type.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/rapidjson_util.h"
+
+namespace paimon {
+
+/// Fixed-size VECTOR<T, N> logical type backed by Arrow FixedSizeList.
+class VectorType : public DataType {
+ public:
+    static constexpr char TYPE[] = "VECTOR";
+
+    VectorType(const std::shared_ptr<arrow::DataType>& type, bool nullable,
+               const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
+        : DataType(type, nullable, metadata) {}
+
+    static bool IsValidElementType(const std::shared_ptr<arrow::DataType>& type) {
+        switch (type->id()) {
+            case arrow::Type::BOOL:
+            case arrow::Type::INT8:
+            case arrow::Type::INT16:
+            case arrow::Type::INT32:
+            case arrow::Type::INT64:
+            case arrow::Type::FLOAT:
+            case arrow::Type::DOUBLE:
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const
+        noexcept(false) override {
+        rapidjson::Value obj(rapidjson::kObjectType);
+        obj.AddMember(
+            rapidjson::StringRef("type"),
+            RapidJsonUtil::SerializeValue(WithNullable(std::string(TYPE)), allocator).Move(),
+            *allocator);
+        auto* type = checked_cast<arrow::FixedSizeListType*>(type_.get());
+        auto value_field = type->value_field();
+        std::shared_ptr<DataType> data_type =
+            DataType::Create(value_field->type(), value_field->nullable(), value_field->metadata());
+        obj.AddMember(rapidjson::StringRef("element"),
+                      RapidJsonUtil::SerializeValue(*data_type, allocator).Move(), *allocator);
+        obj.AddMember(rapidjson::StringRef("length"),
+                      RapidJsonUtil::SerializeValue(type->list_size(), allocator).Move(),
+                      *allocator);
+        return obj;
+    }
+};
+
+}  // namespace paimon
diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp b/src/paimon/common/utils/arrow/arrow_utils.cpp
index 707e888f7..f29e1d11e 100644
--- a/src/paimon/common/utils/arrow/arrow_utils.cpp
+++ b/src/paimon/common/utils/arrow/arrow_utils.cpp
@@ -30,6 +30,7 @@
 #include "arrow/util/compression.h"
 #include "fmt/format.h"
 #include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/arrow/vector_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/string_utils.h"
 
@@ -160,6 +161,28 @@ Result<std::shared_ptr<arrow::ArrayData>> RebaseListLike(
     return rebased;
 }
 
+/// Rebases a fixed size list array, whose child holds `list_size` values per row.
+Result<std::shared_ptr<arrow::ArrayData>> RebaseFixedSizeList(
+    const std::shared_ptr<arrow::ArrayData>& data, arrow::MemoryPool* pool) {
+    if (data->child_data.size() != 1) {
+        return CopyToZeroOffset(data, pool);
+    }
+    const int64_t list_size =
+        checked_cast<const arrow::FixedSizeListType&>(*data->type).list_size();
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Buffer> validity,
+                           RebaseValidityBitmap(*data, pool));
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::ArrayData> child_slice,
+        data->child_data[0]->SliceSafe(data->offset * list_size, data->length * list_size));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::ArrayData> child,
+                           RebaseToZeroOffset(child_slice, pool));
+    std::shared_ptr<arrow::ArrayData> rebased =
+        arrow::ArrayData::Make(data->type, data->length, data->null_count.load(), /*offset=*/0);
+    rebased->buffers = {std::move(validity)};
+    rebased->child_data = {std::move(child)};
+    return rebased;
+}
+
 /// Rebases a struct array, whose slices keep full length children.
 Result<std::shared_ptr<arrow::ArrayData>> RebaseStruct(
     const std::shared_ptr<arrow::ArrayData>& data, arrow::MemoryPool* pool) {
@@ -233,6 +256,8 @@ Result<std::shared_ptr<arrow::ArrayData>> RebaseToZeroOffset(
             return RebaseListLike<int32_t>(data, pool);
         case arrow::Type::LARGE_LIST:
             return RebaseListLike<int64_t>(data, pool);
+        case arrow::Type::FIXED_SIZE_LIST:
+            return RebaseFixedSizeList(data, pool);
         case arrow::Type::STRUCT:
             return RebaseStruct(data, pool);
         case arrow::Type::DICTIONARY:
@@ -320,6 +345,11 @@ void ArrowUtils::TraverseArray(const std::shared_ptr<arrow::Array>& array) {
             TraverseArray(list_array->values());
             return;
         }
+        case arrow::Type::type::FIXED_SIZE_LIST: {
+            auto* vector_array = checked_cast<arrow::FixedSizeListArray*>(array.get());
+            TraverseArray(vector_array->values());
+            return;
+        }
         default:
             return;
     }
@@ -330,6 +360,13 @@ bool ArrowUtils::EqualsIgnoreNullable(const std::shared_ptr<arrow::DataType>& ty
     if (type->id() != other_type->id() || type->num_fields() != other_type->num_fields()) {
         return false;
     }
+    if (type->id() == arrow::Type::FIXED_SIZE_LIST) {
+        const auto& vector_type = checked_cast<const arrow::FixedSizeListType&>(*type);
+        const auto& other_vector_type = checked_cast<const arrow::FixedSizeListType&>(*other_type);
+        if (vector_type.list_size() != other_vector_type.list_size()) {
+            return false;
+        }
+    }
     for (int32_t i = 0; i < type->num_fields(); ++i) {
         const auto& field = type->field(i);
         const auto& other_field = other_type->field(i);
@@ -363,6 +400,12 @@ Status ArrowUtils::InnerCheckNullabilityMatch(const std::shared_ptr<arrow::Field
         auto list_array = checked_pointer_cast<arrow::ListArray>(data);
         PAIMON_RETURN_NOT_OK(
             InnerCheckNullabilityMatch(list_type->value_field(), list_array->values()));
+    } else if (type->id() == arrow::Type::FIXED_SIZE_LIST) {
+        Status status = VectorUtils::ValidateVectorElements(*data);
+        if (!status.ok()) {
+            return Status::Invalid(
+                fmt::format("VECTOR field {} is invalid: {}", field->name(), status.message()));
+        }
     } else if (type->id() == arrow::Type::MAP) {
         auto map_type = checked_pointer_cast<arrow::MapType>(field->type());
         auto map_array = checked_pointer_cast<arrow::MapArray>(data);
diff --git a/src/paimon/common/utils/arrow/arrow_utils_test.cpp b/src/paimon/common/utils/arrow/arrow_utils_test.cpp
index 3680291c9..4e1fdaa0c 100644
--- a/src/paimon/common/utils/arrow/arrow_utils_test.cpp
+++ b/src/paimon/common/utils/arrow/arrow_utils_test.cpp
@@ -249,6 +249,42 @@ TEST(ArrowUtilsTest, TestCheckNullableMatchWithList) {
     }
 }
 
+TEST(ArrowUtilsTest, TestCheckNullableMatchRejectsNullVectorElement) {
+    auto vector_type = arrow::fixed_size_list(arrow::float32(), 3);
+    auto vector_field = arrow::field("embedding", vector_type);
+    arrow::FloatBuilder values_builder;
+    ASSERT_TRUE(values_builder.Append(1.0f).ok());
+    ASSERT_TRUE(values_builder.AppendNull().ok());
+    ASSERT_TRUE(values_builder.Append(3.0f).ok());
+    std::shared_ptr<arrow::Array> values = values_builder.Finish().ValueOrDie();
+    auto vector_data = arrow::ArrayData::Make(vector_type, 1, {nullptr}, {values->data()}, 0);
+    auto vector_array = arrow::MakeArray(vector_data);
+    auto struct_array = arrow::StructArray::Make({vector_array}, {vector_field}).ValueOrDie();
+
+    ASSERT_NOK_WITH_MSG(
+        ArrowUtils::CheckNullabilityMatch(arrow::schema({vector_field}), struct_array),
+        "VECTOR field embedding is invalid: VECTOR cannot contain null elements");
+}
+
+// Arrow accepts a FixedSizeList whose child is shorter than `length * list_size` when importing
+// it over the C data interface, so the nullability check must reject it rather than scan past the
+// end of the child.
+TEST(ArrowUtilsTest, TestCheckNullableMatchRejectsTruncatedVector) {
+    auto vector_type = arrow::fixed_size_list(arrow::float32(), 3);
+    auto vector_field = arrow::field("embedding", vector_type);
+    arrow::FloatBuilder values_builder;
+    ASSERT_TRUE(values_builder.AppendValues({1.0f, 2.0f, 3.0f}).ok());
+    std::shared_ptr<arrow::Array> values = values_builder.Finish().ValueOrDie();
+    auto vector_data = arrow::ArrayData::Make(vector_type, /*length=*/2, {nullptr},
+                                              {values->data()}, /*null_count=*/0);
+    auto vector_array = arrow::MakeArray(vector_data);
+    auto struct_array = arrow::StructArray::Make({vector_array}, {vector_field}).ValueOrDie();
+
+    ASSERT_NOK_WITH_MSG(
+        ArrowUtils::CheckNullabilityMatch(arrow::schema({vector_field}), struct_array),
+        "VECTOR field embedding is invalid: VECTOR holds 3 elements while 2 rows of dimension 3");
+}
+
 TEST(ArrowUtilsTest, TestCheckNullableMatchWithMap) {
     auto key_field = arrow::field("key", arrow::int32(), /*nullable=*/false);
     auto value_field = arrow::field("value", arrow::int32(), /*nullable=*/true);
@@ -523,6 +559,9 @@ std::vector<NormalizeCase> NormalizeCases() {
         {arrow::list(arrow::utf8()),
          R"([["a"], null, ["bb", "ccc"], [], ["d"], null, ["e", "f"], [], ["g"], ["h"]])",
          {{{0}, 2}}},
+        {arrow::fixed_size_list(arrow::int32(), 2),
+         "[[0, 1], null, [2, 3], [4, 5], [6, 7], null, [8, 9], [10, 11], [12, 13], [14, 15]]",
+         {{{0}, 1}}},
         {arrow::struct_({int_field, text_field}),
          R"([{"a": 0, "b": "x"}, null, {"a": 2, "b": null}, {"a": null, "b": "yyy"},
              {"a": 4, "b": "z"}, {"a": 5, "b": ""}, null, {"a": 7, "b": "w"},
@@ -758,6 +797,14 @@ TEST(ArrowUtilsTest, TestEqualsIgnoreNullable) {
         ASSERT_TRUE(ArrowUtils::EqualsIgnoreNullable(struct_type1, struct_type3));
         ASSERT_FALSE(ArrowUtils::EqualsIgnoreNullable(struct_type1, struct_type4));
     }
+    {
+        auto vector3 = arrow::fixed_size_list(arrow::float32(), 3);
+        auto vector3_non_null =
+            arrow::fixed_size_list(arrow::field("item", arrow::float32(), false), 3);
+        auto vector5 = arrow::fixed_size_list(arrow::float32(), 5);
+        ASSERT_TRUE(ArrowUtils::EqualsIgnoreNullable(vector3, vector3_non_null));
+        ASSERT_FALSE(ArrowUtils::EqualsIgnoreNullable(vector3, vector5));
+    }
     {
         // test complex
         auto key_field = arrow::field("key", arrow::int32(), /*nullable=*/false);
diff --git a/src/paimon/common/utils/arrow/vector_utils.cpp b/src/paimon/common/utils/arrow/vector_utils.cpp
new file mode 100644
index 000000000..e5cc8396f
--- /dev/null
+++ b/src/paimon/common/utils/arrow/vector_utils.cpp
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/utils/arrow/vector_utils.h"
+
+#include <cstdint>
+
+#include "arrow/array.h"
+#include "arrow/array/array_nested.h"
+#include "arrow/type.h"
+#include "fmt/format.h"
+#include "paimon/common/utils/checked_cast.h"
+
+namespace paimon {
+namespace {
+
+Status ValidateListVector(const arrow::ListArray& array) {
+    if (array.values()->null_count() == 0) {
+        return Status::OK();
+    }
+    for (int64_t i = 0; i < array.length(); ++i) {
+        if (array.IsNull(i)) {
+            continue;
+        }
+        int64_t value_offset = array.value_offset(i);
+        int64_t value_length = array.value_length(i);
+        for (int64_t j = 0; j < value_length; ++j) {
+            if (array.values()->IsNull(value_offset + j)) {
+                return Status::Invalid(fmt::format(
+                    "VECTOR cannot contain null elements, found one at row {} position {}", i, j));
+            }
+        }
+    }
+    return Status::OK();
+}
+
+Status ValidateFixedSizeListVector(const arrow::FixedSizeListArray& array) {
+    const auto& vector_type = checked_cast<const arrow::FixedSizeListType&>(*array.type());
+    int32_t vector_length = vector_type.list_size();
+    const std::shared_ptr<arrow::Array>& values = array.values();
+    // Arrow does not check this when importing an array over the C data interface, so the
+    // element scan below would otherwise read past the end of the values array.
+    if (values->length() < (array.offset() + array.length()) * vector_length) {
+        return Status::Invalid(fmt::format(
+            "VECTOR holds {} elements while {} rows of dimension {} require {}", values->length(),
+            array.length(), vector_length, (array.offset() + array.length()) * vector_length));
+    }
+    if (values->null_count() == 0) {
+        return Status::OK();
+    }
+    for (int64_t i = 0; i < array.length(); ++i) {
+        if (array.IsNull(i)) {
+            continue;
+        }
+        int64_t value_offset = (array.offset() + i) * vector_length;
+        for (int32_t j = 0; j < vector_length; ++j) {
+            if (values->IsNull(value_offset + j)) {
+                return Status::Invalid(fmt::format(
+                    "VECTOR cannot contain null elements, found one at row {} position {}", i, j));
+            }
+        }
+    }
+    return Status::OK();
+}
+
+}  // namespace
+
+bool VectorUtils::ContainsVectorType(const std::shared_ptr<arrow::DataType>& type) {
+    if (!type) {
+        return false;
+    }
+    if (type->id() == arrow::Type::FIXED_SIZE_LIST) {
+        return true;
+    }
+    for (const auto& field : type->fields()) {
+        if (ContainsVectorType(field->type())) {
+            return true;
+        }
+    }
+    return false;
+}
+
+bool VectorUtils::ContainsVectorField(const std::shared_ptr<arrow::Field>& field) {
+    return field != nullptr && ContainsVectorType(field->type());
+}
+
+bool VectorUtils::ContainsVector(const std::shared_ptr<arrow::Schema>& schema) {
+    if (!schema) {
+        return false;
+    }
+    for (const auto& field : schema->fields()) {
+        if (ContainsVectorField(field)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+Status VectorUtils::ValidateVectorElements(const arrow::Array& array) {
+    switch (array.type_id()) {
+        case arrow::Type::LIST:
+            return ValidateListVector(checked_cast<const arrow::ListArray&>(array));
+        case arrow::Type::FIXED_SIZE_LIST:
+            return ValidateFixedSizeListVector(
+                checked_cast<const arrow::FixedSizeListArray&>(array));
+        default:
+            return Status::Invalid(
+                fmt::format("Cannot validate VECTOR values of type {}", array.type()->ToString()));
+    }
+}
+
+}  // namespace paimon
diff --git a/src/paimon/common/utils/arrow/vector_utils.h b/src/paimon/common/utils/arrow/vector_utils.h
new file mode 100644
index 000000000..0031a5de7
--- /dev/null
+++ b/src/paimon/common/utils/arrow/vector_utils.h
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "paimon/status.h"
+#include "paimon/visibility.h"
+
+namespace arrow {
+class Array;
+class DataType;
+class Field;
+class Schema;
+}  // namespace arrow
+
+namespace paimon {
+
+/// Helpers shared by the schema, read and write paths handling VECTOR values, which are
+/// represented as Arrow FixedSizeList.
+class PAIMON_EXPORT VectorUtils {
+ public:
+    VectorUtils() = delete;
+    ~VectorUtils() = delete;
+
+    static bool ContainsVectorType(const std::shared_ptr<arrow::DataType>& type);
+
+    static bool ContainsVectorField(const std::shared_ptr<arrow::Field>& field);
+
+    static bool ContainsVector(const std::shared_ptr<arrow::Schema>& schema);
+
+    /// Rejects VECTOR values whose elements are not fully materialized or contain nulls.
+    /// `array` must be the List or FixedSizeList array holding the VECTOR values.
+    static Status ValidateVectorElements(const arrow::Array& array);
+};
+
+}  // namespace paimon
diff --git a/src/paimon/common/utils/arrow/vector_utils_test.cpp b/src/paimon/common/utils/arrow/vector_utils_test.cpp
new file mode 100644
index 000000000..1cce6853e
--- /dev/null
+++ b/src/paimon/common/utils/arrow/vector_utils_test.cpp
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/utils/arrow/vector_utils.h"
+
+#include <memory>
+
+#include "arrow/api.h"
+#include "arrow/ipc/json_simple.h"
+#include "gtest/gtest.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+namespace {
+
+std::shared_ptr<arrow::Array> ArrayFromJSON(const std::shared_ptr<arrow::DataType>& type,
+                                            const std::string& json) {
+    arrow::Result<std::shared_ptr<arrow::Array>> result =
+        arrow::ipc::internal::json::ArrayFromJSON(type, json);
+    EXPECT_TRUE(result.ok()) << result.status().ToString();
+    return std::move(result).ValueOrDie();
+}
+
+}  // namespace
+
+TEST(VectorUtilsTest, TestContainsVector) {
+    auto vector_type = arrow::fixed_size_list(arrow::float32(), 3);
+    ASSERT_TRUE(VectorUtils::ContainsVectorType(vector_type));
+    ASSERT_TRUE(VectorUtils::ContainsVectorType(arrow::list(vector_type)));
+    ASSERT_TRUE(VectorUtils::ContainsVectorType(arrow::map(arrow::utf8(), vector_type)));
+    ASSERT_TRUE(VectorUtils::ContainsVectorType(arrow::struct_({arrow::field("v", vector_type)})));
+    ASSERT_FALSE(VectorUtils::ContainsVectorType(arrow::list(arrow::float32())));
+    ASSERT_FALSE(VectorUtils::ContainsVectorType(nullptr));
+
+    ASSERT_TRUE(VectorUtils::ContainsVectorField(arrow::field("v", arrow::list(vector_type))));
+    ASSERT_FALSE(VectorUtils::ContainsVectorField(arrow::field("v", arrow::int32())));
+    ASSERT_FALSE(VectorUtils::ContainsVectorField(nullptr));
+
+    ASSERT_TRUE(VectorUtils::ContainsVector(
+        arrow::schema({arrow::field("id", arrow::int32()), arrow::field("v", vector_type)})));
+    ASSERT_FALSE(VectorUtils::ContainsVector(arrow::schema({arrow::field("id", arrow::int32())})));
+    ASSERT_FALSE(VectorUtils::ContainsVector(nullptr));
+}
+
+TEST(VectorUtilsTest, TestValidateVectorElements) {
+    auto vector_type = arrow::fixed_size_list(arrow::float32(), 3);
+    ASSERT_OK(VectorUtils::ValidateVectorElements(
+        *ArrayFromJSON(vector_type, R"([[1.0, 2.0, 3.0], null, [4.0, 5.0, 6.0]])")));
+    ASSERT_NOK_WITH_MSG(VectorUtils::ValidateVectorElements(
+                            *ArrayFromJSON(vector_type, R"([[1.0, 2.0, 3.0], [4.0, null, 6.0]])")),
+                        "VECTOR cannot contain null elements, found one at row 1 position 1");
+
+    // A sliced array must be validated against its own rows only.
+    std::shared_ptr<arrow::Array> sliced =
+        ArrayFromJSON(vector_type, R"([[1.0, null, 3.0], [4.0, 5.0, 6.0]])")->Slice(1, 1);
+    ASSERT_OK(VectorUtils::ValidateVectorElements(*sliced));
+
+    auto list_type = arrow::list(arrow::float32());
+    ASSERT_OK(VectorUtils::ValidateVectorElements(
+        *ArrayFromJSON(list_type, R"([[1.0, 2.0, 3.0], null])")));
+    ASSERT_NOK_WITH_MSG(
+        VectorUtils::ValidateVectorElements(*ArrayFromJSON(list_type, R"([[1.0, null, 3.0]])")),
+        "VECTOR cannot contain null elements, found one at row 0 position 1");
+
+    ASSERT_NOK_WITH_MSG(
+        VectorUtils::ValidateVectorElements(*ArrayFromJSON(arrow::int32(), "[1, 2]")),
+        "Cannot validate VECTOR values of type int32");
+}
+
+// Arrow does not check that a FixedSizeList child holds `length * list_size` values when
+// importing an array over the C data interface, so the element scan must reject it instead of
+// reading past the end of the child.
+TEST(VectorUtilsTest, TestValidateVectorElementsRejectsTruncatedValues) {
+    auto vector_type = arrow::fixed_size_list(arrow::float32(), 3);
+    std::shared_ptr<arrow::Array> values = ArrayFromJSON(arrow::float32(), "[1.0, null, 3.0]");
+    auto truncated = arrow::MakeArray(arrow::ArrayData::Make(vector_type, /*length=*/2, {nullptr},
+                                                             {values->data()},
+                                                             /*null_count=*/0));
+
+    ASSERT_NOK_WITH_MSG(VectorUtils::ValidateVectorElements(*truncated),
+                        "VECTOR holds 3 elements while 2 rows of dimension 3 require 6");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/common/utils/field_type_utils.h b/src/paimon/common/utils/field_type_utils.h
index 722467699..d2786e26e 100644
--- a/src/paimon/common/utils/field_type_utils.h
+++ b/src/paimon/common/utils/field_type_utils.h
@@ -93,6 +93,8 @@ class FieldTypeUtils {
                 return FieldType::MAP;
             case arrow::Type::type::STRUCT:
                 return FieldType::STRUCT;
+            case arrow::Type::type::FIXED_SIZE_LIST:
+                return FieldType::VECTOR;
             default:
                 return Status::Invalid(
                     fmt::format("Not support arrow type {}", static_cast<int32_t>(arrow_type)));
@@ -135,6 +137,8 @@ class FieldTypeUtils {
                 return "STRUCT";
             case FieldType::VARIANT:
                 return "VARIANT";
+            case FieldType::VECTOR:
+                return "VECTOR";
             default:
                 return "UNKNOWN, type id:" + std::to_string(static_cast<int32_t>(type));
         }
diff --git a/src/paimon/common/utils/field_type_utils_test.cpp b/src/paimon/common/utils/field_type_utils_test.cpp
index 50f602375..c70edb092 100644
--- a/src/paimon/common/utils/field_type_utils_test.cpp
+++ b/src/paimon/common/utils/field_type_utils_test.cpp
@@ -101,6 +101,10 @@ TEST(FieldTypeUtilsTest, ConvertToFieldType) {
     ASSERT_OK_AND_ASSIGN(result, FieldTypeUtils::ConvertToFieldType(arrow::Type::type::STRUCT));
     ASSERT_EQ(result, FieldType::STRUCT);
 
+    ASSERT_OK_AND_ASSIGN(result,
+                         FieldTypeUtils::ConvertToFieldType(arrow::Type::type::FIXED_SIZE_LIST));
+    ASSERT_EQ(result, FieldType::VECTOR);
+
     // Test unsupported Arrow type
     ASSERT_NOK(FieldTypeUtils::ConvertToFieldType(arrow::Type::type::UINT8));
 }
@@ -124,6 +128,7 @@ TEST(FieldTypeUtilsTest, FieldTypeToString) {
     ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::ARRAY), "ARRAY");
     ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::MAP), "MAP");
     ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::STRUCT), "STRUCT");
+    ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::VECTOR), "VECTOR");
 
     // Test UNKNOWN type
     auto unknown_type = static_cast<FieldType>(128);
diff --git a/src/paimon/core/io/vector_file_batch_reader.cpp b/src/paimon/core/io/vector_file_batch_reader.cpp
new file mode 100644
index 000000000..a4573eef9
--- /dev/null
+++ b/src/paimon/core/io/vector_file_batch_reader.cpp
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/io/vector_file_batch_reader.h"
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/c/abi.h"
+#include "arrow/c/bridge.h"
+#include "arrow/compute/api.h"
+#include "arrow/type.h"
+#include "fmt/format.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/arrow/vector_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/status.h"
+
+namespace paimon {
+namespace {
+
+std::shared_ptr<arrow::Field> FindField(const std::shared_ptr<arrow::DataType>& type,
+                                        const std::string& name) {
+    for (const auto& field : type->fields()) {
+        if (field->name() == name) {
+            return field;
+        }
+    }
+    return nullptr;
+}
+
+/// Rebuilds `map_type` with new key and item types, keeping the name and metadata of its
+/// entries field.
+std::shared_ptr<arrow::DataType> MakeMapType(const arrow::MapType& map_type,
+                                             const std::shared_ptr<arrow::Field>& key_field,
+                                             const std::shared_ptr<arrow::Field>& item_field) {
+    return std::make_shared<arrow::MapType>(
+        map_type.value_field()->WithType(arrow::struct_({key_field, item_field})),
+        map_type.keys_sorted());
+}
+
+/// Returns the type to request from the file format plugin. A VECTOR is only read back as a
+/// LIST when the file itself stores it as one: writers such as Paimon Java expose VECTOR
+/// columns as Arrow LIST, while Paimon Rust and Python expose them as FixedSizeList.
+std::shared_ptr<arrow::DataType> GetPhysicalReadType(
+    const std::shared_ptr<arrow::DataType>& logical_type,
+    const std::shared_ptr<arrow::DataType>& file_type) {
+    switch (logical_type->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+            if (!file_type || file_type->id() != arrow::Type::LIST) {
+                return logical_type;
+            }
+            const auto& vector_type = checked_cast<const arrow::FixedSizeListType&>(*logical_type);
+            const auto& list_type = checked_cast<const arrow::ListType&>(*file_type);
+            return arrow::list(vector_type.value_field()->WithType(
+                GetPhysicalReadType(vector_type.value_type(), list_type.value_type())));
+        }
+        case arrow::Type::STRUCT: {
+            if (!file_type || file_type->id() != arrow::Type::STRUCT) {
+                return logical_type;
+            }
+            arrow::FieldVector fields;
+            fields.reserve(logical_type->num_fields());
+            for (const auto& field : logical_type->fields()) {
+                std::shared_ptr<arrow::Field> file_field = FindField(file_type, field->name());
+                fields.push_back(field->WithType(
+                    GetPhysicalReadType(field->type(), file_field ? file_field->type() : nullptr)));
+            }
+            return arrow::struct_(fields);
+        }
+        case arrow::Type::LIST: {
+            if (!file_type || file_type->id() != arrow::Type::LIST) {
+                return logical_type;
+            }
+            return arrow::list(logical_type->field(0)->WithType(
+                GetPhysicalReadType(logical_type->field(0)->type(), file_type->field(0)->type())));
+        }
+        case arrow::Type::MAP: {
+            if (!file_type || file_type->id() != arrow::Type::MAP) {
+                return logical_type;
+            }
+            const auto& map_type = checked_cast<const arrow::MapType&>(*logical_type);
+            const auto& file_map_type = checked_cast<const arrow::MapType&>(*file_type);
+            return MakeMapType(map_type,
+                               map_type.key_field()->WithType(GetPhysicalReadType(
+                                   map_type.key_type(), file_map_type.key_type())),
+                               map_type.item_field()->WithType(GetPhysicalReadType(
+                                   map_type.item_type(), file_map_type.item_type())));
+        }
+        default:
+            return logical_type;
+    }
+}
+
+Result<std::shared_ptr<arrow::Array>> CastListToVector(
+    const std::shared_ptr<arrow::Array>& array,
+    const std::shared_ptr<arrow::FixedSizeListType>& read_type, arrow::MemoryPool* pool) {
+    if (array->type_id() != arrow::Type::LIST) {
+        return Status::Invalid(
+            fmt::format("Cannot restore VECTOR from type {}", array->type()->ToString()));
+    }
+    PAIMON_RETURN_NOT_OK(VectorUtils::ValidateVectorElements(*array));
+    arrow::compute::ExecContext exec_context(pool);
+    arrow::TypeHolder type_holder(read_type.get());
+    arrow::compute::CastOptions options = arrow::compute::CastOptions::Safe();
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::Array> result,
+        arrow::compute::Cast(*array, type_holder, options, &exec_context));
+    return result;
+}
+
+std::shared_ptr<arrow::DataType> RebuildNestedType(
+    const std::shared_ptr<arrow::DataType>& read_type,
+    const std::vector<std::shared_ptr<arrow::ArrayData>>& children) {
+    if (read_type->id() == arrow::Type::STRUCT) {
+        arrow::FieldVector fields;
+        fields.reserve(children.size());
+        for (int32_t i = 0; i < static_cast<int32_t>(children.size()); ++i) {
+            fields.push_back(read_type->field(i)->WithType(children[i]->type));
+        }
+        return arrow::struct_(fields);
+    }
+    if (read_type->id() == arrow::Type::LIST) {
+        return arrow::list(read_type->field(0)->WithType(children[0]->type));
+    }
+
+    const auto& entries_type = checked_cast<const arrow::StructType&>(*children[0]->type);
+    const auto& map_type = checked_cast<const arrow::MapType&>(*read_type);
+    return MakeMapType(map_type, map_type.key_field()->WithType(entries_type.field(0)->type()),
+                       map_type.item_field()->WithType(entries_type.field(1)->type()));
+}
+
+Result<std::shared_ptr<arrow::Array>> ConvertToReadType(
+    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
+    arrow::MemoryPool* pool) {
+    if (!VectorUtils::ContainsVectorType(read_type)) {
+        return array;
+    }
+    switch (read_type->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+            if (array->type_id() == arrow::Type::FIXED_SIZE_LIST) {
+                const auto& source_type =
+                    checked_cast<const arrow::FixedSizeListType&>(*array->type());
+                const auto& vector_type = checked_cast<const arrow::FixedSizeListType&>(*read_type);
+                if (source_type.list_size() != vector_type.list_size() ||
+                    !source_type.value_type()->Equals(vector_type.value_type())) {
+                    return Status::Invalid(fmt::format("VECTOR type mismatch: data {} vs read {}",
+                                                       source_type.ToString(),
+                                                       vector_type.ToString()));
+                }
+                PAIMON_RETURN_NOT_OK(VectorUtils::ValidateVectorElements(*array));
+                // Writers disagree on the element field, for example `element: float not null`
+                // for Paimon Rust against the `item: float` of a Paimon schema. Restore the
+                // requested type so that files storing VECTOR as LIST and files storing it as
+                // FixedSizeList produce batches of one type.
+                std::shared_ptr<arrow::ArrayData> data = array->data()->Copy();
+                data->type = read_type;
+                return arrow::MakeArray(data);
+            }
+            return CastListToVector(
+                array, checked_pointer_cast<arrow::FixedSizeListType>(read_type), pool);
+        }
+        case arrow::Type::STRUCT:
+        case arrow::Type::LIST:
+        case arrow::Type::MAP: {
+            if (array->type_id() != read_type->id()) {
+                return Status::Invalid(fmt::format("Cannot reconcile file type {} with {}",
+                                                   array->type()->ToString(),
+                                                   read_type->ToString()));
+            }
+            if (array->type()->num_fields() != read_type->num_fields() ||
+                array->data()->child_data.size() != static_cast<size_t>(read_type->num_fields())) {
+                return Status::Invalid(
+                    fmt::format("Cannot reconcile file type {} with {}: nested field count differs",
+                                array->type()->ToString(), read_type->ToString()));
+            }
+            std::vector<std::shared_ptr<arrow::ArrayData>> children;
+            children.reserve(read_type->num_fields());
+            for (int32_t i = 0; i < read_type->num_fields(); ++i) {
+                PAIMON_ASSIGN_OR_RAISE(
+                    std::shared_ptr<arrow::Array> child,
+                    ConvertToReadType(arrow::MakeArray(array->data()->child_data[i]),
+                                      read_type->field(i)->type(), pool));
+                children.push_back(child->data());
+            }
+            std::shared_ptr<arrow::ArrayData> data = array->data()->Copy();
+            data->child_data = std::move(children);
+            data->type = RebuildNestedType(read_type, data->child_data);
+            return arrow::MakeArray(data);
+        }
+        default:
+            return array;
+    }
+}
+
+}  // namespace
+
+VectorFileBatchReader::VectorFileBatchReader(std::unique_ptr<FileBatchReader>&& reader,
+                                             const std::shared_ptr<MemoryPool>& pool)
+    : arrow_pool_(GetArrowPool(pool)), reader_(std::move(reader)) {}
+
+bool VectorFileBatchReader::ContainsVector(const std::shared_ptr<arrow::Schema>& schema) {
+    return VectorUtils::ContainsVector(schema);
+}
+
+Status VectorFileBatchReader::SetReadSchema(
+    ::ArrowSchema* read_schema, const std::shared_ptr<Predicate>& predicate,
+    const std::optional<RoaringBitmap32>& selection_bitmap) {
+    if (!read_schema) {
+        return Status::Invalid("SetReadSchema failed: read schema cannot be nullptr");
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> logical_schema,
+                                      arrow::ImportSchema(read_schema));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema, reader_->GetFileSchema());
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> file_schema,
+                                      arrow::ImportSchema(c_file_schema.get()));
+    arrow::FieldVector physical_fields;
+    physical_fields.reserve(logical_schema->num_fields());
+    for (const auto& field : logical_schema->fields()) {
+        std::shared_ptr<arrow::Field> file_field = file_schema->GetFieldByName(field->name());
+        physical_fields.push_back(field->WithType(
+            GetPhysicalReadType(field->type(), file_field ? file_field->type() : nullptr)));
+    }
+    std::shared_ptr<arrow::Schema> physical_schema =
+        arrow::schema(physical_fields, logical_schema->metadata());
+    ArrowSchema c_physical_schema;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*physical_schema, &c_physical_schema));
+    PAIMON_RETURN_NOT_OK(reader_->SetReadSchema(&c_physical_schema, predicate, selection_bitmap));
+    read_type_ = arrow::struct_(logical_schema->fields());
+    return Status::OK();
+}
+
+Result<BatchReader::ReadBatch> VectorFileBatchReader::ConvertBatch(ReadBatch&& batch) const {
+    if (BatchReader::IsEofBatch(batch) || !read_type_) {
+        return std::move(batch);
+    }
+    auto& [c_array, c_schema] = batch;
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
+                                      arrow::ImportArray(c_array.get(), c_schema.get()));
+    PAIMON_ASSIGN_OR_RAISE(array, ConvertToReadType(array, read_type_, arrow_pool_.get()));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate());
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, c_array.get(), c_schema.get()));
+    return std::move(batch);
+}
+
+Result<BatchReader::ReadBatch> VectorFileBatchReader::NextBatch() {
+    PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, reader_->NextBatch());
+    return ConvertBatch(std::move(batch));
+}
+
+Result<BatchReader::ReadBatchWithBitmap> VectorFileBatchReader::NextBatchWithBitmap() {
+    PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch_with_bitmap, reader_->NextBatchWithBitmap());
+    if (BatchReader::IsEofBatch(batch_with_bitmap)) {
+        return std::move(batch_with_bitmap);
+    }
+    PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, ConvertBatch(std::move(batch_with_bitmap.first)));
+    batch_with_bitmap.first = std::move(batch);
+    return std::move(batch_with_bitmap);
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/vector_file_batch_reader.h b/src/paimon/core/io/vector_file_batch_reader.h
new file mode 100644
index 000000000..b7eab263c
--- /dev/null
+++ b/src/paimon/core/io/vector_file_batch_reader.h
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+
+#include "arrow/c/abi.h"
+#include "paimon/reader/file_batch_reader.h"
+
+namespace arrow {
+class DataType;
+class MemoryPool;
+class Schema;
+}  // namespace arrow
+
+namespace paimon {
+class MemoryPool;
+
+/// Reconciles logical VECTOR values with the variable-length LIST representation exposed to file
+/// format plugins.
+class VectorFileBatchReader : public FileBatchReader {
+ public:
+    VectorFileBatchReader(std::unique_ptr<FileBatchReader>&& reader,
+                          const std::shared_ptr<MemoryPool>& pool);
+
+    static bool ContainsVector(const std::shared_ptr<arrow::Schema>& schema);
+
+    Result<std::unique_ptr<::ArrowSchema>> GetFileSchema() const override {
+        return reader_->GetFileSchema();
+    }
+
+    Status SetReadSchema(::ArrowSchema* read_schema, const std::shared_ptr<Predicate>& predicate,
+                         const std::optional<RoaringBitmap32>& selection_bitmap) override;
+
+    Result<ReadBatch> NextBatch() override;
+
+    Result<ReadBatchWithBitmap> NextBatchWithBitmap() override;
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return reader_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        reader_->Close();
+    }
+
+    Result<uint64_t> GetPreviousBatchFileRowId(uint64_t batch_row_id) const override {
+        return reader_->GetPreviousBatchFileRowId(batch_row_id);
+    }
+
+    Result<uint64_t> GetNumberOfRows() const override {
+        return reader_->GetNumberOfRows();
+    }
+
+    bool SupportPreciseBitmapSelection() const override {
+        return reader_->SupportPreciseBitmapSelection();
+    }
+
+ private:
+    Result<ReadBatch> ConvertBatch(ReadBatch&& batch) const;
+
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    std::shared_ptr<arrow::DataType> read_type_;
+    std::unique_ptr<FileBatchReader> reader_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/vector_file_batch_reader_test.cpp b/src/paimon/core/io/vector_file_batch_reader_test.cpp
new file mode 100644
index 000000000..e334e62aa
--- /dev/null
+++ b/src/paimon/core/io/vector_file_batch_reader_test.cpp
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/io/vector_file_batch_reader.h"
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "arrow/ipc/json_simple.h"
+#include "gtest/gtest.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/mock/mock_file_batch_reader.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+namespace {
+
+std::shared_ptr<arrow::StructType> AsStructType(const std::shared_ptr<arrow::DataType>& type) {
+    return checked_pointer_cast<arrow::StructType>(type);
+}
+
+}  // namespace
+
+TEST(VectorFileBatchReaderTest, ConvertSchemaAndNextBatch) {
+    auto physical_type = AsStructType(arrow::struct_({
+        arrow::field("id", arrow::int32()),
+        arrow::field("embedding", arrow::list(arrow::float32())),
+    }));
+    auto logical_type = AsStructType(arrow::struct_({
+        arrow::field("id", arrow::int32()),
+        arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3)),
+    }));
+    const std::string json = R"([
+        [1, [1.0, 2.0, 3.0]],
+        [2, null],
+        [3, [4.0, 5.0, 6.0]]
+    ])";
+    auto physical_array =
+        arrow::ipc::internal::json::ArrayFromJSON(physical_type, json).ValueOrDie();
+    auto mock_reader =
+        std::make_unique<MockFileBatchReader>(physical_array, physical_type, /*batch_size=*/10);
+    mock_reader->EnableRandomizeBatchSize(false);
+    MockFileBatchReader* inner_reader = mock_reader.get();
+    VectorFileBatchReader reader(std::move(mock_reader), GetDefaultPool());
+
+    ASSERT_TRUE(VectorFileBatchReader::ContainsVector(arrow::schema(logical_type->fields())));
+    ASSERT_FALSE(VectorFileBatchReader::ContainsVector(arrow::schema(physical_type->fields())));
+    ArrowSchema c_read_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(logical_type->fields()), &c_read_schema).ok());
+    ASSERT_OK(reader.SetReadSchema(&c_read_schema, /*predicate=*/nullptr,
+                                   /*selection_bitmap=*/std::nullopt));
+    ASSERT_EQ(inner_reader->read_schema_->field(1)->type()->id(), arrow::Type::LIST);
+
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader.NextBatch());
+    arrow::Result<std::shared_ptr<arrow::Array>> actual_result =
+        arrow::ImportArray(batch.first.get(), batch.second.get());
+    ASSERT_TRUE(actual_result.ok()) << actual_result.status().ToString();
+    std::shared_ptr<arrow::Array> actual = std::move(actual_result).ValueOrDie();
+    auto expected = arrow::ipc::internal::json::ArrayFromJSON(logical_type, json).ValueOrDie();
+    ASSERT_TRUE(expected->Equals(actual)) << actual->ToString();
+    ASSERT_OK_AND_ASSIGN(batch, reader.NextBatch());
+    ASSERT_TRUE(BatchReader::IsEofBatch(batch));
+}
+
+TEST(VectorFileBatchReaderTest, KeepFixedSizeListFileSchema) {
+    auto logical_type = AsStructType(arrow::struct_({
+        arrow::field("id", arrow::int32()),
+        arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3)),
+    }));
+    const std::string json = R"([
+        [1, [1.0, 2.0, 3.0]],
+        [2, null],
+        [3, [4.0, 5.0, 6.0]]
+    ])";
+    auto logical_array = arrow::ipc::internal::json::ArrayFromJSON(logical_type, json).ValueOrDie();
+    auto mock_reader =
+        std::make_unique<MockFileBatchReader>(logical_array, logical_type, /*batch_size=*/10);
+    mock_reader->EnableRandomizeBatchSize(false);
+    MockFileBatchReader* inner_reader = mock_reader.get();
+    VectorFileBatchReader reader(std::move(mock_reader), GetDefaultPool());
+
+    ArrowSchema c_read_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(logical_type->fields()), &c_read_schema).ok());
+    ASSERT_OK(reader.SetReadSchema(&c_read_schema, /*predicate=*/nullptr,
+                                   /*selection_bitmap=*/std::nullopt));
+    ASSERT_EQ(inner_reader->read_schema_->field(1)->type()->id(), arrow::Type::FIXED_SIZE_LIST);
+
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader.NextBatch());
+    arrow::Result<std::shared_ptr<arrow::Array>> actual_result =
+        arrow::ImportArray(batch.first.get(), batch.second.get());
+    ASSERT_TRUE(actual_result.ok()) << actual_result.status().ToString();
+    ASSERT_TRUE(logical_array->Equals(std::move(actual_result).ValueOrDie()));
+}
+
+// Paimon Rust names the element field of a VECTOR column `element` and marks it non-nullable,
+// while a Paimon schema names it `item`. Batches must carry the requested type either way,
+// otherwise they cannot be combined with batches read from a file storing VECTOR as LIST.
+TEST(VectorFileBatchReaderTest, NormalizeFixedSizeListElementField) {
+    auto file_vector =
+        arrow::fixed_size_list(arrow::field("element", arrow::float32(), /*nullable=*/false), 3);
+    auto logical_vector = arrow::fixed_size_list(arrow::float32(), 3);
+    auto file_type = AsStructType(arrow::struct_({
+        arrow::field("embedding", file_vector),
+        arrow::field("history", arrow::list(file_vector)),
+    }));
+    auto logical_type = AsStructType(arrow::struct_({
+        arrow::field("embedding", logical_vector),
+        arrow::field("history", arrow::list(logical_vector)),
+    }));
+    const std::string json = R"([
+        [[1.0, 2.0, 3.0], [[4.0, 5.0, 6.0]]],
+        [null, []]
+    ])";
+    auto file_array = arrow::ipc::internal::json::ArrayFromJSON(file_type, json).ValueOrDie();
+    auto mock_reader =
+        std::make_unique<MockFileBatchReader>(file_array, file_type, /*batch_size=*/10);
+    mock_reader->EnableRandomizeBatchSize(false);
+    VectorFileBatchReader reader(std::move(mock_reader), GetDefaultPool());
+
+    ArrowSchema c_read_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(logical_type->fields()), &c_read_schema).ok());
+    ASSERT_OK(reader.SetReadSchema(&c_read_schema, /*predicate=*/nullptr,
+                                   /*selection_bitmap=*/std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader.NextBatch());
+    arrow::Result<std::shared_ptr<arrow::Array>> actual_result =
+        arrow::ImportArray(batch.first.get(), batch.second.get());
+    ASSERT_TRUE(actual_result.ok()) << actual_result.status().ToString();
+    std::shared_ptr<arrow::Array> actual = std::move(actual_result).ValueOrDie();
+    ASSERT_TRUE(actual->type()->Equals(logical_type)) << actual->type()->ToString();
+    auto expected = arrow::ipc::internal::json::ArrayFromJSON(logical_type, json).ValueOrDie();
+    ASSERT_TRUE(expected->Equals(actual)) << actual->ToString();
+}
+
+TEST(VectorFileBatchReaderTest, ConvertNestedVectorsWithBitmap) {
+    auto logical_vector =
+        arrow::fixed_size_list(arrow::field("item", arrow::float64(), /*nullable=*/false), 2);
+    auto physical_vector = arrow::list(arrow::field("item", arrow::float64(), /*nullable=*/false));
+    auto logical_type = AsStructType(arrow::struct_({
+        arrow::field("vectors", arrow::list(logical_vector)),
+        arrow::field("by_name", arrow::map(arrow::utf8(), logical_vector)),
+    }));
+    auto physical_type = AsStructType(arrow::struct_({
+        arrow::field("vectors", arrow::list(physical_vector)),
+        arrow::field("by_name", arrow::map(arrow::utf8(), physical_vector)),
+    }));
+    const std::string json = R"([[[[1.0, 2.0], null], [["a", [3.0, 4.0]]]],
+                                 [null, [["b", null]]]])";
+    auto physical_array =
+        arrow::ipc::internal::json::ArrayFromJSON(physical_type, json).ValueOrDie();
+    RoaringBitmap32 bitmap;
+    bitmap.Add(1);
+    auto mock_reader = std::make_unique<MockFileBatchReader>(physical_array, physical_type, bitmap,
+                                                             /*read_batch_size=*/10);
+    mock_reader->EnableRandomizeBatchSize(false);
+    VectorFileBatchReader reader(std::move(mock_reader), GetDefaultPool());
+    ArrowSchema c_read_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(logical_type->fields()), &c_read_schema).ok());
+    ASSERT_OK(reader.SetReadSchema(&c_read_schema, /*predicate=*/nullptr,
+                                   /*selection_bitmap=*/std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch_with_bitmap,
+                         reader.NextBatchWithBitmap());
+    ASSERT_FALSE(batch_with_bitmap.second.Contains(0));
+    ASSERT_TRUE(batch_with_bitmap.second.Contains(1));
+    arrow::Result<std::shared_ptr<arrow::Array>> actual_result = arrow::ImportArray(
+        batch_with_bitmap.first.first.get(), batch_with_bitmap.first.second.get());
+    ASSERT_TRUE(actual_result.ok()) << actual_result.status().ToString();
+    std::shared_ptr<arrow::Array> actual = std::move(actual_result).ValueOrDie();
+    auto expected = arrow::ipc::internal::json::ArrayFromJSON(logical_type, json).ValueOrDie();
+    ASSERT_TRUE(expected->Equals(actual)) << actual->ToString();
+}
+
+TEST(VectorFileBatchReaderTest, RejectInvalidVectorValues) {
+    auto physical_type =
+        AsStructType(arrow::struct_({arrow::field("embedding", arrow::list(arrow::float32()))}));
+    auto logical_type = AsStructType(
+        arrow::struct_({arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3))}));
+    for (const char* json : {R"([[[1.0, 2.0]]])", R"([[[1.0, null, 3.0]]])"}) {
+        auto physical_array =
+            arrow::ipc::internal::json::ArrayFromJSON(physical_type, json).ValueOrDie();
+        auto mock_reader = std::make_unique<MockFileBatchReader>(physical_array, physical_type,
+                                                                 /*read_batch_size=*/10);
+        mock_reader->EnableRandomizeBatchSize(false);
+        VectorFileBatchReader reader(std::move(mock_reader), GetDefaultPool());
+        ArrowSchema c_read_schema;
+        ASSERT_TRUE(
+            arrow::ExportSchema(*arrow::schema(logical_type->fields()), &c_read_schema).ok());
+        ASSERT_OK(reader.SetReadSchema(&c_read_schema, /*predicate=*/nullptr,
+                                       /*selection_bitmap=*/std::nullopt));
+        ASSERT_NOK(reader.NextBatch());
+    }
+}
+
+TEST(VectorFileBatchReaderTest, RejectInvalidFixedSizeListVectorValues) {
+    auto physical_type = AsStructType(
+        arrow::struct_({arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3))}));
+    auto physical_array =
+        arrow::ipc::internal::json::ArrayFromJSON(physical_type, R"([[[1.0, null, 3.0]]])")
+            .ValueOrDie();
+    auto mock_reader = std::make_unique<MockFileBatchReader>(physical_array, physical_type,
+                                                             /*read_batch_size=*/10);
+    mock_reader->EnableRandomizeBatchSize(false);
+    VectorFileBatchReader reader(std::move(mock_reader), GetDefaultPool());
+    ArrowSchema c_read_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(physical_type->fields()), &c_read_schema).ok());
+    ASSERT_OK(reader.SetReadSchema(&c_read_schema, /*predicate=*/nullptr,
+                                   /*selection_bitmap=*/std::nullopt));
+    ASSERT_NOK(reader.NextBatch());
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp
index 40fb6e5be..bb82c5d82 100644
--- a/src/paimon/core/operation/abstract_split_read.cpp
+++ b/src/paimon/core/operation/abstract_split_read.cpp
@@ -43,6 +43,7 @@
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/io/data_file_path_factory.h"
 #include "paimon/core/io/field_mapping_reader.h"
+#include "paimon/core/io/vector_file_batch_reader.h"
 #include "paimon/core/operation/internal_read_context.h"
 #include "paimon/core/partition/partition_info.h"
 #include "paimon/core/schema/table_schema.h"
@@ -215,6 +216,9 @@ Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::CreateFieldMappingRe
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileBatchReader> file_reader,
                            CreateFileBatchReader(file_format_identifier, data_file_path,
                                                  file_meta->file_size, reader_builder));
+    if (VectorFileBatchReader::ContainsVector(read_schema)) {
+        file_reader = std::make_unique<VectorFileBatchReader>(std::move(file_reader), pool_);
+    }
     std::set<int32_t> skip_map_selected_keys_filter_field_ids;
     if (file_format_identifier != "blob") {
         std::pair<std::unique_ptr<FileBatchReader>, std::set<int32_t>> shared_shredding_result;
diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h
index 94a59fa02..b4f80adb6 100644
--- a/src/paimon/core/operation/data_evolution_split_read.h
+++ b/src/paimon/core/operation/data_evolution_split_read.h
@@ -65,7 +65,7 @@ struct DeletionFile;
 /// ->FieldMappingReader->(ApplyDeletionVectorBatchReader)->(ApplyBitmapIndexBatchReader)
 /// ->(CompleteRowTrackingFieldsBatchReader)->(ShreddingFileReader)
 /// ->(MapSharedShreddingFileReader)
-/// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader
+/// ->(VectorFileBatchReader)->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader
 ///
 ///
 /// A union `SplitRead` to read multiple inner files to merge columns. A single-file row range
diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h
index ac211b257..6a97b9b37 100644
--- a/src/paimon/core/operation/raw_file_split_read.h
+++ b/src/paimon/core/operation/raw_file_split_read.h
@@ -54,8 +54,8 @@ struct DeletionFile;
 /// splits)->CompleteRowKindBatchReader->(PredicateBatchReader)
 /// ->ConcatBatchReader across
 /// files->FieldMappingReader->(ApplyBitmapIndexBatchReader)->(CompleteRowTrackingFieldsBatchReader)
-/// ->(ShreddingFileReader)->(MapSharedShreddingFileReader)->(DelegatingPrefetchReader)
-/// ->(PrefetchFileBatchReader)->FormatReader
+/// ->(ShreddingFileReader)->(MapSharedShreddingFileReader)->(VectorFileBatchReader)
+/// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader
 
 class RawFileSplitRead : public AbstractSplitRead {
  public:
diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp
index f78e15506..0db1145f7 100644
--- a/src/paimon/core/schema/arrow_schema_validator.cpp
+++ b/src/paimon/core/schema/arrow_schema_validator.cpp
@@ -28,6 +28,7 @@
 #include "paimon/common/data/variant/variant_access_utils.h"
 #include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/common/types/vector_type.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/decimal_utils.h"
 #include "paimon/common/utils/string_utils.h"
@@ -41,6 +42,7 @@ namespace paimon {
 
 bool ArrowSchemaValidator::IsNestedType(const std::shared_ptr<arrow::DataType>& data_type) {
     return (data_type->id() == arrow::Type::MAP || data_type->id() == arrow::Type::LIST ||
+            data_type->id() == arrow::Type::FIXED_SIZE_LIST ||
             data_type->id() == arrow::Type::STRUCT);
 }
 
@@ -128,6 +130,14 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId(
                 value_field->type(), value_field->metadata(), /*allow_blob=*/false, field_id_set));
             break;
         }
+        case arrow::Type::type::FIXED_SIZE_LIST: {
+            const auto& vector_type = checked_cast<const arrow::FixedSizeListType&>(*type);
+            if (vector_type.list_size() < 1 ||
+                !VectorType::IsValidElementType(vector_type.value_type())) {
+                return Status::Invalid("Invalid VECTOR type: ", type->ToString());
+            }
+            break;
+        }
         case arrow::Type::type::STRUCT: {
             if (VariantTypeUtils::IsVariantMetadata(key_value_metadata)) {
                 // A variant struct is a leaf type: its value/metadata children carry fixed
@@ -203,6 +213,18 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr<arrow::Field>&
             PAIMON_RETURN_NOT_OK(ValidateField(value_field, /*allow_blob=*/false));
             break;
         }
+        case arrow::Type::type::FIXED_SIZE_LIST: {
+            const auto& vector_type = checked_cast<const arrow::FixedSizeListType&>(*field->type());
+            if (vector_type.list_size() < 1) {
+                return Status::Invalid("Vector length must be positive, but was ",
+                                       vector_type.list_size());
+            }
+            if (!VectorType::IsValidElementType(vector_type.value_type())) {
+                return Status::Invalid("Invalid element type for vector: ",
+                                       vector_type.value_type()->ToString());
+            }
+            break;
+        }
         case arrow::Type::type::STRUCT: {
             if (VariantTypeUtils::IsVariantField(field)) {
                 if (VariantAccessUtils::IsVariantAccessType(field->type())) {
diff --git a/src/paimon/core/schema/arrow_schema_validator_test.cpp b/src/paimon/core/schema/arrow_schema_validator_test.cpp
index 0363dff6a..ed56b6370 100644
--- a/src/paimon/core/schema/arrow_schema_validator_test.cpp
+++ b/src/paimon/core/schema/arrow_schema_validator_test.cpp
@@ -53,14 +53,29 @@ TEST(ArrowSchemaValidatorTest, TestSimple) {
         "col16",
         arrow::struct_({arrow::field("sub1", arrow::int8()), arrow::field("sub2", arrow::int16()),
                         arrow::field("sub3", arrow::int64())}));
+    auto col17_field = arrow::field("col17", arrow::fixed_size_list(arrow::float32(), 3));
 
-    auto arrow_schema = arrow::schema(
-        arrow::FieldVector({col1_field, col2_field, col3_field, col4_field, col5_field, col6_field,
-                            col7_field, col8_field, col9_field, col10_field, col11_field,
-                            col12_field, col13_field, col14_field, col15_field, col16_field}));
+    auto arrow_schema = arrow::schema(arrow::FieldVector(
+        {col1_field, col2_field, col3_field, col4_field, col5_field, col6_field, col7_field,
+         col8_field, col9_field, col10_field, col11_field, col12_field, col13_field, col14_field,
+         col15_field, col16_field, col17_field}));
     ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*arrow_schema));
 }
 
+TEST(ArrowSchemaValidatorTest, TestVectorElementType) {
+    for (const auto& element_type :
+         {arrow::boolean(), arrow::int8(), arrow::int16(), arrow::int32(), arrow::int64(),
+          arrow::float32(), arrow::float64()}) {
+        auto vector = arrow::field("embedding", arrow::fixed_size_list(element_type, 3));
+        ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*arrow::schema({vector})));
+    }
+    for (const auto& element_type : {arrow::utf8()}) {
+        auto vector = arrow::field("embedding", arrow::fixed_size_list(element_type, 3));
+        ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow::schema({vector})),
+                            "Invalid element type for vector");
+    }
+}
+
 TEST(ArrowSchemaValidatorTest, TestValidateNoRedundantFields) {
     auto col1_field = arrow::field("col1", arrow::int64());
     auto col2_field = arrow::field("col2", arrow::int32());
diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp
index 7b8947ea1..7342826d4 100644
--- a/src/paimon/core/schema/schema_validation.cpp
+++ b/src/paimon/core/schema/schema_validation.cpp
@@ -38,6 +38,7 @@
 #include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/arrow/vector_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/object_utils.h"
 #include "paimon/common/utils/preconditions.h"
@@ -98,6 +99,15 @@ Status ValidateSharedShreddingFileFormat(const std::string& option_key,
     return Status::OK();
 }
 
+Status ValidateVectorFileFormat(const std::string& option_key, const std::string& file_format) {
+    if (StringUtils::ToLowerCase(file_format) != "parquet") {
+        return Status::Invalid(
+            fmt::format("VECTOR currently only supports parquet data files, but {} is {}.",
+                        option_key, file_format));
+    }
+    return Status::OK();
+}
+
 Status ValidatePerLevelOption(
     const std::map<std::string, std::string>& options, const std::string& option_key,
     const std::function<Status(const std::string&, const std::string&)>& validator) {
@@ -188,6 +198,7 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) {
     PAIMON_RETURN_NOT_OK(ValidateRowTracking(schema, options));
     PAIMON_RETURN_NOT_OK(ValidateBlobFields(schema, options));
     PAIMON_RETURN_NOT_OK(ValidateMapStorageLayout(schema, options));
+    PAIMON_RETURN_NOT_OK(ValidateVectorFields(schema, options));
     return Status::OK();
 }
 
@@ -622,6 +633,9 @@ Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema,
         if (ContainsBlobField(map_type->item_field())) {
             return Status::Invalid("MAP shared-shredding currently cannot contain BLOB fields.");
         }
+        if (VectorUtils::ContainsVectorField(map_type->item_field())) {
+            return Status::Invalid("MAP shared-shredding currently cannot contain VECTOR fields.");
+        }
         // Validate max-columns config
         PAIMON_RETURN_NOT_OK(options.GetMapSharedShreddingMaxColumns(field_name));
         // Validate placement policy config
@@ -648,4 +662,30 @@ Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema,
     return Status::OK();
 }
 
+Status SchemaValidation::ValidateVectorFields(const TableSchema& schema,
+                                              const CoreOptions& options) {
+    bool has_vector = false;
+    for (const auto& field : schema.Fields()) {
+        if (VectorUtils::ContainsVectorField(field.ArrowField())) {
+            has_vector = true;
+            break;
+        }
+    }
+    if (!has_vector) {
+        return Status::OK();
+    }
+    if (!schema.PrimaryKeys().empty()) {
+        return Status::NotImplemented(
+            "VECTOR fields in primary-key tables are not implemented yet.");
+    }
+    if (options.DataEvolutionEnabled()) {
+        return Status::NotImplemented(
+            "VECTOR fields in data-evolution tables are not implemented yet.");
+    }
+    PAIMON_RETURN_NOT_OK(
+        ValidateVectorFileFormat(Options::FILE_FORMAT, options.GetFileFormat()->Identifier()));
+    return ValidatePerLevelOption(options.ToMap(), Options::FILE_FORMAT_PER_LEVEL,
+                                  ValidateVectorFileFormat);
+}
+
 }  // namespace paimon
diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h
index 613372ff8..abf4d5b02 100644
--- a/src/paimon/core/schema/schema_validation.h
+++ b/src/paimon/core/schema/schema_validation.h
@@ -75,6 +75,8 @@ class SchemaValidation {
 
     static Status ValidateMapStorageLayout(const TableSchema& schema, const CoreOptions& options);
 
+    static Status ValidateVectorFields(const TableSchema& schema, const CoreOptions& options);
+
     static bool IsComplexType(const std::shared_ptr<arrow::Field>& field);
 };
 
diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp
index 4c878113c..47603497b 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -46,6 +46,69 @@ TEST(SchemaValidationTest, TestSimple) {
     ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
 }
 
+TEST(SchemaValidationTest, TestVectorType) {
+    auto vector_field = arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3));
+    auto schema = arrow::schema({arrow::field("id", arrow::int64()), vector_field});
+    std::map<std::string, std::string> parquet_options = {{Options::BUCKET, "-1"},
+                                                          {Options::FILE_FORMAT, "parquet"}};
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
+                         TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                             /*primary_keys=*/{}, parquet_options));
+    ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+
+    std::map<std::string, std::string> orc_options = {{Options::BUCKET, "-1"},
+                                                      {Options::FILE_FORMAT, "orc"}};
+    ASSERT_OK_AND_ASSIGN(table_schema,
+                         TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                             /*primary_keys=*/{}, orc_options));
+    ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                        "VECTOR currently only supports parquet data files");
+
+    std::map<std::string, std::string> primary_key_options = {{Options::BUCKET, "1"}};
+    ASSERT_OK_AND_ASSIGN(table_schema,
+                         TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                             /*primary_keys=*/{"embedding"}, primary_key_options));
+    ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                        "in primary key field embedding is unsupported");
+
+    primary_key_options[Options::FILE_FORMAT] = "parquet";
+    ASSERT_OK_AND_ASSIGN(table_schema,
+                         TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                             /*primary_keys=*/{"id"}, primary_key_options));
+    ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                        "VECTOR fields in primary-key tables are not implemented yet.");
+
+    auto nested_schema = arrow::schema({
+        arrow::field("id", arrow::int64()),
+        arrow::field("payload", arrow::struct_({arrow::field("embedding", vector_field->type())})),
+    });
+    ASSERT_OK_AND_ASSIGN(
+        table_schema,
+        TableSchema::Create(/*schema_id=*/0, nested_schema,
+                            /*partition_keys=*/{}, /*primary_keys=*/{"id"}, primary_key_options));
+    ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                        "VECTOR fields in primary-key tables are not implemented yet.");
+
+    std::map<std::string, std::string> data_evolution_options = {
+        {Options::BUCKET, "-1"},
+        {Options::FILE_FORMAT, "parquet"},
+        {Options::ROW_TRACKING_ENABLED, "true"},
+        {Options::DATA_EVOLUTION_ENABLED, "true"},
+    };
+    ASSERT_OK_AND_ASSIGN(table_schema,
+                         TableSchema::Create(/*schema_id=*/0, schema,
+                                             /*partition_keys=*/{},
+                                             /*primary_keys=*/{}, data_evolution_options));
+    ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                        "VECTOR fields in data-evolution tables are not implemented yet.");
+    ASSERT_OK_AND_ASSIGN(table_schema,
+                         TableSchema::Create(/*schema_id=*/0, nested_schema,
+                                             /*partition_keys=*/{},
+                                             /*primary_keys=*/{}, data_evolution_options));
+    ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                        "VECTOR fields in data-evolution tables are not implemented yet.");
+}
+
 TEST(SchemaValidationTest, TestRowTracking) {
     auto f0 = arrow::field("f0", arrow::utf8());
     auto f1 = arrow::field("f1", arrow::int32());
diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp
index d7be7f115..6e2d8747e 100644
--- a/src/paimon/core/schema/table_schema.cpp
+++ b/src/paimon/core/schema/table_schema.cpp
@@ -118,6 +118,14 @@ Result<std::shared_ptr<arrow::Field>> TableSchema::AssignFieldIdsRecursively(
                                                          /*set_field_id=*/false, field_id));
         return arrow::field(field->name(), arrow::list(new_value_field), field->nullable(),
                             metadata);
+    } else if (type->id() == arrow::Type::FIXED_SIZE_LIST) {
+        auto vector_type = checked_pointer_cast<arrow::FixedSizeListType>(field->type());
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Field> new_value_field,
+                               AssignFieldIdsRecursively(vector_type->value_field(),
+                                                         /*set_field_id=*/false, field_id));
+        return arrow::field(field->name(),
+                            arrow::fixed_size_list(new_value_field, vector_type->list_size()),
+                            field->nullable(), metadata);
     } else if (field->type()->id() == arrow::Type::MAP) {
         auto map_type = checked_pointer_cast<arrow::MapType>(field->type());
         std::shared_ptr<arrow::Field> key_field = map_type->key_field();
diff --git a/src/paimon/core/utils/field_mapping.cpp b/src/paimon/core/utils/field_mapping.cpp
index 447d8d589..be7287dd6 100644
--- a/src/paimon/core/utils/field_mapping.cpp
+++ b/src/paimon/core/utils/field_mapping.cpp
@@ -185,7 +185,7 @@ Result<std::vector<std::shared_ptr<CastExecutor>>> FieldMappingBuilder::CreateDa
         if (!read_fields[i].Type()->Equals(data_fields[i].Type())) {
             auto read_type_id = read_fields[i].Type()->id();
             if (read_type_id == arrow::Type::STRUCT || read_type_id == arrow::Type::LIST ||
-                read_type_id == arrow::Type::MAP) {
+                read_type_id == arrow::Type::MAP || read_type_id == arrow::Type::FIXED_SIZE_LIST) {
                 // Nested type differs by pruning/evolution; the reader's reshape
                 // handles it, no scalar cast.
                 cast_executors.push_back(nullptr);
diff --git a/src/paimon/format/parquet/CMakeLists.txt b/src/paimon/format/parquet/CMakeLists.txt
index a1a566c08..c31e3cc35 100644
--- a/src/paimon/format/parquet/CMakeLists.txt
+++ b/src/paimon/format/parquet/CMakeLists.txt
@@ -20,6 +20,7 @@ set(PAIMON_PARQUET_FILE_FORMAT
     file_reader_wrapper.cpp
     page_filtered_row_group_reader.cpp
     parquet_timestamp_converter.cpp
+    parquet_vector_converter.cpp
     parquet_file_batch_reader.cpp
     parquet_file_format_factory.cpp
     parquet_format_writer.cpp
@@ -55,6 +56,8 @@ if(PAIMON_BUILD_TESTS)
                     file_reader_wrapper_test.cpp
                     page_filtered_row_group_reader_test.cpp
                     parquet_timestamp_converter_test.cpp
+                    parquet_vector_converter_test.cpp
+                    parquet_vector_io_test.cpp
                     parquet_field_id_converter_test.cpp
                     parquet_file_batch_reader_test.cpp
                     parquet_format_writer_test.cpp
diff --git a/src/paimon/format/parquet/parquet_field_id_converter.cpp b/src/paimon/format/parquet/parquet_field_id_converter.cpp
index 56d36d36e..adb271734 100644
--- a/src/paimon/format/parquet/parquet_field_id_converter.cpp
+++ b/src/paimon/format/parquet/parquet_field_id_converter.cpp
@@ -105,6 +105,12 @@ arrow::Result<std::shared_ptr<Field>> ParquetFieldIdConverter::ProcessField(
                               ProcessField(list_type->value_field(), convert_type));
         auto new_type = arrow::list(new_value_field);
         return field->WithType(new_type)->WithMergedMetadata(updated_metadata);
+    } else if (type->id() == arrow::Type::FIXED_SIZE_LIST) {
+        auto vector_type = checked_pointer_cast<arrow::FixedSizeListType>(type);
+        ARROW_ASSIGN_OR_RAISE(auto new_value_field,
+                              ProcessField(vector_type->value_field(), convert_type));
+        auto new_type = arrow::fixed_size_list(new_value_field, vector_type->list_size());
+        return field->WithType(new_type)->WithMergedMetadata(updated_metadata);
     } else if (type->id() == arrow::Type::MAP) {
         auto map_type = checked_pointer_cast<arrow::MapType>(type);
         ARROW_ASSIGN_OR_RAISE(auto new_key_field,
diff --git a/src/paimon/format/parquet/parquet_field_id_converter_test.cpp b/src/paimon/format/parquet/parquet_field_id_converter_test.cpp
index 7f7c114ed..d2053f120 100644
--- a/src/paimon/format/parquet/parquet_field_id_converter_test.cpp
+++ b/src/paimon/format/parquet/parquet_field_id_converter_test.cpp
@@ -186,7 +186,8 @@ TEST_F(ParquetFieldIdConverterTest, TestNestedType) {
                                       arrow::field("sub2", arrow::timestamp(arrow::TimeUnit::NANO)),
                                       arrow::field("sub3", arrow::decimal128(23, 5)),
                                       arrow::field("sub4", arrow::binary()),
-                                      arrow::field("sub5", arrow::binary())})))};
+                                      arrow::field("sub5", arrow::binary())}))),
+        arrow::field("f3", arrow::fixed_size_list(arrow::float32(), 7))};
     auto schema = arrow::schema(fields);
     ASSERT_OK_AND_ASSIGN(
         auto table_schema,
@@ -211,8 +212,11 @@ TEST_F(ParquetFieldIdConverterTest, TestNestedType) {
         {"sub4", arrow::Type::BINARY, "16"},     {"sub5", arrow::Type::BINARY, "17"},
         {"sub1", arrow::Type::DATE32, "18"},     {"sub2", arrow::Type::TIMESTAMP, "19"},
         {"sub3", arrow::Type::DECIMAL128, "20"}, {"sub4", arrow::Type::BINARY, "21"},
-        {"sub5", arrow::Type::BINARY, "22"}};
+        {"sub5", arrow::Type::BINARY, "22"},     {"f3", arrow::Type::FIXED_SIZE_LIST, "23"}};
     ASSERT_EQ(expected_field_infos, field_infos);
+    auto new_vector =
+        checked_pointer_cast<arrow::FixedSizeListType>(new_schema->GetFieldByName("f3")->type());
+    ASSERT_EQ(new_vector->list_size(), 7);
     // convert to paimon.id
     ASSERT_OK_AND_ASSIGN(auto old_schema,
                          ParquetFieldIdConverter::GetPaimonIdsFromParquetIds(new_schema));
@@ -220,6 +224,9 @@ TEST_F(ParquetFieldIdConverterTest, TestNestedType) {
     PrintFieldMetadata(old_schema, ParquetFieldIdConverter::IdConvertType::PARQUET_TO_PAIMON_ID,
                        &old_field_infos);
     ASSERT_EQ(expected_field_infos, old_field_infos);
+    auto old_vector =
+        checked_pointer_cast<arrow::FixedSizeListType>(old_schema->GetFieldByName("f3")->type());
+    ASSERT_EQ(old_vector->list_size(), 7);
 }
 
 }  // namespace paimon::parquet::test
diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp
index 20e5e0ea7..0c9d065e5 100644
--- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp
+++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp
@@ -41,6 +41,7 @@
 #include "fmt/format.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/options_utils.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/common/utils/string_utils.h"
@@ -114,6 +115,12 @@ bool HasSameNestedProjectionShape(const std::shared_ptr<arrow::DataType>& read_t
             const auto& file_list = static_cast<const arrow::ListType&>(*file_type);
             return HasSameNestedProjectionShape(read_list.value_type(), file_list.value_type());
         }
+        case arrow::Type::FIXED_SIZE_LIST: {
+            const auto& read_vector = checked_cast<const arrow::FixedSizeListType&>(*read_type);
+            const auto& file_vector = checked_cast<const arrow::FixedSizeListType&>(*file_type);
+            return read_vector.list_size() == file_vector.list_size() &&
+                   HasSameNestedProjectionShape(read_vector.value_type(), file_vector.value_type());
+        }
         case arrow::Type::MAP: {
             const auto& read_map = static_cast<const arrow::MapType&>(*read_type);
             const auto& file_map = static_cast<const arrow::MapType&>(*file_type);
@@ -754,6 +761,16 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptr<arrow::D
         const auto& file_list = static_cast<const arrow::ListType&>(*file_type);
         PAIMON_RETURN_NOT_OK(CollectLeafIndices(read_list.value_type(), file_list.value_type(),
                                                 leaf_index, indices));
+    } else if (file_type->id() == arrow::Type::FIXED_SIZE_LIST) {
+        if (!HasSameNestedProjectionShape(read_type, file_type)) {
+            return Status::Invalid(fmt::format(
+                "Parquet does not support partial projection inside list/map: src {} vs target {}",
+                file_type->ToString(), read_type->ToString()));
+        }
+        const auto& read_vector = checked_cast<const arrow::FixedSizeListType&>(*read_type);
+        const auto& file_vector = checked_cast<const arrow::FixedSizeListType&>(*file_type);
+        PAIMON_RETURN_NOT_OK(CollectLeafIndices(read_vector.value_type(), file_vector.value_type(),
+                                                leaf_index, indices));
     } else if (file_type->id() == arrow::Type::MAP) {
         if (!HasSameNestedProjectionShape(read_type, file_type)) {
             return Status::Invalid(fmt::format(
@@ -775,8 +792,7 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptr<arrow::D
 
 void ParquetFileBatchReader::SkipLeafIndices(const std::shared_ptr<arrow::DataType>& file_type,
                                              int32_t* leaf_index) {
-    if (file_type->id() == arrow::Type::STRUCT || file_type->id() == arrow::Type::LIST ||
-        file_type->id() == arrow::Type::MAP) {
+    if (ArrowSchemaValidator::IsNestedType(file_type)) {
         for (int32_t i = 0; i < file_type->num_fields(); i++) {
             SkipLeafIndices(file_type->field(i)->type(), leaf_index);
         }
diff --git a/src/paimon/format/parquet/parquet_format_writer.cpp b/src/paimon/format/parquet/parquet_format_writer.cpp
index 0a8e38b43..6e69e6945 100644
--- a/src/paimon/format/parquet/parquet_format_writer.cpp
+++ b/src/paimon/format/parquet/parquet_format_writer.cpp
@@ -23,6 +23,7 @@
 #include <string_view>
 #include <utility>
 
+#include "arrow/array/array_nested.h"
 #include "arrow/c/bridge.h"
 #include "arrow/memory_pool.h"
 #include "arrow/record_batch.h"
@@ -31,7 +32,9 @@
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/utils/arrow/arrow_output_stream_adapter.h"
 #include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
 #include "paimon/format/parquet/parquet_format_defs.h"
+#include "paimon/format/parquet/parquet_vector_converter.h"
 #include "parquet/arrow/writer.h"
 #include "parquet/properties.h"
 
@@ -55,17 +58,33 @@ Result<std::unique_ptr<ParquetFormatWriter>> ParquetFormatWriter::Create(
     ::parquet::ArrowWriterProperties::Builder arrow_properties_builder;
     auto arrow_writer_properties =
         arrow_properties_builder.enable_deprecated_int96_timestamps()->build();
+    auto logical_type = arrow::struct_(schema->fields());
+    auto write_type =
+        checked_pointer_cast<arrow::StructType>(ParquetVectorConverter::GetWriteType(logical_type));
+    auto write_schema = arrow::schema(write_type->fields(), schema->metadata());
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
         std::unique_ptr<::parquet::arrow::FileWriter> file_writer,
-        ::parquet::arrow::FileWriter::Open(*schema, pool.get(), out, writer_properties,
+        ::parquet::arrow::FileWriter::Open(*write_schema, pool.get(), out, writer_properties,
                                            arrow_writer_properties));
-    return std::unique_ptr<ParquetFormatWriter>(
-        new ParquetFormatWriter(std::move(file_writer), out, schema, max_memory_use, pool));
+    return std::unique_ptr<ParquetFormatWriter>(new ParquetFormatWriter(
+        std::move(file_writer), out, schema, max_memory_use,
+        /*needs_vector_conversion=*/!logical_type->Equals(write_type), pool));
 }
 
 Status ParquetFormatWriter::AddBatch(ArrowArray* batch) {
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<::arrow::RecordBatch> record_batch,
                                       arrow::ImportRecordBatch(batch, schema_));
+    if (needs_vector_conversion_) {
+        // TODO(ChaomingZhangCN): Remove this conversion after upgrading Arrow. Arrow 17
+        // mishandles nullable FixedSizeList values when writing them as Parquet LIST.
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> struct_array,
+                                          record_batch->ToStructArray());
+        std::shared_ptr<arrow::Array> array = struct_array;
+        PAIMON_ASSIGN_OR_RAISE(array,
+                               ParquetVectorConverter::ConvertToWriteType(array, pool_.get()));
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(record_batch,
+                                          arrow::RecordBatch::FromStructArray(array, pool_.get()));
+    }
     if (static_cast<uint64_t>(pool_->bytes_allocated()) > max_memory_use_) {
         PAIMON_RETURN_NOT_OK_FROM_ARROW(writer_->NewBufferedRowGroup());
     }
@@ -113,13 +132,14 @@ Result<uint64_t> ParquetFormatWriter::GetEstimateLength() const {
 ParquetFormatWriter::ParquetFormatWriter(std::unique_ptr<::parquet::arrow::FileWriter> writer,
                                          const std::shared_ptr<ArrowOutputStreamAdapter>& out,
                                          const std::shared_ptr<arrow::Schema>& schema,
-                                         uint64_t max_memory_use,
+                                         uint64_t max_memory_use, bool needs_vector_conversion,
                                          const std::shared_ptr<arrow::MemoryPool>& pool)
     : pool_(pool),
       out_(out),
       writer_(std::move(writer)),
       schema_(schema),
       metrics_(std::make_shared<MetricsImpl>()),
-      max_memory_use_(max_memory_use) {}
+      max_memory_use_(max_memory_use),
+      needs_vector_conversion_(needs_vector_conversion) {}
 
 }  // namespace paimon::parquet
diff --git a/src/paimon/format/parquet/parquet_format_writer.h b/src/paimon/format/parquet/parquet_format_writer.h
index 4ab58d73c..f8f441195 100644
--- a/src/paimon/format/parquet/parquet_format_writer.h
+++ b/src/paimon/format/parquet/parquet_format_writer.h
@@ -72,6 +72,7 @@ class ParquetFormatWriter : public FormatWriter {
     ParquetFormatWriter(std::unique_ptr<::parquet::arrow::FileWriter> writer,
                         const std::shared_ptr<ArrowOutputStreamAdapter>& out,
                         const std::shared_ptr<arrow::Schema>& schema, uint64_t max_memory_use,
+                        bool needs_vector_conversion,
                         const std::shared_ptr<arrow::MemoryPool>& pool);
 
     Result<uint64_t> GetEstimateLength() const;
@@ -83,6 +84,7 @@ class ParquetFormatWriter : public FormatWriter {
     std::shared_ptr<Metrics> metrics_;
     int64_t total_records_written_ = 0;
     uint64_t max_memory_use_;
+    bool needs_vector_conversion_;
 };
 
 }  // namespace paimon::parquet
diff --git a/src/paimon/format/parquet/parquet_stats_extractor.cpp b/src/paimon/format/parquet/parquet_stats_extractor.cpp
index 4b8f97f0f..8dfe7faac 100644
--- a/src/paimon/format/parquet/parquet_stats_extractor.cpp
+++ b/src/paimon/format/parquet/parquet_stats_extractor.cpp
@@ -296,7 +296,9 @@ ParquetStatsExtractor::ExtractWithFileInfo(const std::shared_ptr<FileSystem>& fi
             // nested type do not have parquet stats
             const auto& logical_type = node->logical_type();
             FieldType nested_type = FieldType::UNKNOWN;
-            if (logical_type->is_list()) {
+            if (write_schema_->field(field_idx)->type()->id() == arrow::Type::FIXED_SIZE_LIST) {
+                nested_type = FieldType::VECTOR;
+            } else if (logical_type->is_list()) {
                 nested_type = FieldType::ARRAY;
             } else if (logical_type->is_map()) {
                 nested_type = FieldType::MAP;
diff --git a/src/paimon/format/parquet/parquet_stats_extractor_test.cpp b/src/paimon/format/parquet/parquet_stats_extractor_test.cpp
index 4dc7fbe58..65998426d 100644
--- a/src/paimon/format/parquet/parquet_stats_extractor_test.cpp
+++ b/src/paimon/format/parquet/parquet_stats_extractor_test.cpp
@@ -62,7 +62,8 @@ class ParquetStatsExtractorTest : public ::testing::Test {
     void TearDown() override {}
 
     void CheckStats(const arrow::FieldVector& fields, const std::string& input,
-                    const std::vector<std::string>& expected_stats, int64_t expect_row_count) {
+                    const std::vector<std::string>& expected_stats, int64_t expect_row_count,
+                    const std::vector<FieldType>& expected_types = {}) {
         auto arrow_schema = arrow::schema(fields);
         auto struct_type = arrow::struct_(fields);
         std::map<std::string, std::string> options;
@@ -95,6 +96,12 @@ class ParquetStatsExtractorTest : public ::testing::Test {
         for (size_t i = 0; i < expected_stats.size(); i++) {
             ASSERT_EQ(expected_stats[i], col_stats_vec[i]->ToString());
         }
+        if (!expected_types.empty()) {
+            ASSERT_EQ(col_stats_vec.size(), expected_types.size());
+            for (size_t i = 0; i < expected_types.size(); ++i) {
+                ASSERT_EQ(col_stats_vec[i]->GetFieldType(), expected_types[i]);
+            }
+        }
         auto row_count = result.second.GetRowCount();
         ASSERT_EQ(row_count, expect_row_count);
     }
@@ -237,6 +244,13 @@ TEST_F(ParquetStatsExtractorTest, TestExtractStatsComplexType) {
     CheckStats(fields, data_str, expected_stats_str, /*expect_row_count=*/6);
 }
 
+TEST_F(ParquetStatsExtractorTest, TestExtractVectorStats) {
+    arrow::FieldVector fields = {
+        arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3))};
+    CheckStats(fields, R"([[[1.0, 2.0, 3.0]], [null]])", {"min null, max null, null count null"},
+               /*expect_row_count=*/2, {FieldType::VECTOR});
+}
+
 TEST_F(ParquetStatsExtractorTest, TestNullForAllType) {
     auto timezone = DateTimeUtils::GetLocalTimezoneName();
     arrow::FieldVector fields = {
diff --git a/src/paimon/format/parquet/parquet_vector_converter.cpp b/src/paimon/format/parquet/parquet_vector_converter.cpp
new file mode 100644
index 000000000..5b6446d22
--- /dev/null
+++ b/src/paimon/format/parquet/parquet_vector_converter.cpp
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/format/parquet/parquet_vector_converter.h"
+
+#include <cstdint>
+#include <limits>
+#include <memory>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/array/array_nested.h"
+#include "arrow/array/builder_primitive.h"
+#include "arrow/compute/api.h"
+#include "arrow/type.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/arrow/vector_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/status.h"
+
+namespace paimon::parquet {
+namespace {
+
+Result<std::shared_ptr<arrow::Array>> CastToListType(
+    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& write_type,
+    arrow::MemoryPool* pool) {
+    arrow::compute::ExecContext exec_context(pool);
+    arrow::TypeHolder type_holder(write_type.get());
+    arrow::compute::CastOptions options = arrow::compute::CastOptions::Safe();
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::Array> result,
+        arrow::compute::Cast(*array, type_holder, options, &exec_context));
+    return result;
+}
+
+/// Rebuilds a nullable VECTOR as a LIST whose null slots have a zero length, dropping the
+/// values Arrow keeps for them.
+///
+/// TODO(ChaomingZhangCN): Cast the whole array once Arrow is upgraded. Arrow 17 casts a null
+/// FixedSizeList row to a null LIST slot spanning `list_size` values, and the Parquet writer
+/// rejects a LIST with non-zero length null slots.
+Result<std::shared_ptr<arrow::Array>> CompactNullVectorsToList(
+    const arrow::FixedSizeListArray& vector_array,
+    const std::shared_ptr<arrow::DataType>& write_type, arrow::MemoryPool* pool) {
+    const auto& vector_type = checked_cast<const arrow::FixedSizeListType&>(*vector_array.type());
+    const int32_t vector_length = vector_type.list_size();
+    if (vector_array.length() > std::numeric_limits<int32_t>::max() / vector_length) {
+        return Status::Invalid("VECTOR values exceed the maximum Parquet LIST offset");
+    }
+
+    arrow::Int32Builder offsets_builder(pool);
+    arrow::Int64Builder indices_builder(pool);
+    arrow::BooleanBuilder validity_builder(pool);
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Reserve(vector_array.length() + 1));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Reserve(vector_array.length() * vector_length));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Reserve(vector_array.length()));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(0));
+
+    int32_t offset = 0;
+    for (int64_t i = 0; i < vector_array.length(); ++i) {
+        bool valid = !vector_array.IsNull(i);
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Append(valid));
+        if (valid) {
+            int64_t value_offset = (vector_array.offset() + i) * vector_length;
+            for (int32_t j = 0; j < vector_length; ++j) {
+                PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Append(value_offset + j));
+            }
+            offset += vector_length;
+        }
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(offset));
+    }
+
+    std::shared_ptr<arrow::Array> offsets;
+    std::shared_ptr<arrow::Array> indices;
+    std::shared_ptr<arrow::Array> validity;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Finish(&offsets));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Finish(&indices));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Finish(&validity));
+
+    arrow::compute::ExecContext exec_context(pool);
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        arrow::Datum values,
+        arrow::compute::Take(arrow::Datum(vector_array.values()), arrow::Datum(indices),
+                             arrow::compute::TakeOptions::NoBoundsCheck(), &exec_context));
+    return std::make_shared<arrow::ListArray>(
+        write_type, vector_array.length(), offsets->data()->buffers[1], values.make_array(),
+        validity->data()->buffers[1], vector_array.null_count());
+}
+
+}  // namespace
+
+std::shared_ptr<arrow::DataType> ParquetVectorConverter::GetWriteType(
+    const std::shared_ptr<arrow::DataType>& logical_type) {
+    switch (logical_type->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+            const auto& vector_type = checked_cast<const arrow::FixedSizeListType&>(*logical_type);
+            return arrow::list(
+                vector_type.value_field()->WithType(GetWriteType(vector_type.value_type())));
+        }
+        case arrow::Type::STRUCT: {
+            arrow::FieldVector fields;
+            fields.reserve(logical_type->num_fields());
+            for (const auto& field : logical_type->fields()) {
+                fields.push_back(field->WithType(GetWriteType(field->type())));
+            }
+            return arrow::struct_(fields);
+        }
+        case arrow::Type::LIST:
+            return arrow::list(
+                logical_type->field(0)->WithType(GetWriteType(logical_type->field(0)->type())));
+        case arrow::Type::MAP: {
+            const auto& map_type = checked_cast<const arrow::MapType&>(*logical_type);
+            return std::make_shared<arrow::MapType>(
+                map_type.value_field()->WithType(arrow::struct_(
+                    {map_type.key_field()->WithType(GetWriteType(map_type.key_type())),
+                     map_type.item_field()->WithType(GetWriteType(map_type.item_type()))})),
+                map_type.keys_sorted());
+        }
+        default:
+            return logical_type;
+    }
+}
+
+Result<std::shared_ptr<arrow::Array>> ParquetVectorConverter::ConvertToWriteType(
+    const std::shared_ptr<arrow::Array>& array, arrow::MemoryPool* pool) {
+    if (!VectorUtils::ContainsVectorType(array->type())) {
+        return array;
+    }
+    std::shared_ptr<arrow::DataType> write_type = GetWriteType(array->type());
+    switch (array->type_id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+            PAIMON_RETURN_NOT_OK(VectorUtils::ValidateVectorElements(*array));
+            const auto& vector_array = checked_cast<const arrow::FixedSizeListArray&>(*array);
+            if (vector_array.null_count() == 0) {
+                return CastToListType(array, write_type, pool);
+            }
+            return CompactNullVectorsToList(vector_array, write_type, pool);
+        }
+        case arrow::Type::STRUCT:
+        case arrow::Type::LIST:
+        case arrow::Type::MAP: {
+            std::vector<std::shared_ptr<arrow::ArrayData>> children;
+            children.reserve(array->data()->child_data.size());
+            for (const auto& child_data : array->data()->child_data) {
+                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> child,
+                                       ConvertToWriteType(arrow::MakeArray(child_data), pool));
+                children.push_back(child->data());
+            }
+            std::shared_ptr<arrow::ArrayData> data = array->data()->Copy();
+            data->child_data = std::move(children);
+            data->type = write_type;
+            return arrow::MakeArray(data);
+        }
+        default:
+            return array;
+    }
+}
+
+}  // namespace paimon::parquet
diff --git a/src/paimon/format/parquet/parquet_vector_converter.h b/src/paimon/format/parquet/parquet_vector_converter.h
new file mode 100644
index 000000000..a265e2d12
--- /dev/null
+++ b/src/paimon/format/parquet/parquet_vector_converter.h
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "arrow/memory_pool.h"
+#include "paimon/result.h"
+
+namespace arrow {
+class Array;
+class DataType;
+}  // namespace arrow
+
+namespace paimon::parquet {
+
+/// Converts logical FixedSizeList VECTOR arrays to Parquet LIST arrays.
+class ParquetVectorConverter {
+ public:
+    ParquetVectorConverter() = delete;
+    ~ParquetVectorConverter() = delete;
+
+    static Result<std::shared_ptr<arrow::Array>> ConvertToWriteType(
+        const std::shared_ptr<arrow::Array>& array, arrow::MemoryPool* pool);
+
+    static std::shared_ptr<arrow::DataType> GetWriteType(
+        const std::shared_ptr<arrow::DataType>& logical_type);
+};
+
+}  // namespace paimon::parquet
diff --git a/src/paimon/format/parquet/parquet_vector_converter_test.cpp b/src/paimon/format/parquet/parquet_vector_converter_test.cpp
new file mode 100644
index 000000000..6e1c0b0df
--- /dev/null
+++ b/src/paimon/format/parquet/parquet_vector_converter_test.cpp
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/format/parquet/parquet_vector_converter.h"
+
+#include <memory>
+
+#include "arrow/api.h"
+#include "arrow/ipc/json_simple.h"
+#include "gtest/gtest.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::parquet::test {
+
+TEST(ParquetVectorConverterTest, ConvertNullableVectorToList) {
+    auto vector_type = arrow::fixed_size_list(arrow::float32(), 3);
+    auto vector_array = arrow::ipc::internal::json::ArrayFromJSON(
+                            vector_type, R"([[1.0, 2.0, 3.0], null, [4.0, 5.0, 6.0]])")
+                            .ValueOrDie();
+
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<arrow::Array> converted,
+        ParquetVectorConverter::ConvertToWriteType(vector_array, arrow::default_memory_pool()));
+    ASSERT_EQ(converted->type()->id(), arrow::Type::LIST);
+    auto list_array = checked_pointer_cast<arrow::ListArray>(converted);
+    ASSERT_EQ(list_array->value_length(0), 3);
+    ASSERT_TRUE(list_array->IsNull(1));
+    // The Parquet writer rejects a null LIST slot spanning values, so the values Arrow keeps for
+    // a null VECTOR row are dropped.
+    ASSERT_EQ(list_array->value_length(1), 0);
+    ASSERT_EQ(list_array->value_length(2), 3);
+    ASSERT_EQ(list_array->values()->length(), 6);
+    auto values = checked_pointer_cast<arrow::FloatArray>(list_array->values());
+    ASSERT_FLOAT_EQ(values->Value(3), 4.0f);
+}
+
+TEST(ParquetVectorConverterTest, ConvertNestedVectorsToList) {
+    auto vector_type =
+        arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 2);
+    auto nested_type = arrow::struct_({
+        arrow::field("vectors", arrow::list(vector_type)),
+        arrow::field("by_name", arrow::map(arrow::utf8(), vector_type)),
+    });
+    auto nested_array =
+        arrow::ipc::internal::json::ArrayFromJSON(nested_type,
+                                                  R"([[[[1.0, 2.0], null], [["a", [3.0, 4.0]]]],
+                                 [null, [["b", null]]]])")
+            .ValueOrDie();
+
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<arrow::Array> physical_array,
+        ParquetVectorConverter::ConvertToWriteType(nested_array, arrow::default_memory_pool()));
+    auto physical_type = checked_pointer_cast<arrow::StructType>(physical_array->type());
+    auto physical_list = checked_pointer_cast<arrow::ListType>(physical_type->field(0)->type());
+    auto physical_map = checked_pointer_cast<arrow::MapType>(physical_type->field(1)->type());
+    ASSERT_EQ(physical_list->value_type()->id(), arrow::Type::LIST);
+    ASSERT_EQ(physical_map->item_type()->id(), arrow::Type::LIST);
+}
+
+TEST(ParquetVectorConverterTest, ConvertSlicedVectorToList) {
+    auto vector_type = arrow::fixed_size_list(arrow::float64(), 2);
+    auto vector_array =
+        arrow::ipc::internal::json::ArrayFromJSON(vector_type, R"([[1.0, 2.0], [3.0, 4.0], null])")
+            .ValueOrDie()
+            ->Slice(1, 2);
+
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<arrow::Array> converted,
+        ParquetVectorConverter::ConvertToWriteType(vector_array, arrow::default_memory_pool()));
+    auto list_array = checked_pointer_cast<arrow::ListArray>(converted);
+    ASSERT_EQ(list_array->length(), 2);
+    ASSERT_EQ(list_array->value_length(0), 2);
+    ASSERT_TRUE(list_array->IsNull(1));
+    auto values = checked_pointer_cast<arrow::DoubleArray>(list_array->values());
+    ASSERT_DOUBLE_EQ(values->Value(0), 3.0);
+    ASSERT_DOUBLE_EQ(values->Value(1), 4.0);
+}
+
+}  // namespace paimon::parquet::test
diff --git a/src/paimon/format/parquet/parquet_vector_io_test.cpp b/src/paimon/format/parquet/parquet_vector_io_test.cpp
new file mode 100644
index 000000000..e177cc1b6
--- /dev/null
+++ b/src/paimon/format/parquet/parquet_vector_io_test.cpp
@@ -0,0 +1,437 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/c/abi.h"
+#include "arrow/c/bridge.h"
+#include "arrow/ipc/json_simple.h"
+#include "gtest/gtest.h"
+#include "paimon/common/utils/arrow/arrow_input_stream_adapter.h"
+#include "paimon/common/utils/arrow/arrow_output_stream_adapter.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/io/vector_file_batch_reader.h"
+#include "paimon/defs.h"
+#include "paimon/format/parquet/parquet_file_batch_reader.h"
+#include "paimon/format/parquet/parquet_format_defs.h"
+#include "paimon/format/parquet/parquet_format_writer.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/testing/utils/read_result_collector.h"
+#include "paimon/testing/utils/testharness.h"
+#include "parquet/arrow/writer.h"
+#include "parquet/properties.h"
+
+namespace paimon {
+class Predicate;
+}  // namespace paimon
+
+namespace paimon::parquet::test {
+
+class ParquetVectorIoTest : public ::testing::Test {
+ public:
+    void SetUp() override {
+        pool_ = GetDefaultPool();
+        arrow_pool_ = GetArrowPool(pool_);
+        dir_ = paimon::test::UniqueTestDirectory::Create();
+        ASSERT_TRUE(dir_);
+        fs_ = dir_->GetFileSystem();
+    }
+
+    void WriteAndCheck(const std::string& file_name,
+                       const std::shared_ptr<arrow::StructType>& write_type,
+                       const std::shared_ptr<arrow::StructType>& read_type,
+                       const std::string& json) {
+        std::string file_path = dir_->Str() + "/" + file_name;
+        WriteWithFormatWriter(file_path, write_type, json, /*max_row_group_length=*/1024);
+
+        std::shared_ptr<arrow::StructType> file_type;
+        ReadFileType(file_path, &file_type);
+        std::shared_ptr<arrow::DataType> physical_value_type = file_type->field(1)->type();
+        if (physical_value_type->id() == arrow::Type::STRUCT) {
+            physical_value_type = physical_value_type->field(0)->type();
+        }
+        ASSERT_EQ(physical_value_type->id(), arrow::Type::LIST);
+
+        std::unique_ptr<FileBatchReader> vector_reader;
+        CreateVectorReader(file_path, arrow::schema(read_type->fields()), /*predicate=*/nullptr,
+                           /*options=*/{}, /*batch_size=*/10, &vector_reader);
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> actual,
+                             paimon::test::ReadResultCollector::CollectResult(vector_reader.get()));
+
+        arrow::Result<std::shared_ptr<arrow::Array>> expected_result =
+            arrow::ipc::internal::json::ArrayFromJSON(read_type, json);
+        ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString();
+        std::shared_ptr<arrow::Array> expected = std::move(expected_result).ValueOrDie();
+        ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(actual))
+            << actual->ToString();
+    }
+
+    /// Writes the JSON rows through the Paimon Parquet writer, which stores VECTOR values as
+    /// Parquet LIST.
+    void WriteWithFormatWriter(const std::string& file_path,
+                               const std::shared_ptr<arrow::StructType>& write_type,
+                               const std::string& json, int64_t max_row_group_length) {
+        arrow::Result<std::shared_ptr<arrow::Array>> write_array_result =
+            arrow::ipc::internal::json::ArrayFromJSON(write_type, json);
+        ASSERT_TRUE(write_array_result.ok()) << write_array_result.status().ToString();
+        std::shared_ptr<arrow::Array> write_array = std::move(write_array_result).ValueOrDie();
+        auto c_array = std::make_unique<ArrowArray>();
+        ASSERT_TRUE(arrow::ExportArray(*write_array, c_array.get()).ok());
+
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<OutputStream> out,
+                             fs_->Create(file_path, /*overwrite=*/false));
+        ::parquet::WriterProperties::Builder properties_builder;
+        properties_builder.max_row_group_length(max_row_group_length);
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr<ParquetFormatWriter> writer,
+            ParquetFormatWriter::Create(out, arrow::schema(write_type->fields()),
+                                        properties_builder.build(),
+                                        DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE, arrow_pool_));
+        ASSERT_OK(writer->AddBatch(c_array.get()));
+        ASSERT_OK(writer->Finish());
+        ASSERT_OK(out->Close());
+    }
+
+    /// Writes `array` with the plain Arrow Parquet writer, storing the Arrow schema so that
+    /// FixedSizeList columns are read back as FixedSizeList, the way Paimon Rust and Python
+    /// writers store them.
+    void WriteWithArrowWriter(const std::string& file_path,
+                              const std::shared_ptr<arrow::StructType>& type,
+                              const std::string& json) {
+        arrow::Result<std::shared_ptr<arrow::Array>> array_result =
+            arrow::ipc::internal::json::ArrayFromJSON(type, json);
+        ASSERT_TRUE(array_result.ok()) << array_result.status().ToString();
+        arrow::Result<std::shared_ptr<arrow::RecordBatch>> batch_result =
+            arrow::RecordBatch::FromStructArray(std::move(array_result).ValueOrDie());
+        ASSERT_TRUE(batch_result.ok()) << batch_result.status().ToString();
+        arrow::Result<std::shared_ptr<arrow::Table>> table_result =
+            arrow::Table::FromRecordBatches({std::move(batch_result).ValueOrDie()});
+        ASSERT_TRUE(table_result.ok()) << table_result.status().ToString();
+
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<OutputStream> out,
+                             fs_->Create(file_path, /*overwrite=*/false));
+        auto arrow_out = std::make_shared<ArrowOutputStreamAdapter>(out);
+        ::parquet::WriterProperties::Builder properties_builder;
+        std::shared_ptr<::parquet::ArrowWriterProperties> arrow_properties =
+            ::parquet::ArrowWriterProperties::Builder().store_schema()->build();
+        arrow::Status status = ::parquet::arrow::WriteTable(
+            *std::move(table_result).ValueOrDie(), arrow_pool_.get(), arrow_out,
+            /*chunk_size=*/1024, properties_builder.build(), arrow_properties);
+        ASSERT_TRUE(status.ok()) << status.ToString();
+        ASSERT_OK(out->Close());
+    }
+
+    void ReadFileType(const std::string& file_path,
+                      std::shared_ptr<arrow::StructType>* file_type_out) {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_path));
+        ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
+        auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr<ParquetFileBatchReader> reader,
+            ParquetFileBatchReader::Create(std::move(in_stream), /*options=*/{},
+                                           /*batch_size=*/10, /*file_metadata=*/nullptr,
+                                           /*storage_read_bytes=*/nullptr, arrow_pool_));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<ArrowSchema> c_file_schema, reader->GetFileSchema());
+        arrow::Result<std::shared_ptr<arrow::DataType>> file_type_result =
+            arrow::ImportType(c_file_schema.get());
+        ASSERT_TRUE(file_type_result.ok()) << file_type_result.status().ToString();
+        *file_type_out =
+            checked_pointer_cast<arrow::StructType>(std::move(file_type_result).ValueOrDie());
+    }
+
+    void CreateVectorReader(const std::string& file_path,
+                            const std::shared_ptr<arrow::Schema>& read_schema,
+                            const std::shared_ptr<Predicate>& predicate,
+                            const std::map<std::string, std::string>& options, int32_t batch_size,
+                            std::unique_ptr<FileBatchReader>* vector_reader_out) {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_path));
+        ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
+        auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr<ParquetFileBatchReader> reader,
+            ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size,
+                                           /*file_metadata=*/nullptr,
+                                           /*storage_read_bytes=*/nullptr, arrow_pool_));
+        std::unique_ptr<FileBatchReader> vector_reader =
+            std::make_unique<VectorFileBatchReader>(std::move(reader), pool_);
+        auto c_schema = std::make_unique<ArrowSchema>();
+        ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok());
+        ASSERT_OK(vector_reader->SetReadSchema(c_schema.get(), predicate,
+                                               /*selection_bitmap=*/std::nullopt));
+        *vector_reader_out = std::move(vector_reader);
+    }
+
+    void ReadFixtureAndCheck(
+        const std::string& file_name, arrow::Type::type expected_file_vector_type,
+        int32_t vector_length, const std::vector<int32_t>& expected_ids,
+        const std::vector<std::optional<std::vector<float>>>& expected_vectors) {
+        std::string file_path =
+            paimon::test::GetDataDir() + "/parquet/vector_compatibility/" + file_name;
+        std::shared_ptr<arrow::StructType> file_type;
+        ReadFileType(file_path, &file_type);
+        std::shared_ptr<arrow::Field> file_vector_field = file_type->GetFieldByName("embedding");
+        ASSERT_TRUE(file_vector_field);
+        ASSERT_EQ(file_vector_field->type()->id(), expected_file_vector_type);
+        std::shared_ptr<arrow::Field> file_id_field = file_type->GetFieldByName("id");
+        ASSERT_TRUE(file_id_field);
+
+        auto vector_type = arrow::fixed_size_list(
+            arrow::field("element", arrow::float32(), /*nullable=*/false), vector_length);
+        auto logical_schema =
+            arrow::schema({file_id_field, file_vector_field->WithType(vector_type)});
+        std::unique_ptr<FileBatchReader> vector_reader;
+        CreateVectorReader(file_path, logical_schema, /*predicate=*/nullptr, /*options=*/{},
+                           /*batch_size=*/10, &vector_reader);
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> actual,
+                             paimon::test::ReadResultCollector::CollectResult(vector_reader.get()));
+        ASSERT_EQ(actual->num_chunks(), 1);
+        ASSERT_EQ(actual->type()->id(), arrow::Type::STRUCT);
+        auto struct_array = checked_pointer_cast<arrow::StructArray>(actual->chunk(0));
+        std::shared_ptr<arrow::Array> id_field = struct_array->GetFieldByName("id");
+        std::shared_ptr<arrow::Array> vector_field = struct_array->GetFieldByName("embedding");
+        ASSERT_TRUE(id_field);
+        ASSERT_TRUE(vector_field);
+        ASSERT_EQ(id_field->type_id(), arrow::Type::INT32);
+        ASSERT_EQ(vector_field->type_id(), arrow::Type::FIXED_SIZE_LIST);
+        auto ids = checked_pointer_cast<arrow::Int32Array>(id_field);
+        auto vector_array = checked_pointer_cast<arrow::FixedSizeListArray>(vector_field);
+        ASSERT_EQ(ids->length(), static_cast<int64_t>(expected_ids.size()));
+        ASSERT_EQ(vector_array->length(), static_cast<int64_t>(expected_vectors.size()));
+        for (int64_t i = 0; i < ids->length(); ++i) {
+            ASSERT_FALSE(ids->IsNull(i));
+            ASSERT_EQ(ids->Value(i), expected_ids[i]);
+            if (!expected_vectors[i]) {
+                ASSERT_TRUE(vector_array->IsNull(i));
+                continue;
+            }
+            ASSERT_FALSE(vector_array->IsNull(i));
+            ASSERT_EQ(vector_array->value_length(i),
+                      static_cast<int64_t>(expected_vectors[i]->size()));
+            auto values = checked_pointer_cast<arrow::FloatArray>(vector_array->value_slice(i));
+            for (int64_t j = 0; j < values->length(); ++j) {
+                ASSERT_FALSE(values->IsNull(j));
+                ASSERT_FLOAT_EQ(values->Value(j), expected_vectors[i].value()[j]);
+            }
+        }
+    }
+
+ private:
+    std::shared_ptr<MemoryPool> pool_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    std::shared_ptr<FileSystem> fs_;
+    std::unique_ptr<paimon::test::UniqueTestDirectory> dir_;
+};
+
+TEST_F(ParquetVectorIoTest, WriteAndReadVector) {
+    auto vector_type =
+        arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 3);
+    auto struct_type = checked_pointer_cast<arrow::StructType>(arrow::struct_(
+        {arrow::field("id", arrow::int32()), arrow::field("embedding", vector_type)}));
+    WriteAndCheck("vector.parquet", struct_type, struct_type,
+                  R"([[1, [1.0, 2.0, 3.0]], [2, null], [3, [4.0, 5.0, 6.0]]])");
+}
+
+TEST_F(ParquetVectorIoTest, ReadOrdinaryParquetListAsVector) {
+    auto physical_type = checked_pointer_cast<arrow::StructType>(
+        arrow::struct_({arrow::field("id", arrow::int32()),
+                        arrow::field("embedding", arrow::list(arrow::float32()))}));
+    auto logical_type = checked_pointer_cast<arrow::StructType>(arrow::struct_({
+        arrow::field("id", arrow::int32()),
+        arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3)),
+    }));
+    WriteAndCheck("list.parquet", physical_type, logical_type,
+                  R"([[1, [1.0, 2.0, 3.0]], [2, null], [3, [4.0, 5.0, 6.0]]])");
+}
+
+TEST_F(ParquetVectorIoTest, WriteAndReadNestedDoubleVector) {
+    auto vector_type =
+        arrow::fixed_size_list(arrow::field("item", arrow::float64(), /*nullable=*/false), 2);
+    auto struct_type = checked_pointer_cast<arrow::StructType>(arrow::struct_({
+        arrow::field("id", arrow::int32()),
+        arrow::field("payload", arrow::struct_({arrow::field("embedding", vector_type),
+                                                arrow::field("history", arrow::list(vector_type)),
+                                                arrow::field("by_name", arrow::map(arrow::utf8(),
+                                                                                   vector_type))})),
+    }));
+    WriteAndCheck("nested-vector.parquet", struct_type, struct_type,
+                  R"([[1, [[1.0, 2.0], [[3.0, 4.0], null], [["a", [5.0, 6.0]]]]],
+                       [2, [null, null, [["b", null]]]]])");
+}
+
+// Vectors nested in a LIST keep their Arrow type when a third-party writer stores them as
+// FixedSizeList, so the Parquet reader must accept a FixedSizeList read type as well.
+TEST_F(ParquetVectorIoTest, ReadNestedFixedSizeListFile) {
+    auto vector_type =
+        arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 3);
+    auto logical_type = checked_pointer_cast<arrow::StructType>(arrow::struct_({
+        arrow::field("id", arrow::int32()),
+        arrow::field("history", arrow::list(vector_type)),
+    }));
+    const std::string json = R"([[1, [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]], [2, []]])";
+    std::string file_path = dir_->Str() + "/nested-fixed-size-list.parquet";
+    WriteWithArrowWriter(file_path, logical_type, json);
+
+    // Without this the file would expose the column as list<list<float>> and the read would take
+    // the LIST to VECTOR conversion instead of the nested FixedSizeList path under test.
+    std::shared_ptr<arrow::StructType> file_type;
+    ReadFileType(file_path, &file_type);
+    std::shared_ptr<arrow::Field> file_history_field = file_type->GetFieldByName("history");
+    ASSERT_TRUE(file_history_field);
+    ASSERT_EQ(file_history_field->type()->id(), arrow::Type::LIST);
+    ASSERT_EQ(file_history_field->type()->field(0)->type()->id(), arrow::Type::FIXED_SIZE_LIST);
+
+    std::unique_ptr<FileBatchReader> reader;
+    CreateVectorReader(file_path, arrow::schema(logical_type->fields()), /*predicate=*/nullptr,
+                       /*options=*/{}, /*batch_size=*/10, &reader);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> actual,
+                         paimon::test::ReadResultCollector::CollectResult(reader.get()));
+    arrow::Result<std::shared_ptr<arrow::Array>> expected_result =
+        arrow::ipc::internal::json::ArrayFromJSON(logical_type, json);
+    ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString();
+    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(std::move(expected_result).ValueOrDie())
+                    ->Equals(actual))
+        << actual->ToString();
+}
+
+TEST_F(ParquetVectorIoTest, ReadVectorWithPredicatePushdown) {
+    auto vector_type =
+        arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 3);
+    auto logical_type = checked_pointer_cast<arrow::StructType>(arrow::struct_(
+        {arrow::field("id", arrow::int32()), arrow::field("embedding", vector_type)}));
+    // One row per row group, so the predicate on `id` prunes row groups while reading.
+    std::string file_path = dir_->Str() + "/vector-predicate.parquet";
+    WriteWithFormatWriter(file_path, logical_type,
+                          R"([[1, [1.0, 2.0, 3.0]], [2, null], [3, [4.0, 5.0, 6.0]],
+                              [4, [7.0, 8.0, 9.0]]])",
+                          /*max_row_group_length=*/1);
+
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::GreaterThan(
+        /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(2));
+    std::unique_ptr<FileBatchReader> reader;
+    CreateVectorReader(file_path, arrow::schema(logical_type->fields()), predicate,
+                       /*options=*/{}, /*batch_size=*/10, &reader);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> actual,
+                         paimon::test::ReadResultCollector::CollectResult(reader.get()));
+    arrow::Result<std::shared_ptr<arrow::Array>> expected_result =
+        arrow::ipc::internal::json::ArrayFromJSON(
+            logical_type, R"([[3, [4.0, 5.0, 6.0]], [4, [7.0, 8.0, 9.0]]])");
+    ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString();
+    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(std::move(expected_result).ValueOrDie())
+                    ->Equals(actual))
+        << actual->ToString();
+}
+
+TEST_F(ParquetVectorIoTest, ReadJavaFixture) {
+    ReadFixtureAndCheck(
+        "java_vector.parquet", arrow::Type::LIST, /*vector_length=*/2,
+        /*expected_ids=*/{0, 1, 2, 3, 4},
+        /*expected_vectors=*/
+        {{{0.0f, 0.0f}}, {{1.0f, 0.0f}}, {{2.0f, 0.0f}}, {{3.0f, 0.0f}}, {{4.0f, 0.0f}}});
+}
+
+TEST_F(ParquetVectorIoTest, ReadRustFixture) {
+    ReadFixtureAndCheck("rust_vector.parquet", arrow::Type::FIXED_SIZE_LIST,
+                        /*vector_length=*/3, /*expected_ids=*/{1, 2, 3},
+                        /*expected_vectors=*/
+                        {{{1.0f, 2.0f, 3.0f}}, {{7.0f, 8.0f, 9.0f}}, {{4.0f, 5.0f, 6.0f}}});
+}
+
+TEST_F(ParquetVectorIoTest, ReadNullableJavaFixture) {
+    ReadFixtureAndCheck("java_vector_nullable.parquet", arrow::Type::LIST, /*vector_length=*/3,
+                        /*expected_ids=*/{1, 2, 3},
+                        /*expected_vectors=*/
+                        {{{1.0f, 2.0f, 3.0f}}, std::nullopt, {{4.0f, 5.0f, 6.0f}}});
+}
+
+// A table can hold files from several writers, and Paimon Java stores VECTOR as Parquet LIST
+// while Paimon Rust stores it as FixedSizeList. Reading both with the table schema must produce
+// batches of one Arrow type, otherwise they cannot be combined into a single result.
+TEST_F(ParquetVectorIoTest, ReadMixedListAndFixedSizeListFixtures) {
+    // The Arrow type a Paimon schema builds for `id INT, embedding VECTOR<FLOAT, 3>`. The Rust
+    // fixture instead names the element field `element` and marks it non-nullable.
+    auto logical_schema =
+        arrow::schema({arrow::field("id", arrow::int32()),
+                       arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3))});
+    std::shared_ptr<arrow::DataType> logical_type = arrow::struct_(logical_schema->fields());
+
+    // A reader owns the memory pool that its batches are allocated from, so it has to outlive
+    // the chunks collected from it. This mirrors a scan, which holds every split reader until
+    // the whole result has been consumed.
+    std::vector<std::unique_ptr<FileBatchReader>> readers;
+    arrow::ArrayVector chunks;
+    for (const char* file_name : {"java_vector_nullable.parquet", "rust_vector.parquet"}) {
+        std::string file_path =
+            paimon::test::GetDataDir() + "/parquet/vector_compatibility/" + file_name;
+        std::unique_ptr<FileBatchReader> reader;
+        CreateVectorReader(file_path, logical_schema, /*predicate=*/nullptr, /*options=*/{},
+                           /*batch_size=*/10, &reader);
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> actual,
+                             paimon::test::ReadResultCollector::CollectResult(reader.get()));
+        readers.push_back(std::move(reader));
+        ASSERT_TRUE(actual->type()->Equals(logical_type))
+            << file_name << ": " << actual->type()->ToString();
+        chunks.insert(chunks.end(), actual->chunks().begin(), actual->chunks().end());
+    }
+
+    arrow::Result<std::shared_ptr<arrow::ChunkedArray>> merged_result =
+        arrow::ChunkedArray::Make(chunks);
+    ASSERT_TRUE(merged_result.ok()) << merged_result.status().ToString();
+    arrow::Result<std::shared_ptr<arrow::Array>> expected_result =
+        arrow::ipc::internal::json::ArrayFromJSON(
+            logical_type, R"([[1, [1.0, 2.0, 3.0]], [2, null], [3, [4.0, 5.0, 6.0]],
+                              [1, [1.0, 2.0, 3.0]], [2, [7.0, 8.0, 9.0]], [3, [4.0, 5.0, 6.0]]])");
+    ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString();
+    std::shared_ptr<arrow::ChunkedArray> merged = std::move(merged_result).ValueOrDie();
+    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(std::move(expected_result).ValueOrDie())
+                    ->Equals(merged))
+        << merged->ToString();
+}
+
+// A writer that stores the Arrow schema, such as Paimon Rust or Python, exposes the VECTOR column
+// as FixedSizeList. Arrow 17 cannot read a null value from such a column: Parquet stores a null
+// list slot with no values, while FixedSizeListReader::AssembleArray in
+// parquet/arrow/reader.cc requires every slot to span exactly `list_size` values.
+//
+// TODO(ChaomingZhangCN): Turn this into a read check once Arrow is upgraded.
+TEST_F(ParquetVectorIoTest, ReadNullableRustFixtureIsUnsupported) {
+    std::string file_path =
+        paimon::test::GetDataDir() + "/parquet/vector_compatibility/rust_vector_nullable.parquet";
+    std::shared_ptr<arrow::StructType> file_type;
+    ReadFileType(file_path, &file_type);
+    std::shared_ptr<arrow::Field> file_vector_field = file_type->GetFieldByName("embedding");
+    ASSERT_TRUE(file_vector_field);
+    ASSERT_EQ(file_vector_field->type()->id(), arrow::Type::FIXED_SIZE_LIST);
+
+    std::unique_ptr<FileBatchReader> reader;
+    CreateVectorReader(file_path, arrow::schema(file_type->fields()), /*predicate=*/nullptr,
+                       /*options=*/{}, /*batch_size=*/10, &reader);
+    ASSERT_NOK_WITH_MSG(paimon::test::ReadResultCollector::CollectResult(reader.get()),
+                        "Expected all lists to be of size=3");
+}
+
+}  // namespace paimon::parquet::test
diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp
index d41b1a4a1..82eb5257b 100644
--- a/test/inte/write_and_read_inte_test.cpp
+++ b/test/inte/write_and_read_inte_test.cpp
@@ -309,6 +309,183 @@ TEST_P(WriteAndReadInteTest, TestAppendSimple) {
     ASSERT_TRUE(success);
 }
 
+TEST_P(WriteAndReadInteTest, TestAppendVector) {
+    auto [file_format, file_system] = GetParam();
+    if (file_format != "parquet") {
+        return;
+    }
+
+    auto vector_type =
+        arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 3);
+    arrow::FieldVector fields = {arrow::field("id", arrow::int32()),
+                                 arrow::field("embedding", vector_type)};
+    std::map<std::string, std::string> options = {
+        {Options::MANIFEST_FORMAT, "avro"},  {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
+        {Options::FILE_SYSTEM, file_system},
+    };
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+    ASSERT_OK_AND_ASSIGN(auto helper,
+                         TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{},
+                                            /*primary_keys=*/{}, options,
+                                            /*is_streaming_mode=*/false));
+    const std::string data_json = R"([
+        [1, [1.0, 2.0, 3.0]],
+        [2, null],
+        [3, [4.0, 5.0, 6.0]]
+    ])";
+    auto data =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), data_json).ValueOrDie();
+    auto c_array = std::make_unique<ArrowArray>();
+    ASSERT_TRUE(arrow::ExportArray(*data, c_array.get()).ok());
+    RecordBatchBuilder batch_builder(c_array.get());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch, batch_builder.SetBucket(0).Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit_messages,
+                         helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0,
+                                                /*expected_commit_messages=*/std::nullopt));
+    (void)commit_messages;
+
+    arrow::FieldVector result_fields = fields;
+    result_fields.insert(result_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8()));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<Split>> data_splits,
+                         helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> actual,
+                         helper->ReadResult(data_splits));
+    const std::string expected_json = R"([
+        [0, 1, [1.0, 2.0, 3.0]],
+        [0, 2, null],
+        [0, 3, [4.0, 5.0, 6.0]]
+    ])";
+    auto expected =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), expected_json)
+            .ValueOrDie();
+    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(actual));
+}
+
+TEST_P(WriteAndReadInteTest, TestAppendNestedVector) {
+    auto [file_format, file_system] = GetParam();
+    if (file_format != "parquet") {
+        return;
+    }
+
+    auto vector_type =
+        arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 2);
+    arrow::FieldVector fields = {
+        arrow::field("id", arrow::int32()),
+        arrow::field("payload", arrow::struct_({arrow::field("embedding", vector_type)})),
+        arrow::field("history", arrow::list(vector_type)),
+        arrow::field("by_name", arrow::map(arrow::utf8(), vector_type)),
+    };
+    std::map<std::string, std::string> options = {
+        {Options::MANIFEST_FORMAT, "avro"},  {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
+        {Options::FILE_SYSTEM, file_system},
+    };
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+    ASSERT_OK_AND_ASSIGN(auto helper,
+                         TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{},
+                                            /*primary_keys=*/{}, options,
+                                            /*is_streaming_mode=*/false));
+    const std::string data_json = R"([
+        [1, [[1.0, 2.0]], [[3.0, 4.0], null], [["a", [5.0, 6.0]], ["b", null]]],
+        [2, [null], null, []],
+        [3, null, [], [["c", [7.0, 8.0]]]]
+    ])";
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), data_json,
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<Split>> data_splits,
+                         helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt));
+    arrow::FieldVector result_fields = fields;
+    result_fields.insert(result_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8()));
+    const std::string expected_json = R"([
+        [0, 1, [[1.0, 2.0]], [[3.0, 4.0], null], [["a", [5.0, 6.0]], ["b", null]]],
+        [0, 2, [null], null, []],
+        [0, 3, null, [], [["c", [7.0, 8.0]]]]
+    ])";
+    ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(arrow::struct_(result_fields),
+                                                                  data_splits, expected_json));
+    ASSERT_TRUE(success);
+}
+
+// Pushing a predicate down on a non-vector column must not disturb the VECTOR column, whose
+// read schema differs from the type stored in the data file.
+TEST_P(WriteAndReadInteTest, TestAppendVectorWithPredicate) {
+    auto [file_format, file_system] = GetParam();
+    if (file_format != "parquet") {
+        return;
+    }
+
+    auto vector_type =
+        arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 3);
+    arrow::FieldVector fields = {arrow::field("id", arrow::int32()),
+                                 arrow::field("embedding", vector_type)};
+    std::map<std::string, std::string> options = {
+        {Options::MANIFEST_FORMAT, "avro"},
+        {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1048576"},
+        {Options::BUCKET, "-1"},
+        {Options::FILE_SYSTEM, file_system},
+        // One row per row group, so the predicate prunes row groups instead of rows.
+        {"parquet.write.max-row-group-length", "1"},
+    };
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+    ASSERT_OK_AND_ASSIGN(auto helper,
+                         TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{},
+                                            /*primary_keys=*/{}, options,
+                                            /*is_streaming_mode=*/false));
+    std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar");
+    const std::string data_json = R"([
+        [1, [1.0, 2.0, 3.0]],
+        [2, null],
+        [3, [4.0, 5.0, 6.0]],
+        [4, [7.0, 8.0, 9.0]]
+    ])";
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), data_json,
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id",
+                                                   FieldType::INT, Literal(2));
+    ScanContextBuilder scan_context_builder(table_path);
+    scan_context_builder.SetOptions(options)
+        .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString())
+        .SetPredicate(predicate);
+    ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context)));
+    ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan());
+    ASSERT_FALSE(result_plan->Splits().empty());
+
+    ReadContextBuilder read_context_builder(table_path);
+    read_context_builder.SetOptions(options).SetPredicate(predicate);
+    ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context)));
+    ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits()));
+    ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(batch_reader.get()));
+
+    arrow::FieldVector fields_with_row_kind = fields;
+    fields_with_row_kind.insert(fields_with_row_kind.begin(),
+                                arrow::field("_VALUE_KIND", arrow::int8()));
+    auto expected = std::make_shared<arrow::ChunkedArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_with_row_kind), R"([
+        [0, 3, [4.0, 5.0, 6.0]],
+        [0, 4, [7.0, 8.0, 9.0]]
+    ])")
+            .ValueOrDie());
+    ASSERT_TRUE(expected->Equals(actual)) << actual->ToString();
+}
+
 TEST_P(WriteAndReadInteTest, TestPKSimple) {
     arrow::FieldVector fields = {
         arrow::field("pk", arrow::utf8()),
diff --git a/test/test_data/parquet/vector_compatibility/README.md b/test/test_data/parquet/vector_compatibility/README.md
new file mode 100644
index 000000000..15eb2ef30
--- /dev/null
+++ b/test/test_data/parquet/vector_compatibility/README.md
@@ -0,0 +1,38 @@
+# VECTOR Parquet compatibility fixtures
+
+These files pin the two physical Arrow schemas produced by Java and Rust writers for Paimon
+VECTOR columns, with and without null vectors.
+
+- `java_vector.parquet` was copied from Apache Paimon Rust commit
+  `403a2b2e9bfc4ea66cd7e633619f1460efd18bc8`, path
+  `crates/paimon/testdata/pkvector/pk_vector_ivf_flat/bucket-0/data-932a1249-f7e0-4a03-8e1f-ab8c85cbb76f-0.parquet`.
+  The fixture documentation records Apache Paimon Java commit `7234e4c34` and
+  `PkVectorFixtureGenerator` as its source. Its VECTOR column is exposed as Arrow `list`.
+- `java_vector_nullable.parquet` was generated with parquet-mr 1.15.1 (`parquet-avro`
+  `AvroParquetWriter` with `parquet.avro.write-old-list-structure=false`, so the column uses the
+  standard 3-level `list` / `element` layout Paimon Java writes). The rows are `(1, [1, 2, 3])`,
+  `(2, null)` and `(3, [4, 5, 6])`. The file carries no `ARROW:schema` key, so the VECTOR column
+  is exposed as Arrow `list`.
+- `rust_vector.parquet` was generated with Apache Arrow Rust 58.4.0 using
+  `FixedSizeListBuilder<Float32Builder>` and `parquet::arrow::ArrowWriter`, the same Arrow and
+  Parquet representation used by Apache Paimon Rust. Its VECTOR column is exposed as Arrow
+  `fixed_size_list[3]`. The rows are `(1, [1, 2, 3])`, `(2, [7, 8, 9])`, and
+  `(3, [4, 5, 6])`.
+- `rust_vector_nullable.parquet` was generated the same way, with the rows `(1, [1, 2, 3])`,
+  `(2, null)` and `(3, [4, 5, 6])`.
+
+A file that stores the Arrow schema, as the Rust writer does, is read back as
+`fixed_size_list`. Arrow 17 cannot read a null value from such a column, because Parquet stores a
+null list slot with no values while `FixedSizeListReader::AssembleArray` in
+`parquet/arrow/reader.cc` requires every slot to span exactly `list_size` values. Reading
+`rust_vector_nullable.parquet` therefore fails until Arrow is upgraded, which
+`ParquetVectorIoTest.ReadNullableRustFixtureIsUnsupported` pins.
+
+SHA-256 checksums:
+
+```text
+2b2325cc2266301beaa2c78ec666cb5e0ee62283049de2a7231e3c9ae07bf3ca  java_vector.parquet
+42352e11daf5a291e8a8c4cfc8d0f0f6f8c9099cf7dcf24c28d9e159a29e0d8a  java_vector_nullable.parquet
+b5ba47e766ad72fca9c8485aa718ad27709c1fb4d34fb3670aa35e2001cbdbb0  rust_vector.parquet
+f86058b1bc6cf803003446ca0abb7923e928d455fd39a7288c6d3acdd5fd10e9  rust_vector_nullable.parquet
+```
diff --git a/test/test_data/parquet/vector_compatibility/java_vector.parquet b/test/test_data/parquet/vector_compatibility/java_vector.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..5184c7a9f29400d93dec34ea86ee08b9a96b67f4
GIT binary patch
literal 1303
zcmbVMO=uHA6rM~b$#!kA#k^q`7Re<`Dq6FxX(|Lwt0}eBQmZKlg_xhM2AZ_CX^*l{
z=&3?4UPPe>5iK5a(UW-5gC6Wr6zZV|QADUG4@IqScC$&^Uc_Z5JM-qf@B7|+Ga2h2
zH-JC{lJIeJWvfjC8J7}BghZa5{7r_|i2Zo*m*Vi^U^sA%eq%5Q)d%bM<O5+Z2(g40
z#R%aq9(EO19udiqAq3mo8&6mQD4>Hw7P9?n;0!eQymV)&MSzQlx~6t?^EwLtOE|Oh
z{qpYhD>)&sUHUbL9;KhfSrT6yR?=ZQ1_N;S;itC&4T31g4}Lf7u~Qb`Iu#WPdcD46
zs1-pmPVJzoV&{(^@dc{#ni<n^9#xTSm%ZC=l<;Yaa?Sg|#L}{<tfcL&>awg3qBTfV
zjZ08QS}|LMN^Mr9FkGAN(E9!cgIQrkkd>jf*=(^iQ_jriZ<?hn$Y>1{C6F0OUCR^;
z=|8)>y8-Mb+S)T$QkT=I(Q~QHX!_DXYCQc5HxD}jhcXlW7t^WC$c54K;iIt~v7<db
zB6Zk27)=q3kQh~3%u>!Q6pG~=cr5%eswL1R#uPSPoUZ`IlNqK;z??QqX1S8KRxOJW
zE~}b4sS^z)hl!-t^+79_`2;WPQo<>|(+N%@b}O6})&vhS>Lf_T-vl7YU;3bn2t4NC
z9mvfFzRfH8VxrMHrQS~@s5i7>-AlA7h`#&aj7@~XC8jFPb+S3G^Hq;trTRCI=BRCC
zfOR2m2FLL{GEp6Ivz$bJ1c38(e=U((VX(~gn(WYjF3PfmSRY(0Ne%m%+Sllm-5a+j
zh#%ySJp8>!-(dPzudUlkHtTj*MBT5uvvFn#zb{yq128~%m2p{@_;X(nuiWr#cJ9`K
zSvgUf3oV*+^TnBR$mlY9yP}~;ZlO3`2%U;{=X?5$Xuc=jUFbgD8|^WTJ|iACvUxK%
bW#;p7BWF(a8lCE-Mo9BZe&P)t!@tD8O9jc#

literal 0
HcmV?d00001

diff --git a/test/test_data/parquet/vector_compatibility/java_vector_nullable.parquet b/test/test_data/parquet/vector_compatibility/java_vector_nullable.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..fa900ac0dc1c19629f8c58d610eddfa820fdc5bf
GIT binary patch
literal 765
zcmYjP&1%~~5FT%gs+5uvnq4H&!7NxfK^0L_|6+)9F(s4`dQC%-rBy1>Pr8;JhhXrj
z&yic-qU6+LPrdXp3hlLY)^eP%GuoY*Z)Uz3bawpSqd*NjzyALFy=hSmO`Ylh6#yWp
z8>r|z!SmzGpRX@1x`n1jwN*G$fQ`L9fW;BM1}LZt)H~Gsfw@ggqpGUmwJb(V1}4)`
zpbV;13@S9mATe!CH(LZ=fN3$E4w*`}*eRW<7eYR~eLfXIk;{)Vzou1m)xjWf2u)&a
zigYBxFwQP1q1wAXW;CltHHpUsB{&-*pNT}IA}at%Sf*oxg*EFs62ux5y==&aw%#xK
zmgE^Umh4Ll>EIHuFZ6fINr2rGy2HL#Xb)6D-K^tyokuoH1`nOF$rhWjnSF{))ZFU3
zIAWyn#CqCfy%AtPRi6c+17P1OOtW>oc5p$C@#@N#pC_Vl{i)2|aqvl`zHwK%<;BgF
z;5{xykjs!eJo8g!Kkg8HQ7n>h%zNjz58+rrSE+GU@VT=Nt#`aeobzmwBpe~D3|~tB
z2E%b7QY_1(B=@n#g~LM;`IDsJJ(V%Pn1iu>EfFT&G!I4MDt1Oy%>c&9YNXextWAX+
z$9Y!sT(9YRwZ>=?Ct)pUA2i#ePUJ^Xv(@bi?{G8<+8wXe?uNnsVf3aGc;5bjziT=c
N016xvfHVB@{{w+Cp{M`=

literal 0
HcmV?d00001

diff --git a/test/test_data/parquet/vector_compatibility/rust_vector.parquet b/test/test_data/parquet/vector_compatibility/rust_vector.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..761ee734172e3e0204c7f85cf76d19ca91d0d636
GIT binary patch
literal 949
zcmZuw&2AD=6h2(0<Cw6Zr1xefVbO(7)F4_@Ok+sO1uCc{0x>itriRKmlJZj-TDB}*
zyEHC*2;<tN>DmYI5qt!{b7!!L>B&8F&i(E=-}wo&b$3Tv(v@yY+LCfaDIyZkG*psw
zq*K647HZ=meTxv@l+BjRnQ)@jJ|ZhP(BDwfL|Nzx{#WRa3TTS5SU^TX!`vI+QvxO@
ztc~oWR~HU^FDE?L;s#($ec`D!%!L3mw`AD!?Tq&r1mxvaS>8h+dzUHMW*0H9o0-9U
zK3l$t-o6he18D)gO|BRXsU+N+!Q><u4oA~a3Td~>$SGN_H6BF^DKIx8Zv^9D5=<AQ
zY%lK>e9D8hM0VC=5(!uxt7E_HJNdZZxOqi)O!x6^bCbLRc821-&nyh7k7>E3J_}hS
zCUs)cy{nc?8|(5YlvCeQIR0hIe}`JU1pcn}xz^R~np)+%|Mo=cOIj-RQ^*RbPeR$g
z1GB|zb2GZzeR=pK-jw@CJYZQat6J+0l;wriOI-`~bB18G=2w;u?pi-%7tq?`#%^uK
z_*4Gi5f*&Ay1vv03?a|(n<VhZa?m+4jtMzUz=qZ#Q@}zvl=;Fq|9?IcR9*AwdYpum
zkefPU8%r{s32pSE{^8E}WQuthZ}b*Nliq2+F+T6F>m~hOTMxq|QfnF~)OF2sMm(n(
zHhD+qEIf7ftdB3b-q~RO;XGK}pGVsl!8{txrrV{x;%@PwSI86L&@cRS+W5o%1v4(U
AuK)l5

literal 0
HcmV?d00001

diff --git a/test/test_data/parquet/vector_compatibility/rust_vector_nullable.parquet b/test/test_data/parquet/vector_compatibility/rust_vector_nullable.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..8610e46e1065f7f248148c0383b5c2a2c7985043
GIT binary patch
literal 932
zcmZuw&2Cab6h3z#S4>Dlo6h7;!lDbjsRS!cifIf<8K9*qiG>9X#MDr^MN)n$m&%r<
zYoEZC56~xY<rDY}uDdj@{LWmc#`I*)J!if-=Q}@v6Vu$4ap_5~E?r3*L=F)NXbdVz
zCS)RqmlV{+Lo7oG&&zyWCT%#;;sB8)9Owf21^TUo)QrsJ<l0>-f%Ae{W7%vh_UXIO
z8$x?=t_New39WFkF!KVoRDgx{-GqK7C`mf4iZY8p_HG<`#I6+j&2h3jolfR&!*}n4
zQCG$RJ|a&HdQ=eJ!(enC^m>DFUkd5gi^wU&z4&kt&ZNMYh<p$XgHbS^k#cof=X^?o
zwLor4Gl>MO&eWM-^b_f*->7*-+qV1Yw!Tg}hZUhXuP+5d>Zg#K>X##Th)EL>>HgoA
zY#XccD72<NJ2?K0<6oc_FM+>{10}Zqx!qH%`1o#5q<#yj)K?)(q;4JM`U#j#UYnOR
z&F1UV=h3FTTk(`-J}(<%8c>#3#>`EnZ(cG4YYe}#G;lZO1-pP&7B_Y)6ULADgGX5K
z?V0M_95RGF%WspwAIV{(Z5<nOxddy*^qB${!lBI9kMjTLvq9w*pQ%PkNa=G^ZMKmt
zr?a88R@gb+9iERd5977gtUYR7c51`x&Z^$B-+Sv}n59&Xt<yJE!*hl_ryX|rj?P(l
u>he_wU-I--clzNvm_3<>TOWgIIGBvL3i~^IJ9(W;6XDPg{B%z6|NH|iJGlk`

literal 0
HcmV?d00001


From afc8471fbd5cfb3bc9b7c18bf5ec93672e656d91 Mon Sep 17 00:00:00 2001
From: lszskye <57179283+lszskye@users.noreply.github.com>
Date: Wed, 19 Aug 2026 19:44:11 -0700
Subject: [PATCH 05/93] fix: optimize shared-shredding read & fix ORC read-size
 estimation for nested columns (#216)

---
 .../map_shared_shredding_file_reader.cpp      | 232 ++++++++++++------
 .../map_shared_shredding_file_reader_test.cpp | 150 +++++++++++
 .../format/orc/orc_file_batch_reader.cpp      |  16 +-
 .../format/orc/orc_file_batch_reader_test.cpp |  97 +++++++-
 4 files changed, 413 insertions(+), 82 deletions(-)

diff --git a/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp
index b7c84095c..b1cdf963b 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp
+++ b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp
@@ -32,7 +32,6 @@
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
-#include "paimon/core/casting/casting_utils.h"
 #include "paimon/core/utils/nested_projection_utils.h"
 
 namespace paimon {
@@ -110,6 +109,56 @@ class SharedSelectedKeysReadPlan : public MapFieldReadPlan {
     std::vector<SelectedKey> selected_keys_;
 };
 
+Result<std::shared_ptr<arrow::Array>> MaskSinglePhysicalColumn(
+    const std::shared_ptr<arrow::StructArray>& physical_struct_array,
+    const std::shared_ptr<arrow::ListArray>& field_mapping_array,
+    const std::shared_ptr<arrow::Int32Array>& field_mapping_values,
+    const std::shared_ptr<arrow::Array>& physical_column_array, int32_t physical_column_id,
+    int32_t field_id, const std::string& field_name, arrow::MemoryPool* arrow_pool) {
+    int64_t row_count = physical_struct_array->length();
+    if (physical_column_array->length() != row_count) {
+        return Status::Invalid("shared-shredding physical column length does not match row count");
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Buffer> validity,
+                                      arrow::AllocateEmptyBitmap(row_count, arrow_pool));
+    int64_t valid_count = 0;
+    for (int64_t row = 0; row < row_count; ++row) {
+        if (physical_struct_array->IsNull(row)) {
+            continue;
+        }
+        if (field_mapping_array->IsNull(row)) {
+            return Status::Invalid(fmt::format(
+                "__field_mapping cannot be null in non-null shared-shredding row for field {}",
+                field_name));
+        }
+        int32_t mapping_offset = field_mapping_array->value_offset(row);
+        int32_t mapping_length = field_mapping_array->value_length(row);
+        if (physical_column_id < 0 || physical_column_id >= mapping_length) {
+            return Status::Invalid("physical column id is out of __field_mapping range");
+        }
+        int32_t mapping_index = mapping_offset + physical_column_id;
+        if (field_mapping_values->IsNull(mapping_index)) {
+            return Status::Invalid("__field_mapping element cannot be null");
+        }
+        if (field_mapping_values->Value(mapping_index) != field_id ||
+            physical_column_array->IsNull(row)) {
+            continue;
+        }
+        arrow::bit_util::SetBit(validity->mutable_data(), row);
+        ++valid_count;
+    }
+
+    // Replace only the top-level validity; offsets, values, and nested children stay shared.
+    std::shared_ptr<arrow::ArrayData> result_data = physical_column_array->data()->Copy();
+    if (result_data->buffers.empty()) {
+        return Status::Invalid("shared-shredding physical column has no validity buffer slot");
+    }
+    int64_t null_count = row_count - valid_count;
+    result_data->buffers[0] = null_count == 0 ? nullptr : std::move(validity);
+    result_data->SetNullCount(null_count);
+    return arrow::MakeArray(std::move(result_data));
+}
+
 class DefaultSelectedKeysReadPlan : public MapFieldReadPlan {
  public:
     DefaultSelectedKeysReadPlan(const std::shared_ptr<arrow::Field>& logical_field,
@@ -386,12 +435,10 @@ Result<std::shared_ptr<arrow::Array>> FullMapReadPlan::Materialize(
     std::shared_ptr<arrow::MapArray> overflow_array;
     CollectPhysicalColumns(physical_struct_array, &physical_column_name_to_array, &overflow_array);
     for (auto& [_, physical_column_array] : physical_column_name_to_array) {
-        if (physical_column_array->type_id() == arrow::Type::DICTIONARY) {
-            PAIMON_ASSIGN_OR_RAISE(
-                physical_column_array,
-                CastingUtils::Cast(physical_column_array, logical_map_type_->item_type(),
-                                   arrow::compute::CastOptions::Safe(), arrow_pool));
-        }
+        PAIMON_ASSIGN_OR_RAISE(
+            physical_column_array,
+            NestedProjectionUtils::AlignArrayToReadType(
+                physical_column_array, logical_map_type_->item_type(), arrow_pool));
     }
 
     std::shared_ptr<arrow::Int32Array> overflow_keys;
@@ -406,12 +453,9 @@ Result<std::shared_ptr<arrow::Array>> FullMapReadPlan::Materialize(
         if (!overflow_items) {
             return Status::Invalid("__overflow map item array is null");
         }
-        if (overflow_items->type_id() == arrow::Type::DICTIONARY) {
-            PAIMON_ASSIGN_OR_RAISE(
-                overflow_items,
-                CastingUtils::Cast(overflow_items, logical_map_type_->item_type(),
-                                   arrow::compute::CastOptions::Safe(), arrow_pool));
-        }
+        PAIMON_ASSIGN_OR_RAISE(overflow_items,
+                               NestedProjectionUtils::AlignArrayToReadType(
+                                   overflow_items, logical_map_type_->item_type(), arrow_pool));
     }
 
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr<arrow::ArrayBuilder> map_builder_base,
@@ -530,12 +574,9 @@ Result<std::shared_ptr<arrow::Array>> SharedSelectedKeysReadPlan::Materialize(
     std::shared_ptr<arrow::MapArray> overflow_array;
     CollectPhysicalColumns(physical_struct_array, &physical_column_name_to_array, &overflow_array);
     for (auto& [_, physical_column_array] : physical_column_name_to_array) {
-        if (physical_column_array->type_id() == arrow::Type::DICTIONARY) {
-            PAIMON_ASSIGN_OR_RAISE(
-                physical_column_array,
-                CastingUtils::Cast(physical_column_array, value_type,
-                                   arrow::compute::CastOptions::Safe(), arrow_pool));
-        }
+        PAIMON_ASSIGN_OR_RAISE(physical_column_array,
+                               NestedProjectionUtils::AlignArrayToReadType(physical_column_array,
+                                                                           value_type, arrow_pool));
     }
 
     std::shared_ptr<arrow::Int32Array> overflow_keys;
@@ -550,65 +591,89 @@ Result<std::shared_ptr<arrow::Array>> SharedSelectedKeysReadPlan::Materialize(
         if (!overflow_items) {
             return Status::Invalid("__overflow map item array is null");
         }
-        if (overflow_items->type_id() == arrow::Type::DICTIONARY) {
-            PAIMON_ASSIGN_OR_RAISE(
-                overflow_items,
-                CastingUtils::Cast(overflow_items, value_type, arrow::compute::CastOptions::Safe(),
-                                   arrow_pool));
-        }
+        PAIMON_ASSIGN_OR_RAISE(overflow_items, NestedProjectionUtils::AlignArrayToReadType(
+                                                   overflow_items, value_type, arrow_pool));
     }
 
-    std::unique_ptr<arrow::ArrayBuilder> access_builder_base;
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(access_builder_base,
-                                      arrow::MakeBuilder(LogicalField()->type(), arrow_pool));
-    if (!access_builder_base || !access_builder_base->type() ||
-        access_builder_base->type()->id() != arrow::Type::STRUCT) {
-        return Status::Invalid(
-            fmt::format("selected-key MAP field {} is not a STRUCT", LogicalField()->name()));
-    }
-    auto* access_builder = checked_cast<arrow::StructBuilder*>(access_builder_base.get());
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(access_builder->Reserve(physical_struct_array->length()));
-
-    for (int64_t row = 0; row < physical_struct_array->length(); ++row) {
-        if (physical_struct_array->IsNull(row)) {
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(access_builder->AppendNull());
+    int64_t row_count = physical_struct_array->length();
+    arrow::ArrayVector selected_key_arrays;
+    selected_key_arrays.reserve(selected_keys_.size());
+    for (int32_t key_index = 0; key_index < selected_keys_type->num_fields(); ++key_index) {
+        const SelectedKey& selected_key = selected_keys_[key_index];
+        if (selected_key.field_id < 0) {
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> null_array,
+                arrow::MakeArrayOfNull(selected_keys_type->field(key_index)->type(), row_count,
+                                       arrow_pool));
+            selected_key_arrays.push_back(std::move(null_array));
             continue;
         }
-        if (field_mapping_array->IsNull(row)) {
-            return Status::Invalid(fmt::format(
-                "__field_mapping cannot be null in non-null shared-shredding row for field {}",
-                LogicalField()->name()));
+
+        if (selected_key.candidate_columns.size() == 1 && !selected_key.may_use_overflow) {
+            int32_t physical_column_id = selected_key.candidate_columns[0];
+            std::string physical_column_name =
+                MapSharedShreddingDefine::PhysicalColumnName(physical_column_id);
+            auto physical_column_iter = physical_column_name_to_array.find(physical_column_name);
+            if (physical_column_iter == physical_column_name_to_array.end()) {
+                return Status::Invalid(
+                    fmt::format("cannot find selected physical column {} for field {}",
+                                physical_column_name, LogicalField()->name()));
+            }
+            const std::shared_ptr<arrow::Array>& physical_column_array =
+                physical_column_iter->second;
+            if (physical_column_array->offset() == 0) {
+                PAIMON_ASSIGN_OR_RAISE(
+                    std::shared_ptr<arrow::Array> masked_array,
+                    MaskSinglePhysicalColumn(physical_struct_array, field_mapping_array,
+                                             field_mapping_values, physical_column_array,
+                                             physical_column_id, selected_key.field_id,
+                                             LogicalField()->name(), arrow_pool));
+                selected_key_arrays.push_back(std::move(masked_array));
+                continue;
+            } else {
+                return Status::Invalid("paimon only supports arrays with zero offset");
+            }
         }
-        int32_t mapping_offset = field_mapping_array->value_offset(row);
 
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(access_builder->Append());
-        for (int32_t key_index = 0; key_index < selected_keys_type->num_fields(); ++key_index) {
-            arrow::ArrayBuilder* value_builder = access_builder->field_builder(key_index);
-            const SelectedKey& selected_key = selected_keys_[key_index];
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::unique_ptr<arrow::ArrayBuilder> value_builder,
+            arrow::MakeBuilder(selected_keys_type->field(key_index)->type(), arrow_pool));
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Reserve(row_count));
+        for (int64_t row = 0; row < row_count; ++row) {
+            if (physical_struct_array->IsNull(row)) {
+                PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->AppendNull());
+                continue;
+            }
+            if (field_mapping_array->IsNull(row)) {
+                return Status::Invalid(fmt::format(
+                    "__field_mapping cannot be null in non-null shared-shredding row for field {}",
+                    LogicalField()->name()));
+            }
+            int32_t mapping_offset = field_mapping_array->value_offset(row);
             bool appended = false;
-            if (selected_key.field_id >= 0) {
-                for (int32_t physical_column_id : selected_key.candidate_columns) {
-                    int32_t mapping_index = mapping_offset + physical_column_id;
-                    if (field_mapping_values->IsNull(mapping_index)) {
-                        return Status::Invalid("__field_mapping element cannot be null");
-                    }
-                    if (field_mapping_values->Value(mapping_index) != selected_key.field_id) {
-                        continue;
-                    }
-                    std::string physical_column_name =
-                        MapSharedShreddingDefine::PhysicalColumnName(physical_column_id);
-                    auto physical_column_iter =
-                        physical_column_name_to_array.find(physical_column_name);
-                    if (physical_column_iter == physical_column_name_to_array.end()) {
-                        return Status::Invalid(
-                            fmt::format("cannot find selected physical column {} for field {}",
-                                        physical_column_name, LogicalField()->name()));
-                    }
-                    PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->AppendArraySlice(
-                        *physical_column_iter->second->data(), row, 1));
-                    appended = true;
-                    break;
+            for (int32_t physical_column_id : selected_key.candidate_columns) {
+                int32_t mapping_index = mapping_offset + physical_column_id;
+                if (field_mapping_values->IsNull(mapping_index)) {
+                    return Status::Invalid("__field_mapping element cannot be null");
+                }
+                if (field_mapping_values->Value(mapping_index) != selected_key.field_id) {
+                    continue;
                 }
+                std::string physical_column_name =
+                    MapSharedShreddingDefine::PhysicalColumnName(physical_column_id);
+                auto physical_column_iter =
+                    physical_column_name_to_array.find(physical_column_name);
+                if (physical_column_iter == physical_column_name_to_array.end()) {
+                    return Status::Invalid(
+                        fmt::format("cannot find selected physical column {} for field {}",
+                                    physical_column_name, LogicalField()->name()));
+                }
+                const std::shared_ptr<arrow::Array>& physical_column_array =
+                    physical_column_iter->second;
+                PAIMON_RETURN_NOT_OK_FROM_ARROW(
+                    value_builder->AppendArraySlice(*physical_column_array->data(), row, 1));
+                appended = true;
+                break;
             }
 
             if (!appended && selected_key.may_use_overflow && overflow_array &&
@@ -630,9 +695,24 @@ Result<std::shared_ptr<arrow::Array>> SharedSelectedKeysReadPlan::Materialize(
                 PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->AppendNull());
             }
         }
+        std::shared_ptr<arrow::Array> selected_key_array;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Finish(&selected_key_array));
+        selected_key_arrays.push_back(std::move(selected_key_array));
+    }
+
+    std::shared_ptr<arrow::Buffer> parent_validity;
+    int64_t parent_null_count = physical_struct_array->null_count();
+    if (parent_null_count > 0) {
+        if (physical_struct_array->offset() == 0) {
+            parent_validity = physical_struct_array->null_bitmap();
+        } else {
+            return Status::Invalid("paimon only supports arrays with zero offset");
+        }
     }
-    std::shared_ptr<arrow::Array> result;
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(access_builder->Finish(&result));
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::StructArray> result,
+        arrow::StructArray::Make(selected_key_arrays, selected_keys_type->fields(),
+                                 std::move(parent_validity), parent_null_count));
     return result;
 }
 
@@ -646,14 +726,10 @@ Result<std::shared_ptr<arrow::Array>> DefaultSelectedKeysReadPlan::Materialize(
     }
     auto map_array = checked_pointer_cast<arrow::MapArray>(physical_array);
     auto selected_keys_type = checked_pointer_cast<arrow::StructType>(LogicalField()->type());
-    auto physical_map_type = checked_pointer_cast<arrow::MapType>(PhysicalReadField()->type());
 
     std::shared_ptr<arrow::Array> items = map_array->items();
-    if (items->type_id() == arrow::Type::DICTIONARY) {
-        PAIMON_ASSIGN_OR_RAISE(items,
-                               CastingUtils::Cast(items, physical_map_type->item_type(),
-                                                  arrow::compute::CastOptions::Safe(), arrow_pool));
-    }
+    PAIMON_ASSIGN_OR_RAISE(items, NestedProjectionUtils::AlignArrayToReadType(
+                                      items, selected_keys_type->field(0)->type(), arrow_pool));
     std::shared_ptr<arrow::Array> keys = map_array->keys();
     std::unique_ptr<arrow::ArrayBuilder> access_builder_base;
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(access_builder_base,
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp
index 80f5046b9..b2dd9374b 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp
+++ b/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp
@@ -333,6 +333,96 @@ TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjection) {
     AssertChunkedArrayEquals(expected, actual);
 }
 
+TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjectionSharesValueBuffers) {
+    ASSERT_OK_AND_ASSIGN(auto physical_array, PhysicalArray());
+    auto physical_root = checked_pointer_cast<arrow::StructArray>(physical_array);
+    auto physical_tags =
+        checked_pointer_cast<arrow::StructArray>(physical_root->GetFieldByName("tags"));
+    auto physical_column =
+        physical_tags->GetFieldByName(MapSharedShreddingDefine::PhysicalColumnName(1));
+
+    auto selected_type = arrow::struct_(
+        {arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64()),
+         arrow::field("e", arrow::int64()), arrow::field("missing", arrow::int64())});
+    auto selected_field = arrow::field(
+        "tags", selected_type, /*nullable=*/true,
+        arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,b,e,missing"}));
+    ASSERT_OK_AND_ASSIGN(
+        auto field_read_plan,
+        MapFieldReadPlanFactory::CreateSharedSelectedKeysReadPlan(selected_field, TagsMeta()));
+    ASSERT_OK_AND_ASSIGN(auto result,
+                         field_read_plan->Materialize(physical_tags, arrow::default_memory_pool()));
+    auto result_struct = checked_pointer_cast<arrow::StructArray>(result);
+
+    auto expected = arrow::ipc::internal::json::ArrayFromJSON(selected_type, R"([
+        [10, 20, null, null],
+        [40, null, null, null],
+        null,
+        [80, null, 70, null]
+    ])")
+                        .ValueOrDie();
+    ASSERT_TRUE(expected->Equals(result)) << "Expected:\n"
+                                          << expected->ToString() << "\nActual:\n"
+                                          << result->ToString();
+    ASSERT_EQ(physical_column->data()->buffers[1], result_struct->field(1)->data()->buffers[1]);
+    ASSERT_EQ(physical_column->data()->buffers[1], result_struct->field(2)->data()->buffers[1]);
+    ASSERT_NE(physical_column->data()->buffers[0], result_struct->field(1)->data()->buffers[0]);
+    ASSERT_EQ(physical_tags->data()->buffers[0], result_struct->data()->buffers[0]);
+}
+
+TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjectionSharesNestedValueBuffers) {
+    auto item_type = arrow::list(arrow::int64());
+    auto logical_schema =
+        arrow::schema({arrow::field("id", arrow::int32()),
+                       arrow::field("tags", arrow::map(arrow::utf8(), item_type))});
+    ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                                                   logical_schema, {{"tags", 1}}));
+    auto physical_array =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), R"([
+        [1, [[0], [1, 2], null]],
+        [2, [[1], [3, 4, 5], null]],
+        [3, null],
+        [4, [[0], null, null]]
+    ])")
+            .ValueOrDie();
+    auto physical_root = checked_pointer_cast<arrow::StructArray>(physical_array);
+    auto physical_tags =
+        checked_pointer_cast<arrow::StructArray>(physical_root->GetFieldByName("tags"));
+    auto physical_column =
+        physical_tags->GetFieldByName(MapSharedShreddingDefine::PhysicalColumnName(0));
+
+    MapSharedShreddingFieldMeta meta;
+    meta.name_to_id = {{"a", 0}, {"b", 1}};
+    meta.field_to_columns = {{0, {0}}, {1, {0}}};
+    meta.num_columns = 1;
+    meta.max_row_width = 1;
+    auto selected_type = arrow::struct_({arrow::field("b", item_type)});
+    auto selected_field =
+        arrow::field("tags", selected_type, /*nullable=*/true,
+                     arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"b"}));
+    ASSERT_OK_AND_ASSIGN(
+        auto field_read_plan,
+        MapFieldReadPlanFactory::CreateSharedSelectedKeysReadPlan(selected_field, meta));
+    ASSERT_OK_AND_ASSIGN(auto result,
+                         field_read_plan->Materialize(physical_tags, arrow::default_memory_pool()));
+    auto result_struct = checked_pointer_cast<arrow::StructArray>(result);
+    auto result_list = checked_pointer_cast<arrow::ListArray>(result_struct->field(0));
+
+    auto expected = arrow::ipc::internal::json::ArrayFromJSON(selected_type, R"([
+        [null],
+        [[3, 4, 5]],
+        null,
+        [null]
+    ])")
+                        .ValueOrDie();
+    ASSERT_TRUE(expected->Equals(result)) << "Expected:\n"
+                                          << expected->ToString() << "\nActual:\n"
+                                          << result->ToString();
+    ASSERT_EQ(physical_column->data()->buffers[1], result_list->data()->buffers[1]);
+    ASSERT_EQ(physical_column->data()->child_data[0]->buffers[1],
+              result_list->data()->child_data[0]->buffers[1]);
+}
+
 TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjectionFromDefaultMap) {
     auto map_type = checked_pointer_cast<arrow::MapType>(
         arrow::map(arrow::utf8(), arrow::field("value", arrow::int64())));
@@ -674,6 +764,66 @@ TEST_F(MapSharedShreddingFileReaderTest, TestOrcDictionaryEncodedStringValue) {
     AssertChunkedArrayEquals(expected, actual);
 }
 
+TEST_F(MapSharedShreddingFileReaderTest, TestOrcDictionaryEncodedStringListValue) {
+    std::shared_ptr<arrow::Schema> logical_schema = arrow::schema({
+        arrow::field("id", arrow::int32()),
+        arrow::field("tags", arrow::map(arrow::utf8(), arrow::list(arrow::utf8()))),
+    });
+    auto options = options_;
+    std::string format = "orc";
+    options[Options::FILE_FORMAT] = format;
+    options["orc.dictionary-key-size-threshold"] = "1";
+    ASSERT_OK_AND_ASSIGN(auto table_schema,
+                         TableSchema::Create(TableSchema::FIRST_SCHEMA_ID, logical_schema,
+                                             /*partition_keys=*/{}, /*primary_keys=*/{}, options));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options));
+    auto path_factory = CreatePathFactory(dir->Str(), format, core_options);
+    auto compact_manager = std::make_shared<NoopCompactManager>();
+    ASSERT_OK_AND_ASSIGN(
+        auto writer,
+        CreateAppendOnlyWriter(core_options, /*schema_id=*/0, logical_schema,
+                               /*write_cols=*/std::nullopt,
+                               /*max_sequence_number=*/-1, path_factory, compact_manager));
+    auto batch = CreateBatch(logical_schema, R"([
+        [1, [["a", ["red", "blue"]], ["b", ["blue"]]]],
+        [2, [["c", ["green"]], ["a", ["red", null, "blue"]], ["b", ["blue"]]]],
+        [3, null],
+        [4, [["d", ["yellow"]], ["e", ["blue"]], ["c", [null]], ["a", ["red"]]]]
+    ])");
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_OK_AND_ASSIGN(auto inc, writer->PrepareCommit(/*wait_compaction=*/true));
+    ASSERT_OK(writer->Close());
+
+    std::string data_file_path =
+        path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name);
+    std::map<std::string, std::string> reader_options = {{"orc.read.enable-lazy-decoding", "true"}};
+    auto reader = WrapReader(OpenFormatReader(data_file_path, format, reader_options),
+                             /*selected_keys_str=*/"a,c");
+
+    auto read_metadata = std::make_shared<arrow::KeyValueMetadata>();
+    read_metadata->Append("paimon.map.selected-keys", "a,c");
+    arrow::FieldVector read_fields = logical_schema->fields();
+    read_fields[1] = read_fields[1]->WithMetadata(read_metadata);
+    auto read_schema = ExportSchema(arrow::schema(std::move(read_fields)));
+    ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr,
+                                    /*selection_bitmap=*/std::nullopt));
+    ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get()));
+    std::shared_ptr<arrow::ChunkedArray> expected;
+    ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(
+                    arrow::struct_(logical_schema->fields()), {R"([
+                        [1, [["a", ["red", "blue"]]]],
+                        [2, [["a", ["red", null, "blue"]], ["c", ["green"]]]],
+                        [3, null],
+                        [4, [["a", ["red"]], ["c", [null]]]]
+                    ])"},
+                    &expected)
+                    .ok());
+    AssertChunkedArrayEquals(expected, actual);
+}
+
 TEST_F(MapSharedShreddingFileReaderTest, TestReadsRealFormatFile) {
     // TODO(lisizhuo.lsz): support other format
     auto options = options_;
diff --git a/src/paimon/format/orc/orc_file_batch_reader.cpp b/src/paimon/format/orc/orc_file_batch_reader.cpp
index cd627eb5b..a201839b3 100644
--- a/src/paimon/format/orc/orc_file_batch_reader.cpp
+++ b/src/paimon/format/orc/orc_file_batch_reader.cpp
@@ -46,6 +46,16 @@
 #include "paimon/format/orc/predicate_converter.h"
 
 namespace paimon::orc {
+namespace {
+
+void CollectAllColumnIds(const ::orc::Type* type, std::vector<uint64_t>* column_ids) {
+    column_ids->push_back(type->getColumnId());
+    for (uint64_t i = 0; i < type->getSubtypeCount(); ++i) {
+        CollectAllColumnIds(type->getSubtype(i), column_ids);
+    }
+}
+
+}  // namespace
 
 OrcFileBatchReader::OrcFileBatchReader(std::unique_ptr<::orc::ReaderMetrics>&& reader_metrics,
                                        std::unique_ptr<OrcReaderWrapper>&& reader,
@@ -228,13 +238,15 @@ Status OrcFileBatchReader::CollectTargetColumnIds(const ::orc::Type* src_type,
             }
             break;
         }
-        // Do not support partial field recall inside list/map types.
         default: {
+            // Partial field recall inside list/map types is unsupported, so the target must match
+            // the complete source subtree. Include the container and every descendant because all
+            // of their streams are recalled by the ORC reader.
             if (src_type->toString() != target_type->toString()) {
                 return Status::Invalid(fmt::format("type mismatch: src {} vs target {}",
                                                    src_type->toString(), target_type->toString()));
             }
-            target_column_ids->push_back(src_type->getColumnId());
+            CollectAllColumnIds(src_type, target_column_ids);
             break;
         }
     }
diff --git a/src/paimon/format/orc/orc_file_batch_reader_test.cpp b/src/paimon/format/orc/orc_file_batch_reader_test.cpp
index 038c93e2e..c39ba5166 100644
--- a/src/paimon/format/orc/orc_file_batch_reader_test.cpp
+++ b/src/paimon/format/orc/orc_file_batch_reader_test.cpp
@@ -368,8 +368,9 @@ TEST_F(OrcFileBatchReaderTest, TestCreateRowReaderOptions) {
                              OrcFileBatchReader::CreateRowReaderOptions(
                                  src_type.get(), target_type.get(),
                                  /*search_arg=*/nullptr, options, &target_column_ids));
-        // Struct IDs (0, 1) not included. Selected: sub1(2), sub2-list(3), sub3(6), col3-map(8).
-        ASSERT_EQ(target_column_ids, (std::vector<uint64_t>{2, 3, 6, 8}));
+        // Struct IDs (0, 1) are not included. LIST/MAP containers include their complete
+        // subtrees: sub1(2), sub2(3, 4, 5), sub3(6), col3(8, 9, 10).
+        ASSERT_EQ(target_column_ids, (std::vector<uint64_t>{2, 3, 4, 5, 6, 8, 9, 10}));
     }
     {
         // read with type mismatch in nested field
@@ -485,6 +486,98 @@ TEST_F(OrcFileBatchReaderTest, TestCreateRowReaderOptions) {
     }
 }
 
+TEST_F(OrcFileBatchReaderTest, TestCollectTargetColumnIdsPrimitiveList) {
+    std::unique_ptr<::orc::Type> src_type =
+        ::orc::Type::buildTypeFromString("struct<items:array<int>,ignored:string>");
+    std::unique_ptr<::orc::Type> target_type =
+        ::orc::Type::buildTypeFromString("struct<items:array<int>>");
+    std::vector<uint64_t> target_column_ids;
+
+    ASSERT_OK(OrcFileBatchReader::CollectTargetColumnIds(src_type.get(), target_type.get(),
+                                                         &target_column_ids));
+    // root struct(0), items-list(1), element(2), ignored(3)
+    ASSERT_EQ(target_column_ids, (std::vector<uint64_t>{1, 2}));
+}
+
+TEST_F(OrcFileBatchReaderTest, TestCollectTargetColumnIdsDeeplyNestedList) {
+    std::string schema = "struct<items:array<array<struct<value:double,label:string>>>>";
+    std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(schema);
+    std::unique_ptr<::orc::Type> target_type = ::orc::Type::buildTypeFromString(schema);
+    std::vector<uint64_t> target_column_ids;
+
+    ASSERT_OK(OrcFileBatchReader::CollectTargetColumnIds(src_type.get(), target_type.get(),
+                                                         &target_column_ids));
+    // root struct(0), outer list(1), inner list(2), element struct(3), value(4), label(5)
+    ASSERT_EQ(target_column_ids, (std::vector<uint64_t>{1, 2, 3, 4, 5}));
+}
+
+TEST_F(OrcFileBatchReaderTest, TestCollectTargetColumnIdsPrimitiveMap) {
+    std::unique_ptr<::orc::Type> src_type =
+        ::orc::Type::buildTypeFromString("struct<attributes:map<string,int>,ignored:double>");
+    std::unique_ptr<::orc::Type> target_type =
+        ::orc::Type::buildTypeFromString("struct<attributes:map<string,int>>");
+    std::vector<uint64_t> target_column_ids;
+
+    ASSERT_OK(OrcFileBatchReader::CollectTargetColumnIds(src_type.get(), target_type.get(),
+                                                         &target_column_ids));
+    // root struct(0), attributes-map(1), key(2), value(3), ignored(4)
+    ASSERT_EQ(target_column_ids, (std::vector<uint64_t>{1, 2, 3}));
+}
+
+TEST_F(OrcFileBatchReaderTest, TestCollectTargetColumnIdsDeeplyNestedMap) {
+    std::string schema =
+        "struct<attributes:map<string,array<struct<score:double,tags:array<string>>>>>";
+    std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(schema);
+    std::unique_ptr<::orc::Type> target_type = ::orc::Type::buildTypeFromString(schema);
+    std::vector<uint64_t> target_column_ids;
+
+    ASSERT_OK(OrcFileBatchReader::CollectTargetColumnIds(src_type.get(), target_type.get(),
+                                                         &target_column_ids));
+    // root struct(0), map(1), key(2), value-list(3), element struct(4), score(5),
+    // tags-list(6), tag element(7)
+    ASSERT_EQ(target_column_ids, (std::vector<uint64_t>{1, 2, 3, 4, 5, 6, 7}));
+}
+
+TEST_F(OrcFileBatchReaderTest, TestCollectTargetColumnIdsStructProjectionWithListAndMap) {
+    std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(
+        "struct<outer:struct<items:array<int>,plain:double,attributes:map<string,int>>,"
+        "ignored:string>");
+    std::unique_ptr<::orc::Type> target_type = ::orc::Type::buildTypeFromString(
+        "struct<outer:struct<items:array<int>,attributes:map<string,int>>>");
+    std::vector<uint64_t> target_column_ids;
+
+    ASSERT_OK(OrcFileBatchReader::CollectTargetColumnIds(src_type.get(), target_type.get(),
+                                                         &target_column_ids));
+    // root struct(0) and outer struct(1) are not included. Selected: items(2, 3) and
+    // attributes(5, 6, 7). plain(4) and ignored(8) are skipped.
+    ASSERT_EQ(target_column_ids, (std::vector<uint64_t>{2, 3, 5, 6, 7}));
+}
+
+TEST_F(OrcFileBatchReaderTest, TestCollectTargetColumnIdsRejectsPartialListAndMapProjection) {
+    {
+        std::unique_ptr<::orc::Type> src_type =
+            ::orc::Type::buildTypeFromString("struct<items:array<struct<a:int,b:double>>>");
+        std::unique_ptr<::orc::Type> target_type =
+            ::orc::Type::buildTypeFromString("struct<items:array<struct<a:int>>>");
+        std::vector<uint64_t> target_column_ids;
+        ASSERT_NOK_WITH_MSG(OrcFileBatchReader::CollectTargetColumnIds(
+                                src_type.get(), target_type.get(), &target_column_ids),
+                            "type mismatch");
+        ASSERT_TRUE(target_column_ids.empty());
+    }
+    {
+        std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(
+            "struct<attributes:map<string,struct<a:int,b:double>>>");
+        std::unique_ptr<::orc::Type> target_type =
+            ::orc::Type::buildTypeFromString("struct<attributes:map<string,struct<a:int>>>");
+        std::vector<uint64_t> target_column_ids;
+        ASSERT_NOK_WITH_MSG(OrcFileBatchReader::CollectTargetColumnIds(
+                                src_type.get(), target_type.get(), &target_column_ids),
+                            "type mismatch");
+        ASSERT_TRUE(target_column_ids.empty());
+    }
+}
+
 TEST_P(OrcFileBatchReaderTest, TestNextBatchSimple) {
     std::string file_name = paimon::test::GetDataDir() +
                             "/orc/append_09.db/append_09/f1=10/bucket-1/"

From b0e411b2998776a4571175d465aaf944d5a98a0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B0=8F=E6=98=8E=E5=90=8C=E5=AD=A6?=
 <72908278+ChaomingZhangCN@users.noreply.github.com>
Date: Thu, 20 Aug 2026 14:06:29 +0800
Subject: [PATCH 06/93] fix(parquet): pass ReadHints into VECTOR
 ParquetFileBatchReader tests (#223)

---
 src/paimon/format/parquet/parquet_vector_io_test.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/paimon/format/parquet/parquet_vector_io_test.cpp b/src/paimon/format/parquet/parquet_vector_io_test.cpp
index e177cc1b6..f45dad1e5 100644
--- a/src/paimon/format/parquet/parquet_vector_io_test.cpp
+++ b/src/paimon/format/parquet/parquet_vector_io_test.cpp
@@ -155,7 +155,8 @@ class ParquetVectorIoTest : public ::testing::Test {
             std::unique_ptr<ParquetFileBatchReader> reader,
             ParquetFileBatchReader::Create(std::move(in_stream), /*options=*/{},
                                            /*batch_size=*/10, /*file_metadata=*/nullptr,
-                                           /*storage_read_bytes=*/nullptr, arrow_pool_));
+                                           /*storage_read_bytes=*/nullptr, arrow_pool_,
+                                           /*hints=*/std::nullopt));
         ASSERT_OK_AND_ASSIGN(std::unique_ptr<ArrowSchema> c_file_schema, reader->GetFileSchema());
         arrow::Result<std::shared_ptr<arrow::DataType>> file_type_result =
             arrow::ImportType(c_file_schema.get());
@@ -176,7 +177,8 @@ class ParquetVectorIoTest : public ::testing::Test {
             std::unique_ptr<ParquetFileBatchReader> reader,
             ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size,
                                            /*file_metadata=*/nullptr,
-                                           /*storage_read_bytes=*/nullptr, arrow_pool_));
+                                           /*storage_read_bytes=*/nullptr, arrow_pool_,
+                                           /*hints=*/std::nullopt));
         std::unique_ptr<FileBatchReader> vector_reader =
             std::make_unique<VectorFileBatchReader>(std::move(reader), pool_);
         auto c_schema = std::make_unique<ArrowSchema>();

From 8a110d488413adbafca638315a5062cd99dd26e3 Mon Sep 17 00:00:00 2001
From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com>
Date: Thu, 20 Aug 2026 15:24:41 +0800
Subject: [PATCH 07/93] feat(file-index): support writing file indexes (#210)

---
 include/paimon/defs.h                         |   4 +
 include/paimon/file_index/file_index_format.h |  33 ++-
 src/paimon/CMakeLists.txt                     |   6 +
 src/paimon/common/defs.cpp                    |   1 +
 .../common/file_index/file_index_format.cpp   | 135 ++++++++++
 .../file_index/file_index_format_test.cpp     |  28 +-
 .../common/io/byte_array_output_stream.cpp    |  80 ++++++
 .../common/io/byte_array_output_stream.h      |  69 +++++
 .../io/byte_array_output_stream_test.cpp      |  86 ++++++
 .../io/data_input_output_stream_test.cpp      |   8 +-
 .../io/memory_segment_output_stream.cpp       |   6 +-
 .../io/memory_segment_output_stream_test.cpp  |  12 +
 .../core/append/append_only_writer_test.cpp   |  37 +++
 src/paimon/core/core_options.cpp              |   8 +
 src/paimon/core/core_options.h                |   1 +
 src/paimon/core/core_options_test.cpp         |   3 +
 .../io/append_data_file_writer_factory.cpp    |   6 +
 src/paimon/core/io/data_file_index_writer.cpp | 179 +++++++++++++
 src/paimon/core/io/data_file_index_writer.h   | 100 +++++++
 .../core/io/data_file_index_writer_test.cpp   | 253 ++++++++++++++++++
 src/paimon/core/io/data_file_writer.cpp       |  25 +-
 src/paimon/core/io/data_file_writer.h         |  17 +-
 src/paimon/core/io/data_file_writer_base.h    | 145 ++++++++++
 .../core/io/data_file_writer_factory.cpp      |  15 ++
 src/paimon/core/io/data_file_writer_factory.h |   6 +
 src/paimon/core/io/file_index_options.cpp     | 109 ++++++++
 src/paimon/core/io/file_index_options.h       |  63 +++++
 .../core/io/file_index_options_test.cpp       |  58 ++++
 .../core/io/key_value_data_file_writer.cpp    |  29 +-
 .../core/io/key_value_data_file_writer.h      |  19 +-
 .../io/key_value_data_file_writer_factory.cpp |   6 +
 ...edding_append_data_file_writer_factory.cpp |   6 +
 ...ing_key_value_data_file_writer_factory.cpp |   6 +
 src/paimon/core/io/single_file_writer.h       |  25 +-
 test/inte/write_and_read_inte_test.cpp        |  83 ++++++
 35 files changed, 1568 insertions(+), 99 deletions(-)
 create mode 100644 src/paimon/common/io/byte_array_output_stream.cpp
 create mode 100644 src/paimon/common/io/byte_array_output_stream.h
 create mode 100644 src/paimon/common/io/byte_array_output_stream_test.cpp
 create mode 100644 src/paimon/core/io/data_file_index_writer.cpp
 create mode 100644 src/paimon/core/io/data_file_index_writer.h
 create mode 100644 src/paimon/core/io/data_file_index_writer_test.cpp
 create mode 100644 src/paimon/core/io/data_file_writer_base.h
 create mode 100644 src/paimon/core/io/file_index_options.cpp
 create mode 100644 src/paimon/core/io/file_index_options.h
 create mode 100644 src/paimon/core/io/file_index_options_test.cpp

diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index d1ebf507a..e944587f2 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -405,6 +405,10 @@ struct PAIMON_EXPORT Options {
     /// "file-index.read.enabled" - Whether enabled read file index. Default value is "true".
     static const char FILE_INDEX_READ_ENABLED[];
 
+    /// "file-index.in-manifest-threshold" - The threshold to store file index bytes in the
+    /// manifest. Default value is 500B.
+    static const char FILE_INDEX_IN_MANIFEST_THRESHOLD[];
+
     /// "data-file.external-paths" - The external paths where the data of this table will be
     /// written, multiple elements separated by commas.
     static const char DATA_FILE_EXTERNAL_PATHS[];
diff --git a/include/paimon/file_index/file_index_format.h b/include/paimon/file_index/file_index_format.h
index b46dee8c4..3993b6247 100644
--- a/include/paimon/file_index/file_index_format.h
+++ b/include/paimon/file_index/file_index_format.h
@@ -19,6 +19,7 @@
 #pragma once
 
 #include <cstdint>
+#include <map>
 #include <memory>
 #include <string>
 #include <vector>
@@ -32,6 +33,8 @@ struct ArrowSchema;
 namespace paimon {
 class InputStream;
 class MemoryPool;
+class Bytes;
+class OutputStream;
 
 /// Defines the on-disk format and versioning for Paimon file-level indexes.
 /// File index file format. Put all column and offset in the header.
@@ -88,9 +91,15 @@ class MemoryPool;
 class PAIMON_EXPORT FileIndexFormat {
  public:
     class Reader;
+    class Writer;
+
+    /// Serialized file indexes grouped as column name -> index type -> index bytes. A null bytes
+    /// pointer represents an empty index for that column and index type.
+    /// For example, indexes["col1"]["bsi"] = <bytes>;
+    using ColumnIndexes = std::map<std::string, std::map<std::string, std::shared_ptr<Bytes>>>;
+
     /// Creates a `Reader` to parse a index file (may contain multiple indexes) from the given input
     /// stream.
-    ///
     /// @param input_stream Input stream containing serialized index data.
     /// @param pool Memory pool for temporary allocations during reading.
     /// @return A unique pointer to a `Reader` on success, or an error if the stream is invalid
@@ -98,18 +107,38 @@ class PAIMON_EXPORT FileIndexFormat {
     static Result<std::unique_ptr<Reader>> CreateReader(
         const std::shared_ptr<InputStream>& input_stream, const std::shared_ptr<MemoryPool>& pool);
 
+    /// Creates a `Writer` which serializes a complete V1 file index container.
+    ///
+    /// @param output_stream Destination stream for serialized index data.
+    /// @param pool Memory pool for writer-side allocations.
+    /// @return A unique pointer to a `Writer` on success.
+    static Result<std::unique_ptr<Writer>> CreateWriter(
+        const std::shared_ptr<OutputStream>& output_stream,
+        const std::shared_ptr<MemoryPool>& pool);
+
  public:
     static const int64_t MAGIC;
     static const int32_t EMPTY_INDEX_FLAG;
     static const int32_t V_1;
 };
 
+/// Writer for file index file.
+class FileIndexFormat::Writer {
+ public:
+    virtual ~Writer() = default;
+
+    /// Writes all column indexes. This is a terminal, one-shot operation.
+    virtual Status WriteColumnIndexes(const FileIndexFormat::ColumnIndexes& indexes) = 0;
+
+    /// Flushes and closes the output stream supplied to `CreateWriter()`.
+    virtual Status Close() = 0;
+};
+
 /// Reader for file index file.
 class FileIndexFormat::Reader {
  public:
     virtual ~Reader() = default;
     /// Reads index data for a specific column from the index file.
-    ///
     /// @param column_name Name of the column to retrieve index data for.
     /// @param arrow_schema Arrow schema that must contain a field corresponding to `column_name`.
     /// @return A vector of shared pointers to FileIndexReader objects, each corresponding to a
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 9b0807b64..bdb110057 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -85,6 +85,7 @@ set(PAIMON_COMMON_SRCS
     common/global_index/global_indexer_factory.cpp
     common/io/buffered_input_stream.cpp
     common/io/byte_array_input_stream.cpp
+    common/io/byte_array_output_stream.cpp
     common/io/data_input_stream.cpp
     common/io/data_output_stream.cpp
     common/io/memory_segment_output_stream.cpp
@@ -270,6 +271,8 @@ set(PAIMON_CORE_SRCS
     core/io/data_file_meta.cpp
     core/io/data_file_meta_serializer.cpp
     core/io/data_file_path_factory.cpp
+    core/io/data_file_index_writer.cpp
+    core/io/file_index_options.cpp
     core/io/append_data_file_writer_factory.cpp
     core/io/blob_data_file_writer_factory.cpp
     core/io/data_file_writer_factory.cpp
@@ -577,6 +580,7 @@ if(PAIMON_BUILD_TESTS)
                     common/global_index/rangebitmap/range_bitmap_global_index_test.cpp
                     common/global_index/wrap/file_index_reader_wrapper_test.cpp
                     common/io/byte_array_input_stream_test.cpp
+                    common/io/byte_array_output_stream_test.cpp
                     common/io/data_input_output_stream_test.cpp
                     common/io/buffered_input_stream_test.cpp
                     common/io/memory_segment_output_stream_test.cpp
@@ -752,6 +756,8 @@ if(PAIMON_BUILD_TESTS)
                     core/io/complete_row_tracking_fields_reader_test.cpp
                     core/io/vector_file_batch_reader_test.cpp
                     core/io/data_file_meta_test.cpp
+                    core/io/data_file_index_writer_test.cpp
+                    core/io/file_index_options_test.cpp
                     core/io/file_index_evaluator_test.cpp
                     core/io/single_file_writer_test.cpp
                     core/io/rolling_blob_file_writer_test.cpp
diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp
index bac4f16f7..ef35940e9 100644
--- a/src/paimon/common/defs.cpp
+++ b/src/paimon/common/defs.cpp
@@ -100,6 +100,7 @@ const char Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP[] =
 const char Options::SCAN_FALLBACK_BRANCH[] = "scan.fallback-branch";
 const char Options::BRANCH[] = "branch";
 const char Options::FILE_INDEX_READ_ENABLED[] = "file-index.read.enabled";
+const char Options::FILE_INDEX_IN_MANIFEST_THRESHOLD[] = "file-index.in-manifest-threshold";
 const char Options::DATA_FILE_EXTERNAL_PATHS[] = "data-file.external-paths";
 const char Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY[] = "data-file.external-paths.strategy";
 const char Options::DATA_FILE_PREFIX[] = "data-file.prefix";
diff --git a/src/paimon/common/file_index/file_index_format.cpp b/src/paimon/common/file_index/file_index_format.cpp
index 855008450..fab5c7a7b 100644
--- a/src/paimon/common/file_index/file_index_format.cpp
+++ b/src/paimon/common/file_index/file_index_format.cpp
@@ -27,7 +27,9 @@
 #include "arrow/type.h"
 #include "fmt/format.h"
 #include "paimon/common/file_index/empty/empty_file_index_reader.h"
+#include "paimon/common/io/data_output_stream.h"
 #include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/file_index/file_indexer.h"
 #include "paimon/file_index/file_indexer_factory.h"
 #include "paimon/io/byte_array_input_stream.h"
@@ -39,6 +41,128 @@ namespace paimon {
 class InputStream;
 class MemoryPool;
 
+class FileIndexFormatWriterImpl : public FileIndexFormat::Writer {
+ public:
+    explicit FileIndexFormatWriterImpl(const std::shared_ptr<OutputStream>& output_stream)
+        : output_stream_(output_stream) {
+        assert(output_stream_);
+    }
+
+    Status WriteColumnIndexes(const FileIndexFormat::ColumnIndexes& indexes) override {
+        if (written_) {
+            return Status::Invalid("File index column indexes have already been written");
+        }
+
+        PAIMON_RETURN_NOT_OK(WriteHead(indexes));
+        // Write body.
+        DataOutputStream data_output(output_stream_);
+        for (const auto& [column_name, column_indexes] : indexes) {
+            for (const auto& [index_type, bytes] : column_indexes) {
+                if (bytes) {
+                    PAIMON_RETURN_NOT_OK(data_output.WriteBytes(bytes));
+                }
+            }
+        }
+        written_ = true;
+        return Status::OK();
+    }
+
+    Status Close() override {
+        if (closed_) {
+            return Status::OK();
+        }
+        closed_ = true;
+        PAIMON_RETURN_NOT_OK(output_stream_->Flush());
+        return output_stream_->Close();
+    }
+
+ private:
+    static constexpr int32_t kRedundantLength = 0;
+
+    static Result<int32_t> CalculateHeadLength(const FileIndexFormat::ColumnIndexes& indexes) {
+        // magic(8), version(4), header length(4), and column count(4).
+        int64_t head_length = 8 + 4 + 4 + 4;
+        int64_t body_length = 0;
+        PAIMON_RETURN_NOT_OK(
+            ValidateValueInRange<int32_t>(indexes.size(), "file index column count"));
+        for (const auto& [column_name, column_indexes] : indexes) {
+            PAIMON_RETURN_NOT_OK(ValidateValueInRange<uint16_t>(column_name.size(),
+                                                                "file index column name length"));
+            PAIMON_RETURN_NOT_OK(
+                ValidateValueInRange<int32_t>(column_indexes.size(), "column index count"));
+            // column name(2 + N) + index count(4)
+            head_length += 2 + static_cast<int64_t>(column_name.size()) + 4;
+            for (const auto& [index_type, bytes] : column_indexes) {
+                PAIMON_RETURN_NOT_OK(ValidateValueInRange<uint16_t>(index_type.size(),
+                                                                    "file index type name length"));
+                // index type(2 + N) + body offset(4) + body length(4)
+                head_length += 2 + static_cast<int64_t>(index_type.size()) + 4 + 4;
+                if (bytes) {
+                    PAIMON_RETURN_NOT_OK(AddChecked(bytes->size(), "index body", &body_length));
+                }
+            }
+        }
+
+        head_length += 4;  // The trailing redundant-length field(4).
+        PAIMON_RETURN_NOT_OK(
+            ValidateValueInRange<int32_t>(head_length, "file index header length"));
+        int64_t container_length = head_length + body_length;
+        PAIMON_RETURN_NOT_OK(ValidateValueInRange<int32_t>(container_length, "file index size"));
+        return static_cast<int32_t>(head_length);
+    }
+
+    Status WriteHead(const FileIndexFormat::ColumnIndexes& indexes) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t head_length, CalculateHeadLength(indexes));
+        DataOutputStream data_output(output_stream_);
+        // Write magic.
+        PAIMON_RETURN_NOT_OK(data_output.WriteValue<int64_t>(FileIndexFormat::MAGIC));
+        // Write version.
+        PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(FileIndexFormat::V_1));
+        // Write head length.
+        PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(head_length));
+        // Write column count.
+        PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(static_cast<int32_t>(indexes.size())));
+
+        int64_t body_offset = head_length;
+        for (const auto& [column_name, column_indexes] : indexes) {
+            // Write column name.
+            PAIMON_RETURN_NOT_OK(data_output.WriteString(column_name));
+            // Write index count for the column.
+            PAIMON_RETURN_NOT_OK(
+                data_output.WriteValue<int32_t>(static_cast<int32_t>(column_indexes.size())));
+            for (const auto& [index_type, bytes] : column_indexes) {
+                // Write index type.
+                PAIMON_RETURN_NOT_OK(data_output.WriteString(index_type));
+                // Write body offset and length.
+                if (bytes) {
+                    PAIMON_RETURN_NOT_OK(
+                        data_output.WriteValue<int32_t>(static_cast<int32_t>(body_offset)));
+                    PAIMON_RETURN_NOT_OK(
+                        data_output.WriteValue<int32_t>(static_cast<int32_t>(bytes->size())));
+                    body_offset += static_cast<int64_t>(bytes->size());
+                } else {
+                    PAIMON_RETURN_NOT_OK(
+                        data_output.WriteValue<int32_t>(FileIndexFormat::EMPTY_INDEX_FLAG));
+                    PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(0));
+                }
+            }
+        }
+        // Write redundant length for future format extensions.
+        return data_output.WriteValue<int32_t>(kRedundantLength);
+    }
+
+    template <typename T>
+    static Status AddChecked(T value, const char* name, int64_t* total) {
+        PAIMON_RETURN_NOT_OK(ValidateValueInRange<int32_t>(value, name));
+        *total += static_cast<int64_t>(value);
+        return ValidateValueInRange<int32_t>(*total, name);
+    }
+
+    std::shared_ptr<OutputStream> output_stream_;
+    bool written_ = false;
+    bool closed_ = false;
+};
+
 class FileIndexFormatReaderImpl : public FileIndexFormat::Reader {
  public:
     using HeaderType =
@@ -153,4 +277,15 @@ Result<std::unique_ptr<FileIndexFormat::Reader>> FileIndexFormat::CreateReader(
     const std::shared_ptr<InputStream>& input_stream, const std::shared_ptr<MemoryPool>& pool) {
     return FileIndexFormatReaderImpl::Create(input_stream, pool);
 }
+
+Result<std::unique_ptr<FileIndexFormat::Writer>> FileIndexFormat::CreateWriter(
+    const std::shared_ptr<OutputStream>& output_stream, const std::shared_ptr<MemoryPool>& pool) {
+    if (!output_stream) {
+        return Status::Invalid("File index output stream cannot be null");
+    }
+    if (!pool) {
+        return Status::Invalid("File index memory pool cannot be null");
+    }
+    return std::make_unique<FileIndexFormatWriterImpl>(output_stream);
+}
 }  // namespace paimon
diff --git a/src/paimon/common/file_index/file_index_format_test.cpp b/src/paimon/common/file_index/file_index_format_test.cpp
index 7851d57e1..40989f7e9 100644
--- a/src/paimon/common/file_index/file_index_format_test.cpp
+++ b/src/paimon/common/file_index/file_index_format_test.cpp
@@ -24,17 +24,20 @@
 #include "paimon/common/file_index/bloomfilter/bloom_filter_file_index.h"
 #include "paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h"
 #include "paimon/common/file_index/empty/empty_file_index_reader.h"
+#include "paimon/common/io/byte_array_output_stream.h"
 #include "paimon/data/timestamp.h"
 #include "paimon/defs.h"
 #include "paimon/file_index/file_index_result.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/io/byte_array_input_stream.h"
+#include "paimon/memory/bytes.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/predicate/literal.h"
 #include "paimon/status.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
+
 class FileIndexFormatTest : public ::testing::Test {
  public:
     void SetUp() override {
@@ -55,14 +58,25 @@ class FileIndexFormatTest : public ::testing::Test {
     std::shared_ptr<MemoryPool> pool_;
 };
 
-TEST_F(FileIndexFormatTest, TestCreateEmptyFileIndexReader) {
+TEST_F(FileIndexFormatTest, TestWriteAndReadEmptyIndexGoldenBytes) {
+    // the expected bytes are generated from Java Paimon
+    std::vector<char> expected = {0,   5,   78,  78, -48, 26, 53, -82, 0, 0, 0, 1, 0, 0, 0,   47,
+                                  0,   0,   0,   1,  0,   2,  99, 49,  0, 0, 0, 1, 0, 5, 101, 109,
+                                  112, 116, 121, -1, -1,  -1, -1, 0,   0, 0, 0, 0, 0, 0, 0};
+    FileIndexFormat::ColumnIndexes indexes;
+    indexes["c1"]["empty"] = nullptr;
+    auto segment_output = std::make_unique<MemorySegmentOutputStream>(
+        MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_);
+    auto output = std::make_shared<ByteArrayOutputStream>(std::move(segment_output));
+
+    ASSERT_OK_AND_ASSIGN(auto writer, FileIndexFormat::CreateWriter(output, pool_));
+    ASSERT_OK(writer->WriteColumnIndexes(indexes));
+    ASSERT_OK(writer->Close());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> actual, output->Finish(pool_.get()));
+
+    ASSERT_EQ(expected, std::vector<char>(actual->data(), actual->data() + actual->size()));
     auto schema = arrow::schema({arrow::field("c1", arrow::utf8())});
-    std::vector<char> index_file_bytes = {0,  5,  78, 78, -48, 26, 53,  -82, 0,   0,   0,   1,
-                                          0,  0,  0,  47, 0,   0,  0,   1,   0,   2,   99,  49,
-                                          0,  0,  0,  1,  0,   5,  101, 109, 112, 116, 121, -1,
-                                          -1, -1, -1, 0,  0,   0,  0,   0,   0,   0,   0};
-    auto input_stream =
-        std::make_shared<ByteArrayInputStream>(index_file_bytes.data(), index_file_bytes.size());
+    auto input_stream = std::make_shared<ByteArrayInputStream>(actual->data(), actual->size());
     ASSERT_OK_AND_ASSIGN(auto reader, FileIndexFormat::CreateReader(input_stream, pool_));
     ASSERT_OK_AND_ASSIGN(auto index_file_readers,
                          reader->ReadColumnIndex("c1", CreateArrowSchema(schema).get()));
diff --git a/src/paimon/common/io/byte_array_output_stream.cpp b/src/paimon/common/io/byte_array_output_stream.cpp
new file mode 100644
index 000000000..bc6d1f1ab
--- /dev/null
+++ b/src/paimon/common/io/byte_array_output_stream.cpp
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/common/io/byte_array_output_stream.h"
+
+#include <algorithm>
+#include <cassert>
+#include <limits>
+#include <utility>
+#include <vector>
+
+#include "paimon/common/memory/memory_segment_utils.h"
+#include "paimon/common/utils/math.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/memory/memory_pool.h"
+
+namespace paimon {
+
+ByteArrayOutputStream::ByteArrayOutputStream(std::unique_ptr<MemorySegmentOutputStream>&& output)
+    : output_(std::move(output)) {
+    assert(output_);
+}
+
+Result<int64_t> ByteArrayOutputStream::Write(const char* buffer, int64_t size) {
+    if (closed_) {
+        return Status::Invalid("Byte array output stream is closed");
+    }
+    PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "write length"));
+    if (buffer == nullptr && size > 0) {
+        return Status::Invalid("Write buffer must not be null when size is positive");
+    }
+    int64_t remaining = size;
+    while (remaining > 0) {
+        uint32_t to_write = static_cast<uint32_t>(std::min<int64_t>(
+            remaining, static_cast<int64_t>(std::numeric_limits<uint32_t>::max())));
+        output_->Write(buffer, to_write);
+        buffer += to_write;
+        remaining -= to_write;
+    }
+    return size;
+}
+
+Status ByteArrayOutputStream::Close() {
+    closed_ = true;
+    return Status::OK();
+}
+
+Result<std::shared_ptr<Bytes>> ByteArrayOutputStream::Finish(MemoryPool* pool) {
+    assert(pool);
+    PAIMON_RETURN_NOT_OK(Close());
+    if (result_) {
+        return result_;
+    }
+    // TODO(jinli.zjw): Support int64_t lengths in MemorySegmentUtils::CopyToBytes and remove this
+    // limit.
+    const int64_t size = output_->CurrentSize();
+    PAIMON_RETURN_NOT_OK(ValidateValueInRange<int32_t>(size, "byte array output stream size"));
+    const std::vector<MemorySegment>& segments = output_->Segments();
+    result_ = std::make_shared<Bytes>(static_cast<size_t>(size), pool);
+    MemorySegmentUtils::CopyToBytes(segments, /*offset=*/0, result_.get(),
+                                    /*bytes_offset=*/0, static_cast<int32_t>(size));
+    return result_;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/common/io/byte_array_output_stream.h b/src/paimon/common/io/byte_array_output_stream.h
new file mode 100644
index 000000000..9b87ca429
--- /dev/null
+++ b/src/paimon/common/io/byte_array_output_stream.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "paimon/common/io/memory_segment_output_stream.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+class Bytes;
+class MemoryPool;
+
+/// An in-memory output stream backed by segments allocated from a Paimon MemoryPool.
+class ByteArrayOutputStream : public OutputStream {
+ public:
+    /// Takes ownership of an initialized segmented output stream.
+    explicit ByteArrayOutputStream(std::unique_ptr<MemorySegmentOutputStream>&& output);
+
+    ~ByteArrayOutputStream() override = default;
+
+    Result<int64_t> Write(const char* buffer, int64_t size) override;
+
+    Status Flush() override {
+        return Status::OK();
+    }
+
+    Result<int64_t> GetPos() const override {
+        return output_->CurrentSize();
+    }
+
+    Result<std::string> GetUri() const override {
+        return std::string();
+    }
+
+    Status Close() override;
+
+    /// Closes the stream and returns its contents as an exactly-sized contiguous byte array.
+    /// @note The caller must keep `pool` alive until the returned bytes are destroyed.
+    Result<std::shared_ptr<Bytes>> Finish(MemoryPool* pool);
+
+ private:
+    std::unique_ptr<MemorySegmentOutputStream> output_;
+    std::shared_ptr<Bytes> result_;
+    bool closed_ = false;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/common/io/byte_array_output_stream_test.cpp b/src/paimon/common/io/byte_array_output_stream_test.cpp
new file mode 100644
index 000000000..bd185095d
--- /dev/null
+++ b/src/paimon/common/io/byte_array_output_stream_test.cpp
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/common/io/byte_array_output_stream.h"
+
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "gtest/gtest.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+TEST(ByteArrayOutputStreamTest, TestWriteAndFinish) {
+    std::shared_ptr<MemoryPool> pool = GetMemoryPool();
+    auto output = std::make_unique<MemorySegmentOutputStream>(/*segment_size=*/2, pool);
+    std::shared_ptr<ByteArrayOutputStream> stream =
+        std::make_shared<ByteArrayOutputStream>(std::move(output));
+    ASSERT_GT(pool->CurrentUsage(), 0);
+    ASSERT_OK_AND_ASSIGN(int64_t first_write, stream->Write("ab", 2));
+    ASSERT_EQ(2, first_write);
+    ASSERT_OK_AND_ASSIGN(int64_t second_write, stream->Write("cdef", 4));
+    ASSERT_EQ(4, second_write);
+    ASSERT_OK_AND_ASSIGN(int64_t position, stream->GetPos());
+    ASSERT_EQ(6, position);
+    ASSERT_EQ(pool->CurrentUsage(), pool->MaxMemoryUsage());
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> result, stream->Finish(pool.get()));
+    ASSERT_EQ("abcdef", std::string(result->data(), result->size()));
+    ASSERT_NOK_WITH_MSG(stream->Write("x", 1), "closed");
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> repeated, stream->Finish(pool.get()));
+    ASSERT_EQ(result, repeated);
+    stream.reset();
+    ASSERT_EQ(6, pool->CurrentUsage());
+}
+
+TEST(ByteArrayOutputStreamTest, TestWriteValidation) {
+    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
+    auto output = std::make_unique<MemorySegmentOutputStream>(/*segment_size=*/8, pool);
+    std::shared_ptr<ByteArrayOutputStream> stream =
+        std::make_shared<ByteArrayOutputStream>(std::move(output));
+    ASSERT_NOK(stream->Write(nullptr, 1));
+    ASSERT_NOK(stream->Write("", -1));
+    ASSERT_OK_AND_ASSIGN(int64_t written, stream->Write(nullptr, 0));
+    ASSERT_EQ(0, written);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> result, stream->Finish(pool.get()));
+    ASSERT_EQ(0, result->size());
+}
+
+TEST(ByteArrayOutputStreamTest, TestCallerKeepsMemoryPoolAlive) {
+    std::shared_ptr<MemoryPool> pool = GetMemoryPool();
+    auto output = std::make_unique<MemorySegmentOutputStream>(/*segment_size=*/8, pool);
+    std::shared_ptr<ByteArrayOutputStream> stream =
+        std::make_shared<ByteArrayOutputStream>(std::move(output));
+    ASSERT_OK_AND_ASSIGN(int64_t written, stream->Write("data", 4));
+    ASSERT_EQ(4, written);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> result, stream->Finish(pool.get()));
+
+    stream.reset();
+    ASSERT_GT(pool->CurrentUsage(), 0);
+    ASSERT_EQ("data", std::string(result->data(), result->size()));
+
+    result.reset();
+    ASSERT_EQ(0, pool->CurrentUsage());
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/common/io/data_input_output_stream_test.cpp b/src/paimon/common/io/data_input_output_stream_test.cpp
index 4e6063706..0a5dd5748 100644
--- a/src/paimon/common/io/data_input_output_stream_test.cpp
+++ b/src/paimon/common/io/data_input_output_stream_test.cpp
@@ -79,12 +79,8 @@ class DataInputOutputStreamTest : public ::testing::Test,
         (void)data_output_stream->WriteValue(static_cast<int64_t>(9223372036854775805));  // 8 bytes
         (void)data_output_stream->WriteValue(true);                                       // 1 byte
         std::string str1 = "This is a very very very long sentence.";
-        if constexpr (std::is_same_v<T, MemorySegmentOutputStream>) {
-            (void)data_output_stream->WriteString(str1);  // 39 bytes + 2 bytes len
-        } else {
-            (void)data_output_stream->WriteString(str1);  // 39 bytes + 2 bytes len
-        }
-        std::string str2 = "我是一个粉刷匠～";  // 24 bytes
+        (void)data_output_stream->WriteString(str1);  // 39 bytes + 2 bytes len
+        std::string str2 = "我是一个粉刷匠～";        // 24 bytes
         auto bytes = std::make_shared<Bytes>(str2, pool_.get());
         (void)data_output_stream->WriteBytes(bytes);
     }
diff --git a/src/paimon/common/io/memory_segment_output_stream.cpp b/src/paimon/common/io/memory_segment_output_stream.cpp
index 5355f72ba..2d0d274a7 100644
--- a/src/paimon/common/io/memory_segment_output_stream.cpp
+++ b/src/paimon/common/io/memory_segment_output_stream.cpp
@@ -54,11 +54,7 @@ void MemorySegmentOutputStream::WriteString(const std::string& str) {
 }
 
 void MemorySegmentOutputStream::Write(const char* data, uint32_t size) {
-    auto bytes = std::make_shared<Bytes>(size, pool_.get());
-    if (size != 0) {
-        memcpy(bytes->data(), data, size);
-    }
-    auto segment = MemorySegment::Wrap(bytes);
+    MemorySegment segment = MemorySegment::WrapView(data, size);
     Write(segment, 0, segment.Size());
 }
 
diff --git a/src/paimon/common/io/memory_segment_output_stream_test.cpp b/src/paimon/common/io/memory_segment_output_stream_test.cpp
index 61fbfe305..69c207ce9 100644
--- a/src/paimon/common/io/memory_segment_output_stream_test.cpp
+++ b/src/paimon/common/io/memory_segment_output_stream_test.cpp
@@ -82,4 +82,16 @@ TEST_P(MemorySegmentOutputStreamTest, TestSimple) {
     ASSERT_EQ(out.CurrentSize(), input_stream->GetPos().value());
 }
 
+TEST(MemorySegmentOutputStreamTest, TestRawWriteDoesNotAllocateTemporaryBuffer) {
+    std::shared_ptr<MemoryPool> pool = GetMemoryPool();
+    MemorySegmentOutputStream out(/*segment_size=*/8, pool);
+    uint64_t allocated_before_write = pool->CurrentUsage();
+
+    out.Write("abc", 3);
+
+    ASSERT_EQ(allocated_before_write, pool->CurrentUsage());
+    ASSERT_EQ(pool->CurrentUsage(), pool->MaxMemoryUsage());
+    ASSERT_EQ(3, out.CurrentSize());
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/append/append_only_writer_test.cpp b/src/paimon/core/append/append_only_writer_test.cpp
index 8ef6e135f..114016091 100644
--- a/src/paimon/core/append/append_only_writer_test.cpp
+++ b/src/paimon/core/append/append_only_writer_test.cpp
@@ -56,9 +56,11 @@
 #include "paimon/core/stats/simple_stats.h"
 #include "paimon/core/utils/commit_increment.h"
 #include "paimon/defs.h"
+#include "paimon/file_index/file_index_format.h"
 #include "paimon/format/file_format_factory.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/fs/local/local_file_system.h"
+#include "paimon/io/byte_array_input_stream.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/record_batch.h"
 #include "paimon/testing/utils/binary_row_generator.h"
@@ -404,6 +406,41 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndPrepareCommit) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(AppendOnlyWriterTest, TestWritePublishesEmbeddedBitmapIndex) {
+    CoreOptions options = CreateOptions(
+        {{"file-index.bitmap.columns", "f0"}, {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1MB"}});
+    auto schema =
+        arrow::schema({arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::int32())});
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = CreatePathFactory(dir->Str(), "mock_format", options);
+    ASSERT_OK_AND_ASSIGN(
+        auto writer, CreateAppendOnlyWriter(
+                         options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt,
+                         /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_));
+
+    ASSERT_OK(writer->Write(CreateBatch(schema, R"([{"f0": 1, "f1": 10},
+                                                     {"f0": 2, "f1": 20},
+                                                     {"f0": 1, "f1": 30}])")));
+    ASSERT_OK_AND_ASSIGN(CommitIncrement increment,
+                         writer->PrepareCommit(/*wait_compaction=*/true));
+    const auto& files = increment.GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(1, files.size());
+    ASSERT_TRUE(files[0]->embedded_index);
+    ASSERT_TRUE(files[0]->extra_files.empty());
+
+    auto input = std::make_shared<ByteArrayInputStream>(files[0]->embedded_index->data(),
+                                                        files[0]->embedded_index->size());
+    ASSERT_OK_AND_ASSIGN(auto index_reader, FileIndexFormat::CreateReader(input, memory_pool_));
+    ::ArrowSchema c_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
+    ASSERT_OK_AND_ASSIGN(auto column_readers, index_reader->ReadColumnIndex("f0", &c_schema));
+    ASSERT_EQ(1, column_readers.size());
+    ASSERT_OK_AND_ASSIGN(auto result, column_readers[0]->VisitEqual(Literal(1)));
+    ASSERT_EQ("{0,2}", result->ToString());
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(AppendOnlyWriterTest, TestWriteAndClose) {
     std::map<std::string, std::string> raw_options;
     raw_options[Options::FILE_FORMAT] = "orc";
diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp
index 578950fbe..1c2e164bf 100644
--- a/src/paimon/core/core_options.cpp
+++ b/src/paimon/core/core_options.cpp
@@ -386,6 +386,7 @@ struct CoreOptions::Impl {
     int64_t manifest_target_file_size = 8 * 1024 * 1024;
     int64_t deletion_vector_target_file_size = 2 * 1024 * 1024;
     int64_t manifest_full_compaction_file_size = 16 * 1024 * 1024;
+    int64_t file_index_in_manifest_threshold = 500;
     int64_t write_buffer_size = 256 * 1024 * 1024;
     int64_t commit_timeout = std::numeric_limits<int64_t>::max();
     int64_t commit_min_retry_wait = 10;
@@ -838,6 +839,9 @@ struct CoreOptions::Impl {
 
     // Parse index-related configurations: file index, global index.
     Status ParseIndexOptions(const ConfigParser& parser) {
+        // Parse file-index.in-manifest-threshold - max inline file index size, default 500B
+        PAIMON_RETURN_NOT_OK(parser.ParseMemorySize(Options::FILE_INDEX_IN_MANIFEST_THRESHOLD,
+                                                    &file_index_in_manifest_threshold));
         // Parse file-index.read.enabled - whether to enable reading file index, default true
         PAIMON_RETURN_NOT_OK(
             parser.Parse<bool>(Options::FILE_INDEX_READ_ENABLED, &file_index_read_enabled));
@@ -1654,6 +1658,10 @@ bool CoreOptions::FileIndexReadEnabled() const {
     return impl_->file_index_read_enabled;
 }
 
+int64_t CoreOptions::FileIndexInManifestThreshold() const {
+    return impl_->file_index_in_manifest_threshold;
+}
+
 std::optional<std::string> CoreOptions::GetDataFileExternalPaths() const {
     return impl_->data_file_external_paths;
 }
diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h
index 9eb289883..53ef4ad0f 100644
--- a/src/paimon/core/core_options.h
+++ b/src/paimon/core/core_options.h
@@ -207,6 +207,7 @@ class PAIMON_EXPORT CoreOptions {
     bool NeedLookup() const;
     bool PrepareCommitWaitCompaction() const;
     bool FileIndexReadEnabled() const;
+    int64_t FileIndexInManifestThreshold() const;
 
     std::map<std::string, std::string> GetFieldsSequenceGroups() const;
     bool PartialUpdateRemoveRecordOnDelete() const;
diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp
index ebc127edb..0054a5b5f 100644
--- a/src/paimon/core/core_options_test.cpp
+++ b/src/paimon/core/core_options_test.cpp
@@ -134,6 +134,7 @@ TEST(CoreOptionsTest, TestDefaultValue) {
     ASSERT_EQ(std::nullopt, core_options.GetScanFallbackBranch());
     ASSERT_EQ("main", core_options.GetBranch());
     ASSERT_TRUE(core_options.FileIndexReadEnabled());
+    ASSERT_EQ(500, core_options.FileIndexInManifestThreshold());
     ASSERT_EQ(std::nullopt, core_options.GetDataFileExternalPaths());
     ASSERT_EQ(ExternalPathStrategy::NONE, core_options.GetExternalPathStrategy());
     ASSERT_TRUE(core_options.EnableAdaptivePrefetchStrategy());
@@ -248,6 +249,7 @@ TEST(CoreOptionsTest, TestFromMap) {
         {Options::SCAN_FALLBACK_BRANCH, "fallback"},
         {Options::BRANCH, "rt"},
         {Options::FILE_INDEX_READ_ENABLED, "false"},
+        {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "2KB"},
         {Options::DATA_FILE_EXTERNAL_PATHS, "FILE:///tmp/index"},
         {Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY, "round-robin"},
         {Options::FILE_COMPRESSION, "snappy"},
@@ -398,6 +400,7 @@ TEST(CoreOptionsTest, TestFromMap) {
     ASSERT_EQ(core_options.GetScanFallbackBranch(), std::optional<std::string>("fallback"));
     ASSERT_EQ(core_options.GetBranch(), "rt");
     ASSERT_FALSE(core_options.FileIndexReadEnabled());
+    ASSERT_EQ(2 * 1024, core_options.FileIndexInManifestThreshold());
     ASSERT_EQ(core_options.GetDataFileExternalPaths(),
               std::optional<std::string>("FILE:///tmp/index"));
     ASSERT_EQ(core_options.GetExternalPathStrategy(), ExternalPathStrategy::ROUND_ROBIN);
diff --git a/src/paimon/core/io/append_data_file_writer_factory.cpp b/src/paimon/core/io/append_data_file_writer_factory.cpp
index e12374ddb..d0b677a78 100644
--- a/src/paimon/core/io/append_data_file_writer_factory.cpp
+++ b/src/paimon/core/io/append_data_file_writer_factory.cpp
@@ -24,6 +24,7 @@
 #include "arrow/c/abi.h"
 #include "arrow/c/helpers.h"
 #include "paimon/core/core_options.h"
+#include "paimon/core/io/data_file_index_writer.h"
 #include "paimon/core/io/data_file_path_factory.h"
 #include "paimon/fs/file_system.h"
 
@@ -57,6 +58,11 @@ AppendDataFileWriterFactory::CreateWriter() const {
         options_.GetFileCompression(), std::function<Status(::ArrowArray*, ::ArrowArray*)>(),
         schema_id_, seq_num_counter, file_source_, resources.stats_extractor,
         path_factory_->IsExternalPath(), write_cols_, pool_);
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<DataFileIndexWriter> file_index_writer,
+                           CreateFileIndexWriter(write_schema_, path_factory_));
+    if (file_index_writer) {
+        writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_);
+    }
     PAIMON_RETURN_NOT_OK(
         writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder));
     return std::unique_ptr<SingleFileWriter<::ArrowArray*, std::shared_ptr<DataFileMeta>>>(
diff --git a/src/paimon/core/io/data_file_index_writer.cpp b/src/paimon/core/io/data_file_index_writer.cpp
new file mode 100644
index 000000000..5c97bb3da
--- /dev/null
+++ b/src/paimon/core/io/data_file_index_writer.cpp
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/io/data_file_index_writer.h"
+
+#include <cassert>
+#include <utility>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
+#include "fmt/format.h"
+#include "paimon/common/io/byte_array_output_stream.h"
+#include "paimon/common/io/memory_segment_output_stream.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/file_index/file_index_format.h"
+#include "paimon/file_index/file_index_writer.h"
+#include "paimon/file_index/file_indexer.h"
+#include "paimon/file_index/file_indexer_factory.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/status.h"
+
+namespace paimon {
+Result<std::unique_ptr<DataFileIndexWriter>> DataFileIndexWriter::Create(
+    const std::shared_ptr<arrow::Schema>& logical_schema, const FileIndexOptions& options,
+    const std::shared_ptr<FileSystem>& file_system,
+    const std::shared_ptr<DataFilePathFactory>& path_factory,
+    const std::shared_ptr<MemoryPool>& pool) {
+    assert(logical_schema);
+    assert(file_system);
+    assert(path_factory);
+    assert(pool);
+    std::vector<IndexWriterEntry> writers;
+    writers.reserve(options.Definitions().size());
+    for (const FileIndexDefinition& definition : options.Definitions()) {
+        if (SpecialFields::IsSystemField(definition.column_name)) {
+            return Status::Invalid(
+                fmt::format("File index column '{}' is a system field", definition.column_name));
+        }
+        int32_t field_index = logical_schema->GetFieldIndex(definition.column_name);
+        if (field_index < 0) {
+            return Status::Invalid(
+                fmt::format("File index column '{}' does not exist in the write schema",
+                            definition.column_name));
+        }
+        std::shared_ptr<arrow::Field> field = logical_schema->field(field_index);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileIndexer> indexer,
+                               FileIndexerFactory::Get(definition.index_type, definition.options));
+        if (!indexer) {
+            return Status::Invalid(
+                fmt::format("File index type '{}' is not registered", definition.index_type));
+        }
+        ::ArrowSchema c_schema;
+        ArrowSchemaMarkReleased(&c_schema);
+        ScopeGuard schema_guard([&c_schema]() { ArrowSchemaRelease(&c_schema); });
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow::schema({field}), &c_schema));
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileIndexWriter> writer,
+                               indexer->CreateWriter(&c_schema, pool));
+        writers.push_back(
+            {definition.column_name, definition.index_type, field_index, field, std::move(writer)});
+    }
+    return std::unique_ptr<DataFileIndexWriter>(new DataFileIndexWriter(
+        std::move(writers), options.InManifestThreshold(), file_system, path_factory, pool));
+}
+
+DataFileIndexWriter::DataFileIndexWriter(std::vector<IndexWriterEntry>&& writers,
+                                         int64_t in_manifest_threshold,
+                                         const std::shared_ptr<FileSystem>& file_system,
+                                         const std::shared_ptr<DataFilePathFactory>& path_factory,
+                                         const std::shared_ptr<MemoryPool>& pool)
+    : writers_(std::move(writers)),
+      in_manifest_threshold_(in_manifest_threshold),
+      file_system_(file_system),
+      path_factory_(path_factory),
+      pool_(pool) {}
+
+Status DataFileIndexWriter::AddBatch(const std::shared_ptr<arrow::StructArray>& logical_batch) {
+    if (finished_) {
+        return Status::Invalid("Data file index writer has already finished");
+    }
+    for (const IndexWriterEntry& entry : writers_) {
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::StructArray> projected,
+            arrow::StructArray::Make({logical_batch->field(entry.field_index)}, {entry.field}));
+        ::ArrowArray c_array;
+        ArrowArrayMarkReleased(&c_array);
+        ScopeGuard array_guard([&c_array]() { ArrowArrayRelease(&c_array); });
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*projected, &c_array));
+        PAIMON_RETURN_NOT_OK(entry.writer->AddBatch(&c_array));
+    }
+    return Status::OK();
+}
+
+Result<std::shared_ptr<Bytes>> DataFileIndexWriter::SerializeContainer() {
+    FileIndexFormat::ColumnIndexes column_indexes;
+    for (const IndexWriterEntry& entry : writers_) {
+        PAIMON_ASSIGN_OR_RAISE(column_indexes[entry.column_name][entry.index_type],
+                               entry.writer->SerializedBytes());
+    }
+
+    auto segment_output = std::make_unique<MemorySegmentOutputStream>(
+        MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_);
+    std::shared_ptr<ByteArrayOutputStream> output =
+        std::make_shared<ByteArrayOutputStream>(std::move(segment_output));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileIndexFormat::Writer> format_writer,
+                           FileIndexFormat::CreateWriter(output, pool_));
+    PAIMON_RETURN_NOT_OK(format_writer->WriteColumnIndexes(column_indexes));
+    PAIMON_RETURN_NOT_OK(format_writer->Close());
+    return output->Finish(pool_.get());
+}
+
+Result<FileIndexWriteResult> DataFileIndexWriter::Finish(const std::string& data_file_path) {
+    if (finished_) {
+        return Status::Invalid("Data file index writer has already finished");
+    }
+    finished_ = true;
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> bytes, SerializeContainer());
+    if (static_cast<int64_t>(bytes->size()) <= in_manifest_threshold_) {
+        return FileIndexWriteResult{bytes, {}};
+    }
+
+    external_index_path_ = path_factory_->ToFileIndexPath(data_file_path);
+    PAIMON_RETURN_NOT_OK(WriteExternal(external_index_path_.value(), bytes));
+    return FileIndexWriteResult{nullptr, {PathUtil::GetName(external_index_path_.value())}};
+}
+
+Status DataFileIndexWriter::WriteExternal(const std::string& path,
+                                          const std::shared_ptr<Bytes>& bytes) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OutputStream> output,
+                           file_system_->Create(path, /*overwrite=*/false));
+    ScopeGuard guard([this, &output]() {
+        if (output) {
+            [[maybe_unused]] Status _ = output->Close();
+        }
+        Abort();
+    });
+    PAIMON_ASSIGN_OR_RAISE(int64_t written,
+                           output->Write(bytes->data(), static_cast<int64_t>(bytes->size())));
+    if (written != static_cast<int64_t>(bytes->size())) {
+        return Status::IOError(fmt::format("Short write for file index {}: expected {}, wrote {}",
+                                           path, bytes->size(), written));
+    }
+    PAIMON_RETURN_NOT_OK(output->Flush());
+    Status close_status = output->Close();
+    output.reset();
+    PAIMON_RETURN_NOT_OK(close_status);
+    guard.Release();
+    return Status::OK();
+}
+
+void DataFileIndexWriter::Abort() {
+    if (external_index_path_) {
+        [[maybe_unused]] Status _ = file_system_->Delete(external_index_path_.value());
+    }
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/data_file_index_writer.h b/src/paimon/core/io/data_file_index_writer.h
new file mode 100644
index 000000000..883b719fc
--- /dev/null
+++ b/src/paimon/core/io/data_file_index_writer.h
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "paimon/core/io/file_index_options.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+
+namespace arrow {
+class Field;
+class Schema;
+class StructArray;
+}  // namespace arrow
+
+namespace paimon {
+
+class Bytes;
+class DataFilePathFactory;
+class FileIndexWriter;
+class FileSystem;
+class MemoryPool;
+
+struct FileIndexWriteResult {
+    std::shared_ptr<Bytes> embedded_index;
+    std::vector<std::optional<std::string>> extra_files;
+};
+
+/// Builds every configured column index for one data file.
+class DataFileIndexWriter {
+ public:
+    static Result<std::unique_ptr<DataFileIndexWriter>> Create(
+        const std::shared_ptr<arrow::Schema>& logical_schema, const FileIndexOptions& options,
+        const std::shared_ptr<FileSystem>& file_system,
+        const std::shared_ptr<DataFilePathFactory>& path_factory,
+        const std::shared_ptr<MemoryPool>& pool);
+
+    Status AddBatch(const std::shared_ptr<arrow::StructArray>& logical_batch);
+
+    /// Finalizes and publishes all configured indexes. This is a terminal, one-shot operation.
+    ///
+    /// @param data_file_path Path of the data file associated with these indexes.
+    /// @return Embedded index bytes or the external index file name.
+    Result<FileIndexWriteResult> Finish(const std::string& data_file_path);
+
+    void Abort();
+
+    const std::optional<std::string>& ExternalIndexPath() const {
+        return external_index_path_;
+    }
+
+ private:
+    struct IndexWriterEntry {
+        std::string column_name;
+        std::string index_type;
+        int32_t field_index;
+        std::shared_ptr<arrow::Field> field;
+        std::shared_ptr<FileIndexWriter> writer;
+    };
+
+    DataFileIndexWriter(std::vector<IndexWriterEntry>&& writers, int64_t in_manifest_threshold,
+                        const std::shared_ptr<FileSystem>& file_system,
+                        const std::shared_ptr<DataFilePathFactory>& path_factory,
+                        const std::shared_ptr<MemoryPool>& pool);
+
+    Result<std::shared_ptr<Bytes>> SerializeContainer();
+    Status WriteExternal(const std::string& path, const std::shared_ptr<Bytes>& bytes);
+
+    std::vector<IndexWriterEntry> writers_;
+    int64_t in_manifest_threshold_;
+    std::shared_ptr<FileSystem> file_system_;
+    std::shared_ptr<DataFilePathFactory> path_factory_;
+    std::shared_ptr<MemoryPool> pool_;
+    std::optional<std::string> external_index_path_;
+    bool finished_ = false;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/data_file_index_writer_test.cpp b/src/paimon/core/io/data_file_index_writer_test.cpp
new file mode 100644
index 000000000..e9ab7940b
--- /dev/null
+++ b/src/paimon/core/io/data_file_index_writer_test.cpp
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/io/data_file_index_writer.h"
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string>
+
+#include "arrow/c/bridge.h"
+#include "arrow/ipc/json_simple.h"
+#include "arrow/type.h"
+#include "gtest/gtest.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/core/io/file_index_options.h"
+#include "paimon/defs.h"
+#include "paimon/file_index/file_index_format.h"
+#include "paimon/fs/local/local_file_system.h"
+#include "paimon/io/byte_array_input_stream.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/testing/mock/mock_file_system.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+namespace {
+
+struct CloseFailingState {
+    int32_t close_count = 0;
+    int32_t delete_count = 0;
+};
+
+class CloseFailingOutputStream : public MockOutputStream {
+ public:
+    explicit CloseFailingOutputStream(const std::shared_ptr<CloseFailingState>& state)
+        : state_(state) {}
+
+    Result<int64_t> Write(const char*, int64_t size) override {
+        return size;
+    }
+
+    Status Close() override {
+        ++state_->close_count;
+        return Status::IOError("close failed");
+    }
+
+ private:
+    std::shared_ptr<CloseFailingState> state_;
+};
+
+class CloseFailingFileSystem : public MockFileSystem {
+ public:
+    explicit CloseFailingFileSystem(const std::shared_ptr<CloseFailingState>& state)
+        : state_(state) {}
+
+    Result<std::unique_ptr<OutputStream>> Create(const std::string&, bool) const override {
+        return std::unique_ptr<OutputStream>(new CloseFailingOutputStream(state_));
+    }
+
+    Status Delete(const std::string&, bool = true) const override {
+        ++state_->delete_count;
+        return Status::OK();
+    }
+
+ private:
+    std::shared_ptr<CloseFailingState> state_;
+};
+
+}  // namespace
+
+class DataFileIndexWriterTest : public ::testing::Test {
+ public:
+    void SetUp() override {
+        pool_ = GetDefaultPool();
+        file_system_ = std::make_shared<LocalFileSystem>();
+        directory_ = UniqueTestDirectory::Create();
+        ASSERT_TRUE(directory_);
+        path_factory_ = std::make_shared<DataFilePathFactory>();
+        ASSERT_OK(path_factory_->Init(directory_->Str(), "orc", "data-", nullptr));
+        schema_ =
+            arrow::schema({arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::int32())});
+    }
+
+    Result<std::unique_ptr<DataFileIndexWriter>> CreateWriter(
+        const std::map<std::string, std::string>& index_options) const {
+        PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
+                               CoreOptions::FromMap(index_options, file_system_));
+        PAIMON_ASSIGN_OR_RAISE(FileIndexOptions parsed,
+                               FileIndexOptions::FromCoreOptions(core_options));
+        return DataFileIndexWriter::Create(schema_, parsed, file_system_, path_factory_, pool_);
+    }
+
+    std::shared_ptr<arrow::StructArray> CreateBatch(const std::string& json) const {
+        std::shared_ptr<arrow::Array> array =
+            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema_->fields()), json)
+                .ValueOrDie();
+        return checked_pointer_cast<arrow::StructArray>(array);
+    }
+
+    Result<std::unique_ptr<FileIndexFormat::Reader>> CreateReader(
+        const std::shared_ptr<Bytes>& bytes) const {
+        auto input = std::make_shared<ByteArrayInputStream>(bytes->data(), bytes->size());
+        return FileIndexFormat::CreateReader(input, pool_);
+    }
+
+    Result<std::vector<std::shared_ptr<FileIndexReader>>> ReadColumn(
+        FileIndexFormat::Reader* reader, const std::string& column_name) const {
+        ::ArrowSchema c_schema;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, &c_schema));
+        return reader->ReadColumnIndex(column_name, &c_schema);
+    }
+
+ protected:
+    std::shared_ptr<MemoryPool> pool_;
+    std::shared_ptr<LocalFileSystem> file_system_;
+    std::unique_ptr<UniqueTestDirectory> directory_;
+    std::shared_ptr<DataFilePathFactory> path_factory_;
+    std::shared_ptr<arrow::Schema> schema_;
+};
+
+TEST_F(DataFileIndexWriterTest, TestBitmapAndRangeBitmapEmbeddedRoundTrip) {
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         CreateWriter({{"file-index.bitmap.columns", "f0"},
+                                       {"file-index.range-bitmap.columns", "f1"},
+                                       {"file-index.range-bitmap.f1.chunk-size", "1KB"},
+                                       {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1MB"}}));
+    ASSERT_OK(writer->AddBatch(CreateBatch(R"([{"f0": 1, "f1": 10},
+                                                {"f0": 2, "f1": 20}])")));
+    ASSERT_OK(writer->AddBatch(CreateBatch(R"([{"f0": 1, "f1": 30},
+                                                {"f0": null, "f1": 40}])")));
+
+    ASSERT_OK_AND_ASSIGN(FileIndexWriteResult result, writer->Finish("unused.orc"));
+    ASSERT_TRUE(result.embedded_index);
+    ASSERT_TRUE(result.extra_files.empty());
+    ASSERT_OK_AND_ASSIGN(auto reader, CreateReader(result.embedded_index));
+
+    ASSERT_OK_AND_ASSIGN(auto bitmap_readers, ReadColumn(reader.get(), "f0"));
+    ASSERT_EQ(1, bitmap_readers.size());
+    ASSERT_OK_AND_ASSIGN(auto equal_result, bitmap_readers[0]->VisitEqual(Literal(1)));
+    ASSERT_EQ("{0,2}", equal_result->ToString());
+    ASSERT_OK_AND_ASSIGN(auto null_result, bitmap_readers[0]->VisitIsNull());
+    ASSERT_EQ("{3}", null_result->ToString());
+
+    ASSERT_OK_AND_ASSIGN(auto range_readers, ReadColumn(reader.get(), "f1"));
+    ASSERT_EQ(1, range_readers.size());
+    ASSERT_OK_AND_ASSIGN(auto greater_result, range_readers[0]->VisitGreaterThan(Literal(20)));
+    ASSERT_EQ("{2,3}", greater_result->ToString());
+}
+
+TEST_F(DataFileIndexWriterTest, TestExternalIndexAndAbortCleanup) {
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         CreateWriter({{"file-index.bitmap.columns", "f0"},
+                                       {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1B"}}));
+    ASSERT_OK(writer->AddBatch(CreateBatch(R"([{"f0": 1, "f1": 10}])")));
+    std::string data_path = path_factory_->NewPath();
+
+    ASSERT_OK_AND_ASSIGN(FileIndexWriteResult result, writer->Finish(data_path));
+    ASSERT_FALSE(result.embedded_index);
+    ASSERT_EQ(1, result.extra_files.size());
+    ASSERT_TRUE(result.extra_files[0]);
+    ASSERT_EQ(PathUtil::GetName(path_factory_->ToFileIndexPath(data_path)),
+              result.extra_files[0].value());
+    std::string index_path = path_factory_->ToFileIndexPath(data_path);
+    ASSERT_OK_AND_ASSIGN(bool exists, file_system_->Exists(index_path));
+    ASSERT_TRUE(exists);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input, file_system_->Open(index_path));
+    ASSERT_OK_AND_ASSIGN(auto reader, FileIndexFormat::CreateReader(input, pool_));
+    ASSERT_OK_AND_ASSIGN(auto bitmap_readers, ReadColumn(reader.get(), "f0"));
+    ASSERT_EQ(1, bitmap_readers.size());
+    ASSERT_OK_AND_ASSIGN(auto equal_result, bitmap_readers[0]->VisitEqual(Literal(1)));
+    ASSERT_EQ("{0}", equal_result->ToString());
+
+    writer->Abort();
+    ASSERT_OK_AND_ASSIGN(exists, file_system_->Exists(index_path));
+    ASSERT_FALSE(exists);
+}
+
+TEST_F(DataFileIndexWriterTest, TestUnavailableWriterFailsCreation) {
+    ASSERT_NOK_WITH_MSG(CreateWriter({{"file-index.unknown.columns", "f0"}}),
+                        "File index type 'unknown' is not registered");
+    ASSERT_NOK_WITH_MSG(CreateWriter({{"file-index.bloom-filter.columns", "f0"}}),
+                        "do not support index writer in bloom filter");
+}
+
+TEST_F(DataFileIndexWriterTest, TestRejectSystemFieldIndex) {
+    std::shared_ptr<arrow::Schema> key_value_schema =
+        SpecialFields::CompleteSequenceAndValueKindField(schema_);
+    for (const std::string& field_name :
+         {SpecialFields::SequenceNumber().Name(), SpecialFields::ValueKind().Name()}) {
+        ASSERT_OK_AND_ASSIGN(
+            CoreOptions core_options,
+            CoreOptions::FromMap({{"file-index.bitmap.columns", field_name}}, file_system_));
+        ASSERT_OK_AND_ASSIGN(FileIndexOptions options,
+                             FileIndexOptions::FromCoreOptions(core_options));
+        ASSERT_NOK_WITH_MSG(DataFileIndexWriter::Create(key_value_schema, options, file_system_,
+                                                        path_factory_, pool_),
+                            "is a system field");
+    }
+}
+
+TEST_F(DataFileIndexWriterTest, TestFinishIsOneShot) {
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         CreateWriter({{"file-index.bitmap.columns", "f0"},
+                                       {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1MB"}}));
+    ASSERT_OK(writer->AddBatch(CreateBatch(R"([{"f0": 1, "f1": 10}])")));
+    ASSERT_OK(writer->Finish("unused.orc"));
+
+    ASSERT_NOK_WITH_MSG(writer->Finish("unused.orc"), "already finished");
+    ASSERT_NOK_WITH_MSG(writer->AddBatch(CreateBatch(R"([{"f0": 2, "f1": 20}])")),
+                        "already finished");
+}
+
+TEST_F(DataFileIndexWriterTest, TestCloseFailureClosesExternalStreamOnce) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                         CoreOptions::FromMap({{"file-index.bitmap.columns", "f0"},
+                                               {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1B"}},
+                                              file_system_));
+    ASSERT_OK_AND_ASSIGN(FileIndexOptions options, FileIndexOptions::FromCoreOptions(core_options));
+    auto state = std::make_shared<CloseFailingState>();
+    auto close_failing_file_system = std::make_shared<CloseFailingFileSystem>(state);
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         DataFileIndexWriter::Create(schema_, options, close_failing_file_system,
+                                                     path_factory_, pool_));
+    ASSERT_OK(writer->AddBatch(CreateBatch(R"([{"f0": 1, "f1": 10}])")));
+
+    ASSERT_NOK_WITH_MSG(writer->Finish(path_factory_->NewPath()), "close failed");
+    ASSERT_EQ(1, state->close_count);
+    ASSERT_EQ(1, state->delete_count);
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/io/data_file_writer.cpp b/src/paimon/core/io/data_file_writer.cpp
index 4ed3e040c..9275fcf25 100644
--- a/src/paimon/core/io/data_file_writer.cpp
+++ b/src/paimon/core/io/data_file_writer.cpp
@@ -19,6 +19,7 @@
 #include "paimon/core/io/data_file_writer.h"
 
 #include <cassert>
+#include <utility>
 
 #include "arrow/c/abi.h"
 #include "paimon/common/utils/long_counter.h"
@@ -36,7 +37,7 @@ DataFileWriter::DataFileWriter(
     const std::shared_ptr<FormatStatsExtractor>& stats_extractor, bool is_external_path,
     const std::optional<std::vector<std::string>>& write_cols,
     const std::shared_ptr<MemoryPool>& pool)
-    : SingleFileWriter(compression, converter),
+    : DataFileWriterBase(compression, std::move(converter)),
       pool_(pool),
       schema_id_(schema_id),
       is_external_path_(is_external_path),
@@ -45,28 +46,13 @@ DataFileWriter::DataFileWriter(
       stats_extractor_(stats_extractor),
       write_cols_(write_cols) {}
 
-void DataFileWriter::SetMetadataFinalizer(MetadataFinalizer finalizer) {
-    metadata_finalizer_ = std::move(finalizer);
-}
-
 Status DataFileWriter::Write(ArrowArray* batch) {
     int64_t record_count = batch->length;
-    PAIMON_RETURN_NOT_OK(SingleFileWriter::Write(batch));
+    PAIMON_RETURN_NOT_OK(WriteRecordWithFileIndex(batch));
     seq_num_counter_->Add(record_count);
     return Status::OK();
 }
 
-Status DataFileWriter::BeforeFinish() {
-    if (metadata_finalizer_) {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> updated_schema,
-                               metadata_finalizer_());
-        if (updated_schema) {
-            PAIMON_RETURN_NOT_OK(UpdateSchema(updated_schema));
-        }
-    }
-    return Status::OK();
-}
-
 Result<std::shared_ptr<DataFileMeta>> DataFileWriter::GetResult() {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<ColumnStats>> field_stats, GetFieldStats());
     PAIMON_ASSIGN_OR_RAISE(SimpleStats stats,
@@ -77,11 +63,12 @@ Result<std::shared_ptr<DataFileMeta>> DataFileWriter::GetResult() {
         PAIMON_ASSIGN_OR_RAISE(Path external_path, PathUtil::ToPath(path_));
         final_path = external_path.ToString();
     }
+    const FileIndexWriteResult& file_index = GetFileIndexWriteResult();
     return DataFileMeta::ForAppend(
         PathUtil::GetName(path_), output_bytes_, RecordCount(), stats,
         seq_num_counter_->GetValue() - RecordCount(), seq_num_counter_->GetValue() - 1, schema_id_,
-        {}, /*embedded_index=*/nullptr, file_source_, /*value_stats_cols=*/std::nullopt, final_path,
-        /*first_row_id=*/std::nullopt, write_cols_);
+        file_index.extra_files, file_index.embedded_index, file_source_,
+        /*value_stats_cols=*/std::nullopt, final_path, /*first_row_id=*/std::nullopt, write_cols_);
 }
 
 Result<std::vector<std::shared_ptr<ColumnStats>>> DataFileWriter::GetFieldStats() {
diff --git a/src/paimon/core/io/data_file_writer.h b/src/paimon/core/io/data_file_writer.h
index 60cc808a9..f56f34956 100644
--- a/src/paimon/core/io/data_file_writer.h
+++ b/src/paimon/core/io/data_file_writer.h
@@ -28,7 +28,7 @@
 #include "arrow/c/abi.h"
 #include "paimon/common/utils/long_counter.h"
 #include "paimon/core/io/data_file_meta.h"
-#include "paimon/core/io/single_file_writer.h"
+#include "paimon/core/io/data_file_writer_base.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/result.h"
 #include "paimon/status.h"
@@ -44,13 +44,8 @@ class FormatStatsExtractor;
 class LongCounter;
 class MemoryPool;
 
-class DataFileWriter : public SingleFileWriter<::ArrowArray*, std::shared_ptr<DataFileMeta>> {
+class DataFileWriter : public DataFileWriterBase<::ArrowArray*> {
  public:
-    /// Callback invoked during BeforeFinish() to finalize file metadata.
-    /// Produces an updated schema with per-field metadata (e.g. shredding metadata)
-    /// and may perform other finalization work (e.g. reporting stats to cross-file context).
-    using MetadataFinalizer = std::function<Result<std::shared_ptr<arrow::Schema>>()>;
-
     DataFileWriter(const std::string& compression,
                    std::function<Status(::ArrowArray*, ::ArrowArray*)> converter, int64_t schema_id,
                    const std::shared_ptr<LongCounter>& seq_num_counter, FileSource file_source,
@@ -58,17 +53,10 @@ class DataFileWriter : public SingleFileWriter<::ArrowArray*, std::shared_ptr<Da
                    bool is_external_path, const std::optional<std::vector<std::string>>& write_cols,
                    const std::shared_ptr<MemoryPool>& pool);
 
-    /// Sets the metadata finalizer. Called during BeforeFinish() to produce an updated
-    /// schema and perform finalization callbacks. Must be set before Close().
-    void SetMetadataFinalizer(MetadataFinalizer finalizer);
-
     Status Write(::ArrowArray* batch) override;
 
     Result<std::shared_ptr<DataFileMeta>> GetResult() override;
 
- protected:
-    Status BeforeFinish() override;
-
  private:
     Result<std::vector<std::shared_ptr<ColumnStats>>> GetFieldStats();
 
@@ -80,7 +68,6 @@ class DataFileWriter : public SingleFileWriter<::ArrowArray*, std::shared_ptr<Da
     FileSource file_source_;
     std::shared_ptr<FormatStatsExtractor> stats_extractor_;
     std::optional<std::vector<std::string>> write_cols_;
-    MetadataFinalizer metadata_finalizer_;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/io/data_file_writer_base.h b/src/paimon/core/io/data_file_writer_base.h
new file mode 100644
index 000000000..ccea898a7
--- /dev/null
+++ b/src/paimon/core/io/data_file_writer_base.h
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+
+#include "arrow/c/bridge.h"
+#include "arrow/type.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/io/data_file_index_writer.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/single_file_writer.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+struct KeyValueBatch;
+
+/// Common lifecycle for data file writers which may finalize schema metadata and publish a
+/// file-level index. Concrete writers remain responsible for their record-specific state and
+/// DataFileMeta construction.
+template <typename Record>
+class DataFileWriterBase : public SingleFileWriter<Record, std::shared_ptr<DataFileMeta>> {
+ public:
+    using Base = SingleFileWriter<Record, std::shared_ptr<DataFileMeta>>;
+    using AbortExecutor = typename Base::AbortExecutor;
+    /// Callback invoked during BeforeFinish() to finalize file metadata.
+    /// Produces an updated schema with per-field metadata (e.g. shredding metadata)
+    /// and may perform other finalization work (e.g. reporting stats to cross-file context).
+    using MetadataFinalizer = std::function<Result<std::shared_ptr<arrow::Schema>>()>;
+
+    /// Sets the metadata finalizer. Called during BeforeFinish() to produce an updated
+    /// schema and perform finalization callbacks. Must be set before Close().
+    void SetMetadataFinalizer(MetadataFinalizer finalizer) {
+        metadata_finalizer_ = std::move(finalizer);
+    }
+
+    void SetFileIndexWriter(std::unique_ptr<DataFileIndexWriter>&& file_index_writer,
+                            const std::shared_ptr<arrow::Schema>& logical_schema) {
+        file_index_writer_ = std::move(file_index_writer);
+        logical_type_ = arrow::struct_(logical_schema->fields());
+    }
+
+    void Abort() override {
+        if (file_index_writer_) {
+            // The external index uses a path different from the data file path deleted by Base.
+            file_index_writer_->Abort();
+        }
+        Base::Abort();
+    }
+
+    Result<AbortExecutor> GetAbortExecutor() const override {
+        PAIMON_ASSIGN_OR_RAISE(AbortExecutor executor, Base::GetAbortExecutor());
+        if (file_index_writer_ && file_index_writer_->ExternalIndexPath()) {
+            executor.Add(this->fs_, file_index_writer_->ExternalIndexPath().value());
+        }
+        return executor;
+    }
+
+ protected:
+    DataFileWriterBase(const std::string& compression,
+                       std::function<Status(Record, ::ArrowArray*)> converter)
+        : Base(compression, std::move(converter)) {}
+
+    /// Extracts the pre-conversion Arrow batch from record for file index construction, then
+    /// passes record to the underlying data file writer, which may convert it to a physical schema.
+    Status WriteRecordWithFileIndex(Record record) {
+        PAIMON_RETURN_NOT_OK(AddFileIndexBatch(GetFileIndexBatch(record)));
+        return Base::Write(std::move(record));
+    }
+
+    const FileIndexWriteResult& GetFileIndexWriteResult() const {
+        return file_index_result_;
+    }
+
+    Status BeforeFinish() override {
+        if (metadata_finalizer_) {
+            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> updated_schema,
+                                   metadata_finalizer_());
+            if (updated_schema) {
+                PAIMON_RETURN_NOT_OK(this->UpdateSchema(updated_schema));
+            }
+        }
+        return Status::OK();
+    }
+
+    Status BeforeCompletion() override {
+        if (file_index_writer_) {
+            PAIMON_ASSIGN_OR_RAISE(file_index_result_, file_index_writer_->Finish(this->path_));
+        }
+        return Status::OK();
+    }
+
+ private:
+    static ::ArrowArray* GetFileIndexBatch(Record& record) {
+        if constexpr (std::is_same_v<Record, ::ArrowArray*>) {
+            return record;
+        } else {
+            static_assert(std::is_same_v<Record, KeyValueBatch>,
+                          "Unsupported data file record type");
+            return record.batch.get();
+        }
+    }
+
+    Status AddFileIndexBatch(::ArrowArray* batch) {
+        if (!file_index_writer_) {
+            return Status::OK();
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> logical_array,
+                                          arrow::ImportArray(batch, logical_type_));
+        std::shared_ptr<arrow::StructArray> logical_batch =
+            checked_pointer_cast<arrow::StructArray>(logical_array);
+        PAIMON_RETURN_NOT_OK(file_index_writer_->AddBatch(logical_batch));
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*logical_batch, batch));
+        return Status::OK();
+    }
+
+    MetadataFinalizer metadata_finalizer_;
+    std::unique_ptr<DataFileIndexWriter> file_index_writer_;
+    std::shared_ptr<arrow::DataType> logical_type_;
+    FileIndexWriteResult file_index_result_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/data_file_writer_factory.cpp b/src/paimon/core/io/data_file_writer_factory.cpp
index b929dde83..07195ab7f 100644
--- a/src/paimon/core/io/data_file_writer_factory.cpp
+++ b/src/paimon/core/io/data_file_writer_factory.cpp
@@ -24,6 +24,9 @@
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/core_options.h"
+#include "paimon/core/io/data_file_index_writer.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/core/io/file_index_options.h"
 #include "paimon/format/file_format.h"
 #include "paimon/format/writer_builder.h"
 
@@ -58,4 +61,16 @@ Result<DataFileWriterFactory::WriterResources> DataFileWriterFactory::CreateWrit
     return resources;
 }
 
+Result<std::unique_ptr<DataFileIndexWriter>> DataFileWriterFactory::CreateFileIndexWriter(
+    const std::shared_ptr<arrow::Schema>& logical_schema,
+    const std::shared_ptr<DataFilePathFactory>& path_factory) const {
+    PAIMON_ASSIGN_OR_RAISE(FileIndexOptions file_index_options,
+                           FileIndexOptions::FromCoreOptions(options_));
+    if (file_index_options.Empty()) {
+        return std::unique_ptr<DataFileIndexWriter>();
+    }
+    return DataFileIndexWriter::Create(logical_schema, file_index_options, options_.GetFileSystem(),
+                                       path_factory, pool_);
+}
+
 }  // namespace paimon
diff --git a/src/paimon/core/io/data_file_writer_factory.h b/src/paimon/core/io/data_file_writer_factory.h
index c727b47d0..cab942f0e 100644
--- a/src/paimon/core/io/data_file_writer_factory.h
+++ b/src/paimon/core/io/data_file_writer_factory.h
@@ -32,6 +32,8 @@ class Schema;
 namespace paimon {
 
 class FileFormat;
+class DataFileIndexWriter;
+class DataFilePathFactory;
 class FormatStatsExtractor;
 class MemoryPool;
 class WriterBuilder;
@@ -52,6 +54,10 @@ class DataFileWriterFactory {
                                                   const std::shared_ptr<arrow::Schema>& file_schema,
                                                   bool create_stats_extractor) const;
 
+    Result<std::unique_ptr<DataFileIndexWriter>> CreateFileIndexWriter(
+        const std::shared_ptr<arrow::Schema>& logical_schema,
+        const std::shared_ptr<DataFilePathFactory>& path_factory) const;
+
     CoreOptions options_;
     int64_t schema_id_;
     std::shared_ptr<MemoryPool> pool_;
diff --git a/src/paimon/core/io/file_index_options.cpp b/src/paimon/core/io/file_index_options.cpp
new file mode 100644
index 000000000..a9587363a
--- /dev/null
+++ b/src/paimon/core/io/file_index_options.cpp
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/io/file_index_options.h"
+
+#include <cstddef>
+#include <set>
+#include <utility>
+
+#include "fmt/format.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/core_options.h"
+#include "paimon/defs.h"
+#include "paimon/status.h"
+
+namespace paimon {
+namespace {
+
+constexpr char kFileIndexPrefix[] = "file-index.";
+constexpr char kColumnsSuffix[] = ".columns";
+constexpr size_t kFileIndexPrefixLength = sizeof(kFileIndexPrefix) - 1;
+constexpr size_t kColumnsSuffixLength = sizeof(kColumnsSuffix) - 1;
+
+}  // namespace
+
+Result<FileIndexOptions> FileIndexOptions::FromCoreOptions(const CoreOptions& options) {
+    FileIndexOptions result;
+    const std::map<std::string, std::string>& raw_options = options.ToMap();
+    result.in_manifest_threshold_ = options.FileIndexInManifestThreshold();
+
+    std::set<std::pair<std::string, std::string>> declared;
+    for (const auto& [key, value] : raw_options) {
+        if (!StringUtils::StartsWith(key, kFileIndexPrefix) ||
+            !StringUtils::EndsWith(key, kColumnsSuffix)) {
+            continue;
+        }
+        if (key.size() < kFileIndexPrefixLength + kColumnsSuffixLength) {
+            return Status::Invalid(fmt::format("Invalid file index option {}", key));
+        }
+        const size_t index_type_length = key.size() - kFileIndexPrefixLength - kColumnsSuffixLength;
+        const std::string index_type = key.substr(kFileIndexPrefixLength, index_type_length);
+        if (index_type.empty()) {
+            return Status::Invalid(fmt::format("Invalid file index option {}", key));
+        }
+        // TODO(jinli.zjw): Align malformed list option parsing (for example, "f1,f2,,") with Java.
+        // Update this together with ConfigParser::ParseList to keep option parsing consistent.
+        for (std::string column_name : StringUtils::Split(value, ",", /*ignore_empty=*/false)) {
+            StringUtils::Trim(&column_name);
+            if (column_name.empty()) {
+                return Status::Invalid(
+                    fmt::format("Wrong option in {}, should not have empty column", key));
+            }
+            if (column_name.find('[') != std::string::npos &&
+                StringUtils::EndsWith(column_name, "]")) {
+                return Status::NotImplemented(
+                    "Writing file indexes for nested map columns is not supported");
+            }
+            if (declared.emplace(column_name, index_type).second) {
+                result.definitions_.push_back({column_name, index_type, {}});
+            }
+        }
+    }
+
+    for (const auto& [key, value] : raw_options) {
+        if (!StringUtils::StartsWith(key, kFileIndexPrefix) ||
+            StringUtils::EndsWith(key, kColumnsSuffix) ||
+            key == Options::FILE_INDEX_IN_MANIFEST_THRESHOLD) {
+            continue;
+        }
+        std::vector<std::string> parts =
+            StringUtils::Split(key.substr(kFileIndexPrefixLength), ".", /*ignore_empty=*/false);
+        if (parts.size() != 3) {
+            continue;
+        }
+        bool found = false;
+        for (FileIndexDefinition& definition : result.definitions_) {
+            if (definition.index_type == parts[0] && definition.column_name == parts[1]) {
+                definition.options[parts[2]] = value;
+                found = true;
+                break;
+            }
+        }
+        if (!found) {
+            return Status::Invalid(
+                fmt::format("Wrong file index option '{}': column '{}' is not declared in "
+                            "'file-index.{}.columns'",
+                            key, parts[1], parts[0]));
+        }
+    }
+    return result;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/file_index_options.h b/src/paimon/core/io/file_index_options.h
new file mode 100644
index 000000000..7b7c019bf
--- /dev/null
+++ b/src/paimon/core/io/file_index_options.h
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <string>
+#include <vector>
+
+#include "paimon/result.h"
+
+namespace paimon {
+
+class CoreOptions;
+
+struct FileIndexDefinition {
+    std::string column_name;
+    std::string index_type;
+    std::map<std::string, std::string> options;
+};
+
+/// Parsed write-side file index configuration.
+class FileIndexOptions {
+ public:
+    static Result<FileIndexOptions> FromCoreOptions(const CoreOptions& options);
+
+    const std::vector<FileIndexDefinition>& Definitions() const {
+        return definitions_;
+    }
+
+    int64_t InManifestThreshold() const {
+        return in_manifest_threshold_;
+    }
+
+    bool Empty() const {
+        return definitions_.empty();
+    }
+
+ private:
+    FileIndexOptions() = default;
+
+    std::vector<FileIndexDefinition> definitions_;
+    int64_t in_manifest_threshold_ = 0;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/file_index_options_test.cpp b/src/paimon/core/io/file_index_options_test.cpp
new file mode 100644
index 000000000..157203f95
--- /dev/null
+++ b/src/paimon/core/io/file_index_options_test.cpp
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/io/file_index_options.h"
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include "gtest/gtest.h"
+#include "paimon/core/core_options.h"
+#include "paimon/fs/local/local_file_system.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+namespace {
+
+Result<FileIndexOptions> ParseOptions(const std::map<std::string, std::string>& index_options) {
+    std::shared_ptr<LocalFileSystem> file_system = std::make_shared<LocalFileSystem>();
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
+                           CoreOptions::FromMap(index_options, file_system));
+    return FileIndexOptions::FromCoreOptions(core_options);
+}
+
+}  // namespace
+
+TEST(FileIndexOptionsTest, TestRejectOverlappingPrefixAndSuffix) {
+    ASSERT_NOK_WITH_MSG(ParseOptions({{"file-index.columns", "f0"}}),
+                        "Invalid file index option file-index.columns");
+}
+
+TEST(FileIndexOptionsTest, TestNestedMapColumnSyntax) {
+    ASSERT_OK_AND_ASSIGN(FileIndexOptions options,
+                         ParseOptions({{"file-index.bitmap.columns", "col[key"}}));
+    ASSERT_EQ(1, options.Definitions().size());
+    ASSERT_EQ("col[key", options.Definitions()[0].column_name);
+
+    ASSERT_NOK_WITH_MSG(ParseOptions({{"file-index.bitmap.columns", "col[key]"}}),
+                        "nested map columns is not supported");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/io/key_value_data_file_writer.cpp b/src/paimon/core/io/key_value_data_file_writer.cpp
index 9393c7c3c..9c32e0674 100644
--- a/src/paimon/core/io/key_value_data_file_writer.cpp
+++ b/src/paimon/core/io/key_value_data_file_writer.cpp
@@ -25,7 +25,6 @@
 #include <utility>
 #include <variant>
 
-#include "arrow/type.h"
 #include "fmt/format.h"
 #include "paimon/common/data/binary_array.h"
 #include "paimon/common/data/binary_array_writer.h"
@@ -53,7 +52,7 @@ KeyValueDataFileWriter::KeyValueDataFileWriter(
     const std::shared_ptr<FormatStatsExtractor>& stats_extractor,
     const std::shared_ptr<arrow::Schema>& write_schema, bool is_external_path,
     const std::shared_ptr<MemoryPool>& pool)
-    : SingleFileWriter(compression, converter),
+    : DataFileWriterBase(compression, std::move(converter)),
       pool_(pool),
       schema_id_(schema_id),
       level_(level),
@@ -64,10 +63,6 @@ KeyValueDataFileWriter::KeyValueDataFileWriter(
       is_external_path_(is_external_path),
       disable_stats_(stats_extractor == nullptr) {}
 
-void KeyValueDataFileWriter::SetMetadataFinalizer(MetadataFinalizer finalizer) {
-    metadata_finalizer_ = std::move(finalizer);
-}
-
 Status KeyValueDataFileWriter::Write(KeyValueBatch batch) {
     // update min and max key
     if (!min_key_) {
@@ -80,19 +75,7 @@ Status KeyValueDataFileWriter::Write(KeyValueBatch batch) {
     // update delete row count
     delete_row_count_ += batch.delete_row_count;
 
-    PAIMON_RETURN_NOT_OK(SingleFileWriter::Write(std::move(batch)));
-    return Status::OK();
-}
-
-Status KeyValueDataFileWriter::BeforeFinish() {
-    if (metadata_finalizer_) {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> updated_schema,
-                               metadata_finalizer_());
-        if (updated_schema) {
-            PAIMON_RETURN_NOT_OK(UpdateSchema(updated_schema));
-        }
-    }
-    return Status::OK();
+    return WriteRecordWithFileIndex(std::move(batch));
 }
 
 Result<std::shared_ptr<DataFileMeta>> KeyValueDataFileWriter::GetResult() {
@@ -120,14 +103,14 @@ Result<std::shared_ptr<DataFileMeta>> KeyValueDataFileWriter::GetResult() {
         final_path = external_path.ToString();
     }
     PAIMON_ASSIGN_OR_RAISE(int64_t local_micro, DateTimeUtils::GetCurrentLocalTimeUs());
+    const FileIndexWriteResult& file_index = GetFileIndexWriteResult();
     return std::make_shared<DataFileMeta>(
         PathUtil::GetName(path_), output_bytes_, RecordCount(), min_key, max_key, key_stats,
         value_stats, min_sequence_number_, max_sequence_number_, schema_id_, level_,
-        /*extra_files=*/std::vector<std::optional<std::string>>(),
+        file_index.extra_files,
         Timestamp(/*millisecond=*/local_micro / 1000, /*nano_of_millisecond=*/0), delete_row_count_,
-        /*embedded_index=*/nullptr, file_source_,
-        /*value_stats_cols=*/std::nullopt, final_path, /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        file_index.embedded_index, file_source_, /*value_stats_cols=*/std::nullopt, final_path,
+        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
 }
 
 Status KeyValueDataFileWriter::GenerateMinMaxKey(BinaryRow* min_key, BinaryRow* max_key) const {
diff --git a/src/paimon/core/io/key_value_data_file_writer.h b/src/paimon/core/io/key_value_data_file_writer.h
index e1e3fd92c..eb7a2efca 100644
--- a/src/paimon/core/io/key_value_data_file_writer.h
+++ b/src/paimon/core/io/key_value_data_file_writer.h
@@ -17,6 +17,7 @@
  */
 
 #pragma once
+
 #include <cstdint>
 #include <functional>
 #include <limits>
@@ -25,7 +26,7 @@
 #include <vector>
 
 #include "paimon/core/io/data_file_meta.h"
-#include "paimon/core/io/single_file_writer.h"
+#include "paimon/core/io/data_file_writer_base.h"
 #include "paimon/core/key_value.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/result.h"
@@ -44,14 +45,8 @@ class InternalRow;
 class MemoryPool;
 class SimpleStats;
 
-class KeyValueDataFileWriter
-    : public SingleFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>> {
+class KeyValueDataFileWriter : public DataFileWriterBase<KeyValueBatch> {
  public:
-    /// Callback invoked during BeforeFinish() to finalize file metadata.
-    /// Produces an updated schema with per-field metadata (e.g. shredding metadata)
-    /// and may perform other finalization work (e.g. reporting stats to cross-file context).
-    using MetadataFinalizer = std::function<Result<std::shared_ptr<arrow::Schema>>()>;
-
     KeyValueDataFileWriter(const std::string& compression,
                            std::function<Status(KeyValueBatch&&, ::ArrowArray*)> converter,
                            int64_t schema_id, int32_t level, FileSource file_source,
@@ -60,17 +55,10 @@ class KeyValueDataFileWriter
                            const std::shared_ptr<arrow::Schema>& write_schema,
                            bool is_external_path, const std::shared_ptr<MemoryPool>& pool);
 
-    /// Sets the metadata finalizer. Called during BeforeFinish() to produce an updated
-    /// schema and perform finalization callbacks. Must be set before Close().
-    void SetMetadataFinalizer(MetadataFinalizer finalizer);
-
     Status Write(KeyValueBatch batch) override;
 
     Result<std::shared_ptr<DataFileMeta>> GetResult() override;
 
- protected:
-    Status BeforeFinish() override;
-
  private:
     Result<std::vector<std::shared_ptr<ColumnStats>>> GetFieldStats();
 
@@ -96,7 +84,6 @@ class KeyValueDataFileWriter
     int64_t max_sequence_number_ = std::numeric_limits<int64_t>::min();
     std::shared_ptr<InternalRow> min_key_;
     std::shared_ptr<InternalRow> max_key_;
-    MetadataFinalizer metadata_finalizer_;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/io/key_value_data_file_writer_factory.cpp b/src/paimon/core/io/key_value_data_file_writer_factory.cpp
index 07d50b980..8f3885591 100644
--- a/src/paimon/core/io/key_value_data_file_writer_factory.cpp
+++ b/src/paimon/core/io/key_value_data_file_writer_factory.cpp
@@ -24,6 +24,7 @@
 
 #include "arrow/c/helpers.h"
 #include "paimon/core/core_options.h"
+#include "paimon/core/io/data_file_index_writer.h"
 #include "paimon/core/io/data_file_path_factory.h"
 #include "paimon/core/io/key_value_data_file_writer.h"
 #include "paimon/format/file_format.h"
@@ -60,6 +61,11 @@ KeyValueDataFileWriterFactory::CreateWriter() const {
         options_.GetWriteFileCompression(level_), std::move(converter), schema_id_, level_,
         file_source_, primary_keys_, resources.stats_extractor, write_schema_,
         path_factory_->IsExternalPath(), pool_);
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<DataFileIndexWriter> file_index_writer,
+                           CreateFileIndexWriter(write_schema_, path_factory_));
+    if (file_index_writer) {
+        writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_);
+    }
     PAIMON_RETURN_NOT_OK(
         writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder));
     return std::unique_ptr<SingleFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>(
diff --git a/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp
index 6e4843bbb..0e4e82199 100644
--- a/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp
+++ b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp
@@ -23,6 +23,7 @@
 
 #include "arrow/c/helpers.h"
 #include "paimon/core/core_options.h"
+#include "paimon/core/io/data_file_index_writer.h"
 #include "paimon/core/io/data_file_path_factory.h"
 #include "paimon/core/io/data_file_writer.h"
 #include "paimon/core/io/infer_shredding_file_writer.h"
@@ -89,6 +90,11 @@ ShreddingAppendDataFileWriterFactory::CreateShreddedWriter(
         options_.GetFileCompression(), std::move(batch_converter), schema_id_, seq_num_counter,
         file_source_, resources.stats_extractor, path_factory_->IsExternalPath(), write_cols_,
         pool_);
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<DataFileIndexWriter> file_index_writer,
+                           CreateFileIndexWriter(write_schema_, path_factory_));
+    if (file_index_writer) {
+        writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_);
+    }
     PAIMON_RETURN_NOT_OK(
         writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder));
     ShreddingWritePlanFactory::MetadataFinalizer finalizer =
diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp
index 8ac583ee0..30d4c9fce 100644
--- a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp
+++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp
@@ -23,6 +23,7 @@
 
 #include "arrow/c/helpers.h"
 #include "paimon/core/core_options.h"
+#include "paimon/core/io/data_file_index_writer.h"
 #include "paimon/core/io/data_file_path_factory.h"
 #include "paimon/core/io/infer_shredding_file_writer.h"
 #include "paimon/core/io/key_value_data_file_writer.h"
@@ -88,6 +89,11 @@ ShreddingKeyValueDataFileWriterFactory::CreateShreddedWriter(
         options_.GetWriteFileCompression(level_), std::move(batch_converter), schema_id_, level_,
         file_source_, primary_keys_, resources.stats_extractor, file_schema,
         path_factory_->IsExternalPath(), pool_);
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<DataFileIndexWriter> file_index_writer,
+                           CreateFileIndexWriter(write_schema_, path_factory_));
+    if (file_index_writer) {
+        writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_);
+    }
     PAIMON_RETURN_NOT_OK(
         writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder));
     ShreddingWritePlanFactory::MetadataFinalizer finalizer =
diff --git a/src/paimon/core/io/single_file_writer.h b/src/paimon/core/io/single_file_writer.h
index 99507b579..6db3a699c 100644
--- a/src/paimon/core/io/single_file_writer.h
+++ b/src/paimon/core/io/single_file_writer.h
@@ -25,6 +25,7 @@
 #include <memory>
 #include <string>
 #include <utility>
+#include <vector>
 
 #include "arrow/c/abi.h"
 #include "arrow/c/bridge.h"
@@ -64,21 +65,27 @@ class SingleFileWriter : public FileWriter<T, R> {
     class AbortExecutor {
      public:
         AbortExecutor(const std::shared_ptr<FileSystem>& fs, const std::string& path)
-            : fs_(fs), path_(path), logger_(Logger::GetLogger("AbortExecutor")) {}
+            : paths_({{fs, path}}), logger_(Logger::GetLogger("AbortExecutor")) {}
+
+        void Add(const std::shared_ptr<FileSystem>& fs, const std::string& path) {
+            paths_.emplace_back(fs, path);
+        }
 
         void Abort() {
-            if (fs_) {
-                auto status = fs_->Delete(path_);
+            for (const auto& [fs, path] : paths_) {
+                if (!fs) {
+                    continue;
+                }
+                auto status = fs->Delete(path);
                 if (!status.ok()) {
-                    PAIMON_LOG_WARN(logger_, "Exception occurs when deleting %s: %s", path_.c_str(),
+                    PAIMON_LOG_WARN(logger_, "Exception occurs when deleting %s: %s", path.c_str(),
                                     status.ToString().c_str());
                 }
             }
         }
 
      private:
-        std::shared_ptr<FileSystem> fs_;
-        std::string path_;
+        std::vector<std::pair<std::shared_ptr<FileSystem>, std::string>> paths_;
         std::shared_ptr<Logger> logger_;
     };
 
@@ -132,6 +139,11 @@ class SingleFileWriter : public FileWriter<T, R> {
         return Status::OK();
     }
 
+    /// Hook called after the data file is closed and before its completion callback is published.
+    virtual Status BeforeCompletion() {
+        return Status::OK();
+    }
+
     /// Serializes schema and forwards it as file metadata to FormatWriter.
     Status UpdateSchema(const std::shared_ptr<arrow::Schema>& schema);
 
@@ -239,6 +251,7 @@ Status SingleFileWriter<T, R>::Close() {
     // guard still removes the file on a callback error, while a repeated Close() does not publish
     // the same file again.
     closed_ = true;
+    PAIMON_RETURN_NOT_OK(BeforeCompletion());
     if (completion_callback_) {
         PAIMON_RETURN_NOT_OK(completion_callback_());
     }
diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp
index 82eb5257b..5916fc97c 100644
--- a/test/inte/write_and_read_inte_test.cpp
+++ b/test/inte/write_and_read_inte_test.cpp
@@ -486,6 +486,89 @@ TEST_P(WriteAndReadInteTest, TestAppendVectorWithPredicate) {
     ASSERT_TRUE(expected->Equals(actual)) << actual->ToString();
 }
 
+// TODO(jinli.zjw): move to a single file for a file index inte test
+TEST_P(WriteAndReadInteTest, TestAppendWithExternalBitmapAndRangeBitmapIndexes) {
+    arrow::FieldVector fields = {arrow::field("name", arrow::utf8()),
+                                 arrow::field("score", arrow::int32())};
+    auto [file_format, file_system] = GetParam();
+    std::map<std::string, std::string> options = {
+        {Options::MANIFEST_FORMAT, "avro"},
+        {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1MB"},
+        {Options::BUCKET, "-1"},
+        {Options::FILE_SYSTEM, file_system},
+        {"file-index.bitmap.columns", "name"},
+        {"file-index.range-bitmap.columns", "score"},
+        {"file-index.range-bitmap.score.chunk-size", "1KB"},
+        {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1B"},
+    };
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+
+    auto schema = arrow::schema(fields);
+    ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(test_dir_, schema, /*partition_keys=*/{},
+                                                         /*primary_keys=*/{}, options,
+                                                         /*is_streaming_mode=*/false));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([
+            ["Alice", 10],
+            ["Bob", 20],
+            ["Alice", 30],
+            ["Lucy", 40]
+        ])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(auto data_files, CurrentDataFiles(options));
+    ASSERT_EQ(1, data_files.size());
+    const auto& [bucket_path, data_file] = data_files[0];
+    ASSERT_FALSE(data_file->embedded_index);
+    ASSERT_EQ(1, data_file->extra_files.size());
+    ASSERT_TRUE(data_file->extra_files[0]);
+    ASSERT_EQ(data_file->file_name + ".index", data_file->extra_files[0].value());
+    std::string index_path = PathUtil::JoinPath(bucket_path, data_file->extra_files[0].value());
+    ASSERT_OK_AND_ASSIGN(bool index_exists, dir_->GetFileSystem()->Exists(index_path));
+    ASSERT_TRUE(index_exists);
+
+    std::string indexed_name = "Alice";
+    auto name_predicate = PredicateBuilder::Equal(
+        /*field_index=*/0, /*field_name=*/"name", FieldType::STRING,
+        Literal(FieldType::STRING, indexed_name.data(), indexed_name.size()));
+    auto score_predicate = PredicateBuilder::GreaterThan(
+        /*field_index=*/1, /*field_name=*/"score", FieldType::INT, Literal(20));
+    ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({name_predicate, score_predicate}));
+
+    std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar");
+    ScanContextBuilder scan_context_builder(table_path);
+    scan_context_builder.SetOptions(options)
+        .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString())
+        .SetPredicate(predicate);
+    ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context)));
+    ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan());
+    ASSERT_EQ(1, plan->Splits().size());
+
+    // Keep precise post-read filtering disabled. The exact result therefore verifies that the
+    // bitmap and range-bitmap indexes produced by the write path are consumed by the read path.
+    ReadContextBuilder read_context_builder(table_path);
+    read_context_builder.SetOptions(options).SetPredicate(predicate);
+    ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context)));
+    ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits()));
+    ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(batch_reader.get()));
+
+    arrow::FieldVector fields_with_row_kind = fields;
+    fields_with_row_kind.insert(fields_with_row_kind.begin(),
+                                arrow::field("_VALUE_KIND", arrow::int8()));
+    auto expected_result = arrow::ipc::internal::json::ArrayFromJSON(
+        arrow::struct_(fields_with_row_kind), R"([[0, "Alice", 30]])");
+    ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString();
+    auto expected = std::make_shared<arrow::ChunkedArray>(expected_result.ValueOrDie());
+    ASSERT_TRUE(expected->Equals(actual)) << actual->ToString();
+}
+
 TEST_P(WriteAndReadInteTest, TestPKSimple) {
     arrow::FieldVector fields = {
         arrow::field("pk", arrow::utf8()),

From dbdce8fd646394833586a324599811a135116f6e Mon Sep 17 00:00:00 2001
From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com>
Date: Thu, 20 Aug 2026 16:28:44 +0800
Subject: [PATCH 08/93] perf(parquet): reuse leaf column index set across
 fields in page-filtered reads (#207)

---
 cmake_modules/arrow.diff                               |  8 ++++----
 .../format/parquet/page_filtered_row_group_reader.cpp  | 10 ++++++----
 .../format/parquet/page_filtered_row_group_reader.h    |  9 ++++++---
 3 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff
index ce63af352..75e3bb51e 100644
--- a/cmake_modules/arrow.diff
+++ b/cmake_modules/arrow.diff
@@ -57,7 +57,7 @@ index 285e2a5973..db919d7ef8 100644
    }
 
 +  ::arrow::Status GetColumn(
-+      int i, const std::vector<int>& column_indices,
++      int i, const std::shared_ptr<std::unordered_set<int>>& column_indices,
 +      FileColumnIteratorFactory iterator_factory,
 +      std::unique_ptr<ColumnReader>* out) override;
 +
@@ -235,7 +235,7 @@ index 285e2a5973..db919d7ef8 100644
  }
 
 +::arrow::Status FileReaderImpl::GetColumn(
-+    int i, const std::vector<int>& column_indices,
++    int i, const std::shared_ptr<std::unordered_set<int>>& column_indices,
 +    FileColumnIteratorFactory iterator_factory,
 +    std::unique_ptr<ColumnReader>* out) {
 +  RETURN_NOT_OK(BoundsCheckColumn(i));
@@ -244,7 +244,7 @@ index 285e2a5973..db919d7ef8 100644
 +  ctx->pool = pool_;
 +  ctx->iterator_factory = iterator_factory;
 +  ctx->filter_leaves = true;
-+  ctx->included_leaves = VectorToSharedSet(column_indices);
++  ctx->included_leaves = column_indices;
 +  std::unique_ptr<ColumnReaderImpl> result;
 +  RETURN_NOT_OK(GetReader(manifest_.schema_fields[i], ctx, &result));
 +  *out = std::move(result);
@@ -298,7 +298,7 @@ index 6e46ca43f7..e86ff0ef52 100644
 +  /// \param iterator_factory factory to create FileColumnIterator per leaf
 +  /// \param[out] out the ColumnReader (may be nullptr if all leaves are pruned)
 +  virtual ::arrow::Status GetColumn(
-+      int i, const std::vector<int>& column_indices,
++      int i, const std::shared_ptr<std::unordered_set<int>>& column_indices,
 +      FileColumnIteratorFactory iterator_factory,
 +      std::unique_ptr<ColumnReader>* out) {
 +    return ::arrow::Status::NotImplemented(
diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp
index 1b4bdd30d..f20f224fe 100644
--- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp
+++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp
@@ -284,9 +284,9 @@ Status PageFilteredRowGroupReader::WaitForPreBuffer(
 
 Result<std::shared_ptr<arrow::ChunkedArray>> PageFilteredRowGroupReader::ReadFilteredField(
     const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader,
-    int32_t row_group_index, int32_t field_index, const std::vector<int32_t>& column_indices,
-    const RowRanges& row_ranges, int64_t row_group_row_count,
-    ::parquet::arrow::FileReader* arrow_file_reader) {
+    int32_t row_group_index, int32_t field_index,
+    std::shared_ptr<std::unordered_set<int>> column_indices, const RowRanges& row_ranges,
+    int64_t row_group_row_count, ::parquet::arrow::FileReader* arrow_file_reader) {
     // Factory: set a direct data page read plan on every leaf (per-leaf OffsetIndex).
     // The plan lets Arrow jump over unselected page headers as well as page bodies.
     auto factory =
@@ -397,12 +397,14 @@ Result<std::unique_ptr<arrow::RecordBatchReader>> PageFilteredRowGroupReader::Re
     std::vector<std::shared_ptr<arrow::ChunkedArray>> result_arrays;
     result_arrays.reserve(field_indices.size());
 
+    std::shared_ptr<std::unordered_set<int>> col_indices_set =
+        std::make_shared<std::unordered_set<int>>(column_indices.begin(), column_indices.end());
     // TODO(zhouhongfeng.zhf): This loop could be parallelized.
     for (int field_idx : field_indices) {
         PAIMON_ASSIGN_OR_RAISE(
             std::shared_ptr<arrow::ChunkedArray> chunked_array,
             ReadFilteredField(row_group_page_index_reader, row_group_index, field_idx,
-                              column_indices, row_ranges, row_group_row_count, arrow_file_reader));
+                              col_indices_set, row_ranges, row_group_row_count, arrow_file_reader));
 
         if (chunked_array->length() != expected_rows) {
             return Status::Invalid(
diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h
index 683bde71e..a143ae5a2 100644
--- a/src/paimon/format/parquet/page_filtered_row_group_reader.h
+++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h
@@ -23,6 +23,7 @@
 #include <limits>
 #include <memory>
 #include <optional>
+#include <unordered_set>
 #include <utility>
 #include <vector>
 
@@ -117,11 +118,13 @@ class PageFilteredRowGroupReader {
     /// Sets a direct page read plan on all leaves via factory, then drives each leaf
     /// independently via ResetLeaf/SkipRecords/ReadRecords using its own
     /// compressed_ranges.
+    /// `column_indices` holds `int` rather than `int32_t` because the set is
+    /// handed straight to Arrow's `FileReader::GetColumn` (to avoid reconstruction and deep copy)
     static Result<std::shared_ptr<arrow::ChunkedArray>> ReadFilteredField(
         const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader,
-        int32_t row_group_index, int32_t field_index, const std::vector<int32_t>& column_indices,
-        const RowRanges& row_ranges, int64_t row_group_row_count,
-        ::parquet::arrow::FileReader* arrow_file_reader);
+        int32_t row_group_index, int32_t field_index,
+        std::shared_ptr<std::unordered_set<int>> column_indices, const RowRanges& row_ranges,
+        int64_t row_group_row_count, ::parquet::arrow::FileReader* arrow_file_reader);
 };
 
 }  // namespace paimon::parquet

From aa60634a318420c819a5357ea13ef3991fb0685f Mon Sep 17 00:00:00 2001
From: gripleaf <425797155@qq.com>
Date: Thu, 20 Aug 2026 22:48:44 +0800
Subject: [PATCH 09/93] perf(scan): lazily decode manifest bucket entries
 (#212)

---
 include/paimon/defs.h                         |   4 +
 src/paimon/common/defs.cpp                    |   2 +
 src/paimon/core/core_options.cpp              |   7 +
 src/paimon/core/core_options.h                |   1 +
 src/paimon/core/core_options_test.cpp         |   3 +
 .../manifest/manifest_entry_serializer.cpp    |  27 ++--
 .../core/manifest/manifest_entry_serializer.h |   6 +
 .../manifest_entry_serializer_test.cpp        |  10 ++
 src/paimon/core/manifest/manifest_file.cpp    |  19 +++
 src/paimon/core/manifest/manifest_file.h      |   4 +
 .../core/manifest/manifest_file_test.cpp      | 120 +++++++++++++++++-
 .../append_only_file_store_scan_test.cpp      |  40 +++++-
 src/paimon/core/operation/file_store_scan.cpp |  27 ++++
 src/paimon/core/utils/objects_file.h          |  47 ++++---
 14 files changed, 282 insertions(+), 35 deletions(-)

diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index e944587f2..338eda30a 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -200,6 +200,10 @@ struct PAIMON_EXPORT Options {
     /// cache. Default value is 0.
     static const char SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS[];
 
+    /// "scan.manifest-entry.lazy-decode.enabled" - Whether to deserialize only manifest entries
+    /// for the target bucket when rebuilding the cache. Default value is true.
+    static const char SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED[];
+
     /// "read.batch-size" - Read batch size for any file format if it supports.
     /// The default value is 1024.
     static const char READ_BATCH_SIZE[];
diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp
index ef35940e9..8c5336e19 100644
--- a/src/paimon/common/defs.cpp
+++ b/src/paimon/common/defs.cpp
@@ -59,6 +59,8 @@ const char Options::SCAN_SNAPSHOT_ID[] = "scan.snapshot-id";
 const char Options::SCAN_MODE[] = "scan.mode";
 const char Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS[] =
     "scan.manifest-entry-cache.max-snapshots";
+const char Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED[] =
+    "scan.manifest-entry.lazy-decode.enabled";
 const char Options::READ_BATCH_SIZE[] = "read.batch-size";
 const char Options::WRITE_BATCH_SIZE[] = "write.batch-size";
 const char Options::WRITE_BUFFER_SIZE[] = "write-buffer-size";
diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp
index 1c2e164bf..1e8203a57 100644
--- a/src/paimon/core/core_options.cpp
+++ b/src/paimon/core/core_options.cpp
@@ -426,6 +426,7 @@ struct CoreOptions::Impl {
 
     int32_t manifest_merge_min_count = 30;
     int32_t scan_manifest_entry_cache_max_snapshots = 0;
+    bool scan_manifest_entry_lazy_decode_enabled = true;
     int32_t read_batch_size = 1024;
     int32_t write_batch_size = 1024;
     int32_t local_sort_max_num_file_handles = 128;
@@ -828,6 +829,8 @@ struct CoreOptions::Impl {
             return Status::Invalid(fmt::format("{} must be non-negative",
                                                Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS));
         }
+        PAIMON_RETURN_NOT_OK(parser.Parse<bool>(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED,
+                                                &scan_manifest_entry_lazy_decode_enabled));
         // Parse scan.fallback-branch - fallback branch when partition not found
         PAIMON_RETURN_NOT_OK(parser.Parse(Options::SCAN_FALLBACK_BRANCH, &scan_fallback_branch));
         // Parse branch - branch name, default "main"
@@ -1170,6 +1173,10 @@ int32_t CoreOptions::GetScanManifestEntryCacheMaxSnapshots() const {
     return impl_->scan_manifest_entry_cache_max_snapshots;
 }
 
+bool CoreOptions::ScanManifestEntryLazyDecodeEnabled() const {
+    return impl_->scan_manifest_entry_lazy_decode_enabled;
+}
+
 int64_t CoreOptions::GetManifestTargetFileSize() const {
     return impl_->manifest_target_file_size;
 }
diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h
index 53ef4ad0f..3bb17d6f6 100644
--- a/src/paimon/core/core_options.h
+++ b/src/paimon/core/core_options.h
@@ -108,6 +108,7 @@ class PAIMON_EXPORT CoreOptions {
     std::optional<int64_t> GetScanTimestampMillis() const;
     int64_t GetRealtimeReadViewTtlMillis() const;
     int32_t GetScanManifestEntryCacheMaxSnapshots() const;
+    bool ScanManifestEntryLazyDecodeEnabled() const;
 
     int64_t GetManifestTargetFileSize() const;
     std::shared_ptr<Cache> GetCache() const;
diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp
index 0054a5b5f..f057206dd 100644
--- a/src/paimon/core/core_options_test.cpp
+++ b/src/paimon/core/core_options_test.cpp
@@ -65,6 +65,7 @@ TEST(CoreOptionsTest, TestDefaultValue) {
     ASSERT_EQ(30, core_options.GetManifestMergeMinCount());
     ASSERT_FALSE(core_options.ManifestDeleteFileDropStats());
     ASSERT_EQ(0, core_options.GetScanManifestEntryCacheMaxSnapshots());
+    ASSERT_TRUE(core_options.ScanManifestEntryLazyDecodeEnabled());
     ASSERT_EQ(nullptr, core_options.GetCache());
     ASSERT_EQ(128 * 1024 * 1024L, core_options.GetSourceSplitTargetSize());
     ASSERT_EQ(4 * 1024 * 1024L, core_options.GetSourceSplitOpenFileCost());
@@ -218,6 +219,7 @@ TEST(CoreOptionsTest, TestFromMap) {
         {Options::SCAN_SNAPSHOT_ID, "5"},
         {Options::SCAN_MODE, "from-snapshot-full"},
         {Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "7"},
+        {Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, "false"},
         {Options::SNAPSHOT_NUM_RETAINED_MIN, "15"},
         {Options::SNAPSHOT_NUM_RETAINED_MAX, "30"},
         {Options::SNAPSHOT_EXPIRE_LIMIT, "20"},
@@ -355,6 +357,7 @@ TEST(CoreOptionsTest, TestFromMap) {
     ASSERT_TRUE(core_options.CommitDiscardDuplicateFiles());
     ASSERT_EQ(5, core_options.GetScanSnapshotId().value_or(-1));
     ASSERT_EQ(7, core_options.GetScanManifestEntryCacheMaxSnapshots());
+    ASSERT_FALSE(core_options.ScanManifestEntryLazyDecodeEnabled());
     ExpireConfig expire_config = core_options.GetExpireConfig();
     ASSERT_EQ(15, expire_config.GetSnapshotRetainMin());
     ASSERT_EQ(30, expire_config.GetSnapshotRetainMax());
diff --git a/src/paimon/core/manifest/manifest_entry_serializer.cpp b/src/paimon/core/manifest/manifest_entry_serializer.cpp
index 053405b89..2389cd8d7 100644
--- a/src/paimon/core/manifest/manifest_entry_serializer.cpp
+++ b/src/paimon/core/manifest/manifest_entry_serializer.cpp
@@ -31,17 +31,26 @@ namespace paimon {
 class MemoryPool;
 struct DataFileMeta;
 
+Status ManifestEntrySerializer::ValidateVersion(int32_t version) {
+    if (version == VERSION_2) {
+        return Status::OK();
+    }
+    if (version == VERSION_1) {
+        return Status::Invalid(
+            fmt::format("The current version {} is not compatible with the version {}, "
+                        "please recreate the table.",
+                        VERSION_2, version));
+    }
+    return Status::Invalid(fmt::format("Unsupported version: {}", version));
+}
+
+int32_t ManifestEntrySerializer::GetBucket(const InternalRow& row) {
+    return row.GetInt(3);
+}
+
 Result<ManifestEntry> ManifestEntrySerializer::ConvertFrom(int32_t version,
                                                            const InternalRow& row) const {
-    if (version != VERSION_2) {
-        if (version == VERSION_1) {
-            return Status::Invalid(
-                fmt::format("The current version {} is not compatible with the version {}, "
-                            "please recreate the table.",
-                            GetVersion(), version));
-        }
-        return Status::Invalid("Unsupported version", std::to_string(version));
-    }
+    PAIMON_RETURN_NOT_OK(ValidateVersion(version));
     auto kind = row.GetByte(0);
     PAIMON_ASSIGN_OR_RAISE(FileKind file_kind, FileKind::FromByteValue(kind));
     auto partition_bytes = row.GetBinary(1);
diff --git a/src/paimon/core/manifest/manifest_entry_serializer.h b/src/paimon/core/manifest/manifest_entry_serializer.h
index 7438895f5..4a71a1b68 100644
--- a/src/paimon/core/manifest/manifest_entry_serializer.h
+++ b/src/paimon/core/manifest/manifest_entry_serializer.h
@@ -50,6 +50,12 @@ class ManifestEntrySerializer : public VersionedObjectSerializer<ManifestEntry>
         return VERSION_2;
     }
 
+    /// Validate the serialization version before reading fields that may vary by version.
+    static Status ValidateVersion(int32_t version);
+
+    /// Get the bucket from a versioned manifest entry row without fully deserializing it.
+    static int32_t GetBucket(const InternalRow& row);
+
     Result<ManifestEntry> ConvertFrom(int32_t version, const InternalRow& row) const override;
 
     Result<BinaryRow> ToRow(const ManifestEntry& record) const override;
diff --git a/src/paimon/core/manifest/manifest_entry_serializer_test.cpp b/src/paimon/core/manifest/manifest_entry_serializer_test.cpp
index 2aa2db524..2d8cffc37 100644
--- a/src/paimon/core/manifest/manifest_entry_serializer_test.cpp
+++ b/src/paimon/core/manifest/manifest_entry_serializer_test.cpp
@@ -55,12 +55,22 @@ TEST_F(ManifestEntrySerializerTest, TestToFromRow) {
     ManifestEntrySerializer serializer(pool);
     for (const auto& entry : entries) {
         ASSERT_OK_AND_ASSIGN(auto row, serializer.ToRow(entry));
+        ASSERT_EQ(entry.Bucket(), ManifestEntrySerializer::GetBucket(row));
         ASSERT_OK_AND_ASSIGN(auto result_entry, serializer.FromRow(row));
         ASSERT_EQ(entry, result_entry);
         ASSERT_EQ(entry.ToString(), result_entry.ToString());
     }
 }
 
+TEST_F(ManifestEntrySerializerTest, TestValidateVersion) {
+    ASSERT_OK(ManifestEntrySerializer::ValidateVersion(/*version=*/2));
+    ASSERT_NOK_WITH_MSG(ManifestEntrySerializer::ValidateVersion(/*version=*/1),
+                        "The current version 2 is not compatible with the version 1, please "
+                        "recreate the table.");
+    ASSERT_NOK_WITH_MSG(ManifestEntrySerializer::ValidateVersion(/*version=*/3),
+                        "Unsupported version: 3");
+}
+
 TEST_F(ManifestEntrySerializerTest, TestNullableRecordCount) {
     std::vector<ManifestEntry> empty_entries;
     ASSERT_FALSE(ManifestEntry::NullableRecordCount(empty_entries).has_value());
diff --git a/src/paimon/core/manifest/manifest_file.cpp b/src/paimon/core/manifest/manifest_file.cpp
index 22f2681f6..1be49d0b5 100644
--- a/src/paimon/core/manifest/manifest_file.cpp
+++ b/src/paimon/core/manifest/manifest_file.cpp
@@ -24,6 +24,7 @@
 
 #include "arrow/c/abi.h"
 #include "arrow/c/bridge.h"
+#include "paimon/common/data/columnar/columnar_row.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/core/io/rolling_file_writer.h"
 #include "paimon/core/manifest/manifest_entry.h"
@@ -86,6 +87,24 @@ Result<std::unique_ptr<ManifestFile>> ManifestFile::Create(
                          manifest_file_factory, target_file_size, pool, options, partition_type));
 }
 
+Status ManifestFile::ReadBucketEntries(const std::string& file_name, int32_t bucket,
+                                       std::vector<ManifestEntry>* entries) const {
+    return ReadArrowBatches(
+        file_name,
+        [this, bucket, entries](const std::shared_ptr<arrow::StructArray>& batch) -> Status {
+            for (int64_t i = 0; i < batch->length(); i++) {
+                ColumnarRow row(batch->fields(), pool_, i);
+                PAIMON_RETURN_NOT_OK(ManifestEntrySerializer::ValidateVersion(row.GetInt(0)));
+                if (ManifestEntrySerializer::GetBucket(row) != bucket) {
+                    continue;
+                }
+                PAIMON_ASSIGN_OR_RAISE(ManifestEntry entry, serializer_->FromRow(row));
+                entries->push_back(std::move(entry));
+            }
+            return Status::OK();
+        });
+}
+
 Result<std::vector<ManifestFileMeta>> ManifestFile::Write(
     const std::vector<ManifestEntry>& entries) {
     if (entries.empty()) {
diff --git a/src/paimon/core/manifest/manifest_file.h b/src/paimon/core/manifest/manifest_file.h
index d34764b5f..0211e14d1 100644
--- a/src/paimon/core/manifest/manifest_file.h
+++ b/src/paimon/core/manifest/manifest_file.h
@@ -62,6 +62,10 @@ class ManifestFile : public ObjectsFile<ManifestEntry> {
     /// @note This method is atomic.
     Result<std::vector<ManifestFileMeta>> Write(const std::vector<ManifestEntry>& entries);
 
+    /// Read a manifest file and deserialize only entries for the specified bucket.
+    Status ReadBucketEntries(const std::string& file_name, int32_t bucket,
+                             std::vector<ManifestEntry>* entries) const;
+
  private:
     ManifestFile(const std::shared_ptr<FileSystem>& file_system,
                  const std::shared_ptr<ReaderBuilder>& reader_builder,
diff --git a/src/paimon/core/manifest/manifest_file_test.cpp b/src/paimon/core/manifest/manifest_file_test.cpp
index 8f6e0b2ea..a34f41524 100644
--- a/src/paimon/core/manifest/manifest_file_test.cpp
+++ b/src/paimon/core/manifest/manifest_file_test.cpp
@@ -23,7 +23,6 @@
 #include <optional>
 #include <string>
 #include <utility>
-#include <variant>
 
 #include "arrow/api.h"
 #include "gtest/gtest.h"
@@ -100,10 +99,10 @@ class CountingFileSystem : public FileSystem {
 
 class ManifestFileTest : public testing::Test {
  public:
-    std::vector<ManifestEntry> ReadManifestEntry(const std::string& file_format_str,
-                                                 const std::string& root_path,
-                                                 const std::string& file_name,
-                                                 const std::shared_ptr<MemoryPool>& pool) const {
+    std::vector<ManifestEntry> ReadManifestEntry(
+        const std::string& file_format_str, const std::string& root_path,
+        const std::string& file_name, const std::shared_ptr<MemoryPool>& pool,
+        const std::optional<int32_t>& bucket = std::nullopt) const {
         std::shared_ptr<FileSystem> file_system = std::make_shared<LocalFileSystem>();
         EXPECT_OK_AND_ASSIGN(std::shared_ptr<FileFormat> file_format,
                              FileFormatFactory::Get(file_format_str, {}));
@@ -124,7 +123,12 @@ class ManifestFileTest : public testing::Test {
             ManifestFile::Create(file_system, file_format, "zstd", path_factory,
                                  /*target_file_size=*/1024, pool, options, unused_schema));
         std::vector<ManifestEntry> manifest_entries;
-        EXPECT_OK(manifest_file->Read(file_name, /*filter=*/nullptr, &manifest_entries));
+        if (bucket) {
+            EXPECT_OK(
+                manifest_file->ReadBucketEntries(file_name, bucket.value(), &manifest_entries));
+        } else {
+            EXPECT_OK(manifest_file->Read(file_name, /*filter=*/nullptr, &manifest_entries));
+        }
 
         return manifest_entries;
     }
@@ -316,6 +320,104 @@ TEST_F(ManifestFileTest, TestManifestCacheReusesCachedBytes) {
     ASSERT_EQ(1, manifest_cache->Size());
 }
 
+TEST_F(ManifestFileTest, TestReadBucketEntriesMaterializesOnlySelectedBucket) {
+    auto pool = GetDefaultPool();
+    auto counting_file_system = std::make_shared<CountingFileSystem>();
+    auto manifest_cache =
+        std::make_shared<CountingRoutingCache>(CacheKind::MANIFEST, 64 * 1024 * 1024);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileFormat> file_format,
+                         FileFormatFactory::Get("orc", {}));
+    std::string root_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
+    auto unused_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", arrow::utf8())}));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<FileStorePathFactory> path_factory,
+        FileStorePathFactory::Create(root_path, unused_schema, /*partition_keys=*/{},
+                                     /*default_part_value=*/"", file_format->Identifier(),
+                                     /*data_file_prefix=*/"data-",
+                                     /*legacy_partition_name_enabled=*/true, /*external_paths=*/{},
+                                     /*global_index_external_path=*/std::nullopt,
+                                     /*index_file_in_data_file_dir=*/false, pool));
+    ASSERT_OK_AND_ASSIGN(
+        CoreOptions options,
+        CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}));
+    options.WithCache(manifest_cache);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<ManifestFile> manifest_file,
+        ManifestFile::Create(counting_file_system, file_format, "zstd", path_factory,
+                             /*target_file_size=*/1024, pool, options, unused_schema));
+
+    const std::string manifest_name = "manifest-3a44a0da-1008-463c-914e-28d271375e24-0";
+    std::vector<ManifestEntry> all_entries;
+    ASSERT_OK(manifest_file->Read(manifest_name, /*filter=*/nullptr, &all_entries));
+    ASSERT_EQ(2, all_entries.size());
+
+    std::vector<ManifestEntry> bucket_one_entries;
+    ASSERT_OK(manifest_file->ReadBucketEntries(manifest_name, /*bucket=*/1, &bucket_one_entries));
+    ASSERT_EQ(std::vector<ManifestEntry>({all_entries[0]}), bucket_one_entries);
+
+    std::vector<ManifestEntry> bucket_zero_entries;
+    ASSERT_OK(manifest_file->ReadBucketEntries(manifest_name, /*bucket=*/0, &bucket_zero_entries));
+    ASSERT_EQ(std::vector<ManifestEntry>({all_entries[1]}), bucket_zero_entries);
+
+    std::vector<ManifestEntry> missing_bucket_entries;
+    ASSERT_OK(
+        manifest_file->ReadBucketEntries(manifest_name, /*bucket=*/2, &missing_bucket_entries));
+    ASSERT_TRUE(missing_bucket_entries.empty());
+
+    ASSERT_EQ(1, counting_file_system->open_count);
+    ASSERT_EQ(4, manifest_cache->GetCount());
+    ASSERT_EQ(1, manifest_cache->SupplierCallCount());
+}
+
+TEST_F(ManifestFileTest, TestReadBucketEntriesSkipsDeserializingOtherBuckets) {
+    auto pool = GetDefaultPool();
+    std::vector<ManifestEntry> source_entries =
+        ReadManifestEntry("orc", paimon::test::GetDataDir() + "/orc/append_09.db/append_09",
+                          "manifest-3a44a0da-1008-463c-914e-28d271375e24-0", pool);
+    ASSERT_EQ(2, source_entries.size());
+
+    auto test_dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(test_dir);
+    std::shared_ptr<FileSystem> file_system = test_dir->GetFileSystem();
+    ASSERT_OK(file_system->Mkdirs(FileStorePathFactory::ManifestPath(test_dir->Str())));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileFormat> file_format,
+                         FileFormatFactory::Get("orc", {}));
+    auto unused_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", arrow::utf8())}));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<FileStorePathFactory> path_factory,
+        FileStorePathFactory::Create(test_dir->Str(), unused_schema, /*partition_keys=*/{},
+                                     /*default_part_value=*/"", file_format->Identifier(),
+                                     /*data_file_prefix=*/"data-",
+                                     /*legacy_partition_name_enabled=*/true, /*external_paths=*/{},
+                                     /*global_index_external_path=*/std::nullopt,
+                                     /*index_file_in_data_file_dir=*/false, pool));
+    ASSERT_OK_AND_ASSIGN(
+        CoreOptions options,
+        CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<ManifestFile> manifest_file,
+        ManifestFile::Create(file_system, file_format, "zstd", path_factory,
+                             /*target_file_size=*/1024, pool, options, unused_schema));
+
+    ManifestEntry invalid_other_bucket(FileKind(static_cast<int8_t>(2)),
+                                       source_entries[0].Partition(), /*bucket=*/1,
+                                       /*total_buckets=*/2, source_entries[0].File());
+    ManifestEntry valid_target_bucket(FileKind::Add(), source_entries[1].Partition(), /*bucket=*/0,
+                                      /*total_buckets=*/2, source_entries[1].File());
+    using WrittenFile = std::pair<std::string, int64_t>;
+    ASSERT_OK_AND_ASSIGN(
+        WrittenFile written_file,
+        manifest_file->WriteWithoutRolling({invalid_other_bucket, valid_target_bucket}));
+
+    std::vector<ManifestEntry> all_entries;
+    ASSERT_NOK_WITH_MSG(manifest_file->Read(written_file.first, /*filter=*/nullptr, &all_entries),
+                        "Unsupported byte value 2 for file kind.");
+
+    std::vector<ManifestEntry> bucket_entries;
+    ASSERT_OK(manifest_file->ReadBucketEntries(written_file.first, /*bucket=*/0, &bucket_entries));
+    ASSERT_EQ(std::vector<ManifestEntry>({valid_target_bucket}), bucket_entries);
+}
+
 TEST_F(ManifestFileTest, TestWithNullCount) {
     auto pool = GetDefaultPool();
     auto manifest_entries =
@@ -406,6 +508,9 @@ TEST_F(ManifestFileTest, TestManifestFileCompatibleWithJavaPaimon09) {
     std::vector<ManifestEntry> expected_manifest_entries;
     expected_manifest_entries.emplace_back(manifest_entry);
     ASSERT_EQ(expected_manifest_entries, manifest_entries);
+    ASSERT_EQ(expected_manifest_entries,
+              ReadManifestEntry("avro", paimon::test::GetDataDir() + "/avro", "avro_manifest_09",
+                                pool, /*bucket=*/0));
 }
 
 TEST_F(ManifestFileTest, TestManifestFileCompatibleWithJavaPaimon11) {
@@ -442,6 +547,9 @@ TEST_F(ManifestFileTest, TestManifestFileCompatibleWithJavaPaimon11) {
     std::vector<ManifestEntry> expected_manifest_entries;
     expected_manifest_entries.emplace_back(manifest_entry);
     ASSERT_EQ(expected_manifest_entries, manifest_entries);
+    ASSERT_EQ(expected_manifest_entries,
+              ReadManifestEntry("avro", paimon::test::GetDataDir() + "/avro", "avro_manifest_11",
+                                pool, /*bucket=*/0));
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/append_only_file_store_scan_test.cpp b/src/paimon/core/operation/append_only_file_store_scan_test.cpp
index f319498a4..e1fb5a43a 100644
--- a/src/paimon/core/operation/append_only_file_store_scan_test.cpp
+++ b/src/paimon/core/operation/append_only_file_store_scan_test.cpp
@@ -29,6 +29,7 @@
 #include "paimon/common/data/binary_row.h"
 #include "paimon/common/data/binary_row_writer.h"
 #include "paimon/common/io/cache/lru_cache.h"
+#include "paimon/core/manifest/manifest_entry.h"
 #include "paimon/core/manifest/partition_entry.h"
 #include "paimon/core/operation/metrics/scan_metrics.h"
 #include "paimon/core/schema/schema_manager.h"
@@ -186,11 +187,14 @@ namespace {
 std::shared_ptr<FileStoreScan> BuildScan(const std::string& table_path,
                                          const std::shared_ptr<Cache>& cache,
                                          const std::optional<int32_t>& bucket = std::nullopt,
-                                         const std::shared_ptr<Predicate>& predicate = nullptr) {
+                                         const std::shared_ptr<Predicate>& predicate = nullptr,
+                                         bool manifest_entry_lazy_decode_enabled = true) {
     ScanContextBuilder context_builder(table_path);
     context_builder.AddOption(Options::FILE_FORMAT, "orc")
         .AddOption(Options::MANIFEST_FORMAT, "orc")
         .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "8")
+        .AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED,
+                   manifest_entry_lazy_decode_enabled ? "true" : "false")
         .WithCache(cache);
     if (bucket) {
         context_builder.SetBucketFilter(bucket.value());
@@ -205,6 +209,16 @@ std::shared_ptr<FileStoreScan> BuildScan(const std::string& table_path,
     return typed_table_scan->snapshot_reader_->scan_;
 }
 
+std::vector<std::string> SortedFileNames(std::vector<ManifestEntry>&& entries) {
+    std::vector<std::string> file_names;
+    file_names.reserve(entries.size());
+    for (const auto& entry : entries) {
+        file_names.push_back(entry.FileName());
+    }
+    std::sort(file_names.begin(), file_names.end());
+    return file_names;
+}
+
 }  // namespace
 
 TEST(AppendOnlyFileStoreScanTest, TestDropStatsAfterFiltering) {
@@ -253,13 +267,13 @@ TEST(AppendOnlyFileStoreScanTest, TestSnapshotLiveManifestCachePath) {
                          scan_first->GetSnapshotManager()->LoadSnapshot(/*snapshot_id=*/5));
     scan_first->WithSnapshot(snapshot_5);
     ASSERT_OK_AND_ASSIGN(auto plan_first, scan_first->CreatePlan());
-    size_t first_size = plan_first->Files().size();
+    std::vector<std::string> first_file_names = SortedFileNames(plan_first->Files());
 
     // Second scan on the same snapshot should read the same bucket live entries from cache.
     auto scan_second = BuildScan(table_path, cache, /*bucket=*/0);
     scan_second->WithSnapshot(snapshot_5);
     ASSERT_OK_AND_ASSIGN(auto plan_second, scan_second->CreatePlan());
-    ASSERT_EQ(first_size, plan_second->Files().size());
+    ASSERT_EQ(first_file_names, SortedFileNames(plan_second->Files()));
 }
 
 TEST(AppendOnlyFileStoreScanTest, TestSnapshotLiveManifestCacheRebuildOnMiss) {
@@ -285,6 +299,24 @@ TEST(AppendOnlyFileStoreScanTest, TestSnapshotLiveManifestCacheRebuildOnMiss) {
     auto scan_expected = BuildScan(table_path, /*cache=*/nullptr, /*bucket=*/0);
     scan_expected->WithSnapshot(snapshot_5);
     ASSERT_OK_AND_ASSIGN(auto plan_expected, scan_expected->CreatePlan());
-    ASSERT_EQ(plan_expected->Files().size(), plan_next->Files().size());
+    ASSERT_EQ(SortedFileNames(plan_expected->Files()), SortedFileNames(plan_next->Files()));
+}
+
+TEST(AppendOnlyFileStoreScanTest, TestSnapshotLiveManifestCacheFallbackWithoutLazyDecode) {
+    TimezoneGuard guard("Asia/Shanghai");
+    std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/";
+    auto cache = std::make_shared<LruCache>(/*max_weight=*/16 * 1024 * 1024);
+
+    auto scan_fallback = BuildScan(table_path, cache, /*bucket=*/0, /*predicate=*/nullptr,
+                                   /*manifest_entry_lazy_decode_enabled=*/false);
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot_5,
+                         scan_fallback->GetSnapshotManager()->LoadSnapshot(/*snapshot_id=*/5));
+    scan_fallback->WithSnapshot(snapshot_5);
+    ASSERT_OK_AND_ASSIGN(auto plan_fallback, scan_fallback->CreatePlan());
+
+    auto scan_expected = BuildScan(table_path, /*cache=*/nullptr, /*bucket=*/0);
+    scan_expected->WithSnapshot(snapshot_5);
+    ASSERT_OK_AND_ASSIGN(auto plan_expected, scan_expected->CreatePlan());
+    ASSERT_EQ(SortedFileNames(plan_expected->Files()), SortedFileNames(plan_fallback->Files()));
 }
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp
index 865e006ff..f21b0bb7f 100644
--- a/src/paimon/core/operation/file_store_scan.cpp
+++ b/src/paimon/core/operation/file_store_scan.cpp
@@ -365,6 +365,33 @@ Status FileStoreScan::StoreSnapshotLiveManifestEntries(
 Status FileStoreScan::ReadAndMergeBucketFileEntries(
     const std::vector<ManifestFileMeta>& manifest_metas, int32_t bucket,
     std::vector<ManifestEntry>* merged_entries) const {
+    if (core_options_.ScanManifestEntryLazyDecodeEnabled()) {
+        std::vector<std::future<Result<std::vector<ManifestEntry>>>> futures;
+        futures.reserve(manifest_metas.size());
+        for (const auto& meta : manifest_metas) {
+            auto read_meta_task = [this, meta, bucket]() -> Result<std::vector<ManifestEntry>> {
+                std::vector<ManifestEntry> bucket_entries;
+                PAIMON_RETURN_NOT_OK(
+                    manifest_file_->ReadBucketEntries(meta.FileName(), bucket, &bucket_entries));
+                return bucket_entries;
+            };
+            futures.push_back(Via(executor_.get(), read_meta_task));
+        }
+
+        std::vector<ManifestEntry> bucket_entries;
+        std::vector<Result<std::vector<ManifestEntry>>> entry_lists = CollectAll(futures);
+        for (auto& entry_list : entry_lists) {
+            if (!entry_list.ok()) {
+                return entry_list.status();
+            }
+            bucket_entries.reserve(bucket_entries.size() + entry_list.value().size());
+            for (auto& entry : entry_list.value()) {
+                bucket_entries.emplace_back(std::move(entry));
+            }
+        }
+        return MergeLiveEntries(bucket_entries, merged_entries);
+    }
+
     std::vector<ManifestEntry> unmerged_entries;
     std::vector<ManifestEntry> entries;
     PAIMON_RETURN_NOT_OK(ReadFileEntries(manifest_metas, &entries, /*apply_scan_filter=*/false));
diff --git a/src/paimon/core/utils/objects_file.h b/src/paimon/core/utils/objects_file.h
index f8509fe23..a56952ae2 100644
--- a/src/paimon/core/utils/objects_file.h
+++ b/src/paimon/core/utils/objects_file.h
@@ -19,7 +19,6 @@
 #pragma once
 
 #include <functional>
-#include <limits>
 #include <memory>
 #include <string>
 #include <utility>
@@ -78,6 +77,10 @@ class ObjectsFile {
     Result<std::pair<std::string, int64_t>> WriteWithoutRolling(const std::vector<T>& records);
 
  protected:
+    Status ReadArrowBatches(
+        const std::string& file_name,
+        const std::function<Status(const std::shared_ptr<arrow::StructArray>&)>& consumer) const;
+
     std::shared_ptr<PathFactory> path_factory_;
     std::shared_ptr<MemoryPool> pool_;
     std::unique_ptr<ObjectSerializer<T>> serializer_;
@@ -127,6 +130,30 @@ template <typename T>
 Status ObjectsFile<T>::Read(const std::string& file_name,
                             const std::function<Result<bool>(const T&)>& filter,
                             std::vector<T>* result) const {
+    return ReadArrowBatches(
+        file_name,
+        [this, &filter, result](const std::shared_ptr<arrow::StructArray>& struct_array) -> Status {
+            result->reserve(result->size() + struct_array->length());
+            for (int64_t i = 0; i < struct_array->length(); i++) {
+                ColumnarRow row(struct_array->fields(), pool_, i);
+                PAIMON_ASSIGN_OR_RAISE(T obj, serializer_->FromRow(row));
+                if (filter) {
+                    PAIMON_ASSIGN_OR_RAISE(bool filter_res, filter(obj));
+                    if (filter_res) {
+                        result->push_back(std::move(obj));
+                    }
+                } else {
+                    result->push_back(std::move(obj));
+                }
+            }
+            return Status::OK();
+        });
+}
+
+template <typename T>
+Status ObjectsFile<T>::ReadArrowBatches(
+    const std::string& file_name,
+    const std::function<Status(const std::shared_ptr<arrow::StructArray>&)>& consumer) const {
     std::string file_path = path_factory_->ToPath(file_name);
     std::shared_ptr<InputStream> file_input_stream;
     std::shared_ptr<Bytes> cached_bytes;
@@ -171,22 +198,10 @@ Status ObjectsFile<T>::Read(const std::string& file_name,
         if (!typed_array || typed_array->type_id() != arrow::Type::STRUCT) {
             return Status::Invalid(fmt::format("file {}, cannot cast to struct array", file_name));
         }
-        auto* struct_array = checked_cast<arrow::StructArray*>(typed_array.get());
-        result->reserve(struct_array->length());
-        for (int64_t i = 0; i < struct_array->length(); i++) {
-            ColumnarRow row(struct_array->fields(), pool_, i);
-            PAIMON_ASSIGN_OR_RAISE(T obj, serializer_->FromRow(row));
-            if (filter) {
-                PAIMON_ASSIGN_OR_RAISE(bool filter_res, filter(obj));
-                if (filter_res) {
-                    result->push_back(std::move(obj));
-                }
-            } else {
-                result->push_back(std::move(obj));
-            }
-        }
+        std::shared_ptr<arrow::StructArray> struct_array =
+            checked_pointer_cast<arrow::StructArray>(typed_array);
+        PAIMON_RETURN_NOT_OK(consumer(struct_array));
     }
-    reader->Close();
     return Status::OK();
 }
 

From eafe14e0208c9d6e815cdebf826298c404949fbc Mon Sep 17 00:00:00 2001
From: kid <19265318+u70b3@users.noreply.github.com>
Date: Fri, 21 Aug 2026 17:48:25 +0800
Subject: [PATCH 10/93] fix(build): repair singleton double-checked locking
 race and harden aarch64 portability (#203)

---
 include/paimon/factories/singleton.h          |  39 ++++-
 src/paimon/CMakeLists.txt                     |   3 +
 src/paimon/common/factories/io_hook.cpp       |  65 +++++---
 src/paimon/common/factories/io_hook.h         |   7 +-
 src/paimon/common/factories/io_hook_test.cpp  |  81 +++++++++
 src/paimon/common/factories/singleton.cpp     |  18 +-
 .../common/factories/singleton_test.cpp       | 108 ++++++++++++
 src/paimon/common/io/cache/cache_manager.h    |   9 +-
 .../common/io/cache/cache_manager_test.cpp    | 155 ++++++++++++++++++
 src/paimon/common/sst/sst_file_writer.cpp     |   5 +-
 .../common/utils/read_ahead_cache_test.cpp    |   5 +-
 src/paimon/common/utils/saturating_cast.h     |  53 ++++++
 .../common/utils/saturating_cast_test.cpp     |  74 +++++++++
 src/paimon/common/utils/serialization_utils.h |   4 +-
 .../common/utils/serialization_utils_test.cpp |  48 ++++++
 15 files changed, 629 insertions(+), 45 deletions(-)
 create mode 100644 src/paimon/common/factories/singleton_test.cpp
 create mode 100644 src/paimon/common/io/cache/cache_manager_test.cpp
 create mode 100644 src/paimon/common/utils/saturating_cast.h
 create mode 100644 src/paimon/common/utils/saturating_cast_test.cpp

diff --git a/include/paimon/factories/singleton.h b/include/paimon/factories/singleton.h
index 6e12d4561..a3030e5ea 100644
--- a/include/paimon/factories/singleton.h
+++ b/include/paimon/factories/singleton.h
@@ -19,7 +19,9 @@
 
 #pragma once
 
+#include <atomic>
 #include <memory>
+#include <mutex>
 
 #include "paimon/macros.h"
 #include "paimon/visibility.h"
@@ -30,9 +32,9 @@ class PAIMON_EXPORT LazyInstantiation {
  protected:
     template <typename T>
     static void Create(T*& ptr) {
-        T* tmp = new T;
-        MEMORY_BARRIER();
-        ptr = tmp;
+        // Publication ordering is handled by the release store in
+        // Singleton<T, InstPolicy>::GetInstance(), so no barrier is needed here.
+        ptr = new T;
         static std::shared_ptr<T> destroyer(ptr);
     }
 };
@@ -56,4 +58,35 @@ class PAIMON_EXPORT Singleton : private InstPolicy {
     static T* GetInstance();
 };
 
+template <typename T, typename InstPolicy>
+T* Singleton<T, InstPolicy>::GetInstance() {
+    static std::atomic<T*> ptr{nullptr};
+    static std::mutex mutex;
+    T* p = ptr.load(std::memory_order_acquire);
+    if (PAIMON_UNLIKELY(p == nullptr)) {
+        std::lock_guard<std::mutex> lg(mutex);
+        // Re-check under the mutex with a relaxed load; the mutex already
+        // synchronizes with the creating thread.
+        p = ptr.load(std::memory_order_relaxed);
+        if (p == nullptr) {
+            InstPolicy::Create(p);
+            ptr.store(p, std::memory_order_release);
+        }
+    }
+    return p;
+}
+
+// FactoryCreator and IOHook are instantiated exactly once in singleton.cpp, and the
+// extern declarations below suppress implicit instantiation everywhere else. The
+// file-format/file-system plugins are separate shared libraries linked with
+// -Bsymbolic, so a per-library copy of GetInstance()'s function-local static state
+// would never be interposed: factory registrations would land in a different
+// instance than lookups. Do not replace these with implicit instantiation. Types local to a single
+// translation unit (e.g. test-only types) can still instantiate Singleton<T>
+// implicitly because they cannot span library boundaries.
+class FactoryCreator;
+class IOHook;
+extern template class Singleton<FactoryCreator>;
+extern template class Singleton<IOHook>;
+
 }  // namespace paimon
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index bdb110057..adfd968dc 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -651,7 +651,9 @@ if(PAIMON_BUILD_TESTS)
                     common/utils/range_helper_test.cpp
                     common/utils/read_ahead_cache_test.cpp
                     common/io/cache/lru_cache_test.cpp
+                    common/io/cache/cache_manager_test.cpp
                     common/utils/byte_range_combiner_test.cpp
+                    common/utils/saturating_cast_test.cpp
                     common/utils/scope_guard_test.cpp
                     common/utils/sensitive_config_utils_test.cpp
                     common/utils/serialization_utils_test.cpp
@@ -682,6 +684,7 @@ if(PAIMON_BUILD_TESTS)
 
     add_paimon_test(common_factories_test
                     SOURCES
+                    common/factories/singleton_test.cpp
                     common/factories/factory_creator_test.cpp
                     common/factories/io_hook_test.cpp
                     STATIC_LINK_LIBS
diff --git a/src/paimon/common/factories/io_hook.cpp b/src/paimon/common/factories/io_hook.cpp
index a0576b469..394dc8ea5 100644
--- a/src/paimon/common/factories/io_hook.cpp
+++ b/src/paimon/common/factories/io_hook.cpp
@@ -19,9 +19,12 @@
 #include "paimon/common/factories/io_hook.h"
 
 #include <atomic>
+#include <mutex>
+#include <shared_mutex>
 #include <stdexcept>
 
 #include "fmt/format.h"
+#include "paimon/macros.h"
 #include "paimon/status.h"
 
 namespace paimon {
@@ -29,42 +32,66 @@ namespace paimon {
 class IOHook::Impl {
  public:
     Status Try(const std::string& path) {
-        if (io_count_.fetch_add(1) < pos_.load()) {
-            return Status::OK();
-        } else {
-            switch (mode_) {
-                case IOHook::Mode::SILENT:
-                    return Status::OK();
-                case IOHook::Mode::RETURN_ERROR:
-                    return Status::IOError(fmt::format(
-                        "io hook triggered io error at position {}, path {}", pos_.load(), path));
-                case IOHook::Mode::THROW_EXCEPTION:
-                    throw std::runtime_error(fmt::format(
-                        "io hook throw io exception at position {}, path {}", pos_.load(), path));
-                    return Status::OK();
-                default:
-                    return Status::OK();
-            }
+        // Fast path: the hook is disabled, which is always the case in production;
+        // writers (Reset()/Clear()) only exist in tests. This keeps Try() a single
+        // atomic load on the IO path instead of a shared_mutex acquisition per IO.
+        if (PAIMON_UNLIKELY(armed_.load(std::memory_order_acquire))) {
+            return TryArmed(path);
         }
+        return Status::OK();
     }
 
     inline void Reset(int64_t pos, IOHook::Mode mode) {
+        std::unique_lock<std::shared_mutex> lock(mutex_);
+        mode_ = mode;
         pos_ = pos;
         io_count_ = 0;
-        mode_ = mode;
+        // Arm only after the configuration is complete: TryArmed() reads mode_/pos_
+        // under mutex_, which synchronizes with this store, so an observed armed state
+        // always implies a complete configuration.
+        armed_.store(true, std::memory_order_release);
     }
 
     int64_t IOCount() const {
+        std::shared_lock<std::shared_mutex> lock(mutex_);
         return io_count_.load();
     }
 
     void Clear() {
-        Reset(-1, IOHook::Mode::SILENT);
+        std::unique_lock<std::shared_mutex> lock(mutex_);
+        // Disarm first so IO threads stop taking the lock as soon as possible.
+        armed_.store(false, std::memory_order_release);
+        mode_ = IOHook::Mode::SILENT;
+        pos_ = -1;
+        io_count_ = 0;
     }
 
  private:
+    Status TryArmed(const std::string& path) {
+        std::shared_lock<std::shared_mutex> lock(mutex_);
+        if (io_count_.fetch_add(1) < pos_) {
+            return Status::OK();
+        } else {
+            switch (mode_) {
+                case IOHook::Mode::SILENT:
+                    return Status::OK();
+                case IOHook::Mode::RETURN_ERROR:
+                    return Status::IOError(fmt::format(
+                        "io hook triggered io error at position {}, path {}", pos_, path));
+                case IOHook::Mode::THROW_EXCEPTION:
+                    throw std::runtime_error(fmt::format(
+                        "io hook throw io exception at position {}, path {}", pos_, path));
+                    return Status::OK();
+                default:
+                    return Status::OK();
+            }
+        }
+    }
+
+    mutable std::shared_mutex mutex_;
+    std::atomic<bool> armed_ = {false};
     std::atomic<int64_t> io_count_ = {0};
-    std::atomic<int64_t> pos_ = {-1};
+    int64_t pos_ = -1;
     IOHook::Mode mode_ = IOHook::Mode::SILENT;
 };
 
diff --git a/src/paimon/common/factories/io_hook.h b/src/paimon/common/factories/io_hook.h
index e0a2f68be..0c66381b9 100644
--- a/src/paimon/common/factories/io_hook.h
+++ b/src/paimon/common/factories/io_hook.h
@@ -45,7 +45,8 @@ class PAIMON_EXPORT IOHook : public Singleton<IOHook> {
     };
 
     /// Reset the IO exception position and behavior mode to handle the exception.
-    /// IOCount will be reset to 0.
+    /// IOCount will be reset to 0. Arms the hook: Try() switches from its lock-free
+    /// disabled fast path to the synchronized armed path.
     ///
     /// @params pos The position where the IO exception occurs.
     /// @params mode The mode of behavior for handling the exception.
@@ -56,12 +57,14 @@ class PAIMON_EXPORT IOHook : public Singleton<IOHook> {
     Status Try(const std::string& path);
 
     /// Get the count of IO operations that have already occurred.
+    /// IOs are only counted while the hook is armed (after Reset(), before Clear());
+    /// the disabled fast path does not count.
     ///
     /// @return The number of IO operations executed.
     int64_t IOCount() const;
 
     /// Clear the state of the IOHook, including resetting IO count and
-    /// any stored exception state.
+    /// any stored exception state. Disarms the hook back to the lock-free fast path.
     void Clear();
 
  private:
diff --git a/src/paimon/common/factories/io_hook_test.cpp b/src/paimon/common/factories/io_hook_test.cpp
index 9bbb1b342..653dc73b8 100644
--- a/src/paimon/common/factories/io_hook_test.cpp
+++ b/src/paimon/common/factories/io_hook_test.cpp
@@ -19,9 +19,13 @@
 
 #include "paimon/common/factories/io_hook.h"
 
+#include <atomic>
 #include <stdexcept>
+#include <thread>
+#include <vector>
 
 #include "gtest/gtest.h"
+#include "paimon/status.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
@@ -64,4 +68,81 @@ TEST(IOHookTest, TestThrowExceptionMode) {
     hook->Clear();
 }
 
+// The disabled state is the production default: Try() must take the lock-free fast
+// path, always return OK, and not count IOs (see IOCount()'s contract). Clear() first
+// so the test does not depend on execution order.
+TEST(IOHookTest, TestDisabledFastPath) {
+    auto hook = IOHook::GetInstance();
+    hook->Clear();
+    ASSERT_OK(hook->Try("path"));
+    ASSERT_OK(hook->Try("path"));
+    ASSERT_EQ(0, hook->IOCount());
+
+    // Re-arming and disarming must restore the exact disabled behavior.
+    hook->Reset(0, IOHook::Mode::RETURN_ERROR);
+    ASSERT_NOK(hook->Try("path"));
+    ASSERT_EQ(1, hook->IOCount());
+    hook->Clear();
+    ASSERT_OK(hook->Try("path"));
+    ASSERT_OK(hook->Try("path"));
+    ASSERT_EQ(0, hook->IOCount());
+}
+
+// Regression test for torn IOHook configurations: Reset()/Clear() run on one thread
+// while other threads call Try() concurrently. A shared start barrier releases all
+// threads together, and the reset thread keeps hammering until every worker has
+// finished, so overlap is structural rather than timing-dependent. The continuous
+// arm/disarm cycling also keeps workers switching between the disabled fast path and
+// the synchronized armed path. Under a ThreadSanitizer build this deterministically
+// reports any unsynchronized access; functionally every Try() must return OK.
+TEST(IOHookTest, TestConcurrentResetAndTry) {
+    auto hook = IOHook::GetInstance();
+
+    constexpr int32_t kTryIterations = 50000;
+    constexpr int32_t kNumWorkers = 4;
+
+    std::atomic<bool> start{false};
+    std::atomic<int32_t> workers_done{0};
+    std::atomic<bool> observed_error{false};
+
+    std::thread reset_thread([hook, &start, &workers_done]() {
+        while (!start.load(std::memory_order_acquire)) {
+            std::this_thread::yield();
+        }
+        while (workers_done.load(std::memory_order_relaxed) < kNumWorkers) {
+            hook->Reset(INT64_MAX, IOHook::Mode::RETURN_ERROR);
+            hook->Clear();
+        }
+    });
+
+    std::vector<std::thread> workers;
+    workers.reserve(kNumWorkers);
+    for (int32_t t = 0; t < kNumWorkers; t++) {
+        workers.emplace_back([hook, &start, &workers_done, &observed_error]() {
+            while (!start.load(std::memory_order_acquire)) {
+                std::this_thread::yield();
+            }
+            for (int32_t i = 0; i < kTryIterations; i++) {
+                Status status = hook->Try("concurrent_path");
+                // Reset() arms an unreachable position, while Clear() uses SILENT mode,
+                // so both complete states return OK. An IOError exposes a torn state.
+                if (!status.ok()) {
+                    observed_error.store(true, std::memory_order_relaxed);
+                }
+            }
+            workers_done.fetch_add(1, std::memory_order_relaxed);
+        });
+    }
+
+    start.store(true, std::memory_order_release);
+    reset_thread.join();
+    for (auto& worker : workers) {
+        worker.join();
+    }
+
+    ASSERT_FALSE(observed_error.load(std::memory_order_relaxed));
+    // Leave the process-wide singleton in its default SILENT state for later tests.
+    hook->Clear();
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/common/factories/singleton.cpp b/src/paimon/common/factories/singleton.cpp
index a97322597..2a5daa894 100644
--- a/src/paimon/common/factories/singleton.cpp
+++ b/src/paimon/common/factories/singleton.cpp
@@ -19,26 +19,14 @@
 
 #include "paimon/factories/singleton.h"
 
-#include <mutex>
-
 #include "paimon/common/factories/io_hook.h"
 #include "paimon/factories/factory_creator.h"
 
 namespace paimon {
 
-template <typename T, typename InstPolicy>
-T* Singleton<T, InstPolicy>::GetInstance() {
-    static T* ptr;
-    static std::mutex mutex;
-    if (PAIMON_UNLIKELY(!ptr)) {
-        std::lock_guard<std::mutex> lg(mutex);
-        if (!ptr) {
-            InstPolicy::Create(ptr);
-        }
-    }
-    return const_cast<T*>(ptr);
-}
-
+// The single definition point for the two cross-library singletons. See the
+// extern template declarations in singleton.h for why implicit instantiation
+// must stay suppressed for these types.
 template class Singleton<FactoryCreator>;
 template class Singleton<IOHook>;
 
diff --git a/src/paimon/common/factories/singleton_test.cpp b/src/paimon/common/factories/singleton_test.cpp
new file mode 100644
index 000000000..efaf921cc
--- /dev/null
+++ b/src/paimon/common/factories/singleton_test.cpp
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/factories/singleton.h"
+
+#include <array>
+#include <atomic>
+#include <cstdint>
+#include <thread>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+namespace paimon::test {
+
+namespace {
+
+constexpr int32_t kNumThreads = 32;
+
+// Runs `worker(i)` on kNumThreads threads that are all blocked on a shared start
+// flag and released at (nearly) the same time, so that they race on the first
+// Singleton::GetInstance() publication. Joins all threads before returning.
+template <typename Worker>
+void RunStorm(const Worker& worker) {
+    std::atomic<bool> start{false};
+    std::vector<std::thread> threads;
+    threads.reserve(kNumThreads);
+    for (int32_t i = 0; i < kNumThreads; ++i) {
+        threads.emplace_back([&start, &worker, i]() {
+            while (!start.load(std::memory_order_acquire)) {
+                std::this_thread::yield();
+            }
+            worker(i);
+        });
+    }
+    start.store(true, std::memory_order_release);
+    for (auto& thread : threads) {
+        thread.join();
+    }
+}
+
+// Local to this translation unit, so nothing else in the test binary can have
+// instantiated Singleton<FirstPublicationTarget> before this test runs: the storm
+// below is guaranteed to race on the *first* publication regardless of link order,
+// --gtest_shuffle, or --gtest_filter. GetInstance() is defined in the header, so a
+// translation-unit-local type can instantiate it.
+class FirstPublicationTarget {
+ public:
+    FirstPublicationTarget() {
+        for (size_t i = 0; i < payload_.size(); ++i) {
+            payload_[i] = kMagic ^ (i * 0x9E3779B97F4A7C15ULL);
+        }
+    }
+
+    // The publication race let a reader observe the instance pointer before the
+    // constructor's stores were visible; this checks every word the ctor wrote.
+    bool IsFullyConstructed() const {
+        for (size_t i = 0; i < payload_.size(); ++i) {
+            if (payload_[i] != (kMagic ^ (i * 0x9E3779B97F4A7C15ULL))) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+ private:
+    static constexpr uint64_t kMagic = 0xA5A5F00D12345678ULL;
+    std::array<uint64_t, 64> payload_{};
+};
+
+}  // namespace
+
+// Regression gate for the Singleton double-checked-locking publication race: 32
+// threads race the first GetInstance() of a type local to this file, so the gate
+// cannot silently degrade into exercising only the already-published fast path.
+TEST(SingletonTest, TestConcurrentFirstPublication) {
+    std::array<FirstPublicationTarget*, kNumThreads> instances{};
+    std::array<bool, kNumThreads> fully_constructed{};
+    RunStorm([&instances, &fully_constructed](int32_t i) {
+        instances[i] = Singleton<FirstPublicationTarget>::GetInstance();
+        fully_constructed[i] = instances[i]->IsFullyConstructed();
+    });
+
+    FirstPublicationTarget* expected = instances[0];
+    ASSERT_NE(expected, nullptr);
+    for (int32_t i = 0; i < kNumThreads; ++i) {
+        ASSERT_EQ(expected, instances[i]);
+        ASSERT_TRUE(fully_constructed[i]);
+    }
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/common/io/cache/cache_manager.h b/src/paimon/common/io/cache/cache_manager.h
index f899d46ce..6fafefb5c 100644
--- a/src/paimon/common/io/cache/cache_manager.h
+++ b/src/paimon/common/io/cache/cache_manager.h
@@ -25,6 +25,7 @@
 #include "paimon/cache/cache.h"
 #include "paimon/common/io/cache/cache_key.h"
 #include "paimon/common/io/cache/lru_cache.h"
+#include "paimon/common/utils/saturating_cast.h"
 #include "paimon/memory/memory_segment.h"
 #include "paimon/result.h"
 
@@ -59,9 +60,13 @@ class PAIMON_EXPORT CacheManager {
     /// @param high_priority_pool_ratio Ratio of capacity reserved for index cache [0.0, 1.0).
     ///        If 0, index and data share the same cache.
     CacheManager(int64_t max_memory_bytes, double high_priority_pool_ratio) {
-        auto index_cache_bytes = static_cast<int64_t>(max_memory_bytes * high_priority_pool_ratio);
+        // Both factors are config-validated non-negative values, so the products are finite;
+        // saturation is only a defense against the undefined double->int64_t conversion when
+        // max_memory_bytes is close enough to INT64_MAX that the product rounds to 2^63.
+        auto index_cache_bytes =
+            SaturatingDoubleToInteger<int64_t>(max_memory_bytes * high_priority_pool_ratio);
         auto data_cache_bytes =
-            static_cast<int64_t>(max_memory_bytes * (1.0 - high_priority_pool_ratio));
+            SaturatingDoubleToInteger<int64_t>(max_memory_bytes * (1.0 - high_priority_pool_ratio));
         data_cache_ = std::make_shared<LruCache>(data_cache_bytes);
         if (high_priority_pool_ratio == 0.0) {
             index_cache_ = data_cache_;
diff --git a/src/paimon/common/io/cache/cache_manager_test.cpp b/src/paimon/common/io/cache/cache_manager_test.cpp
new file mode 100644
index 000000000..4d7c180a3
--- /dev/null
+++ b/src/paimon/common/io/cache/cache_manager_test.cpp
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/common/io/cache/cache_manager.h"
+
+#include <cstdint>
+#include <cstring>
+#include <limits>
+#include <memory>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "paimon/cache/cache.h"
+#include "paimon/common/io/cache/cache_key.h"
+#include "paimon/common/io/cache/lru_cache.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/memory/memory_segment.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+class CacheManagerTest : public ::testing::Test {
+ public:
+    void SetUp() override {
+        pool_ = GetDefaultPool();
+    }
+
+    std::shared_ptr<CacheKey> MakeKey(int64_t position, bool is_index = false) const {
+        return CacheKey::ForPosition("test_file", position, 64, is_index);
+    }
+
+    MemorySegment MakeSegment(int32_t size, char fill_byte) const {
+        auto segment = MemorySegment::AllocateHeapMemory(size, pool_.get());
+        std::memset(segment.MutableData(), fill_byte, size);
+        return segment;
+    }
+
+    std::shared_ptr<LruCache> DataLru(const CacheManager& manager) const {
+        return std::dynamic_pointer_cast<LruCache>(manager.DataCache());
+    }
+
+    std::shared_ptr<LruCache> IndexLru(const CacheManager& manager) const {
+        return std::dynamic_pointer_cast<LruCache>(manager.IndexCache());
+    }
+
+ private:
+    std::shared_ptr<MemoryPool> pool_;
+};
+
+/// Regression test for the double->int64_t conversions in the CacheManager constructor:
+/// (double)INT64_MAX rounds to 2^63, which is not representable as int64_t, so casting the
+/// product back is undefined behavior (x86 cvttsd2si yields INT64_MIN, aarch64 fcvtzs
+/// saturates to INT64_MAX). The conversion must saturate, keeping the capacity non-negative.
+TEST_F(CacheManagerTest, TestCapacitySaturatesAtInt64Max) {
+    CacheManager manager(std::numeric_limits<int64_t>::max(), /*high_priority_pool_ratio=*/0.0);
+
+    std::shared_ptr<LruCache> data_lru = DataLru(manager);
+    ASSERT_NE(data_lru, nullptr);
+    ASSERT_GE(data_lru->GetMaxWeight(), 0);
+    ASSERT_EQ(data_lru->GetMaxWeight(), std::numeric_limits<int64_t>::max());
+
+    // A ratio of 0.0 means index and data share the same cache.
+    ASSERT_EQ(manager.DataCache(), manager.IndexCache());
+
+    // The saturated capacity accepts entries instead of rejecting every insert.
+    std::shared_ptr<CacheKey> key = MakeKey(0);
+    auto reader = [&](const std::shared_ptr<CacheKey>&) -> Result<MemorySegment> {
+        return MakeSegment(64, 'A');
+    };
+    ASSERT_OK_AND_ASSIGN(MemorySegment segment, manager.GetPage(key, reader, {}));
+    ASSERT_EQ(segment.Size(), 64);
+    ASSERT_EQ(segment.Get(0), 'A');
+}
+
+/// Verifies the exact capacity split between the data and index caches for a normal
+/// configuration, plus a Get/Invalidate smoke path through CacheManager::GetPage.
+TEST_F(CacheManagerTest, TestNormalSplitAndSmokePath) {
+    CacheManager manager(/*max_memory_bytes=*/1024, /*high_priority_pool_ratio=*/0.5);
+
+    std::shared_ptr<LruCache> data_lru = DataLru(manager);
+    std::shared_ptr<LruCache> index_lru = IndexLru(manager);
+    ASSERT_NE(data_lru, nullptr);
+    ASSERT_NE(index_lru, nullptr);
+    ASSERT_EQ(data_lru->GetMaxWeight(), 512);
+    ASSERT_EQ(index_lru->GetMaxWeight(), 512);
+
+    std::shared_ptr<CacheKey> key = MakeKey(0);
+    int32_t reader_calls = 0;
+    auto reader = [&](const std::shared_ptr<CacheKey>&) -> Result<MemorySegment> {
+        reader_calls++;
+        return MakeSegment(128, 'B');
+    };
+
+    // The first GetPage is a miss and invokes the reader; the second is a cache hit.
+    ASSERT_OK_AND_ASSIGN(MemorySegment first, manager.GetPage(key, reader, {}));
+    ASSERT_EQ(first.Get(0), 'B');
+    ASSERT_EQ(reader_calls, 1);
+    ASSERT_OK_AND_ASSIGN(MemorySegment second, manager.GetPage(key, reader, {}));
+    ASSERT_EQ(second.Get(0), 'B');
+    ASSERT_EQ(reader_calls, 1);
+
+    // After InvalidPage the reader is invoked again.
+    manager.InvalidPage(key);
+    ASSERT_OK_AND_ASSIGN(MemorySegment third, manager.GetPage(key, reader, {}));
+    ASSERT_EQ(third.Get(0), 'B');
+    ASSERT_EQ(reader_calls, 2);
+}
+
+/// Verifies weight-based eviction through GetPage: inserting beyond the data cache capacity
+/// evicts the least recently used page and runs its eviction callback.
+TEST_F(CacheManagerTest, TestGetPageEviction) {
+    // The data cache capacity is 512 * (1.0 - 0.5) = 256 bytes.
+    CacheManager manager(/*max_memory_bytes=*/512, /*high_priority_pool_ratio=*/0.5);
+
+    std::vector<int64_t> evicted;
+    auto callback_for = [&evicted](int64_t position) -> CacheCallback {
+        return
+            [&evicted, position](const std::shared_ptr<CacheKey>&) { evicted.push_back(position); };
+    };
+    auto reader = [&](const std::shared_ptr<CacheKey>&) -> Result<MemorySegment> {
+        return MakeSegment(128, 'C');
+    };
+
+    std::shared_ptr<CacheKey> key0 = MakeKey(0);
+    std::shared_ptr<CacheKey> key1 = MakeKey(1);
+    std::shared_ptr<CacheKey> key2 = MakeKey(2);
+    ASSERT_OK_AND_ASSIGN(MemorySegment segment0, manager.GetPage(key0, reader, callback_for(0)));
+    ASSERT_EQ(segment0.Get(0), 'C');
+    ASSERT_OK_AND_ASSIGN(MemorySegment segment1, manager.GetPage(key1, reader, callback_for(1)));
+    ASSERT_EQ(segment1.Get(0), 'C');
+    ASSERT_TRUE(evicted.empty());
+
+    // 128 + 128 + 128 > 256: inserting key2 evicts key0, the least recently used page.
+    ASSERT_OK_AND_ASSIGN(MemorySegment segment2, manager.GetPage(key2, reader, callback_for(2)));
+    ASSERT_EQ(segment2.Get(0), 'C');
+    ASSERT_EQ(evicted, std::vector<int64_t>({0}));
+    ASSERT_EQ(manager.DataCache()->Size(), 2);
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/common/sst/sst_file_writer.cpp b/src/paimon/common/sst/sst_file_writer.cpp
index ec736e33c..f2b3b2dee 100644
--- a/src/paimon/common/sst/sst_file_writer.cpp
+++ b/src/paimon/common/sst/sst_file_writer.cpp
@@ -20,6 +20,7 @@
 
 #include "paimon/common/utils/crc32c.h"
 #include "paimon/common/utils/murmurhash_utils.h"
+#include "paimon/common/utils/saturating_cast.h"
 
 namespace paimon {
 SstFileWriter::SstFileWriter(const std::shared_ptr<OutputStream>& out,
@@ -27,8 +28,10 @@ SstFileWriter::SstFileWriter(const std::shared_ptr<OutputStream>& out,
                              const std::shared_ptr<BlockCompressionFactory>& factory,
                              const std::shared_ptr<MemoryPool>& pool)
     : pool_(pool), out_(out), bloom_filter_(bloom_filter), block_size_(block_size) {
+    // block_size * 1.1 exceeds INT32_MAX for block_size above ~1.9GB; saturate instead of
+    // relying on the undefined double->int32_t conversion.
     data_block_writer_ =
-        std::make_unique<BlockWriter>(static_cast<int32_t>(block_size * 1.1), pool);
+        std::make_unique<BlockWriter>(SaturatingDoubleToInteger<int32_t>(block_size * 1.1), pool);
     index_block_writer_ =
         std::make_unique<BlockWriter>(BlockHandle::MAX_ENCODED_LENGTH * 1024, pool);
     compression_type_ = factory->GetCompressionType();
diff --git a/src/paimon/common/utils/read_ahead_cache_test.cpp b/src/paimon/common/utils/read_ahead_cache_test.cpp
index ab1d0ed3d..e7900b7b6 100644
--- a/src/paimon/common/utils/read_ahead_cache_test.cpp
+++ b/src/paimon/common/utils/read_ahead_cache_test.cpp
@@ -464,7 +464,8 @@ TEST(TestReadAheadCache, TestPreBufferWindowLimit) {
 
     auto io_hook = paimon::IOHook::GetInstance();
     paimon::ScopeGuard guard([&io_hook]() { io_hook->Clear(); });
-    io_hook->Clear();
+    // IOCount() only counts while armed; INT64_MAX never triggers the error mode.
+    io_hook->Reset(INT64_MAX, paimon::IOHook::Mode::RETURN_ERROR);
 
     AssertReadEquals({0, 10}, "abcdefghij", &cache);
     // The second range did not fit into the window: only one prefetch IO.
@@ -475,7 +476,7 @@ TEST(TestReadAheadCache, TestPreBufferWindowLimit) {
     ASSERT_EQ(io_hook->IOCount(), 2);
 
     // The range is cached now: re-reading it issues no IO at all.
-    io_hook->Clear();
+    io_hook->Reset(INT64_MAX, paimon::IOHook::Mode::RETURN_ERROR);
     AssertReadEquals({16, 10}, "qrstuvwxyz", &cache);
     ASSERT_EQ(io_hook->IOCount(), 0);
 }
diff --git a/src/paimon/common/utils/saturating_cast.h b/src/paimon/common/utils/saturating_cast.h
new file mode 100644
index 000000000..cb9c6039f
--- /dev/null
+++ b/src/paimon/common/utils/saturating_cast.h
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cmath>
+#include <cstdint>
+#include <limits>
+#include <type_traits>
+
+namespace paimon {
+
+/// Converts a double to int32_t or int64_t with Java's float-to-int or float-to-long saturation
+/// policy: NaN converts to 0 and an out-of-range value saturates at the bounds of TargetType.
+/// Narrower Java integer conversions require a subsequent narrowing step and are not supported by
+/// this helper. A bare static_cast of an unrepresentable double is undefined behavior and diverges
+/// across architectures (x86 cvttsd2si yields the "integer indefinite" value, while aarch64 fcvtzs
+/// saturates), so doubles that are not provably in range must go through this helper.
+template <typename TargetType>
+inline TargetType SaturatingDoubleToInteger(double value) {
+    static_assert(std::is_same_v<TargetType, int32_t> || std::is_same_v<TargetType, int64_t>,
+                  "TargetType must be int32_t or int64_t");
+    if (std::isnan(value)) {
+        return 0;
+    }
+    // Comparing against the bounds converted to double keeps the final truncation defined:
+    // (double)INT64_MAX rounds up to 2^63, so every value that reaches the truncation is
+    // representable in TargetType.
+    if (value >= static_cast<double>(std::numeric_limits<TargetType>::max())) {
+        return std::numeric_limits<TargetType>::max();
+    }
+    if (value <= static_cast<double>(std::numeric_limits<TargetType>::lowest())) {
+        return std::numeric_limits<TargetType>::lowest();
+    }
+    return static_cast<TargetType>(value);
+}
+
+}  // namespace paimon
diff --git a/src/paimon/common/utils/saturating_cast_test.cpp b/src/paimon/common/utils/saturating_cast_test.cpp
new file mode 100644
index 000000000..cc6b74cf1
--- /dev/null
+++ b/src/paimon/common/utils/saturating_cast_test.cpp
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/common/utils/saturating_cast.h"
+
+#include <cstdint>
+#include <limits>
+
+#include "gtest/gtest.h"
+
+namespace paimon::test {
+
+TEST(SaturatingCastTest, TestInt64InRangeTruncatesTowardZero) {
+    ASSERT_EQ(SaturatingDoubleToInteger<int64_t>(0.0), 0);
+    ASSERT_EQ(SaturatingDoubleToInteger<int64_t>(1.9), 1);
+    ASSERT_EQ(SaturatingDoubleToInteger<int64_t>(-1.9), -1);
+    // 2^63 - 1024 is the largest double below 2^63: it stays on the truncation path.
+    ASSERT_EQ(SaturatingDoubleToInteger<int64_t>(9223372036854774784.0), 9223372036854774784LL);
+}
+
+TEST(SaturatingCastTest, TestInt64Saturation) {
+    // (double)INT64_MAX rounds up to 2^63, so the boundary double already saturates.
+    ASSERT_EQ(SaturatingDoubleToInteger<int64_t>(
+                  static_cast<double>(std::numeric_limits<int64_t>::max())),
+              std::numeric_limits<int64_t>::max());
+    ASSERT_EQ(SaturatingDoubleToInteger<int64_t>(1e300), std::numeric_limits<int64_t>::max());
+    ASSERT_EQ(SaturatingDoubleToInteger<int64_t>(-1e300), std::numeric_limits<int64_t>::lowest());
+    ASSERT_EQ(SaturatingDoubleToInteger<int64_t>(std::numeric_limits<double>::infinity()),
+              std::numeric_limits<int64_t>::max());
+    ASSERT_EQ(SaturatingDoubleToInteger<int64_t>(-std::numeric_limits<double>::infinity()),
+              std::numeric_limits<int64_t>::lowest());
+    // The lowest bound is exactly representable and must survive as a value.
+    ASSERT_EQ(SaturatingDoubleToInteger<int64_t>(
+                  static_cast<double>(std::numeric_limits<int64_t>::lowest())),
+              std::numeric_limits<int64_t>::lowest());
+}
+
+TEST(SaturatingCastTest, TestInt64NaNBecomesZero) {
+    // Java's (long)Double.NaN == 0.
+    ASSERT_EQ(SaturatingDoubleToInteger<int64_t>(std::numeric_limits<double>::quiet_NaN()), 0);
+}
+
+TEST(SaturatingCastTest, TestInt32Path) {
+    // SstFileWriter converts through the int32_t instantiation.
+    ASSERT_EQ(SaturatingDoubleToInteger<int32_t>(42.7), 42);
+    ASSERT_EQ(SaturatingDoubleToInteger<int32_t>(-42.7), -42);
+    // The int32_t bounds are exactly representable as doubles and saturate inclusively.
+    ASSERT_EQ(SaturatingDoubleToInteger<int32_t>(2147483647.0),
+              std::numeric_limits<int32_t>::max());
+    ASSERT_EQ(SaturatingDoubleToInteger<int32_t>(2147483648.0),
+              std::numeric_limits<int32_t>::max());
+    ASSERT_EQ(SaturatingDoubleToInteger<int32_t>(-2147483648.0),
+              std::numeric_limits<int32_t>::lowest());
+    ASSERT_EQ(SaturatingDoubleToInteger<int32_t>(-2147483649.0),
+              std::numeric_limits<int32_t>::lowest());
+    ASSERT_EQ(SaturatingDoubleToInteger<int32_t>(std::numeric_limits<double>::quiet_NaN()), 0);
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/common/utils/serialization_utils.h b/src/paimon/common/utils/serialization_utils.h
index c1e97d033..449766ca6 100644
--- a/src/paimon/common/utils/serialization_utils.h
+++ b/src/paimon/common/utils/serialization_utils.h
@@ -78,7 +78,9 @@ class SerializationUtils {
         if (PAIMON_UNLIKELY(bytes->size() < 4)) {
             return Status::Invalid(fmt::format("bytes size {} is less than 4", bytes->size()));
         }
-        int32_t arity = *(reinterpret_cast<int32_t*>(bytes->data()));
+        // The buffer is byte-filled, so memcpy avoids the strict-aliasing UB of reinterpret_cast.
+        int32_t arity;
+        memcpy(&arity, bytes->data(), sizeof(int32_t));
         if (SystemByteOrder() == ByteOrder::PAIMON_LITTLE_ENDIAN) {
             arity = EndianSwapValue(arity);
         }
diff --git a/src/paimon/common/utils/serialization_utils_test.cpp b/src/paimon/common/utils/serialization_utils_test.cpp
index 5e612ff2e..56a39a295 100644
--- a/src/paimon/common/utils/serialization_utils_test.cpp
+++ b/src/paimon/common/utils/serialization_utils_test.cpp
@@ -19,7 +19,20 @@
 
 #include "paimon/common/utils/serialization_utils.h"
 
+#include <cstdint>
+#include <memory>
+#include <string>
+
 #include "gtest/gtest.h"
+#include "paimon/common/data/binary_row_writer.h"
+#include "paimon/common/data/binary_string.h"
+#include "paimon/common/io/memory_segment_output_stream.h"
+#include "paimon/common/memory/memory_segment_utils.h"
+#include "paimon/io/byte_array_input_stream.h"
+#include "paimon/io/data_input_stream.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
 
@@ -37,4 +50,39 @@ TEST_F(SerializationUtilsTest, TestSerializeBinaryRow) {
     ASSERT_TRUE(bytes);
 }
 
+TEST_F(SerializationUtilsTest, TestDeserializeBinaryRowFromStream) {
+    std::shared_ptr<MemoryPool> memory_pool = GetDefaultPool();
+    // a row with mixed field types, including negative integers and a string
+    BinaryRow row(3);
+    BinaryRowWriter writer(&row, 0, memory_pool.get());
+    writer.WriteInt(0, -123456);
+    writer.WriteLong(1, static_cast<int64_t>(-9000000000));
+    writer.WriteString(2, BinaryString::FromString("hello paimon!", memory_pool.get()));
+    writer.Complete();
+
+    // the first 4 bytes on the wire are the big-endian arity (Java-compatible format)
+    std::shared_ptr<Bytes> bytes = SerializationUtils::SerializeBinaryRow(row, memory_pool.get());
+    ASSERT_TRUE(bytes);
+    ASSERT_GE(bytes->size(), 4);
+    ASSERT_EQ(static_cast<uint8_t>(bytes->data()[0]), 0x00);
+    ASSERT_EQ(static_cast<uint8_t>(bytes->data()[1]), 0x00);
+    ASSERT_EQ(static_cast<uint8_t>(bytes->data()[2]), 0x00);
+    ASSERT_EQ(static_cast<uint8_t>(bytes->data()[3]), 0x03);
+
+    // round-trip through the stream overloads, which fill a fresh byte buffer on deserialize
+    MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, memory_pool);
+    ASSERT_OK(SerializationUtils::SerializeBinaryRow(row, &out));
+    auto stream_bytes =
+        MemorySegmentUtils::CopyToBytes(out.Segments(), 0, out.CurrentSize(), memory_pool.get());
+    auto input_stream =
+        std::make_shared<ByteArrayInputStream>(stream_bytes->data(), stream_bytes->size());
+    DataInputStream data_input_stream(input_stream);
+    ASSERT_OK_AND_ASSIGN(BinaryRow de_row, SerializationUtils::DeserializeBinaryRow(
+                                               &data_input_stream, memory_pool.get()));
+    ASSERT_EQ(de_row.GetFieldCount(), 3);
+    ASSERT_EQ(de_row.GetInt(0), -123456);
+    ASSERT_EQ(de_row.GetLong(1), static_cast<int64_t>(-9000000000));
+    ASSERT_EQ(de_row.GetString(2).ToString(), "hello paimon!");
+}
+
 }  // namespace paimon::test

From 05d6497bdd8111fbec45eba731ea5b2c06ce0d0b Mon Sep 17 00:00:00 2001
From: lxy <38709059+lxy-9602@users.noreply.github.com>
Date: Fri, 21 Aug 2026 19:41:52 +0800
Subject: [PATCH 11/93] fix(parquet): support nullable fixed-size lists for
 vector (#231)

---
 cmake_modules/arrow.diff                      | 194 +++++++++++++++++-
 docs/source/user_guide/data_types.rst         |   8 +-
 .../core/io/vector_file_batch_reader.cpp      |   5 +
 src/paimon/format/parquet/CMakeLists.txt      |   2 -
 .../format/parquet/parquet_format_writer.cpp  |  30 +--
 .../format/parquet/parquet_format_writer.h    |   2 -
 .../parquet/parquet_vector_converter.cpp      | 174 ----------------
 .../format/parquet/parquet_vector_converter.h |  46 -----
 .../parquet/parquet_vector_converter_test.cpp |  95 ---------
 .../format/parquet/parquet_vector_io_test.cpp |  72 ++++---
 .../parquet/vector_compatibility/README.md    |   8 +-
 11 files changed, 251 insertions(+), 385 deletions(-)
 delete mode 100644 src/paimon/format/parquet/parquet_vector_converter.cpp
 delete mode 100644 src/paimon/format/parquet/parquet_vector_converter.h
 delete mode 100644 src/paimon/format/parquet/parquet_vector_converter_test.cpp

diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff
index 75e3bb51e..b8b83517b 100644
--- a/cmake_modules/arrow.diff
+++ b/cmake_modules/arrow.diff
@@ -48,10 +48,71 @@ index b36c38c6d4..f974a33073 100644
 
    /// \brief Return zero-copy string_view to upcoming bytes.
    ///
+diff --git a/cpp/src/arrow/util/bit_run_reader.h b/cpp/src/arrow/util/bit_run_reader.h
+index a436a503a0..27d483978c 100644
+--- a/cpp/src/arrow/util/bit_run_reader.h
++++ b/cpp/src/arrow/util/bit_run_reader.h
+@@ -168,6 +168,26 @@ class ARROW_EXPORT BitRunReader {
+ using BitRunReader = BitRunReaderLinear;
+ #endif
+
++template <typename Visit>
++inline Status VisitBitRuns(const uint8_t* bitmap, int64_t offset, int64_t length,
++                           Visit&& visit) {
++  if (bitmap == NULLPTR) {
++    // Assuming all set (as in a null bitmap)
++    return visit(static_cast<int64_t>(0), length, true);
++  }
++  BitRunReader reader(bitmap, offset, length);
++  int64_t position = 0;
++  while (true) {
++    const auto run = reader.NextRun();
++    if (run.length == 0) {
++      break;
++    }
++    ARROW_RETURN_NOT_OK(visit(position, run.length, run.set));
++    position += run.length;
++  }
++  return Status::OK();
++}
++
+ struct SetBitRun {
+   int64_t position;
+   int64_t length;
 diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc
-index 285e2a5973..db919d7ef8 100644
+index 285e2a5973..52f42cf5b3 100644
 --- a/cpp/src/parquet/arrow/reader.cc
 +++ b/cpp/src/parquet/arrow/reader.cc
+@@ -19,12 +19,14 @@
+
+ #include <algorithm>
+ #include <cstring>
++#include <iterator>
+ #include <memory>
+ #include <unordered_set>
+ #include <utility>
+ #include <vector>
+
+ #include "arrow/array.h"
++#include "arrow/array/concatenate.h"
+ #include "arrow/buffer.h"
+ #include "arrow/extension_type.h"
+ #include "arrow/io/memory.h"
+@@ -32,12 +34,14 @@
+ #include "arrow/table.h"
+ #include "arrow/type.h"
+ #include "arrow/util/async_generator.h"
++#include "arrow/util/bit_run_reader.h"
+ #include "arrow/util/bit_util.h"
+ #include "arrow/util/future.h"
+ #include "arrow/util/iterator.h"
+ #include "arrow/util/logging.h"
+ #include "arrow/util/parallel.h"
+ #include "arrow/util/range.h"
++#include "arrow/util/span.h"
+ #include "arrow/util/tracing_internal.h"
+ #include "parquet/arrow/reader_internal.h"
+ #include "parquet/column_reader.h"
 @@ -254,6 +254,11 @@ class FileReaderImpl : public FileReader {
      return GetColumn(i, AllRowGroupsFactory(), out);
    }
@@ -151,7 +212,87 @@ index 285e2a5973..db919d7ef8 100644
    virtual ::arrow::Result<std::shared_ptr<ChunkedArray>> AssembleArray(
        std::shared_ptr<ArrayData> data) {
      if (field_->type()->id() == ::arrow::Type::MAP) {
-@@ -709,6 +776,39 @@ class PARQUET_NO_EXPORT StructReader : public ColumnReaderImpl {
+@@ -642,8 +713,10 @@ class ListReader : public ColumnReaderImpl {
+
+   const std::shared_ptr<Field> field() override { return field_; }
+
+- private:
++ protected:
+   std::shared_ptr<ReaderContext> ctx_;
++
++ private:
+   std::shared_ptr<Field> field_;
+   ::parquet::internal::LevelInfo level_info_;
+   std::unique_ptr<ColumnReaderImpl> item_reader_;
+@@ -662,12 +735,62 @@ class PARQUET_NO_EXPORT FixedSizeListReader : public ListReader<int32_t> {
+     DCHECK_EQ(field()->type()->id(), ::arrow::Type::FIXED_SIZE_LIST);
+     const auto& type = checked_cast<::arrow::FixedSizeListType&>(*field()->type());
+     const int32_t* offsets = reinterpret_cast<const int32_t*>(data->buffers[1]->data());
+-    for (int x = 1; x <= data->length; x++) {
+-      int32_t size = offsets[x] - offsets[x - 1];
+-      if (size != type.list_size()) {
+-        return Status::Invalid("Expected all lists to be of size=", type.list_size(),
+-                               " but index ", x, " had size=", size);
++    const int32_t list_size = type.list_size();
++    auto validate_offsets = [&](int64_t start, int64_t length,
++                                bool has_elements) -> Status {
++      const int32_t expected_size = has_elements ? list_size : 0;
++      ::arrow::util::span<const int32_t> run_offsets(
++          offsets + start, static_cast<size_t>(length + 1));
++      const auto first_invalid_offset = std::adjacent_find(
++          run_offsets.begin(), run_offsets.end(),
++          [&](int32_t left, int32_t right) { return right - left != expected_size; });
++      if (first_invalid_offset != run_offsets.end()) {
++        const int64_t x =
++            start + std::distance(run_offsets.begin(), first_invalid_offset);
++        const int32_t size = offsets[x + 1] - offsets[x];
++        if (has_elements) {
++          return Status::Invalid("Expected all lists to be of size=", list_size,
++                                 " but index ", x + 1, " had size=", size);
++        }
++        return Status::Invalid("Expected null fixed-size list at index ", x + 1,
++                               " to have no child values but had size=", size);
+       }
++      return Status::OK();
++    };
++    if (data->GetNullCount() != 0) {
++      // Rebuild the child array run-by-run so null fixed-size list slots still
++      // contribute list_size child values in the final layout.
++      ::arrow::ArrayVector child_arrays;
++
++      auto visit_run = [&](int64_t start, int64_t length, bool has_elements) -> Status {
++        RETURN_NOT_OK(validate_offsets(start, length, has_elements));
++
++        const int64_t child_length = length * list_size;
++        // Valid runs reuse the decoded child slice; null runs materialize null
++        // children to preserve the fixed-size list shape.
++        if (!has_elements) {
++          ARROW_ASSIGN_OR_RAISE(
++              auto null_array,
++              ::arrow::MakeArrayOfNull(type.value_type(), child_length, ctx_->pool));
++          child_arrays.push_back(std::move(null_array));
++          return Status::OK();
++        }
++        child_arrays.push_back(
++            ::arrow::MakeArray(data->child_data[0]->Slice(offsets[start], child_length)));
++        return Status::OK();
++      };
++
++      DCHECK_NE(data->buffers[0], nullptr);
++      RETURN_NOT_OK(::arrow::internal::VisitBitRuns(
++          data->buffers[0]->data(), data->offset, data->length, visit_run));
++
++      // TODO(GH-50271): Build one padded child array directly instead of creating
++      // one temporary Array/ArrayData per validity run and concatenating them.
++      ARROW_ASSIGN_OR_RAISE(auto child_array_with_padding,
++                            ::arrow::Concatenate(child_arrays, ctx_->pool));
++      data->child_data[0] = child_array_with_padding->data();
++    } else {
++      RETURN_NOT_OK(validate_offsets(/*start=*/0, data->length, /*valid=*/true));
+     }
+     data->buffers.resize(1);
+     std::shared_ptr<Array> result = ::arrow::MakeArray(data);
+@@ -709,6 +832,39 @@ class PARQUET_NO_EXPORT StructReader : public ColumnReaderImpl {
      }
      return Status::OK();
    }
@@ -191,7 +332,7 @@ index 285e2a5973..db919d7ef8 100644
    Status BuildArray(int64_t length_upper_bound,
                      std::shared_ptr<ChunkedArray>* out) override;
    Status GetDefLevels(const int16_t** data, int64_t* length) override;
-@@ -1013,25 +1113,32 @@ Status FileReaderImpl::GetRecordBatchReader(const std::vector<int>& row_groups,
+@@ -1013,25 +1169,32 @@ Status FileReaderImpl::GetRecordBatchReader(const std::vector<int>& row_groups,
      return Status::OK();
    }
 
@@ -230,7 +371,7 @@ index 285e2a5973..db919d7ef8 100644
 
          RETURN_NOT_OK(::arrow::internal::OptionalParallelFor(
              reader_properties_.use_threads(), static_cast<int>(readers.size()),
-@@ -1224,6 +1331,23 @@ Status FileReaderImpl::GetColumn(int i, FileColumnIteratorFactory iterator_facto
+@@ -1224,6 +1387,23 @@ Status FileReaderImpl::GetColumn(int i, FileColumnIteratorFactory iterator_facto
    return Status::OK();
  }
 
@@ -400,10 +541,49 @@ index ec3890a41f..943f69bb6c 100644
      return Status::OK();
    }
 diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc
-index 4fd7ef1b47..87326a54f1 100644
+index 4fd7ef1b47..feff99c99b 100644
 --- a/cpp/src/parquet/arrow/writer.cc
 +++ b/cpp/src/parquet/arrow/writer.cc
-@@ -314,6 +314,14 @@ class FileWriterImpl : public FileWriter {
+@@ -26,6 +26,7 @@
+ #include <vector>
+
+ #include "arrow/array.h"
++#include "arrow/array/concatenate.h"
+ #include "arrow/extension_type.h"
+ #include "arrow/ipc/writer.h"
+ #include "arrow/record_batch.h"
+@@ -142,13 +143,24 @@ class ArrowColumnWriterV2 {
+             leaf_idx, ctx, [&](const MultipathLevelBuilderResult& result) {
+               size_t visited_component_size = result.post_list_visited_elements.size();
+               DCHECK_GT(visited_component_size, 0);
+-              if (visited_component_size != 1) {
+-                return Status::NotImplemented(
+-                    "Lists with non-zero length null components are not supported");
++              std::shared_ptr<Array> values_array;
++              if (visited_component_size == 1) {
++                const ElementRange& range = result.post_list_visited_elements[0];
++                values_array = result.leaf_array->Slice(range.start, range.Size());
++              } else {
++                // Multiple leaf ranges can be produced when child values are
++                // skipped, such as null fixed-size-list slots, or when
++                // list-view ranges are non-contiguous. Concatenate the slices
++                // in logical write order.
++                ::arrow::ArrayVector arrays;
++                arrays.reserve(visited_component_size);
++                for (const auto& range : result.post_list_visited_elements) {
++                  DCHECK(!range.Empty());
++                  arrays.push_back(result.leaf_array->Slice(range.start, range.Size()));
++                }
++                ARROW_ASSIGN_OR_RAISE(values_array,
++                                      ::arrow::Concatenate(arrays, ctx->memory_pool));
+               }
+-              const ElementRange& range = result.post_list_visited_elements[0];
+-              std::shared_ptr<Array> values_array =
+-                  result.leaf_array->Slice(range.start, range.Size());
+
+               return column_writer->WriteArrow(result.def_levels, result.rep_levels,
+                                                result.def_rep_level_count, *values_array,
+@@ -314,6 +326,14 @@ class FileWriterImpl : public FileWriter {
      return Status::OK();
    }
 
@@ -418,7 +598,7 @@ index 4fd7ef1b47..87326a54f1 100644
    Status Close() override {
      if (!closed_) {
        // Make idempotent
-@@ -418,10 +426,13 @@ class FileWriterImpl : public FileWriter {
+@@ -418,10 +438,13 @@ class FileWriterImpl : public FileWriter {
 
      // Max number of rows allowed in a row group.
      const int64_t max_row_group_length = this->properties().max_row_group_length();
diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst
index 9fdecf6e5..add537adb 100644
--- a/docs/source/user_guide/data_types.rst
+++ b/docs/source/user_guide/data_types.rst
@@ -201,11 +201,9 @@ and `Arrow DataTypes <https://arrow.apache.org/docs/format/Columnar.html#data-ty
        VECTOR fields are rejected. VECTOR columns also cannot be partition or
        bucket keys. Dedicated vector storage is not included yet.
 
-       **Note:** A data file written by another engine that records the column as
-       Arrow ``FixedSizeList`` instead of ``LIST``, such as Paimon Rust or Python,
-       can only be read while it holds no NULL vector. Parquet stores a NULL list
-       slot with no values, which the Arrow 17 Parquet reader rejects for a
-       ``FixedSizeList`` column.
+       Paimon C++ also reads Parquet files written by Paimon Rust or Python whose
+       embedded Arrow schema restores VECTOR columns as ``FixedSizeList``,
+       including NULL vector values.
 
    * - ``MAP<kt, vt>``
      - Map
diff --git a/src/paimon/core/io/vector_file_batch_reader.cpp b/src/paimon/core/io/vector_file_batch_reader.cpp
index a4573eef9..f16025c42 100644
--- a/src/paimon/core/io/vector_file_batch_reader.cpp
+++ b/src/paimon/core/io/vector_file_batch_reader.cpp
@@ -119,6 +119,11 @@ Result<std::shared_ptr<arrow::Array>> CastListToVector(
             fmt::format("Cannot restore VECTOR from type {}", array->type()->ToString()));
     }
     PAIMON_RETURN_NOT_OK(VectorUtils::ValidateVectorElements(*array));
+    if (array->null_count() == array->length()) {
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> result,
+                                          arrow::MakeArrayOfNull(read_type, array->length(), pool));
+        return result;
+    }
     arrow::compute::ExecContext exec_context(pool);
     arrow::TypeHolder type_holder(read_type.get());
     arrow::compute::CastOptions options = arrow::compute::CastOptions::Safe();
diff --git a/src/paimon/format/parquet/CMakeLists.txt b/src/paimon/format/parquet/CMakeLists.txt
index c31e3cc35..968546581 100644
--- a/src/paimon/format/parquet/CMakeLists.txt
+++ b/src/paimon/format/parquet/CMakeLists.txt
@@ -20,7 +20,6 @@ set(PAIMON_PARQUET_FILE_FORMAT
     file_reader_wrapper.cpp
     page_filtered_row_group_reader.cpp
     parquet_timestamp_converter.cpp
-    parquet_vector_converter.cpp
     parquet_file_batch_reader.cpp
     parquet_file_format_factory.cpp
     parquet_format_writer.cpp
@@ -56,7 +55,6 @@ if(PAIMON_BUILD_TESTS)
                     file_reader_wrapper_test.cpp
                     page_filtered_row_group_reader_test.cpp
                     parquet_timestamp_converter_test.cpp
-                    parquet_vector_converter_test.cpp
                     parquet_vector_io_test.cpp
                     parquet_field_id_converter_test.cpp
                     parquet_file_batch_reader_test.cpp
diff --git a/src/paimon/format/parquet/parquet_format_writer.cpp b/src/paimon/format/parquet/parquet_format_writer.cpp
index 6e69e6945..0a8e38b43 100644
--- a/src/paimon/format/parquet/parquet_format_writer.cpp
+++ b/src/paimon/format/parquet/parquet_format_writer.cpp
@@ -23,7 +23,6 @@
 #include <string_view>
 #include <utility>
 
-#include "arrow/array/array_nested.h"
 #include "arrow/c/bridge.h"
 #include "arrow/memory_pool.h"
 #include "arrow/record_batch.h"
@@ -32,9 +31,7 @@
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/utils/arrow/arrow_output_stream_adapter.h"
 #include "paimon/common/utils/arrow/status_utils.h"
-#include "paimon/common/utils/checked_cast.h"
 #include "paimon/format/parquet/parquet_format_defs.h"
-#include "paimon/format/parquet/parquet_vector_converter.h"
 #include "parquet/arrow/writer.h"
 #include "parquet/properties.h"
 
@@ -58,33 +55,17 @@ Result<std::unique_ptr<ParquetFormatWriter>> ParquetFormatWriter::Create(
     ::parquet::ArrowWriterProperties::Builder arrow_properties_builder;
     auto arrow_writer_properties =
         arrow_properties_builder.enable_deprecated_int96_timestamps()->build();
-    auto logical_type = arrow::struct_(schema->fields());
-    auto write_type =
-        checked_pointer_cast<arrow::StructType>(ParquetVectorConverter::GetWriteType(logical_type));
-    auto write_schema = arrow::schema(write_type->fields(), schema->metadata());
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
         std::unique_ptr<::parquet::arrow::FileWriter> file_writer,
-        ::parquet::arrow::FileWriter::Open(*write_schema, pool.get(), out, writer_properties,
+        ::parquet::arrow::FileWriter::Open(*schema, pool.get(), out, writer_properties,
                                            arrow_writer_properties));
-    return std::unique_ptr<ParquetFormatWriter>(new ParquetFormatWriter(
-        std::move(file_writer), out, schema, max_memory_use,
-        /*needs_vector_conversion=*/!logical_type->Equals(write_type), pool));
+    return std::unique_ptr<ParquetFormatWriter>(
+        new ParquetFormatWriter(std::move(file_writer), out, schema, max_memory_use, pool));
 }
 
 Status ParquetFormatWriter::AddBatch(ArrowArray* batch) {
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<::arrow::RecordBatch> record_batch,
                                       arrow::ImportRecordBatch(batch, schema_));
-    if (needs_vector_conversion_) {
-        // TODO(ChaomingZhangCN): Remove this conversion after upgrading Arrow. Arrow 17
-        // mishandles nullable FixedSizeList values when writing them as Parquet LIST.
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> struct_array,
-                                          record_batch->ToStructArray());
-        std::shared_ptr<arrow::Array> array = struct_array;
-        PAIMON_ASSIGN_OR_RAISE(array,
-                               ParquetVectorConverter::ConvertToWriteType(array, pool_.get()));
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(record_batch,
-                                          arrow::RecordBatch::FromStructArray(array, pool_.get()));
-    }
     if (static_cast<uint64_t>(pool_->bytes_allocated()) > max_memory_use_) {
         PAIMON_RETURN_NOT_OK_FROM_ARROW(writer_->NewBufferedRowGroup());
     }
@@ -132,14 +113,13 @@ Result<uint64_t> ParquetFormatWriter::GetEstimateLength() const {
 ParquetFormatWriter::ParquetFormatWriter(std::unique_ptr<::parquet::arrow::FileWriter> writer,
                                          const std::shared_ptr<ArrowOutputStreamAdapter>& out,
                                          const std::shared_ptr<arrow::Schema>& schema,
-                                         uint64_t max_memory_use, bool needs_vector_conversion,
+                                         uint64_t max_memory_use,
                                          const std::shared_ptr<arrow::MemoryPool>& pool)
     : pool_(pool),
       out_(out),
       writer_(std::move(writer)),
       schema_(schema),
       metrics_(std::make_shared<MetricsImpl>()),
-      max_memory_use_(max_memory_use),
-      needs_vector_conversion_(needs_vector_conversion) {}
+      max_memory_use_(max_memory_use) {}
 
 }  // namespace paimon::parquet
diff --git a/src/paimon/format/parquet/parquet_format_writer.h b/src/paimon/format/parquet/parquet_format_writer.h
index f8f441195..4ab58d73c 100644
--- a/src/paimon/format/parquet/parquet_format_writer.h
+++ b/src/paimon/format/parquet/parquet_format_writer.h
@@ -72,7 +72,6 @@ class ParquetFormatWriter : public FormatWriter {
     ParquetFormatWriter(std::unique_ptr<::parquet::arrow::FileWriter> writer,
                         const std::shared_ptr<ArrowOutputStreamAdapter>& out,
                         const std::shared_ptr<arrow::Schema>& schema, uint64_t max_memory_use,
-                        bool needs_vector_conversion,
                         const std::shared_ptr<arrow::MemoryPool>& pool);
 
     Result<uint64_t> GetEstimateLength() const;
@@ -84,7 +83,6 @@ class ParquetFormatWriter : public FormatWriter {
     std::shared_ptr<Metrics> metrics_;
     int64_t total_records_written_ = 0;
     uint64_t max_memory_use_;
-    bool needs_vector_conversion_;
 };
 
 }  // namespace paimon::parquet
diff --git a/src/paimon/format/parquet/parquet_vector_converter.cpp b/src/paimon/format/parquet/parquet_vector_converter.cpp
deleted file mode 100644
index 5b6446d22..000000000
--- a/src/paimon/format/parquet/parquet_vector_converter.cpp
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "paimon/format/parquet/parquet_vector_converter.h"
-
-#include <cstdint>
-#include <limits>
-#include <memory>
-#include <vector>
-
-#include "arrow/array.h"
-#include "arrow/array/array_nested.h"
-#include "arrow/array/builder_primitive.h"
-#include "arrow/compute/api.h"
-#include "arrow/type.h"
-#include "paimon/common/utils/arrow/status_utils.h"
-#include "paimon/common/utils/arrow/vector_utils.h"
-#include "paimon/common/utils/checked_cast.h"
-#include "paimon/status.h"
-
-namespace paimon::parquet {
-namespace {
-
-Result<std::shared_ptr<arrow::Array>> CastToListType(
-    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& write_type,
-    arrow::MemoryPool* pool) {
-    arrow::compute::ExecContext exec_context(pool);
-    arrow::TypeHolder type_holder(write_type.get());
-    arrow::compute::CastOptions options = arrow::compute::CastOptions::Safe();
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-        std::shared_ptr<arrow::Array> result,
-        arrow::compute::Cast(*array, type_holder, options, &exec_context));
-    return result;
-}
-
-/// Rebuilds a nullable VECTOR as a LIST whose null slots have a zero length, dropping the
-/// values Arrow keeps for them.
-///
-/// TODO(ChaomingZhangCN): Cast the whole array once Arrow is upgraded. Arrow 17 casts a null
-/// FixedSizeList row to a null LIST slot spanning `list_size` values, and the Parquet writer
-/// rejects a LIST with non-zero length null slots.
-Result<std::shared_ptr<arrow::Array>> CompactNullVectorsToList(
-    const arrow::FixedSizeListArray& vector_array,
-    const std::shared_ptr<arrow::DataType>& write_type, arrow::MemoryPool* pool) {
-    const auto& vector_type = checked_cast<const arrow::FixedSizeListType&>(*vector_array.type());
-    const int32_t vector_length = vector_type.list_size();
-    if (vector_array.length() > std::numeric_limits<int32_t>::max() / vector_length) {
-        return Status::Invalid("VECTOR values exceed the maximum Parquet LIST offset");
-    }
-
-    arrow::Int32Builder offsets_builder(pool);
-    arrow::Int64Builder indices_builder(pool);
-    arrow::BooleanBuilder validity_builder(pool);
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Reserve(vector_array.length() + 1));
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Reserve(vector_array.length() * vector_length));
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Reserve(vector_array.length()));
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(0));
-
-    int32_t offset = 0;
-    for (int64_t i = 0; i < vector_array.length(); ++i) {
-        bool valid = !vector_array.IsNull(i);
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Append(valid));
-        if (valid) {
-            int64_t value_offset = (vector_array.offset() + i) * vector_length;
-            for (int32_t j = 0; j < vector_length; ++j) {
-                PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Append(value_offset + j));
-            }
-            offset += vector_length;
-        }
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(offset));
-    }
-
-    std::shared_ptr<arrow::Array> offsets;
-    std::shared_ptr<arrow::Array> indices;
-    std::shared_ptr<arrow::Array> validity;
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Finish(&offsets));
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Finish(&indices));
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Finish(&validity));
-
-    arrow::compute::ExecContext exec_context(pool);
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-        arrow::Datum values,
-        arrow::compute::Take(arrow::Datum(vector_array.values()), arrow::Datum(indices),
-                             arrow::compute::TakeOptions::NoBoundsCheck(), &exec_context));
-    return std::make_shared<arrow::ListArray>(
-        write_type, vector_array.length(), offsets->data()->buffers[1], values.make_array(),
-        validity->data()->buffers[1], vector_array.null_count());
-}
-
-}  // namespace
-
-std::shared_ptr<arrow::DataType> ParquetVectorConverter::GetWriteType(
-    const std::shared_ptr<arrow::DataType>& logical_type) {
-    switch (logical_type->id()) {
-        case arrow::Type::FIXED_SIZE_LIST: {
-            const auto& vector_type = checked_cast<const arrow::FixedSizeListType&>(*logical_type);
-            return arrow::list(
-                vector_type.value_field()->WithType(GetWriteType(vector_type.value_type())));
-        }
-        case arrow::Type::STRUCT: {
-            arrow::FieldVector fields;
-            fields.reserve(logical_type->num_fields());
-            for (const auto& field : logical_type->fields()) {
-                fields.push_back(field->WithType(GetWriteType(field->type())));
-            }
-            return arrow::struct_(fields);
-        }
-        case arrow::Type::LIST:
-            return arrow::list(
-                logical_type->field(0)->WithType(GetWriteType(logical_type->field(0)->type())));
-        case arrow::Type::MAP: {
-            const auto& map_type = checked_cast<const arrow::MapType&>(*logical_type);
-            return std::make_shared<arrow::MapType>(
-                map_type.value_field()->WithType(arrow::struct_(
-                    {map_type.key_field()->WithType(GetWriteType(map_type.key_type())),
-                     map_type.item_field()->WithType(GetWriteType(map_type.item_type()))})),
-                map_type.keys_sorted());
-        }
-        default:
-            return logical_type;
-    }
-}
-
-Result<std::shared_ptr<arrow::Array>> ParquetVectorConverter::ConvertToWriteType(
-    const std::shared_ptr<arrow::Array>& array, arrow::MemoryPool* pool) {
-    if (!VectorUtils::ContainsVectorType(array->type())) {
-        return array;
-    }
-    std::shared_ptr<arrow::DataType> write_type = GetWriteType(array->type());
-    switch (array->type_id()) {
-        case arrow::Type::FIXED_SIZE_LIST: {
-            PAIMON_RETURN_NOT_OK(VectorUtils::ValidateVectorElements(*array));
-            const auto& vector_array = checked_cast<const arrow::FixedSizeListArray&>(*array);
-            if (vector_array.null_count() == 0) {
-                return CastToListType(array, write_type, pool);
-            }
-            return CompactNullVectorsToList(vector_array, write_type, pool);
-        }
-        case arrow::Type::STRUCT:
-        case arrow::Type::LIST:
-        case arrow::Type::MAP: {
-            std::vector<std::shared_ptr<arrow::ArrayData>> children;
-            children.reserve(array->data()->child_data.size());
-            for (const auto& child_data : array->data()->child_data) {
-                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> child,
-                                       ConvertToWriteType(arrow::MakeArray(child_data), pool));
-                children.push_back(child->data());
-            }
-            std::shared_ptr<arrow::ArrayData> data = array->data()->Copy();
-            data->child_data = std::move(children);
-            data->type = write_type;
-            return arrow::MakeArray(data);
-        }
-        default:
-            return array;
-    }
-}
-
-}  // namespace paimon::parquet
diff --git a/src/paimon/format/parquet/parquet_vector_converter.h b/src/paimon/format/parquet/parquet_vector_converter.h
deleted file mode 100644
index a265e2d12..000000000
--- a/src/paimon/format/parquet/parquet_vector_converter.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include <memory>
-
-#include "arrow/memory_pool.h"
-#include "paimon/result.h"
-
-namespace arrow {
-class Array;
-class DataType;
-}  // namespace arrow
-
-namespace paimon::parquet {
-
-/// Converts logical FixedSizeList VECTOR arrays to Parquet LIST arrays.
-class ParquetVectorConverter {
- public:
-    ParquetVectorConverter() = delete;
-    ~ParquetVectorConverter() = delete;
-
-    static Result<std::shared_ptr<arrow::Array>> ConvertToWriteType(
-        const std::shared_ptr<arrow::Array>& array, arrow::MemoryPool* pool);
-
-    static std::shared_ptr<arrow::DataType> GetWriteType(
-        const std::shared_ptr<arrow::DataType>& logical_type);
-};
-
-}  // namespace paimon::parquet
diff --git a/src/paimon/format/parquet/parquet_vector_converter_test.cpp b/src/paimon/format/parquet/parquet_vector_converter_test.cpp
deleted file mode 100644
index 6e1c0b0df..000000000
--- a/src/paimon/format/parquet/parquet_vector_converter_test.cpp
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "paimon/format/parquet/parquet_vector_converter.h"
-
-#include <memory>
-
-#include "arrow/api.h"
-#include "arrow/ipc/json_simple.h"
-#include "gtest/gtest.h"
-#include "paimon/common/utils/checked_cast.h"
-#include "paimon/testing/utils/testharness.h"
-
-namespace paimon::parquet::test {
-
-TEST(ParquetVectorConverterTest, ConvertNullableVectorToList) {
-    auto vector_type = arrow::fixed_size_list(arrow::float32(), 3);
-    auto vector_array = arrow::ipc::internal::json::ArrayFromJSON(
-                            vector_type, R"([[1.0, 2.0, 3.0], null, [4.0, 5.0, 6.0]])")
-                            .ValueOrDie();
-
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<arrow::Array> converted,
-        ParquetVectorConverter::ConvertToWriteType(vector_array, arrow::default_memory_pool()));
-    ASSERT_EQ(converted->type()->id(), arrow::Type::LIST);
-    auto list_array = checked_pointer_cast<arrow::ListArray>(converted);
-    ASSERT_EQ(list_array->value_length(0), 3);
-    ASSERT_TRUE(list_array->IsNull(1));
-    // The Parquet writer rejects a null LIST slot spanning values, so the values Arrow keeps for
-    // a null VECTOR row are dropped.
-    ASSERT_EQ(list_array->value_length(1), 0);
-    ASSERT_EQ(list_array->value_length(2), 3);
-    ASSERT_EQ(list_array->values()->length(), 6);
-    auto values = checked_pointer_cast<arrow::FloatArray>(list_array->values());
-    ASSERT_FLOAT_EQ(values->Value(3), 4.0f);
-}
-
-TEST(ParquetVectorConverterTest, ConvertNestedVectorsToList) {
-    auto vector_type =
-        arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 2);
-    auto nested_type = arrow::struct_({
-        arrow::field("vectors", arrow::list(vector_type)),
-        arrow::field("by_name", arrow::map(arrow::utf8(), vector_type)),
-    });
-    auto nested_array =
-        arrow::ipc::internal::json::ArrayFromJSON(nested_type,
-                                                  R"([[[[1.0, 2.0], null], [["a", [3.0, 4.0]]]],
-                                 [null, [["b", null]]]])")
-            .ValueOrDie();
-
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<arrow::Array> physical_array,
-        ParquetVectorConverter::ConvertToWriteType(nested_array, arrow::default_memory_pool()));
-    auto physical_type = checked_pointer_cast<arrow::StructType>(physical_array->type());
-    auto physical_list = checked_pointer_cast<arrow::ListType>(physical_type->field(0)->type());
-    auto physical_map = checked_pointer_cast<arrow::MapType>(physical_type->field(1)->type());
-    ASSERT_EQ(physical_list->value_type()->id(), arrow::Type::LIST);
-    ASSERT_EQ(physical_map->item_type()->id(), arrow::Type::LIST);
-}
-
-TEST(ParquetVectorConverterTest, ConvertSlicedVectorToList) {
-    auto vector_type = arrow::fixed_size_list(arrow::float64(), 2);
-    auto vector_array =
-        arrow::ipc::internal::json::ArrayFromJSON(vector_type, R"([[1.0, 2.0], [3.0, 4.0], null])")
-            .ValueOrDie()
-            ->Slice(1, 2);
-
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<arrow::Array> converted,
-        ParquetVectorConverter::ConvertToWriteType(vector_array, arrow::default_memory_pool()));
-    auto list_array = checked_pointer_cast<arrow::ListArray>(converted);
-    ASSERT_EQ(list_array->length(), 2);
-    ASSERT_EQ(list_array->value_length(0), 2);
-    ASSERT_TRUE(list_array->IsNull(1));
-    auto values = checked_pointer_cast<arrow::DoubleArray>(list_array->values());
-    ASSERT_DOUBLE_EQ(values->Value(0), 3.0);
-    ASSERT_DOUBLE_EQ(values->Value(1), 4.0);
-}
-
-}  // namespace paimon::parquet::test
diff --git a/src/paimon/format/parquet/parquet_vector_io_test.cpp b/src/paimon/format/parquet/parquet_vector_io_test.cpp
index f45dad1e5..ad60caef3 100644
--- a/src/paimon/format/parquet/parquet_vector_io_test.cpp
+++ b/src/paimon/format/parquet/parquet_vector_io_test.cpp
@@ -258,6 +258,45 @@ TEST_F(ParquetVectorIoTest, WriteAndReadVector) {
                   R"([[1, [1.0, 2.0, 3.0]], [2, null], [3, [4.0, 5.0, 6.0]]])");
 }
 
+TEST_F(ParquetVectorIoTest, WriteAndReadAllNullVector) {
+    auto vector_type =
+        arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 3);
+    auto struct_type = checked_pointer_cast<arrow::StructType>(arrow::struct_(
+        {arrow::field("id", arrow::int32()), arrow::field("embedding", vector_type)}));
+    WriteAndCheck("all-null-vector-list.parquet", struct_type, struct_type,
+                  R"([[1, null], [2, null], [3, null]])");
+}
+
+TEST_F(ParquetVectorIoTest, WriteAndReadAllNullFixedSizeListWithArrowSchema) {
+    auto vector_type =
+        arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 3);
+    auto logical_type = checked_pointer_cast<arrow::StructType>(arrow::struct_({
+        arrow::field("id", arrow::int32()),
+        arrow::field("embedding", vector_type),
+    }));
+    const std::string json = R"([[1, null], [2, null], [3, null]])";
+    std::string file_path = dir_->Str() + "/all-null-vector.parquet";
+    WriteWithArrowWriter(file_path, logical_type, json);
+
+    std::shared_ptr<arrow::StructType> file_type;
+    ReadFileType(file_path, &file_type);
+    std::shared_ptr<arrow::Field> file_vector_field = file_type->GetFieldByName("embedding");
+    ASSERT_TRUE(file_vector_field);
+    ASSERT_EQ(file_vector_field->type()->id(), arrow::Type::FIXED_SIZE_LIST);
+
+    std::unique_ptr<FileBatchReader> reader;
+    CreateVectorReader(file_path, arrow::schema(logical_type->fields()), /*predicate=*/nullptr,
+                       /*options=*/{}, /*batch_size=*/10, &reader);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> actual,
+                         paimon::test::ReadResultCollector::CollectResult(reader.get()));
+    arrow::Result<std::shared_ptr<arrow::Array>> expected_result =
+        arrow::ipc::internal::json::ArrayFromJSON(logical_type, json);
+    ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString();
+    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(std::move(expected_result).ValueOrDie())
+                    ->Equals(actual))
+        << actual->ToString();
+}
+
 TEST_F(ParquetVectorIoTest, ReadOrdinaryParquetListAsVector) {
     auto physical_type = checked_pointer_cast<arrow::StructType>(
         arrow::struct_({arrow::field("id", arrow::int32()),
@@ -370,6 +409,13 @@ TEST_F(ParquetVectorIoTest, ReadNullableJavaFixture) {
                         {{{1.0f, 2.0f, 3.0f}}, std::nullopt, {{4.0f, 5.0f, 6.0f}}});
 }
 
+TEST_F(ParquetVectorIoTest, ReadNullableRustFixture) {
+    ReadFixtureAndCheck("rust_vector_nullable.parquet", arrow::Type::FIXED_SIZE_LIST,
+                        /*vector_length=*/3, /*expected_ids=*/{1, 2, 3},
+                        /*expected_vectors=*/
+                        {{{1.0f, 2.0f, 3.0f}}, std::nullopt, {{4.0f, 5.0f, 6.0f}}});
+}
+
 // A table can hold files from several writers, and Paimon Java stores VECTOR as Parquet LIST
 // while Paimon Rust stores it as FixedSizeList. Reading both with the table schema must produce
 // batches of one Arrow type, otherwise they cannot be combined into a single result.
@@ -386,7 +432,7 @@ TEST_F(ParquetVectorIoTest, ReadMixedListAndFixedSizeListFixtures) {
     // the whole result has been consumed.
     std::vector<std::unique_ptr<FileBatchReader>> readers;
     arrow::ArrayVector chunks;
-    for (const char* file_name : {"java_vector_nullable.parquet", "rust_vector.parquet"}) {
+    for (const char* file_name : {"java_vector_nullable.parquet", "rust_vector_nullable.parquet"}) {
         std::string file_path =
             paimon::test::GetDataDir() + "/parquet/vector_compatibility/" + file_name;
         std::unique_ptr<FileBatchReader> reader;
@@ -406,7 +452,7 @@ TEST_F(ParquetVectorIoTest, ReadMixedListAndFixedSizeListFixtures) {
     arrow::Result<std::shared_ptr<arrow::Array>> expected_result =
         arrow::ipc::internal::json::ArrayFromJSON(
             logical_type, R"([[1, [1.0, 2.0, 3.0]], [2, null], [3, [4.0, 5.0, 6.0]],
-                              [1, [1.0, 2.0, 3.0]], [2, [7.0, 8.0, 9.0]], [3, [4.0, 5.0, 6.0]]])");
+                              [1, [1.0, 2.0, 3.0]], [2, null], [3, [4.0, 5.0, 6.0]]])");
     ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString();
     std::shared_ptr<arrow::ChunkedArray> merged = std::move(merged_result).ValueOrDie();
     ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(std::move(expected_result).ValueOrDie())
@@ -414,26 +460,4 @@ TEST_F(ParquetVectorIoTest, ReadMixedListAndFixedSizeListFixtures) {
         << merged->ToString();
 }
 
-// A writer that stores the Arrow schema, such as Paimon Rust or Python, exposes the VECTOR column
-// as FixedSizeList. Arrow 17 cannot read a null value from such a column: Parquet stores a null
-// list slot with no values, while FixedSizeListReader::AssembleArray in
-// parquet/arrow/reader.cc requires every slot to span exactly `list_size` values.
-//
-// TODO(ChaomingZhangCN): Turn this into a read check once Arrow is upgraded.
-TEST_F(ParquetVectorIoTest, ReadNullableRustFixtureIsUnsupported) {
-    std::string file_path =
-        paimon::test::GetDataDir() + "/parquet/vector_compatibility/rust_vector_nullable.parquet";
-    std::shared_ptr<arrow::StructType> file_type;
-    ReadFileType(file_path, &file_type);
-    std::shared_ptr<arrow::Field> file_vector_field = file_type->GetFieldByName("embedding");
-    ASSERT_TRUE(file_vector_field);
-    ASSERT_EQ(file_vector_field->type()->id(), arrow::Type::FIXED_SIZE_LIST);
-
-    std::unique_ptr<FileBatchReader> reader;
-    CreateVectorReader(file_path, arrow::schema(file_type->fields()), /*predicate=*/nullptr,
-                       /*options=*/{}, /*batch_size=*/10, &reader);
-    ASSERT_NOK_WITH_MSG(paimon::test::ReadResultCollector::CollectResult(reader.get()),
-                        "Expected all lists to be of size=3");
-}
-
 }  // namespace paimon::parquet::test
diff --git a/test/test_data/parquet/vector_compatibility/README.md b/test/test_data/parquet/vector_compatibility/README.md
index 15eb2ef30..8a2fe25c9 100644
--- a/test/test_data/parquet/vector_compatibility/README.md
+++ b/test/test_data/parquet/vector_compatibility/README.md
@@ -22,11 +22,9 @@ VECTOR columns, with and without null vectors.
   `(2, null)` and `(3, [4, 5, 6])`.
 
 A file that stores the Arrow schema, as the Rust writer does, is read back as
-`fixed_size_list`. Arrow 17 cannot read a null value from such a column, because Parquet stores a
-null list slot with no values while `FixedSizeListReader::AssembleArray` in
-`parquet/arrow/reader.cc` requires every slot to span exactly `list_size` values. Reading
-`rust_vector_nullable.parquet` therefore fails until Arrow is upgraded, which
-`ParquetVectorIoTest.ReadNullableRustFixtureIsUnsupported` pins.
+`fixed_size_list`. The bundled Arrow 17 patch backports the Arrow community fix that pads the
+decoded child array for null fixed-size-list slots, so `rust_vector_nullable.parquet` is readable
+as a nullable VECTOR.
 
 SHA-256 checksums:
 

From 3b93d6a0a87cd3fae4ac717fbb6db740e597c4d6 Mon Sep 17 00:00:00 2001
From: wangyong9999 <81852543+wangyong9999@users.noreply.github.com>
Date: Mon, 24 Aug 2026 10:23:52 +0800
Subject: [PATCH 12/93] fix(rest): support libcurl versions before 7.49 (#237)

---
 src/paimon/rest/rest_http_client.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/paimon/rest/rest_http_client.cpp b/src/paimon/rest/rest_http_client.cpp
index b17b0a281..52e6417e7 100644
--- a/src/paimon/rest/rest_http_client.cpp
+++ b/src/paimon/rest/rest_http_client.cpp
@@ -123,7 +123,9 @@ bool IsRetriableTransportError(CURLcode code) {
         case CURLE_RECV_ERROR:
         case CURLE_PARTIAL_FILE:
         case CURLE_HTTP2:
+#if CURL_AT_LEAST_VERSION(7, 49, 0)
         case CURLE_HTTP2_STREAM:
+#endif
             return true;
         default:
             return false;

From d602c2c509f2495f6f5d721be1c5de995f4b5fcd Mon Sep 17 00:00:00 2001
From: gripleaf <425797155@qq.com>
Date: Mon, 24 Aug 2026 14:45:59 +0800
Subject: [PATCH 13/93] perf: avoid shared pointer contention in manifest and
 Avro decode (#239)

---
 src/paimon/core/manifest/manifest_file.cpp    |  3 +-
 src/paimon/core/utils/objects_file.h          |  3 +-
 .../format/avro/avro_direct_decoder.cpp       | 28 +++++++++++++++----
 src/paimon/format/avro/avro_direct_decoder.h  | 20 +++++++++++++
 .../avro/avro_direct_encoder_decoder_test.cpp | 23 +++++++++++++++
 .../format/avro/avro_file_batch_reader.cpp    |  1 +
 6 files changed, 70 insertions(+), 8 deletions(-)

diff --git a/src/paimon/core/manifest/manifest_file.cpp b/src/paimon/core/manifest/manifest_file.cpp
index 1be49d0b5..9f9c8aee2 100644
--- a/src/paimon/core/manifest/manifest_file.cpp
+++ b/src/paimon/core/manifest/manifest_file.cpp
@@ -92,8 +92,9 @@ Status ManifestFile::ReadBucketEntries(const std::string& file_name, int32_t buc
     return ReadArrowBatches(
         file_name,
         [this, bucket, entries](const std::shared_ptr<arrow::StructArray>& batch) -> Status {
+            const arrow::ArrayVector& fields = batch->fields();
             for (int64_t i = 0; i < batch->length(); i++) {
-                ColumnarRow row(batch->fields(), pool_, i);
+                ColumnarRow row(fields, pool_, i);
                 PAIMON_RETURN_NOT_OK(ManifestEntrySerializer::ValidateVersion(row.GetInt(0)));
                 if (ManifestEntrySerializer::GetBucket(row) != bucket) {
                     continue;
diff --git a/src/paimon/core/utils/objects_file.h b/src/paimon/core/utils/objects_file.h
index a56952ae2..b3135b312 100644
--- a/src/paimon/core/utils/objects_file.h
+++ b/src/paimon/core/utils/objects_file.h
@@ -134,8 +134,9 @@ Status ObjectsFile<T>::Read(const std::string& file_name,
         file_name,
         [this, &filter, result](const std::shared_ptr<arrow::StructArray>& struct_array) -> Status {
             result->reserve(result->size() + struct_array->length());
+            const arrow::ArrayVector& fields = struct_array->fields();
             for (int64_t i = 0; i < struct_array->length(); i++) {
-                ColumnarRow row(struct_array->fields(), pool_, i);
+                ColumnarRow row(fields, pool_, i);
                 PAIMON_ASSIGN_OR_RAISE(T obj, serializer_->FromRow(row));
                 if (filter) {
                     PAIMON_ASSIGN_OR_RAISE(bool filter_res, filter(obj));
diff --git a/src/paimon/format/avro/avro_direct_decoder.cpp b/src/paimon/format/avro/avro_direct_decoder.cpp
index f837eed0d..f9c8a9a41 100644
--- a/src/paimon/format/avro/avro_direct_decoder.cpp
+++ b/src/paimon/format/avro/avro_direct_decoder.cpp
@@ -33,6 +33,22 @@
 
 namespace paimon::avro {
 
+const AvroDirectDecoder::DecodeContext::BuilderMetadata&
+AvroDirectDecoder::DecodeContext::GetBuilderMetadata(const arrow::ArrayBuilder* builder) {
+    auto iter = builder_metadata_.find(builder);
+    if (iter != builder_metadata_.end()) {
+        return iter->second;
+    }
+
+    std::shared_ptr<arrow::DataType> data_type = builder->type();
+    BuilderMetadata metadata{data_type->id(), std::nullopt};
+    if (data_type->id() == arrow::Type::TIMESTAMP) {
+        metadata.timestamp_unit =
+            checked_cast<const arrow::TimestampType*>(data_type.get())->unit();
+    }
+    return builder_metadata_.emplace(builder, metadata).first->second;
+}
+
 namespace {
 
 /// Forward declaration for mutual recursion.
@@ -266,8 +282,8 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node,
 
         case ::avro::AVRO_INT: {
             int32_t value = decoder->decodeInt();
-            auto arrow_type = array_builder->type();
-            switch (arrow_type->id()) {
+            const auto& builder_metadata = ctx->GetBuilderMetadata(array_builder);
+            switch (builder_metadata.type) {
                 case arrow::Type::INT8: {
                     auto* builder = checked_cast<arrow::Int8Builder*>(array_builder);
                     PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append(value));
@@ -287,7 +303,7 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node,
                     if (logical_type.type() != ::avro::LogicalType::Type::DATE) {
                         return Status::TypeError(
                             fmt::format("Unexpected avro type [{}] with arrow type [{}].",
-                                        ::avro::toString(type), arrow_type->ToString()));
+                                        ::avro::toString(type), array_builder->type()->ToString()));
                     }
                     auto* builder = checked_cast<arrow::Date32Builder*>(array_builder);
                     PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append(value));
@@ -296,7 +312,7 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node,
                 default:
                     return Status::TypeError(
                         fmt::format("Unexpected avro type [{}] with arrow type [{}].",
-                                    ::avro::toString(type), arrow_type->ToString()));
+                                    ::avro::toString(type), array_builder->type()->ToString()));
             }
         }
 
@@ -315,9 +331,9 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node,
                 case ::avro::LogicalType::Type::LOCAL_TIMESTAMP_MICROS:
                 case ::avro::LogicalType::Type::LOCAL_TIMESTAMP_NANOS: {
                     auto* builder = checked_cast<arrow::TimestampBuilder*>(array_builder);
-                    auto ts_type = checked_cast<arrow::TimestampType*>(builder->type().get());
                     // for arrow second, we need to convert it from avro millisecond
-                    if (ts_type->unit() == arrow::TimeUnit::type::SECOND) {
+                    const auto& builder_metadata = ctx->GetBuilderMetadata(builder);
+                    if (builder_metadata.timestamp_unit == arrow::TimeUnit::type::SECOND) {
                         value /= DateTimeUtils::CONVERSION_FACTORS[DateTimeUtils::MILLISECOND];
                     }
                     PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append(value));
diff --git a/src/paimon/format/avro/avro_direct_decoder.h b/src/paimon/format/avro/avro_direct_decoder.h
index c507091a7..6422f9154 100644
--- a/src/paimon/format/avro/avro_direct_decoder.h
+++ b/src/paimon/format/avro/avro_direct_decoder.h
@@ -22,7 +22,11 @@
 
 #pragma once
 
+#include <optional>
 #include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
 
 #include "arrow/array/builder_base.h"
 #include "avro/Decoder.hh"
@@ -41,10 +45,26 @@ class AvroDirectDecoder {
     /// Avoids frequent small allocations by reusing temporary buffers across multiple decode
     /// operations. This is particularly important for string, binary, and decimal data types.
     struct DecodeContext {
+        struct BuilderMetadata {
+            arrow::Type::type type;
+            std::optional<arrow::TimeUnit::type> timestamp_unit;
+        };
+
+        /// Returns immutable type metadata without repeatedly copying the builder's DataType.
+        const BuilderMetadata& GetBuilderMetadata(const arrow::ArrayBuilder* builder);
+
+        /// Clears metadata before the builder tree is replaced or destroyed.
+        void ClearBuilderMetadata() {
+            builder_metadata_.clear();
+        }
+
         // Scratch buffer for string decoding (reused across rows)
         std::string string_scratch;
         // Scratch buffer for binary/decimal data (reused across rows)
         std::vector<uint8_t> bytes_scratch;
+
+     private:
+        std::unordered_map<const arrow::ArrayBuilder*, BuilderMetadata> builder_metadata_;
     };
 
     /// Directly decode Avro data to Arrow array builders without GenericDatum
diff --git a/src/paimon/format/avro/avro_direct_encoder_decoder_test.cpp b/src/paimon/format/avro/avro_direct_encoder_decoder_test.cpp
index f276d9469..78f4ca483 100644
--- a/src/paimon/format/avro/avro_direct_encoder_decoder_test.cpp
+++ b/src/paimon/format/avro/avro_direct_encoder_decoder_test.cpp
@@ -62,6 +62,7 @@ class AvroDirectEncoderDecoderTest : public ::testing::Test {
         auto decoder = ::avro::binaryDecoder();
         decoder->init(*input_stream);
 
+        decode_ctx_.ClearBuilderMetadata();
         for (int32_t i = 0; i < expected_count; ++i) {
             PAIMON_RETURN_NOT_OK(AvroDirectDecoder::DecodeAvroToBuilder(
                 avro_node, projection, decoder.get(), builder, &decode_ctx_));
@@ -157,6 +158,18 @@ TEST_F(AvroDirectEncoderDecoderTest, TestIntegerTypes) {
         CheckResult(schema_json, input_array, &builder);
     }
 
+    // Test INT16
+    {
+        std::string schema_json = R"({"type": "int"})";
+        arrow::Int16Builder builder;
+        ASSERT_TRUE(builder.Append(1).ok());
+        ASSERT_TRUE(builder.Append(-32768).ok());
+        ASSERT_TRUE(builder.Append(32767).ok());
+        std::shared_ptr<arrow::Array> input_array;
+        ASSERT_TRUE(builder.Finish(&input_array).ok());
+        CheckResult(schema_json, input_array, &builder);
+    }
+
     // Test INT32
     {
         std::string schema_json = R"({"type": "int"})";
@@ -182,6 +195,16 @@ TEST_F(AvroDirectEncoderDecoderTest, TestIntegerTypes) {
     }
 }
 
+TEST_F(AvroDirectEncoderDecoderTest, TestDecodeContextBuilderMetadataLifecycle) {
+    arrow::Int8Builder int8_builder;
+    ASSERT_EQ(decode_ctx_.GetBuilderMetadata(&int8_builder).type, arrow::Type::INT8);
+
+    decode_ctx_.ClearBuilderMetadata();
+
+    arrow::Int16Builder int16_builder;
+    ASSERT_EQ(decode_ctx_.GetBuilderMetadata(&int16_builder).type, arrow::Type::INT16);
+}
+
 TEST_F(AvroDirectEncoderDecoderTest, TestFloatingPointTypes) {
     // Test FLOAT
     {
diff --git a/src/paimon/format/avro/avro_file_batch_reader.cpp b/src/paimon/format/avro/avro_file_batch_reader.cpp
index f48ec4cc4..1e217f5c4 100644
--- a/src/paimon/format/avro/avro_file_batch_reader.cpp
+++ b/src/paimon/format/avro/avro_file_batch_reader.cpp
@@ -172,6 +172,7 @@ Status AvroFileBatchReader::SetReadSchema(::ArrowSchema* read_schema,
     }
     reader_ = std::move(reader);
     array_builder_ = std::move(array_builder);
+    decode_context_.ClearBuilderMetadata();
     previous_first_row_ = std::numeric_limits<uint64_t>::max();
     previous_batch_row_count_ = 0;
     next_row_to_read_ = std::numeric_limits<uint64_t>::max();

From 15d079aa83641f321d10fbf6dc0fd4b8e2fce2fa Mon Sep 17 00:00:00 2001
From: lxy <38709059+lxy-9602@users.noreply.github.com>
Date: Mon, 24 Aug 2026 14:51:42 +0800
Subject: [PATCH 14/93] feat(realtime): improve append table lifecycle and
 query support (#213)

---
 include/paimon/api.h                          |    1 +
 include/paimon/defs.h                         |    8 +
 include/paimon/file_store_commit.h            |   25 +-
 include/paimon/file_store_write.h             |    6 +
 .../realtime/arrow_realtime_store_factory.h   |    2 +-
 include/paimon/realtime/realtime_context.h    |    5 +
 include/paimon/realtime/realtime_store.h      |    6 +-
 include/paimon/scan_context.h                 |    2 +-
 include/paimon/statistics_mode.h              |   32 +
 include/paimon/write_context.h                |    2 +-
 src/paimon/CMakeLists.txt                     |    1 +
 src/paimon/common/defs.cpp                    |    2 +
 .../utils/binary_row_partition_computer.cpp   |   55 +-
 .../utils/binary_row_partition_computer.h     |    6 +
 .../binary_row_partition_computer_test.cpp    |   30 +
 src/paimon/core/core_options.cpp              |   46 +-
 src/paimon/core/core_options.h                |    4 +
 src/paimon/core/core_options_test.cpp         |   19 +
 .../append_only_file_store_write.cpp          |    6 +-
 .../append_only_file_store_write_test.cpp     |    1 +
 .../core/operation/commit/commit_scanner.cpp  |   12 +-
 .../commit/realtime_commit_properties.cpp     |   93 +-
 .../commit/realtime_commit_properties.h       |   20 +
 .../realtime_commit_properties_test.cpp       |  142 +-
 .../core/operation/expire_snapshots.cpp       |   34 +-
 src/paimon/core/operation/expire_snapshots.h  |    3 +-
 .../core/operation/expire_snapshots_test.cpp  |   16 +-
 .../core/operation/file_store_commit.cpp      |    2 +-
 .../core/operation/file_store_commit_impl.cpp |   72 +-
 .../core/operation/file_store_commit_impl.h   |   32 +-
 .../core/operation/file_store_write.cpp       |    3 +
 .../operation/orphan_files_cleaner_impl.cpp   |   14 +-
 .../core/realtime/arrow_realtime_store.cpp    |  154 +-
 .../core/realtime/arrow_realtime_store.h      |   13 +
 .../realtime/arrow_realtime_store_factory.cpp |    7 +-
 .../realtime/arrow_realtime_store_test.cpp    |   86 +-
 .../realtime/realtime_append_only_writer.cpp  |    9 +-
 .../realtime/realtime_append_only_writer.h    |    2 +-
 .../core/realtime/realtime_context_impl.cpp   |   35 +-
 .../core/realtime/realtime_context_impl.h     |    8 +-
 .../core/realtime/realtime_context_test.cpp   |  104 +-
 .../core/table/source/append_count_reader.cpp |   11 +
 .../table/source/append_only_table_read.cpp   |   49 +-
 src/paimon/core/table/source/realtime_split.h |    5 +-
 .../core/table/source/realtime_table_scan.cpp |   12 +-
 src/paimon/core/table/source/table_read.cpp   |    3 +
 src/paimon/core/table/source/table_scan.cpp   |    3 +
 src/paimon/core/utils/partition_utils.h       |   67 +
 .../core/utils/partition_utils_test.cpp       |   71 +
 test/inte/realtime_write_inte_test.cpp        | 1278 ++++++++++++++++-
 50 files changed, 2433 insertions(+), 186 deletions(-)
 create mode 100644 include/paimon/statistics_mode.h
 create mode 100644 src/paimon/core/utils/partition_utils.h
 create mode 100644 src/paimon/core/utils/partition_utils_test.cpp

diff --git a/include/paimon/api.h b/include/paimon/api.h
index d75666850..81f236bc4 100644
--- a/include/paimon/api.h
+++ b/include/paimon/api.h
@@ -33,6 +33,7 @@
 #include "paimon/record_batch.h"             // IWYU pragma: export
 #include "paimon/result.h"                   // IWYU pragma: export
 #include "paimon/scan_context.h"             // IWYU pragma: export
+#include "paimon/statistics_mode.h"          // IWYU pragma: export
 #include "paimon/status.h"                   // IWYU pragma: export
 #include "paimon/table/source/table_read.h"  // IWYU pragma: export
 #include "paimon/table/source/table_scan.h"  // IWYU pragma: export
diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index 338eda30a..8062d3d2d 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -562,10 +562,18 @@ struct PAIMON_EXPORT Options {
     /// "scan.timestamp" can be used as an alternative string input for the same mode.
     static const char SCAN_TIMESTAMP_MILLIS[];
 
+    /// "realtime.enabled" - Whether real-time write, commit, and read operations are enabled.
+    /// Default value is "false".
+    static const char REALTIME_ENABLED[];
+
     /// "realtime.read-view-ttl" - Lifetime of a real-time memory view pinned by scan planning
     /// before reader creation. Default value is "5 min".
     static const char REALTIME_READ_VIEW_TTL[];
 
+    /// "realtime.store.stats-mode" - Statistics collected by the default real-time store.
+    /// Supported values are "none" and "full". Default value is "none".
+    static const char REALTIME_STORE_STATS_MODE[];
+
     /// "scan.timestamp" - Optional timestamp string used in case of "from-timestamp" scan mode,
     /// as an alternative to "scan.timestamp-millis".
     /// It will be automatically converted to timestamp in unix milliseconds, using local time zone.
diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h
index 63efb5621..8af776959 100644
--- a/include/paimon/file_store_commit.h
+++ b/include/paimon/file_store_commit.h
@@ -79,11 +79,17 @@ class PAIMON_EXPORT FileStoreCommit {
     /// orders them by partition, bucket, and offset before validating continuity. The resulting
     /// snapshot atomically publishes the data files and the updated offset map.
     ///
+    /// If this method returns an error, the caller may retry with the same arguments. Each call
+    /// reloads the latest committed state. As in `FilterAndCommit`, a retry's identifier is
+    /// considered committed when it is not newer than the latest identifier for `commit_user`.
+    /// The requested offset ranges must also be covered by the latest committed progress.
+    ///
     /// @param realtime_commits Commit messages and left-closed, right-open offset ranges to
     /// commit.
     /// @param commit_identifier Identifier of the streaming commit operation.
     /// @param watermark Optional event-time watermark.
-    /// @return The id of the final snapshot produced by this commit.
+    /// @return The id of the latest snapshot containing the committed progress. On retry, this may
+    /// be a snapshot produced by a later commit and is suitable for refreshing a real-time context.
     virtual Result<int64_t> CommitWithProgress(
         const std::vector<RealtimeCommitProgress>& realtime_commits, int64_t commit_identifier,
         std::optional<int64_t> watermark) = 0;
@@ -117,6 +123,10 @@ class PAIMON_EXPORT FileStoreCommit {
     /// @param watermark An optional event-time watermark used to indicate the progress of data
     ///     processing. Default is std::nullopt.
     /// @return Result of the operation.
+    /// @note A full-table overwrite clears all committed real-time progress. A partition
+    ///     overwrite removes progress only for matching partitions. In either case, active
+    ///     real-time writers and their `RealtimeContext` instances must be recreated before
+    ///     further real-time operations.
     virtual Status Overwrite(const std::map<std::string, std::string>& partition,
                              const std::vector<std::shared_ptr<CommitMessage>>& commit_messages,
                              int64_t commit_identifier,
@@ -131,6 +141,10 @@ class PAIMON_EXPORT FileStoreCommit {
     /// @param watermark An optional event-time watermark used to indicate the progress of data
     ///     processing. Default is std::nullopt.
     /// @return Result of the operation.
+    /// @note A full-table overwrite clears all committed real-time progress. A partition
+    ///     overwrite removes progress only for matching partitions. In either case, active
+    ///     real-time writers and their `RealtimeContext` instances must be recreated before
+    ///     further real-time operations.
     virtual Result<int32_t> FilterAndOverwrite(
         const std::map<std::string, std::string>& partition,
         const std::vector<std::shared_ptr<CommitMessage>>& commit_messages,
@@ -157,6 +171,9 @@ class PAIMON_EXPORT FileStoreCommit {
     /// @param partitions A vector of partitions to be dropped.
     /// @param commit_identifier An identifier for the commit operation.
     /// @return Status indicating the success or failure of the drop partition operation.
+    /// @note A partition drop removes committed real-time progress only for matching partitions.
+    ///     Active real-time writers and their `RealtimeContext` instances must be recreated before
+    ///     further real-time operations.
     virtual Status DropPartition(const std::vector<std::map<std::string, std::string>>& partitions,
                                  int64_t commit_identifier) = 0;
 
@@ -165,6 +182,9 @@ class PAIMON_EXPORT FileStoreCommit {
     ///
     /// @param commit_identifier An identifier for the commit operation.
     /// @return Status indicating the success or failure of the truncate operation.
+    /// @note Truncation clears all committed real-time progress. Active real-time writers and
+    ///     their `RealtimeContext` instances must be recreated before further real-time
+    ///     operations.
     virtual Status TruncateTable(int64_t commit_identifier) = 0;
 
     /// Abort an unsuccessful commit. The data and index files described by the given commit
@@ -182,6 +202,9 @@ class PAIMON_EXPORT FileStoreCommit {
     /// @param target_snapshot_id The snapshot id to roll back to.
     /// @return Result<bool>; true if the atomic commit succeeded. Returns an error status if
     ///     there is no latest snapshot or the target snapshot does not exist.
+    /// @note Rollback restores the real-time progress recorded by the target snapshot. Active
+    ///     real-time writers and their `RealtimeContext` instances must be recreated before
+    ///     further real-time operations.
     virtual Result<bool> RollbackToAsLatest(int64_t target_snapshot_id) = 0;
 
     /// Configure row-id conflict checking from a specific snapshot id.
diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h
index fdc172c3a..1d7ca0888 100644
--- a/include/paimon/file_store_write.h
+++ b/include/paimon/file_store_write.h
@@ -107,6 +107,12 @@ class PAIMON_EXPORT FileStoreWrite {
     ///
     /// The writer loads the snapshot's partition-bucket offsets and releases sealed memory that is
     /// fully covered by disk. Calling this method on a non-real-time writer returns an error.
+    /// If the snapshot overwrites table contents or moves committed progress backwards, such as
+    /// after a partition drop, overwrite, or rollback, this method returns an error and the caller
+    /// must recreate the `RealtimeContext` and writer. These operations are not fenced against an
+    /// active writer and do not clear its process-local state automatically. The caller must
+    /// coordinate them with active writers; skipping the resetting snapshot and continuing to use
+    /// an old context is unsupported.
     virtual Status RefreshCommittedSnapshot(int64_t snapshot_id);
 
     virtual std::shared_ptr<Metrics> GetMetrics() const = 0;
diff --git a/include/paimon/realtime/arrow_realtime_store_factory.h b/include/paimon/realtime/arrow_realtime_store_factory.h
index 3dc257fc8..4d65743ab 100644
--- a/include/paimon/realtime/arrow_realtime_store_factory.h
+++ b/include/paimon/realtime/arrow_realtime_store_factory.h
@@ -28,7 +28,7 @@ class PAIMON_EXPORT ArrowRealtimeStoreFactory : public RealtimeStoreFactory {
  public:
     /// Creates an Arrow-backed store for one partition and bucket.
     Result<std::shared_ptr<RealtimeStore>> Create(
-        std::unique_ptr<::ArrowSchema> write_schema,
+        std::unique_ptr<::ArrowSchema> write_schema, StatisticsMode statistics_mode,
         const std::map<std::string, std::string>& options,
         const std::shared_ptr<MemoryPool>& memory_pool) override;
 };
diff --git a/include/paimon/realtime/realtime_context.h b/include/paimon/realtime/realtime_context.h
index 9bc870fe8..200e4ba4c 100644
--- a/include/paimon/realtime/realtime_context.h
+++ b/include/paimon/realtime/realtime_context.h
@@ -73,6 +73,11 @@ using RealtimeOffsetMap = std::map<RealtimePartitionBucket, int64_t>;
 /// reads. `RealtimeContext` itself is not a customization interface and must not be implemented by
 /// applications. Customize real-time storage and retrieval through `RealtimeStoreFactory` and
 /// `RealtimeStore` instead.
+///
+/// A context is valid only for one uninterrupted committed-progress history. Overwrite, truncate,
+/// partition drop, and rollback operations do not automatically clear process-local real-time
+/// state. Applications must coordinate these operations with active real-time writers and recreate
+/// the `RealtimeContext` and writers before continuing.
 class PAIMON_EXPORT RealtimeContext {
  public:
     /// Creates a context backed by Paimon's default in-memory Arrow `RealtimeStore`.
diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 9177fc06b..d02952acd 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -31,6 +31,7 @@
 #include "paimon/realtime/offset_range.h"
 #include "paimon/record_batch.h"
 #include "paimon/result.h"
+#include "paimon/statistics_mode.h"
 #include "paimon/visibility.h"
 
 struct ArrowSchema;
@@ -156,13 +157,14 @@ class PAIMON_EXPORT RealtimeStoreFactory {
  public:
     virtual ~RealtimeStoreFactory() = default;
 
-    /// Creates a store configured with the supplied schema, options, and memory pool.
+    /// Creates a store configured with the supplied schema, statistics, options, and memory pool.
     /// @param write_schema Complete table write schema whose ownership is transferred to the
     /// factory. The factory may consume it or retain it in the created store.
+    /// @param statistics_mode Framework-parsed statistics collection mode.
     /// @param options Effective table options available to the store.
     /// @param memory_pool Memory pool provided by the write context.
     virtual Result<std::shared_ptr<RealtimeStore>> Create(
-        std::unique_ptr<::ArrowSchema> write_schema,
+        std::unique_ptr<::ArrowSchema> write_schema, StatisticsMode statistics_mode,
         const std::map<std::string, std::string>& options,
         const std::shared_ptr<MemoryPool>& memory_pool) = 0;
 };
diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h
index dc780ad7e..9c0b9d470 100644
--- a/include/paimon/scan_context.h
+++ b/include/paimon/scan_context.h
@@ -170,7 +170,7 @@ class PAIMON_EXPORT ScanContextBuilder {
     ScanContextBuilder& SetGlobalIndexResult(
         const std::shared_ptr<GlobalIndexResult>& global_index_result);
 
-    /// Enables process-local union reads with the memory indexers owned by `realtime_context`.
+    /// Enables process-local union reads with the real-time stores owned by `realtime_context`.
     ScanContextBuilder& WithRealtimeContext(
         const std::shared_ptr<RealtimeContext>& realtime_context);
 
diff --git a/include/paimon/statistics_mode.h b/include/paimon/statistics_mode.h
new file mode 100644
index 000000000..0be0e9104
--- /dev/null
+++ b/include/paimon/statistics_mode.h
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+namespace paimon {
+
+/// Controls the amount of statistics collected for metadata pruning.
+enum class StatisticsMode {
+    /// Do not collect statistics.
+    NONE,
+    /// Collect statistics for all supported fields.
+    FULL,
+};
+
+}  // namespace paimon
diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h
index fa549eef1..a9dd367ec 100644
--- a/include/paimon/write_context.h
+++ b/include/paimon/write_context.h
@@ -218,7 +218,7 @@ class PAIMON_EXPORT WriteContextBuilder {
     WriteContextBuilder& WithFileSystem(const std::shared_ptr<FileSystem>& file_system);
 
     /// Enables the real-time write path with the provided shared context.
-    /// @param realtime_context Non-null context that owns the real-time indexers.
+    /// @param realtime_context Non-null context that owns the real-time stores.
     /// @return Reference to this builder for method chaining.
     WriteContextBuilder& WithRealtimeContext(
         const std::shared_ptr<RealtimeContext>& realtime_context);
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index adfd968dc..a9810424a 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -908,6 +908,7 @@ if(PAIMON_BUILD_TESTS)
                     core/utils/file_utils_test.cpp
                     core/utils/manifest_meta_reader_test.cpp
                     core/utils/offset_row_test.cpp
+                    core/utils/partition_utils_test.cpp
                     core/utils/partition_path_utils_test.cpp
                     core/utils/snapshot_manager_test.cpp
                     core/utils/tag_manager_test.cpp
diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp
index 8c5336e19..36b9c1c99 100644
--- a/src/paimon/common/defs.cpp
+++ b/src/paimon/common/defs.cpp
@@ -150,7 +150,9 @@ const char Options::AGGREGATION_REMOVE_RECORD_ON_DELETE[] = "aggregation.remove-
 const char Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED[] = "table-read.sequence-number.enabled";
 const char Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED[] = "key-value.sequence_number.enabled";
 const char Options::SCAN_TIMESTAMP_MILLIS[] = "scan.timestamp-millis";
+const char Options::REALTIME_ENABLED[] = "realtime.enabled";
 const char Options::REALTIME_READ_VIEW_TTL[] = "realtime.read-view-ttl";
+const char Options::REALTIME_STORE_STATS_MODE[] = "realtime.store.stats-mode";
 const char Options::SCAN_TIMESTAMP[] = "scan.timestamp";
 const char Options::SCAN_TAG_NAME[] = "scan.tag-name";
 const char Options::WRITE_ONLY[] = "write-only";
diff --git a/src/paimon/common/utils/binary_row_partition_computer.cpp b/src/paimon/common/utils/binary_row_partition_computer.cpp
index 43ec7d40d..ec773e32c 100644
--- a/src/paimon/common/utils/binary_row_partition_computer.cpp
+++ b/src/paimon/common/utils/binary_row_partition_computer.cpp
@@ -77,31 +77,68 @@ Result<std::unique_ptr<BinaryRowPartitionComputer>> BinaryRowPartitionComputer::
 
 Result<BinaryRow> BinaryRowPartitionComputer::ToBinaryRow(
     const std::map<std::string, std::string>& partition) const {
+    return ConvertToBinaryRow(partition, /*included_fields=*/nullptr);
+}
+
+Result<BinaryRow> BinaryRowPartitionComputer::ConvertToBinaryRow(
+    const std::map<std::string, std::string>& partition, std::vector<bool>* included_fields) const {
     BinaryRow binary_row(partition_converters_.size());
     BinaryRowWriter writer(&binary_row, /*initial_size=*/0, memory_pool_.get());
-    for (size_t field_idx = 0; field_idx < partition_converters_.size(); field_idx++) {
-        const auto& partition_extractor = partition_converters_[field_idx];
-        const auto& partition_key = partition_extractor.partition_key;
-        const auto& to_binary_row = partition_extractor.converter;
-        auto input_iter = partition.find(partition_key);
+    if (included_fields != nullptr) {
+        included_fields->assign(partition_converters_.size(), false);
+    }
+    for (size_t field_idx = 0; field_idx < partition_converters_.size(); ++field_idx) {
+        const PartitionConverter& partition_converter = partition_converters_[field_idx];
+        auto input_iter = partition.find(partition_converter.partition_key);
         if (input_iter == partition.end()) {
+            if (included_fields != nullptr) {
+                writer.SetNullAt(field_idx);
+                continue;
+            }
             return Status::Invalid(
                 fmt::format("can not find partition key '{}' in input partition '{}'",
-                            partition_key, partition));
+                            partition_converter.partition_key, partition));
         }
-        const auto& value_str = input_iter->second;
-        if (value_str == default_part_value_) {
+        if (included_fields != nullptr) {
+            (*included_fields)[field_idx] = true;
+        }
+        if (input_iter->second == default_part_value_) {
             // TODO(yonghao.fyh): when support decimal/ timestamp in partition, use
             // WriteTimestamp(null) for non compact precision
             writer.SetNullAt(field_idx);
         } else {
-            PAIMON_RETURN_NOT_OK(to_binary_row(value_str, field_idx, &writer));
+            PAIMON_RETURN_NOT_OK(
+                partition_converter.converter(input_iter->second, field_idx, &writer));
         }
     }
     writer.Complete();
     return binary_row;
 }
 
+Result<std::map<std::string, std::string>> BinaryRowPartitionComputer::NormalizePartitionSpec(
+    const std::map<std::string, std::string>& partition) const {
+    for (const auto& [partition_key, _] : partition) {
+        if (std::find(partition_keys_.begin(), partition_keys_.end(), partition_key) ==
+            partition_keys_.end()) {
+            return Status::Invalid(
+                fmt::format("field {} does not exist in partition keys", partition_key));
+        }
+    }
+
+    std::vector<bool> included_fields;
+    PAIMON_ASSIGN_OR_RAISE(BinaryRow binary_row, ConvertToBinaryRow(partition, &included_fields));
+
+    std::vector<std::pair<std::string, std::string>> normalized_values;
+    PAIMON_ASSIGN_OR_RAISE(normalized_values, GeneratePartitionVector(binary_row));
+    std::map<std::string, std::string> normalized_partition;
+    for (size_t field_idx = 0; field_idx < normalized_values.size(); ++field_idx) {
+        if (included_fields[field_idx]) {
+            normalized_partition.insert(normalized_values[field_idx]);
+        }
+    }
+    return normalized_partition;
+}
+
 Result<std::vector<std::pair<std::string, std::string>>>
 BinaryRowPartitionComputer::GeneratePartitionVector(const BinaryRow& partition) const {
     if (static_cast<size_t>(partition.GetFieldCount()) != partition_converters_.size()) {
diff --git a/src/paimon/common/utils/binary_row_partition_computer.h b/src/paimon/common/utils/binary_row_partition_computer.h
index 63aa7549e..2e96ad6f1 100644
--- a/src/paimon/common/utils/binary_row_partition_computer.h
+++ b/src/paimon/common/utils/binary_row_partition_computer.h
@@ -55,6 +55,8 @@ class BinaryRowPartitionComputer {
         bool legacy_partition_name_enabled, const std::shared_ptr<MemoryPool>& memory_pool);
 
     Result<BinaryRow> ToBinaryRow(const std::map<std::string, std::string>& partition) const;
+    Result<std::map<std::string, std::string>> NormalizePartitionSpec(
+        const std::map<std::string, std::string>& partition) const;
     Result<std::vector<std::pair<std::string, std::string>>> GeneratePartitionVector(
         const BinaryRow& partition) const;
     const std::vector<std::string>& GetPartitionKeys() const {
@@ -73,6 +75,10 @@ class BinaryRowPartitionComputer {
                                const std::vector<PartitionConverter>& partition_converters,
                                const std::shared_ptr<MemoryPool>& memory_pool);
 
+    /// A non-null `included_fields` enables partial partitions and records present fields.
+    Result<BinaryRow> ConvertToBinaryRow(const std::map<std::string, std::string>& partition,
+                                         std::vector<bool>* included_fields) const;
+
     static Result<arrow::Type::type> GetTypeFromArrowSchema(
         const std::shared_ptr<arrow::Schema>& schema, const std::string& field_name);
 
diff --git a/src/paimon/common/utils/binary_row_partition_computer_test.cpp b/src/paimon/common/utils/binary_row_partition_computer_test.cpp
index 974c4d7af..18c421780 100644
--- a/src/paimon/common/utils/binary_row_partition_computer_test.cpp
+++ b/src/paimon/common/utils/binary_row_partition_computer_test.cpp
@@ -268,6 +268,36 @@ TEST(BinaryRowPartitionComputerTest, TestNullOrWhitespaceOnlyStr) {
     ASSERT_EQ(partition_key_values, expected);
 }
 
+TEST(BinaryRowPartitionComputerTest, TestNormalizePartialPartitionSpec) {
+    using PartitionSpec = std::map<std::string, std::string>;
+
+    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
+    std::shared_ptr<arrow::Schema> schema =
+        arrow::schema({arrow::field("pt", arrow::date32()), arrow::field("region", arrow::utf8())});
+    std::vector<std::string> partition_keys = {"pt", "region"};
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<BinaryRowPartitionComputer> legacy_computer,
+        BinaryRowPartitionComputer::Create(partition_keys, schema, "__DEFAULT_PARTITION__",
+                                           /*legacy_partition_name_enabled=*/true, pool));
+    ASSERT_OK_AND_ASSIGN(PartitionSpec legacy_partition,
+                         legacy_computer->NormalizePartitionSpec({{"pt", "2024-01-01"}}));
+    PartitionSpec expected_legacy_partition = {{"pt", "19723"}};
+    ASSERT_EQ(expected_legacy_partition, legacy_partition);
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<BinaryRowPartitionComputer> non_legacy_computer,
+        BinaryRowPartitionComputer::Create(partition_keys, schema, "__DEFAULT_PARTITION__",
+                                           /*legacy_partition_name_enabled=*/false, pool));
+    ASSERT_OK_AND_ASSIGN(PartitionSpec non_legacy_partition,
+                         non_legacy_computer->NormalizePartitionSpec({{"pt", "2024-01-01"}}));
+    PartitionSpec expected_non_legacy_partition = {{"pt", "2024-01-01"}};
+    ASSERT_EQ(expected_non_legacy_partition, non_legacy_partition);
+
+    ASSERT_NOK_WITH_MSG(non_legacy_computer->NormalizePartitionSpec({{"unknown", "value"}}),
+                        "field unknown does not exist in partition keys");
+}
+
 TEST(BinaryRowPartitionComputerTest, TestPartToSimpleString) {
     auto pool = GetDefaultPool();
     {
diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp
index 1e8203a57..6320ec577 100644
--- a/src/paimon/core/core_options.cpp
+++ b/src/paimon/core/core_options.cpp
@@ -38,6 +38,7 @@
 #include "paimon/defs.h"
 #include "paimon/format/file_format.h"
 #include "paimon/format/file_format_factory.h"
+#include "paimon/statistics_mode.h"
 #include "paimon/status.h"
 
 namespace paimon {
@@ -391,7 +392,9 @@ struct CoreOptions::Impl {
     int64_t commit_timeout = std::numeric_limits<int64_t>::max();
     int64_t commit_min_retry_wait = 10;
     int64_t commit_max_retry_wait = 10 * 1000;
+    bool realtime_enabled = false;
     int64_t realtime_read_view_ttl_millis = 5 * 60 * 1000;
+    StatisticsMode realtime_store_statistics_mode = StatisticsMode::NONE;
 
     std::shared_ptr<FileFormat> file_format;
     std::shared_ptr<FileSystem> file_system;
@@ -805,12 +808,6 @@ struct CoreOptions::Impl {
         std::string scan_timestamp_str;
         PAIMON_RETURN_NOT_OK(parser.Parse(Options::SCAN_TIMESTAMP, &scan_timestamp_str));
         PAIMON_RETURN_NOT_OK(parser.Parse(Options::SCAN_TIMESTAMP_MILLIS, &scan_timestamp_millis));
-        PAIMON_RETURN_NOT_OK(parser.ParseTimeDuration(Options::REALTIME_READ_VIEW_TTL,
-                                                      &realtime_read_view_ttl_millis));
-        if (realtime_read_view_ttl_millis <= 0) {
-            return Status::Invalid(
-                fmt::format("{} must be positive", Options::REALTIME_READ_VIEW_TTL));
-        }
         if (scan_timestamp_millis != std::nullopt && !scan_timestamp_str.empty()) {
             return Status::Invalid(
                 "scan.timestamp-millis and scan.timestamp cannot be set at the same time");
@@ -840,6 +837,34 @@ struct CoreOptions::Impl {
         return Status::OK();
     }
 
+    // Parse statistics collected by real-time stores.
+    Status ParseRealtimeStoreStatisticsMode(const ConfigParser& parser) {
+        std::string statistics_mode = "none";
+        PAIMON_RETURN_NOT_OK(parser.Parse(Options::REALTIME_STORE_STATS_MODE, &statistics_mode));
+        statistics_mode = StringUtils::ToLowerCase(statistics_mode);
+        if (statistics_mode == "none") {
+            realtime_store_statistics_mode = StatisticsMode::NONE;
+        } else if (statistics_mode == "full") {
+            realtime_store_statistics_mode = StatisticsMode::FULL;
+        } else {
+            return Status::Invalid(
+                fmt::format("{} must be 'none' or 'full'", Options::REALTIME_STORE_STATS_MODE));
+        }
+        return Status::OK();
+    }
+
+    // Parse real-time write and read configurations.
+    Status ParseRealtimeOptions(const ConfigParser& parser) {
+        PAIMON_RETURN_NOT_OK(parser.Parse<bool>(Options::REALTIME_ENABLED, &realtime_enabled));
+        PAIMON_RETURN_NOT_OK(parser.ParseTimeDuration(Options::REALTIME_READ_VIEW_TTL,
+                                                      &realtime_read_view_ttl_millis));
+        if (realtime_read_view_ttl_millis <= 0) {
+            return Status::Invalid(
+                fmt::format("{} must be positive", Options::REALTIME_READ_VIEW_TTL));
+        }
+        return ParseRealtimeStoreStatisticsMode(parser);
+    }
+
     // Parse index-related configurations: file index, global index.
     Status ParseIndexOptions(const ConfigParser& parser) {
         // Parse file-index.in-manifest-threshold - max inline file index size, default 500B
@@ -1053,6 +1078,7 @@ Result<CoreOptions> CoreOptions::FromMap(
     PAIMON_RETURN_NOT_OK(impl->ParseCommitOptions(parser));
     PAIMON_RETURN_NOT_OK(impl->ParseMergeAndSequenceOptions(parser));
     PAIMON_RETURN_NOT_OK(impl->ParseDeletionVectorOptions(parser));
+    PAIMON_RETURN_NOT_OK(impl->ParseRealtimeOptions(parser));
     PAIMON_RETURN_NOT_OK(impl->ParseScanAndBranchOptions(parser));
     PAIMON_RETURN_NOT_OK(impl->ParseIndexOptions(parser));
     PAIMON_RETURN_NOT_OK(impl->ParseCompactionOptions(parser));
@@ -1165,10 +1191,18 @@ std::optional<int64_t> CoreOptions::GetScanTimestampMillis() const {
     return impl_->scan_timestamp_millis;
 }
 
+bool CoreOptions::RealtimeEnabled() const {
+    return impl_->realtime_enabled;
+}
+
 int64_t CoreOptions::GetRealtimeReadViewTtlMillis() const {
     return impl_->realtime_read_view_ttl_millis;
 }
 
+StatisticsMode CoreOptions::GetRealtimeStoreStatisticsMode() const {
+    return impl_->realtime_store_statistics_mode;
+}
+
 int32_t CoreOptions::GetScanManifestEntryCacheMaxSnapshots() const {
     return impl_->scan_manifest_entry_cache_max_snapshots;
 }
diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h
index 3bb17d6f6..85a4a7fdf 100644
--- a/src/paimon/core/core_options.h
+++ b/src/paimon/core/core_options.h
@@ -40,6 +40,7 @@
 #include "paimon/format/file_format.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/result.h"
+#include "paimon/statistics_mode.h"
 #include "paimon/table/source/startup_mode.h"
 #include "paimon/type_fwd.h"
 #include "paimon/visibility.h"
@@ -106,7 +107,10 @@ class PAIMON_EXPORT CoreOptions {
     int64_t GetSourceSplitOpenFileCost() const;
     std::optional<int64_t> GetScanSnapshotId() const;
     std::optional<int64_t> GetScanTimestampMillis() const;
+    bool RealtimeEnabled() const;
     int64_t GetRealtimeReadViewTtlMillis() const;
+    /// Returns the statistics mode used by the real-time store.
+    StatisticsMode GetRealtimeStoreStatisticsMode() const;
     int32_t GetScanManifestEntryCacheMaxSnapshots() const;
     bool ScanManifestEntryLazyDecodeEnabled() const;
 
diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp
index f057206dd..c110623d7 100644
--- a/src/paimon/core/core_options_test.cpp
+++ b/src/paimon/core/core_options_test.cpp
@@ -29,6 +29,7 @@
 #include "paimon/core/options/expire_config.h"
 #include "paimon/defs.h"
 #include "paimon/fs/local/local_file_system.h"
+#include "paimon/statistics_mode.h"
 #include "paimon/testing/mock/mock_file_system.h"
 #include "paimon/testing/utils/testharness.h"
 #include "paimon/testing/utils/timezone_guard.h"
@@ -54,6 +55,8 @@ TEST(CoreOptionsTest, TestDefaultValue) {
     ASSERT_EQ("__DEFAULT_PARTITION__", core_options.GetPartitionDefaultName());
     ASSERT_EQ(std::nullopt, core_options.GetScanSnapshotId());
     ASSERT_EQ(5 * 60 * 1000, core_options.GetRealtimeReadViewTtlMillis());
+    ASSERT_FALSE(core_options.RealtimeEnabled());
+    ASSERT_EQ(StatisticsMode::NONE, core_options.GetRealtimeStoreStatisticsMode());
     ASSERT_EQ("zstd", core_options.GetFileCompression());
     ASSERT_EQ("zstd", core_options.GetWriteFileCompression(0));
     ASSERT_EQ("zstd", core_options.GetWriteFileCompression(3));
@@ -308,6 +311,7 @@ TEST(CoreOptionsTest, TestFromMap) {
         {Options::LOOKUP_REMOTE_LEVEL_THRESHOLD, "2"},
         {Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, "true"},
         {Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"},
+        {Options::REALTIME_ENABLED, "true"},
         {Options::BUCKET_FUNCTION_TYPE, "mod"},
         {"fields.metrics.map.storage-layout", "shared-shredding"},
         {"fields.metrics.map.shared-shredding.max-columns", "128"},
@@ -469,6 +473,7 @@ TEST(CoreOptionsTest, TestFromMap) {
     ASSERT_EQ(10L * 1024 * 1024 * 1024, core_options.GetLookupCacheMaxDiskSize());
     ASSERT_TRUE(core_options.TableReadSequenceNumberEnabled());
     ASSERT_TRUE(core_options.KeyValueSequenceNumberEnabled());
+    ASSERT_TRUE(core_options.RealtimeEnabled());
     ASSERT_TRUE(core_options.LookupRemoteFileEnabled());
     ASSERT_EQ(core_options.GetLookupRemoteLevelThreshold(), 2);
     ASSERT_EQ(BucketFunctionType::MOD, core_options.GetBucketFunctionType());
@@ -498,6 +503,7 @@ TEST(CoreOptionsTest, TestInvalidCase) {
                         "invalid lookup mode: invalid");
     ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::LOOKUP_COMPACT_MAX_INTERVAL, "invalid"}}),
                         "Invalid Config [lookup-compact.max-interval: invalid]");
+    ASSERT_NOK(CoreOptions::FromMap({{Options::REALTIME_ENABLED, "invalid"}}));
     ASSERT_NOK_WITH_MSG(
         CoreOptions::FromMap({{Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "-1"}}),
         "scan.manifest-entry-cache.max-snapshots must be non-negative");
@@ -810,6 +816,19 @@ TEST(CoreOptionsTest, TestRealtimeReadViewTtlMillis) {
                         "realtime.read-view-ttl must be positive");
 }
 
+TEST(CoreOptionsTest, TestRealtimeStoreStatisticsMode) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions full_options,
+                         CoreOptions::FromMap({{Options::REALTIME_STORE_STATS_MODE, "full"}}));
+    ASSERT_EQ(StatisticsMode::FULL, full_options.GetRealtimeStoreStatisticsMode());
+
+    ASSERT_OK_AND_ASSIGN(CoreOptions none_options,
+                         CoreOptions::FromMap({{Options::REALTIME_STORE_STATS_MODE, "none"}}));
+    ASSERT_EQ(StatisticsMode::NONE, none_options.GetRealtimeStoreStatisticsMode());
+
+    ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::REALTIME_STORE_STATS_MODE, "invalid"}}),
+                        "realtime.store.stats-mode must be 'none' or 'full'");
+}
+
 TEST(CoreOptionsTest, TestScanTimestampMillisExplicitMode) {
     ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
                          CoreOptions::FromMap({{Options::SCAN_MODE, "from-timestamp"},
diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp
index b2946e83c..5d6c2c930 100644
--- a/src/paimon/core/operation/append_only_file_store_write.cpp
+++ b/src/paimon/core/operation/append_only_file_store_write.cpp
@@ -255,9 +255,9 @@ Result<std::shared_ptr<BatchWriter>> AppendOnlyFileStoreWrite::CreateWriter(
                                                      partition_values.end());
     auto c_write_schema = std::make_unique<ArrowSchema>();
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*write_schema_, c_write_schema.get()));
-    return RealtimeAppendOnlyWriter::Create(partition_map, bucket, std::move(c_write_schema),
-                                            realtime_context_, writer, write_schema_,
-                                            options_.ToMap(), pool_);
+    return RealtimeAppendOnlyWriter::Create(
+        partition_map, bucket, std::move(c_write_schema), realtime_context_, writer, write_schema_,
+        options_.GetRealtimeStoreStatisticsMode(), options_.ToMap(), pool_);
 }
 
 Result<AppendOnlyFileStoreWrite::WriterFactory> AppendOnlyFileStoreWrite::GetDataFileWriterFactory(
diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp
index 26411c198..4e1cef501 100644
--- a/src/paimon/core/operation/append_only_file_store_write_test.cpp
+++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp
@@ -273,6 +273,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksInternalOffsetRange)
         {"write-only", "true"},
         {"bucket", "1"},
         {"bucket-key", "id"},
+        {Options::REALTIME_ENABLED, "true"},
     };
     auto logical_schema =
         arrow::schema({arrow::field("id", arrow::int32()), arrow::field("name", arrow::utf8())});
diff --git a/src/paimon/core/operation/commit/commit_scanner.cpp b/src/paimon/core/operation/commit/commit_scanner.cpp
index 5590dd02f..448bd7e47 100644
--- a/src/paimon/core/operation/commit/commit_scanner.cpp
+++ b/src/paimon/core/operation/commit/commit_scanner.cpp
@@ -38,6 +38,7 @@
 #include "paimon/core/operation/commit/overwrite_changes_provider.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/table/bucket_mode.h"
+#include "paimon/core/utils/partition_utils.h"
 #include "paimon/scan_context.h"
 
 namespace paimon {
@@ -183,14 +184,9 @@ Result<std::vector<IndexManifestEntry>> CommitScanner::ReadAllIndexEntriesFromPa
         }
 
         for (const auto& partition_spec : partitions) {
-            bool matched = true;
-            for (const auto& [key, value] : partition_spec) {
-                auto iter = partition.find(key);
-                if (iter == partition.end() || iter->second != value) {
-                    matched = false;
-                    break;
-                }
-            }
+            PAIMON_ASSIGN_OR_RAISE(bool matched,
+                                   PartitionUtils::MatchPartitionSpec(partition, partition_spec,
+                                                                      *partition_computer_));
             if (matched) {
                 return true;
             }
diff --git a/src/paimon/core/operation/commit/realtime_commit_properties.cpp b/src/paimon/core/operation/commit/realtime_commit_properties.cpp
index 262fbbeff..c79fc968e 100644
--- a/src/paimon/core/operation/commit/realtime_commit_properties.cpp
+++ b/src/paimon/core/operation/commit/realtime_commit_properties.cpp
@@ -31,6 +31,7 @@
 #include "paimon/common/utils/rapidjson_util.h"
 #include "paimon/common/utils/uuid.h"
 #include "paimon/core/utils/branch_manager.h"
+#include "paimon/core/utils/partition_utils.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/macros.h"
 
@@ -159,24 +160,71 @@ std::string RealtimeCommitProperties::OffsetsDirectory(const std::string& table_
     return PathUtil::JoinPath(BranchManager::BranchPath(table_root, branch), "metadata");
 }
 
+std::optional<std::string> RealtimeCommitProperties::GetOffsetsPath(const Snapshot& snapshot) {
+    if (!snapshot.Properties()) {
+        return std::nullopt;
+    }
+    const std::map<std::string, std::string>& properties = snapshot.Properties().value();
+    auto iter = properties.find(kOffsetsKey);
+    if (iter == properties.end()) {
+        return std::nullopt;
+    }
+    return iter->second;
+}
+
 Result<RealtimeOffsetMap> RealtimeCommitProperties::ReadOffsets(
     const std::optional<Snapshot>& snapshot, const std::shared_ptr<FileSystem>& file_system) {
-    if (!snapshot || !snapshot->Properties()) {
+    if (!snapshot) {
         return RealtimeOffsetMap{};
     }
-    const std::map<std::string, std::string>& properties = snapshot->Properties().value();
-    auto iter = properties.find(kOffsetsKey);
-    if (iter == properties.end()) {
+    std::optional<std::string> offsets_path = GetOffsetsPath(snapshot.value());
+    if (!offsets_path) {
         return RealtimeOffsetMap{};
     }
     if (file_system == nullptr) {
         return Status::Invalid("file system is null when reading real-time offsets");
     }
     std::string content;
-    PAIMON_RETURN_NOT_OK(file_system->ReadFile(iter->second, &content));
+    PAIMON_RETURN_NOT_OK(file_system->ReadFile(offsets_path.value(), &content));
     return ParseOffsets(content);
 }
 
+Result<bool> RealtimeCommitProperties::AreRangesCommitted(
+    const RealtimeOffsetMap& committed_offsets,
+    const std::map<RealtimePartitionBucket, OffsetRange>& realtime_ranges) {
+    std::optional<bool> all_committed;
+    for (const auto& [partition_bucket, offset_range] : realtime_ranges) {
+        if (partition_bucket.bucket < 0) {
+            return Status::Invalid(
+                fmt::format("real-time commit bucket {} is invalid", partition_bucket.bucket));
+        }
+        if (offset_range.begin < 0 || offset_range.begin >= offset_range.end) {
+            return Status::Invalid("real-time commit offset range is invalid");
+        }
+
+        auto offset_iter = committed_offsets.find(partition_bucket);
+        int64_t committed_end_offset =
+            offset_iter == committed_offsets.end() ? 0 : offset_iter->second;
+        bool range_committed = offset_range.end <= committed_end_offset;
+        if (!range_committed && offset_range.begin < committed_end_offset) {
+            return Status::Invalid(fmt::format(
+                "real-time commit offset range partially overlaps committed offset for bucket {}",
+                partition_bucket.bucket));
+        }
+        if (!range_committed && offset_range.begin != committed_end_offset) {
+            return Status::Invalid(
+                fmt::format("real-time commit offsets for bucket {} are not contiguous",
+                            partition_bucket.bucket));
+        }
+        if (all_committed && all_committed.value() != range_committed) {
+            return Status::Invalid(
+                "real-time commit ranges are only partially covered by committed offsets");
+        }
+        all_committed = range_committed;
+    }
+    return all_committed.value_or(false);
+}
+
 Result<std::string> RealtimeCommitProperties::SerializeOffsets(const RealtimeOffsetMap& offsets) {
     std::string result;
     PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(OffsetsJson(offsets), &result));
@@ -187,11 +235,17 @@ Result<std::map<std::string, std::string>> RealtimeCommitProperties::Build(
     const std::map<std::string, std::string>& properties,
     const std::optional<Snapshot>& latest_snapshot,
     const std::map<RealtimePartitionBucket, OffsetRange>& realtime_ranges,
+    bool reset_all_realtime_progress,
+    const std::vector<std::map<std::string, std::string>>& removed_realtime_partitions,
+    const BinaryRowPartitionComputer& partition_computer,
     const std::shared_ptr<FileSystem>& file_system, const std::string& table_root,
     const std::string& branch) {
     std::map<std::string, std::string> merged_properties = properties;
-    if (realtime_ranges.empty()) {
-        if (latest_snapshot && latest_snapshot->Properties()) {
+    if (reset_all_realtime_progress || !removed_realtime_partitions.empty()) {
+        merged_properties.erase(kOffsetsKey);
+    }
+    if (realtime_ranges.empty() && removed_realtime_partitions.empty()) {
+        if (!reset_all_realtime_progress && latest_snapshot && latest_snapshot->Properties()) {
             const std::map<std::string, std::string>& latest_properties =
                 latest_snapshot->Properties().value();
             auto offsets_iter = latest_properties.find(kOffsetsKey);
@@ -202,8 +256,25 @@ Result<std::map<std::string, std::string>> RealtimeCommitProperties::Build(
         return merged_properties;
     }
 
-    PAIMON_ASSIGN_OR_RAISE(RealtimeOffsetMap merged_offsets,
-                           ReadOffsets(latest_snapshot, file_system));
+    PAIMON_ASSIGN_OR_RAISE(
+        RealtimeOffsetMap merged_offsets,
+        ReadOffsets(reset_all_realtime_progress ? std::nullopt : latest_snapshot, file_system));
+    for (auto iter = merged_offsets.begin(); iter != merged_offsets.end();) {
+        bool removed = false;
+        for (const auto& partition_spec : removed_realtime_partitions) {
+            PAIMON_ASSIGN_OR_RAISE(
+                removed, PartitionUtils::MatchPartitionSpec(iter->first.partition, partition_spec,
+                                                            partition_computer));
+            if (removed) {
+                break;
+            }
+        }
+        if (removed) {
+            iter = merged_offsets.erase(iter);
+        } else {
+            ++iter;
+        }
+    }
     for (const auto& [partition_bucket, offset_range] : realtime_ranges) {
         if (partition_bucket.bucket < 0) {
             return Status::Invalid(
@@ -221,6 +292,10 @@ Result<std::map<std::string, std::string>> RealtimeCommitProperties::Build(
         }
         merged_offsets[partition_bucket] = offset_range.end;
     }
+    if (merged_offsets.empty()) {
+        merged_properties.erase(kOffsetsKey);
+        return merged_properties;
+    }
     PAIMON_ASSIGN_OR_RAISE(
         merged_properties[kOffsetsKey],
         WriteOffsets(merged_offsets, file_system, OffsetsDirectory(table_root, branch)));
diff --git a/src/paimon/core/operation/commit/realtime_commit_properties.h b/src/paimon/core/operation/commit/realtime_commit_properties.h
index 31a014b4b..4c4069200 100644
--- a/src/paimon/core/operation/commit/realtime_commit_properties.h
+++ b/src/paimon/core/operation/commit/realtime_commit_properties.h
@@ -33,6 +33,7 @@
 
 namespace paimon {
 
+class BinaryRowPartitionComputer;
 class FileSystem;
 
 class RealtimeCommitProperties {
@@ -46,16 +47,35 @@ class RealtimeCommitProperties {
 
     static std::string OffsetsDirectory(const std::string& table_root, const std::string& branch);
 
+    /// Returns the offset file referenced by `snapshot`, if present.
+    static std::optional<std::string> GetOffsetsPath(const Snapshot& snapshot);
+
     static Result<RealtimeOffsetMap> ReadOffsets(const std::optional<Snapshot>& snapshot,
                                                  const std::shared_ptr<FileSystem>& file_system);
 
+    /// Returns whether all ranges are already covered by committed offsets.
+    ///
+    /// Ranges must either all immediately follow committed offsets or all be fully covered.
+    /// Mixed states, gaps, and partial overlaps are rejected.
+    static Result<bool> AreRangesCommitted(
+        const RealtimeOffsetMap& committed_offsets,
+        const std::map<RealtimePartitionBucket, OffsetRange>& realtime_ranges);
+
     static Result<std::string> SerializeOffsets(const RealtimeOffsetMap& offsets);
 
     /// Builds snapshot properties against `latest_snapshot` and applies real-time progress.
+    ///
+    /// A full-table replacement sets `reset_all_realtime_progress`. A partition overwrite or
+    /// drop lists only the affected partition specs in `removed_realtime_partitions`; offsets for
+    /// all other partitions are retained. The two reset forms are independent of the snapshot's
+    /// commit kind because an ordinary commit may also use `OVERWRITE` for conflict handling.
     static Result<std::map<std::string, std::string>> Build(
         const std::map<std::string, std::string>& properties,
         const std::optional<Snapshot>& latest_snapshot,
         const std::map<RealtimePartitionBucket, OffsetRange>& realtime_ranges,
+        bool reset_all_realtime_progress,
+        const std::vector<std::map<std::string, std::string>>& removed_realtime_partitions,
+        const BinaryRowPartitionComputer& partition_computer,
         const std::shared_ptr<FileSystem>& file_system, const std::string& table_root,
         const std::string& branch);
 
diff --git a/src/paimon/core/operation/commit/realtime_commit_properties_test.cpp b/src/paimon/core/operation/commit/realtime_commit_properties_test.cpp
index 68b159451..afe529e26 100644
--- a/src/paimon/core/operation/commit/realtime_commit_properties_test.cpp
+++ b/src/paimon/core/operation/commit/realtime_commit_properties_test.cpp
@@ -28,9 +28,12 @@
 #include <utility>
 #include <vector>
 
+#include "arrow/type.h"
 #include "gtest/gtest.h"
+#include "paimon/common/utils/binary_row_partition_computer.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/macros.h"
+#include "paimon/memory/memory_pool.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
@@ -71,6 +74,13 @@ class RealtimeCommitPropertiesTest : public testing::Test {
         ASSERT_NE(nullptr, directory_);
         file_system_ = directory_->GetFileSystem();
         ASSERT_NE(nullptr, file_system_);
+        std::shared_ptr<arrow::Schema> schema = arrow::schema(
+            {arrow::field("dt", arrow::utf8()), arrow::field("region", arrow::utf8())});
+        ASSERT_OK_AND_ASSIGN(partition_computer_,
+                             BinaryRowPartitionComputer::Create(
+                                 /*partition_keys=*/{"dt", "region"}, schema,
+                                 /*default_part_value=*/"__DEFAULT_PARTITION__",
+                                 /*legacy_partition_name_enabled=*/true, GetDefaultPool()));
     }
 
     Snapshot MakeSnapshot(
@@ -120,6 +130,7 @@ class RealtimeCommitPropertiesTest : public testing::Test {
 
     std::unique_ptr<UniqueTestDirectory> directory_;
     std::shared_ptr<FileSystem> file_system_;
+    std::unique_ptr<BinaryRowPartitionComputer> partition_computer_;
     int32_t next_file_id_ = 0;
 };
 
@@ -256,6 +267,52 @@ TEST_F(RealtimeCommitPropertiesTest, SortProgress) {
     ASSERT_EQ(OffsetRange(5, 7), commits[2].offset_range);
 }
 
+TEST_F(RealtimeCommitPropertiesTest, CheckRangesCommitted) {
+    RealtimePartitionBucket bucket0({{"dt", "2"}}, /*bucket=*/0);
+    RealtimePartitionBucket bucket1({{"dt", "2"}}, /*bucket=*/1);
+    RealtimeOffsetMap committed_offsets = {{bucket0, 4}, {bucket1, 9}};
+
+    std::map<RealtimePartitionBucket, OffsetRange> pending = {{bucket0, OffsetRange(4, 6)},
+                                                              {bucket1, OffsetRange(9, 11)}};
+    ASSERT_OK_AND_ASSIGN(bool pending_committed,
+                         RealtimeCommitProperties::AreRangesCommitted(committed_offsets, pending));
+    ASSERT_FALSE(pending_committed);
+
+    std::map<RealtimePartitionBucket, OffsetRange> covered = {{bucket0, OffsetRange(0, 4)},
+                                                              {bucket1, OffsetRange(5, 8)}};
+    ASSERT_OK_AND_ASSIGN(bool covered_committed,
+                         RealtimeCommitProperties::AreRangesCommitted(committed_offsets, covered));
+    ASSERT_TRUE(covered_committed);
+
+    std::map<RealtimePartitionBucket, OffsetRange> single_bucket_covered = {
+        {bucket0, OffsetRange(0, 4)}};
+    ASSERT_OK_AND_ASSIGN(
+        bool single_bucket_covered_committed,
+        RealtimeCommitProperties::AreRangesCommitted(committed_offsets, single_bucket_covered));
+    ASSERT_TRUE(single_bucket_covered_committed);
+
+    std::map<RealtimePartitionBucket, OffsetRange> single_bucket_pending = {
+        {bucket1, OffsetRange(9, 11)}};
+    ASSERT_OK_AND_ASSIGN(
+        bool single_bucket_pending_committed,
+        RealtimeCommitProperties::AreRangesCommitted(committed_offsets, single_bucket_pending));
+    ASSERT_FALSE(single_bucket_pending_committed);
+
+    std::map<RealtimePartitionBucket, OffsetRange> partial_overlap = {{bucket0, OffsetRange(3, 5)}};
+    ASSERT_NOK_WITH_MSG(
+        RealtimeCommitProperties::AreRangesCommitted(committed_offsets, partial_overlap),
+        "partially overlaps");
+
+    std::map<RealtimePartitionBucket, OffsetRange> gap = {{bucket0, OffsetRange(5, 7)}};
+    ASSERT_NOK_WITH_MSG(RealtimeCommitProperties::AreRangesCommitted(committed_offsets, gap),
+                        "are not contiguous");
+
+    std::map<RealtimePartitionBucket, OffsetRange> mixed = {{bucket0, OffsetRange(0, 4)},
+                                                            {bucket1, OffsetRange(9, 11)}};
+    ASSERT_NOK_WITH_MSG(RealtimeCommitProperties::AreRangesCommitted(committed_offsets, mixed),
+                        "only partially covered");
+}
+
 TEST_F(RealtimeCommitPropertiesTest, BuildRejectsInvalidProgress) {
     RealtimePartitionBucket bucket0({{"dt", "2"}}, /*bucket=*/0);
     RealtimeOffsetMap committed_offsets = {{bucket0, 1}};
@@ -265,20 +322,28 @@ TEST_F(RealtimeCommitPropertiesTest, BuildRejectsInvalidProgress) {
         {RealtimePartitionBucket({{"dt", "2"}}, /*bucket=*/-1), OffsetRange(0, 1)}};
     ASSERT_NOK_WITH_MSG(
         RealtimeCommitProperties::Build(/*properties=*/{}, /*latest_snapshot=*/std::nullopt,
-                                        invalid_bucket, file_system_, directory_->Str(), "main"),
+                                        invalid_bucket, /*reset_all_realtime_progress=*/false,
+                                        /*removed_realtime_partitions=*/{}, *partition_computer_,
+                                        file_system_, directory_->Str(), "main"),
         "bucket -1 is invalid");
 
     std::map<RealtimePartitionBucket, OffsetRange> gap = {
         {RealtimePartitionBucket({{"dt", "2"}}, /*bucket=*/0), OffsetRange(3, 5)}};
-    ASSERT_NOK_WITH_MSG(RealtimeCommitProperties::Build(/*properties=*/{}, latest_snapshot, gap,
-                                                        file_system_, directory_->Str(), "main"),
-                        "are not contiguous");
+    ASSERT_NOK_WITH_MSG(
+        RealtimeCommitProperties::Build(/*properties=*/{}, latest_snapshot, gap,
+                                        /*reset_all_realtime_progress=*/false,
+                                        /*removed_realtime_partitions=*/{}, *partition_computer_,
+                                        file_system_, directory_->Str(), "main"),
+        "are not contiguous");
 
     std::map<RealtimePartitionBucket, OffsetRange> overlap = {
         {RealtimePartitionBucket({{"dt", "2"}}, /*bucket=*/0), OffsetRange(0, 2)}};
-    ASSERT_NOK_WITH_MSG(RealtimeCommitProperties::Build(/*properties=*/{}, latest_snapshot, overlap,
-                                                        file_system_, directory_->Str(), "main"),
-                        "are not contiguous");
+    ASSERT_NOK_WITH_MSG(
+        RealtimeCommitProperties::Build(/*properties=*/{}, latest_snapshot, overlap,
+                                        /*reset_all_realtime_progress=*/false,
+                                        /*removed_realtime_partitions=*/{}, *partition_computer_,
+                                        file_system_, directory_->Str(), "main"),
+        "are not contiguous");
 
     RealtimeOffsetMap exhausted_offsets = {{bucket0, std::numeric_limits<int64_t>::max()}};
     ASSERT_OK_AND_ASSIGN(Snapshot exhausted_snapshot, MakeSnapshotWithOffsets(exhausted_offsets));
@@ -287,6 +352,8 @@ TEST_F(RealtimeCommitPropertiesTest, BuildRejectsInvalidProgress) {
          OffsetRange(std::numeric_limits<int64_t>::max(), std::numeric_limits<int64_t>::max())}};
     ASSERT_NOK_WITH_MSG(
         RealtimeCommitProperties::Build(/*properties=*/{}, exhausted_snapshot, after_max,
+                                        /*reset_all_realtime_progress=*/false,
+                                        /*removed_realtime_partitions=*/{}, *partition_computer_,
                                         file_system_, directory_->Str(), "main"),
         "offset range is invalid");
 }
@@ -300,16 +367,54 @@ TEST_F(RealtimeCommitPropertiesTest, BuildWithoutProgress) {
     ASSERT_OK_AND_ASSIGN(Properties inherited,
                          RealtimeCommitProperties::Build(
                              properties, std::optional<Snapshot>(MakeSnapshot(latest_properties)),
-                             /*realtime_ranges=*/{}, /*file_system=*/nullptr,
+                             /*realtime_ranges=*/{}, /*reset_all_realtime_progress=*/false,
+                             /*removed_realtime_partitions=*/{}, *partition_computer_,
+                             /*file_system=*/nullptr,
                              /*table_root=*/"", /*branch=*/"main"));
     ASSERT_EQ("value", inherited.at("custom"));
     ASSERT_EQ(latest_offsets_path, inherited.at(RealtimeCommitProperties::kOffsetsKey));
 
-    ASSERT_OK_AND_ASSIGN(Properties unchanged, RealtimeCommitProperties::Build(
-                                                   properties, /*latest_snapshot=*/std::nullopt,
-                                                   /*realtime_ranges=*/{}, /*file_system=*/nullptr,
-                                                   /*table_root=*/"", /*branch=*/"main"));
+    ASSERT_OK_AND_ASSIGN(
+        Properties unchanged,
+        RealtimeCommitProperties::Build(properties, /*latest_snapshot=*/std::nullopt,
+                                        /*realtime_ranges=*/{},
+                                        /*reset_all_realtime_progress=*/false,
+                                        /*removed_realtime_partitions=*/{}, *partition_computer_,
+                                        /*file_system=*/nullptr,
+                                        /*table_root=*/"", /*branch=*/"main"));
     ASSERT_EQ(properties, unchanged);
+
+    Properties properties_with_stale_offset = properties;
+    properties_with_stale_offset[RealtimeCommitProperties::kOffsetsKey] = "stale.offsets";
+    ASSERT_OK_AND_ASSIGN(
+        Properties overwritten,
+        RealtimeCommitProperties::Build(
+            properties_with_stale_offset, std::optional<Snapshot>(MakeSnapshot(latest_properties)),
+            /*realtime_ranges=*/{}, /*reset_all_realtime_progress=*/true,
+            /*removed_realtime_partitions=*/{}, *partition_computer_,
+            /*file_system=*/nullptr, /*table_root=*/"", /*branch=*/"main"));
+    ASSERT_EQ("value", overwritten.at("custom"));
+    ASSERT_EQ(0, overwritten.count(RealtimeCommitProperties::kOffsetsKey));
+}
+
+TEST_F(RealtimeCommitPropertiesTest, BuildRemovesOnlyOverwrittenPartitions) {
+    RealtimePartitionBucket dt2_bucket0({{"dt", "2"}}, /*bucket=*/0);
+    RealtimePartitionBucket dt2_bucket1({{"dt", "2"}}, /*bucket=*/1);
+    RealtimePartitionBucket dt3_bucket0({{"dt", "3"}}, /*bucket=*/0);
+    RealtimeOffsetMap committed_offsets = {{dt2_bucket0, 3}, {dt2_bucket1, 4}, {dt3_bucket0, 5}};
+    ASSERT_OK_AND_ASSIGN(Snapshot latest_snapshot, MakeSnapshotWithOffsets(committed_offsets));
+    std::vector<std::map<std::string, std::string>> removed_partitions = {{{"dt", "2"}}};
+
+    ASSERT_OK_AND_ASSIGN(Properties properties,
+                         RealtimeCommitProperties::Build(
+                             /*properties=*/{}, latest_snapshot, /*realtime_ranges=*/{},
+                             /*reset_all_realtime_progress=*/false, removed_partitions,
+                             *partition_computer_, file_system_, directory_->Str(), "main"));
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap actual,
+                         RealtimeCommitProperties::ReadOffsets(
+                             std::optional<Snapshot>(MakeSnapshot(properties)), file_system_));
+    RealtimeOffsetMap expected = {{dt3_bucket0, 5}};
+    ASSERT_EQ(expected, actual);
 }
 
 TEST_F(RealtimeCommitPropertiesTest, BuildWritesMergedProgress) {
@@ -322,10 +427,13 @@ TEST_F(RealtimeCommitPropertiesTest, BuildWritesMergedProgress) {
         {RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0), OffsetRange(7, 9)}};
     std::map<std::string, std::string> properties = {{"custom", "value"}};
 
-    ASSERT_OK_AND_ASSIGN(Properties merged,
-                         RealtimeCommitProperties::Build(
-                             properties, std::optional<Snapshot>(MakeSnapshot(latest_properties)),
-                             ranges, file_system_, directory_->Str(), "main"));
+    ASSERT_OK_AND_ASSIGN(
+        Properties merged,
+        RealtimeCommitProperties::Build(
+            properties, std::optional<Snapshot>(MakeSnapshot(latest_properties)), ranges,
+            /*reset_all_realtime_progress=*/false,
+            /*removed_realtime_partitions=*/{}, *partition_computer_, file_system_,
+            directory_->Str(), "main"));
     ASSERT_EQ("value", merged.at("custom"));
     ASSERT_NE(latest_offsets_path, merged.at(RealtimeCommitProperties::kOffsetsKey));
 
@@ -344,6 +452,8 @@ TEST_F(RealtimeCommitPropertiesTest, BuildRequiresFileSystem) {
         {RealtimePartitionBucket({{"dt", "2"}}, /*bucket=*/0), OffsetRange(0, 2)}};
     ASSERT_NOK_WITH_MSG(RealtimeCommitProperties::Build(
                             /*properties=*/{}, /*latest_snapshot=*/std::nullopt, ranges,
+                            /*reset_all_realtime_progress=*/false,
+                            /*removed_realtime_partitions=*/{}, *partition_computer_,
                             /*file_system=*/nullptr, directory_->Str(), "main"),
                         "file system is null");
 }
diff --git a/src/paimon/core/operation/expire_snapshots.cpp b/src/paimon/core/operation/expire_snapshots.cpp
index d6f4b1212..a5bd25053 100644
--- a/src/paimon/core/operation/expire_snapshots.cpp
+++ b/src/paimon/core/operation/expire_snapshots.cpp
@@ -38,6 +38,7 @@
 #include "paimon/core/manifest/manifest_file.h"
 #include "paimon/core/manifest/manifest_file_meta.h"
 #include "paimon/core/manifest/manifest_list.h"
+#include "paimon/core/operation/commit/realtime_commit_properties.h"
 #include "paimon/core/snapshot.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/snapshot_manager.h"
@@ -50,13 +51,14 @@ ExpireSnapshots::ExpireSnapshots(const std::shared_ptr<SnapshotManager>& snapsho
                                  const std::shared_ptr<ManifestList>& manifest_list,
                                  const std::shared_ptr<ManifestFile>& manifest_file,
                                  const std::shared_ptr<FileSystem>& fs, const ExpireConfig& config,
-                                 const std::shared_ptr<Executor>& executor)
+                                 bool realtime_enabled, const std::shared_ptr<Executor>& executor)
     : snapshot_manager_(snapshot_manager),
       path_factory_(path_factory),
       manifest_list_(manifest_list),
       manifest_file_(manifest_file),
       fs_(fs),
       config_(config),
+      realtime_enabled_(realtime_enabled),
       executor_(executor),
       logger_(Logger::GetLogger("ExpireSnapshots")) {}
 
@@ -157,8 +159,24 @@ Result<int32_t> ExpireSnapshots::ExpireUntil(int64_t earliest_snapshot_id,
     std::vector<Snapshot> retained_snapshots;
     PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, snapshot_manager_->LoadSnapshot(end_exclusive_id));
     retained_snapshots.push_back(snapshot);
+    std::set<std::string> retained_offset_files;
+    if (realtime_enabled_) {
+        PAIMON_ASSIGN_OR_RAISE(std::vector<Snapshot> all_snapshots,
+                               snapshot_manager_->GetAllSnapshots());
+        for (const Snapshot& retained_snapshot : all_snapshots) {
+            if (retained_snapshot.Id() < end_exclusive_id) {
+                continue;
+            }
+            std::optional<std::string> offsets_path =
+                RealtimeCommitProperties::GetOffsetsPath(retained_snapshot);
+            if (offsets_path) {
+                retained_offset_files.insert(offsets_path.value());
+            }
+        }
+    }
     std::set<std::string> skipping_sets;
     PAIMON_RETURN_NOT_OK(GetManifestSkippingSet(retained_snapshots, &skipping_sets));
+    std::set<std::string> expired_offset_files;
     for (int64_t id = begin_inclusive_id; id < end_exclusive_id; id++) {
         PAIMON_LOG_DEBUG(logger_, "Ready to delete manifests in snapshot #%ld", id);
         PAIMON_ASSIGN_OR_RAISE(bool exist, snapshot_manager_->SnapshotExists(id));
@@ -169,10 +187,24 @@ Result<int32_t> ExpireSnapshots::ExpireUntil(int64_t earliest_snapshot_id,
         PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, snapshot_manager_->LoadSnapshot(id));
         PAIMON_RETURN_NOT_OK(CleanUnusedManifests(snapshot.BaseManifestList(), skipping_sets));
         PAIMON_RETURN_NOT_OK(CleanUnusedManifests(snapshot.DeltaManifestList(), skipping_sets));
+        if (realtime_enabled_) {
+            std::optional<std::string> offsets_path =
+                RealtimeCommitProperties::GetOffsetsPath(snapshot);
+            if (offsets_path) {
+                expired_offset_files.insert(offsets_path.value());
+            }
+        }
         auto status = fs_->Delete(snapshot_manager_->SnapshotPath(id));
         // delete quietly will ignore any status error
         (void)status;
     }
+    for (const std::string& offsets_path : expired_offset_files) {
+        if (retained_offset_files.count(offsets_path) == 0) {
+            auto status = fs_->Delete(offsets_path);
+            // Orphan cleanup can retry offset files that fail to delete here.
+            (void)status;
+        }
+    }
     PAIMON_RETURN_NOT_OK(snapshot_manager_->CommitEarliestHint(end_exclusive_id));
     return end_exclusive_id - begin_inclusive_id;
 }
diff --git a/src/paimon/core/operation/expire_snapshots.h b/src/paimon/core/operation/expire_snapshots.h
index 75b5fff59..238f599ed 100644
--- a/src/paimon/core/operation/expire_snapshots.h
+++ b/src/paimon/core/operation/expire_snapshots.h
@@ -51,7 +51,7 @@ class ExpireSnapshots {
                     const std::shared_ptr<ManifestList>& manifest_list,
                     const std::shared_ptr<ManifestFile>& manifest_file,
                     const std::shared_ptr<FileSystem>& fs, const ExpireConfig& config,
-                    const std::shared_ptr<Executor>& executor);
+                    bool realtime_enabled, const std::shared_ptr<Executor>& executor);
 
     Result<int32_t> Expire();
 
@@ -74,6 +74,7 @@ class ExpireSnapshots {
     std::shared_ptr<ManifestFile> manifest_file_;
     std::shared_ptr<FileSystem> fs_;
     ExpireConfig config_;
+    bool realtime_enabled_;
     std::shared_ptr<Executor> executor_;
     std::unordered_map<BinaryRow, std::set<std::int32_t>> deletion_buckets_;
 
diff --git a/src/paimon/core/operation/expire_snapshots_test.cpp b/src/paimon/core/operation/expire_snapshots_test.cpp
index d6b965ba5..6a9e46ba7 100644
--- a/src/paimon/core/operation/expire_snapshots_test.cpp
+++ b/src/paimon/core/operation/expire_snapshots_test.cpp
@@ -170,7 +170,7 @@ TEST_F(ExpireSnapshotsTest, TestInvalidInput) {
     {
         ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({}));
         ExpireSnapshots expire(mgr, path_factory_, manifest_list_, manifest_file_, fs_,
-                               options.GetExpireConfig(), executor_);
+                               options.GetExpireConfig(), options.RealtimeEnabled(), executor_);
         ASSERT_OK_AND_ASSIGN(int32_t count, expire.Expire());
         ASSERT_EQ(count, 0);
     }
@@ -178,7 +178,7 @@ TEST_F(ExpireSnapshotsTest, TestInvalidInput) {
         ASSERT_OK_AND_ASSIGN(CoreOptions options,
                              CoreOptions::FromMap({{Options::SNAPSHOT_NUM_RETAINED_MIN, "0"}}));
         ExpireSnapshots expire(mgr, path_factory_, manifest_list_, manifest_file_, fs_,
-                               options.GetExpireConfig(), executor_);
+                               options.GetExpireConfig(), options.RealtimeEnabled(), executor_);
         ASSERT_NOK(expire.Expire());
     }
     {
@@ -186,7 +186,7 @@ TEST_F(ExpireSnapshotsTest, TestInvalidInput) {
                              CoreOptions::FromMap({{Options::SNAPSHOT_NUM_RETAINED_MIN, "10"},
                                                    {Options::SNAPSHOT_NUM_RETAINED_MAX, "9"}}));
         ExpireSnapshots expire(mgr, path_factory_, manifest_list_, manifest_file_, fs_,
-                               options.GetExpireConfig(), executor_);
+                               options.GetExpireConfig(), options.RealtimeEnabled(), executor_);
         ASSERT_NOK(expire.Expire());
     }
     {
@@ -194,7 +194,7 @@ TEST_F(ExpireSnapshotsTest, TestInvalidInput) {
                              CoreOptions::FromMap({{Options::SNAPSHOT_NUM_RETAINED_MIN, "10"},
                                                    {Options::SNAPSHOT_NUM_RETAINED_MAX, "10"}}));
         ExpireSnapshots expire(mgr, path_factory_, manifest_list_, manifest_file_, fs_,
-                               options.GetExpireConfig(), executor_);
+                               options.GetExpireConfig(), options.RealtimeEnabled(), executor_);
         ASSERT_OK_AND_ASSIGN(int32_t count, expire.Expire());
         ASSERT_EQ(count, 0);
     }
@@ -204,7 +204,7 @@ TEST_F(ExpireSnapshotsTest, TestInvalidInput) {
                                                    {Options::SNAPSHOT_NUM_RETAINED_MIN, "10"},
                                                    {Options::SNAPSHOT_NUM_RETAINED_MAX, "10"}}));
         ExpireSnapshots expire(mgr, path_factory_, manifest_list_, manifest_file_, fs_,
-                               options.GetExpireConfig(), executor_);
+                               options.GetExpireConfig(), options.RealtimeEnabled(), executor_);
         ASSERT_NOK(expire.Expire());
     }
     {
@@ -212,7 +212,7 @@ TEST_F(ExpireSnapshotsTest, TestInvalidInput) {
                              CoreOptions::FromMap({{Options::SNAPSHOT_NUM_RETAINED_MIN, "10"},
                                                    {Options::SNAPSHOT_NUM_RETAINED_MAX, "10"}}));
         ExpireSnapshots expire(nullptr, path_factory_, manifest_list_, manifest_file_, fs_,
-                               options.GetExpireConfig(), executor_);
+                               options.GetExpireConfig(), options.RealtimeEnabled(), executor_);
         ASSERT_NOK(expire.Expire());
     }
 }
@@ -222,7 +222,7 @@ TEST_F(ExpireSnapshotsTest, TestGetDataFileToDelete) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({}));
     {
         ExpireSnapshots expire(mgr, path_factory_, manifest_list_, manifest_file_, fs_,
-                               options.GetExpireConfig(), executor_);
+                               options.GetExpireConfig(), options.RealtimeEnabled(), executor_);
         std::map<std::string, ManifestEntry> data_file_to_delete;
         std::vector<ManifestEntry> data_file_entries;
         data_file_entries.push_back(CreateManifestEntry("file1", /*bucket=*/0, FileKind::Delete()));
@@ -237,7 +237,7 @@ TEST_F(ExpireSnapshotsTest, TestGetDataFileToDelete) {
     }
     {
         ExpireSnapshots expire(mgr, path_factory_, manifest_list_, manifest_file_, fs_,
-                               options.GetExpireConfig(), executor_);
+                               options.GetExpireConfig(), options.RealtimeEnabled(), executor_);
         std::map<std::string, ManifestEntry> data_file_to_delete;
         std::vector<ManifestEntry> data_file_entries;
         data_file_entries.push_back(CreateManifestEntry("file1", /*bucket=*/0, FileKind::Add()));
diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp
index b2ba17fa0..ad0942ade 100644
--- a/src/paimon/core/operation/file_store_commit.cpp
+++ b/src/paimon/core/operation/file_store_commit.cpp
@@ -175,7 +175,7 @@ Result<std::unique_ptr<FileStoreCommit>> FileStoreCommit::Create(
 
     auto expire_snapshots = std::make_shared<ExpireSnapshots>(
         snapshot_manager, path_factory, manifest_list, manifest_file, options.GetFileSystem(),
-        options.GetExpireConfig(), ctx->GetExecutor());
+        options.GetExpireConfig(), options.RealtimeEnabled(), ctx->GetExecutor());
 
     CommitScanner::ScanSupplier scan_supplier;
     if (table_schema.value()->PrimaryKeys().empty()) {
diff --git a/src/paimon/core/operation/file_store_commit_impl.cpp b/src/paimon/core/operation/file_store_commit_impl.cpp
index 5388b9e85..8f2b3c732 100644
--- a/src/paimon/core/operation/file_store_commit_impl.cpp
+++ b/src/paimon/core/operation/file_store_commit_impl.cpp
@@ -80,6 +80,7 @@
 #include "paimon/core/table/sink/commit_message_impl.h"
 #include "paimon/core/utils/duration.h"
 #include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/core/utils/partition_utils.h"
 #include "paimon/core/utils/snapshot_manager.h"
 #include "paimon/file_store_write.h"
 #include "paimon/fs/file_system.h"
@@ -96,17 +97,6 @@ constexpr const char* kCommitStrictModeLastSafeSnapshot = "commit.strict-mode.la
 constexpr const char* kSequenceSnapshotOrdering = "sequence.snapshot-ordering";
 constexpr const char* kPkClusteringOverride = "pk-clustering-override";
 
-bool MatchPartitionSpec(const std::map<std::string, std::string>& partition,
-                        const std::map<std::string, std::string>& partition_spec) {
-    for (const auto& [key, value] : partition_spec) {
-        auto iter = partition.find(key);
-        if (iter == partition.end() || iter->second != value) {
-            return false;
-        }
-    }
-    return true;
-}
-
 }  // namespace
 
 Status FileStoreCommitImpl::ValidateCommitOptions(const CoreOptions& options) {
@@ -337,7 +327,6 @@ Result<bool> FileStoreCommitImpl::RollbackToAsLatest(int64_t target_snapshot_id)
     // snapshots between the target and the previous latest, breaking the global uniqueness of
     // _ROW_ID. Keep the larger of the previous latest and the target nextRowId.
     std::optional<int64_t> next_row_id = std::max(latest.NextRowId(), target_snapshot.NextRowId());
-
     int64_t delta_record_count =
         ManifestEntry::RecordCountAdd(delta_files) - ManifestEntry::RecordCountDelete(delta_files);
     Snapshot new_snapshot(
@@ -668,7 +657,10 @@ Status FileStoreCommitImpl::ExecuteOverwrite(
             PAIMON_ASSIGN_OR_RAISE(partition_map, PartitionToMap(entry.Partition()));
             bool belongs_to_overwrite_partition = false;
             for (const auto& partition_spec : partitions) {
-                if (MatchPartitionSpec(partition_map, partition_spec)) {
+                PAIMON_ASSIGN_OR_RAISE(
+                    bool matched, PartitionUtils::MatchPartitionSpec(partition_map, partition_spec,
+                                                                     *partition_computer_));
+                if (matched) {
                     belongs_to_overwrite_partition = true;
                     break;
                 }
@@ -708,6 +700,8 @@ Status FileStoreCommitImpl::ExecuteOverwrite(
                                          changes->compact_index_files, identifier, watermark,
                                          committable->Properties(), /*realtime_ranges=*/{},
                                          Snapshot::CommitKind::Compact(),
+                                         /*reset_all_realtime_progress=*/false,
+                                         /*removed_realtime_partitions=*/{},
                                          /*detect_conflicts=*/true,
                                          /*retry_on_conflict=*/true));
         *attempt += cnt;
@@ -809,8 +803,12 @@ Result<int32_t> FileStoreCommitImpl::TryOverwrite(
     const std::map<std::string, std::string>& properties) {
     std::shared_ptr<CommitChangesProvider> changes_provider =
         commit_scanner_->OverwriteChangesProvider(partitions, changes, index_entries);
+    // ExecuteOverwrite has already resolved dynamic overwrite to the concrete affected
+    // partitions. Only an empty final partition list denotes a full-table replacement.
+    const bool reset_all_realtime_progress = partitions.empty();
     return TryCommit(changes_provider, commit_identifier, watermark, properties,
                      /*realtime_ranges=*/{}, Snapshot::CommitKind::Overwrite(),
+                     reset_all_realtime_progress, partitions,
                      /*detect_conflicts=*/true,
                      /*retry_on_conflict=*/true);
 }
@@ -859,7 +857,9 @@ Status FileStoreCommitImpl::Commit(
             TryCommit(changes.append_table_files, changes.append_changelog,
                       changes.append_index_files, committable->Identifier(),
                       committable->Watermark(), committable->Properties(), realtime_ranges,
-                      commit_kind, check_append_files, retry_on_conflict));
+                      commit_kind,
+                      /*reset_all_realtime_progress=*/false,
+                      /*removed_realtime_partitions=*/{}, check_append_files, retry_on_conflict));
         attempt += cnt;
         generated_snapshot += 1;
     }
@@ -870,6 +870,8 @@ Status FileStoreCommitImpl::Commit(
                                          changes.compact_index_files, committable->Identifier(),
                                          committable->Watermark(), committable->Properties(),
                                          /*realtime_ranges=*/{}, Snapshot::CommitKind::Compact(),
+                                         /*reset_all_realtime_progress=*/false,
+                                         /*removed_realtime_partitions=*/{},
                                          /*detect_conflicts=*/true, retry_on_conflict));
         attempt += cnt;
         generated_snapshot += 1;
@@ -889,6 +891,9 @@ Status FileStoreCommitImpl::Commit(
 Result<int64_t> FileStoreCommitImpl::CommitWithProgress(
     const std::vector<RealtimeCommitProgress>& realtime_commits, int64_t identifier,
     std::optional<int64_t> watermark) {
+    if (!options_.RealtimeEnabled()) {
+        return Status::Invalid("CommitWithProgress requires realtime.enabled=true");
+    }
     if (realtime_commits.empty()) {
         return Status::Invalid("real-time commits must not be empty");
     }
@@ -931,6 +936,30 @@ Result<int64_t> FileStoreCommitImpl::CommitWithProgress(
 
     std::shared_ptr<ManifestCommittable> committable =
         CreateManifestCommittable(identifier, commit_messages, watermark, /*properties=*/{});
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<ManifestCommittable>> pending_committables,
+                           FilterCommitted({committable}));
+    const bool identifier_committed = pending_committables.empty();
+
+    PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
+                           snapshot_manager_->LatestSnapshot());
+    PAIMON_ASSIGN_OR_RAISE(RealtimeOffsetMap committed_offsets,
+                           RealtimeCommitProperties::ReadOffsets(latest_snapshot, fs_));
+    PAIMON_ASSIGN_OR_RAISE(bool ranges_committed, RealtimeCommitProperties::AreRangesCommitted(
+                                                      committed_offsets, realtime_ranges));
+    if (ranges_committed != identifier_committed) {
+        return Status::Invalid(
+            ranges_committed
+                ? "real-time offset ranges were committed by another commit user or identifier"
+                : "real-time commit identifier was committed without the requested offset ranges");
+    }
+    if (ranges_committed) {
+        if (!latest_snapshot) {
+            return Status::Invalid("real-time commit ranges are covered without a snapshot");
+        }
+        return latest_snapshot->Id();
+    }
+
+    PAIMON_RETURN_NOT_OK(CheckFilesExistence(pending_committables));
     const int64_t previous_snapshot_id = last_committed_snapshot_id_;
     PAIMON_RETURN_NOT_OK(Commit(committable, /*check_append_files=*/false,
                                 /*retry_on_conflict=*/false, realtime_ranges));
@@ -946,18 +975,23 @@ Result<int32_t> FileStoreCommitImpl::TryCommit(
     const std::vector<IndexManifestEntry>& index_entries, int64_t identifier,
     std::optional<int64_t> watermark, const std::map<std::string, std::string>& properties,
     const std::map<RealtimePartitionBucket, OffsetRange>& realtime_ranges,
-    Snapshot::CommitKind commit_kind, bool detect_conflicts, bool retry_on_conflict) {
+    Snapshot::CommitKind commit_kind, bool reset_all_realtime_progress,
+    const std::vector<std::map<std::string, std::string>>& removed_realtime_partitions,
+    bool detect_conflicts, bool retry_on_conflict) {
     std::shared_ptr<CommitChangesProvider> changes_provider =
         CommitChangesProvider::Provider(delta_files, changelog_files, index_entries);
     return TryCommit(changes_provider, identifier, watermark, properties, realtime_ranges,
-                     commit_kind, detect_conflicts, retry_on_conflict);
+                     commit_kind, reset_all_realtime_progress, removed_realtime_partitions,
+                     detect_conflicts, retry_on_conflict);
 }
 
 Result<int32_t> FileStoreCommitImpl::TryCommit(
     const std::shared_ptr<CommitChangesProvider>& changes_provider, int64_t identifier,
     std::optional<int64_t> watermark, const std::map<std::string, std::string>& properties,
     const std::map<RealtimePartitionBucket, OffsetRange>& realtime_ranges,
-    Snapshot::CommitKind commit_kind, bool detect_conflicts, bool retry_on_conflict) {
+    Snapshot::CommitKind commit_kind, bool reset_all_realtime_progress,
+    const std::vector<std::map<std::string, std::string>>& removed_realtime_partitions,
+    bool detect_conflicts, bool retry_on_conflict) {
     int32_t retry_count = 0;
     int64_t start_millis = DateTimeUtils::GetCurrentUTCTimeUs() / 1000;
     while (true) {
@@ -968,7 +1002,9 @@ Result<int32_t> FileStoreCommitImpl::TryCommit(
         using SnapshotProperties = std::map<std::string, std::string>;
         PAIMON_ASSIGN_OR_RAISE(
             SnapshotProperties snapshot_properties,
-            RealtimeCommitProperties::Build(properties, latest_snapshot, realtime_ranges, fs_,
+            RealtimeCommitProperties::Build(properties, latest_snapshot, realtime_ranges,
+                                            reset_all_realtime_progress,
+                                            removed_realtime_partitions, *partition_computer_, fs_,
                                             root_path_, snapshot_manager_->Branch()));
         PAIMON_ASSIGN_OR_RAISE(
             bool commit_success,
diff --git a/src/paimon/core/operation/file_store_commit_impl.h b/src/paimon/core/operation/file_store_commit_impl.h
index 7c8044148..ca4de22f9 100644
--- a/src/paimon/core/operation/file_store_commit_impl.h
+++ b/src/paimon/core/operation/file_store_commit_impl.h
@@ -185,21 +185,23 @@ class FileStoreCommitImpl : public FileStoreCommit {
     void ReportCommit(const ManifestEntryChanges& changes, int64_t commit_duration,
                       int32_t generated_snapshot, int32_t attempt);
 
-    Result<int32_t> TryCommit(const std::vector<ManifestEntry>& delta_files,
-                              const std::vector<ManifestEntry>& changelog_files,
-                              const std::vector<IndexManifestEntry>& index_entries,
-                              int64_t identifier, std::optional<int64_t> watermark,
-                              const std::map<std::string, std::string>& properties,
-                              const std::map<RealtimePartitionBucket, OffsetRange>& realtime_ranges,
-                              Snapshot::CommitKind commit_kind, bool detect_conflicts,
-                              bool retry_on_conflict);
-
-    Result<int32_t> TryCommit(const std::shared_ptr<CommitChangesProvider>& changes_provider,
-                              int64_t identifier, std::optional<int64_t> watermark,
-                              const std::map<std::string, std::string>& properties,
-                              const std::map<RealtimePartitionBucket, OffsetRange>& realtime_ranges,
-                              Snapshot::CommitKind commit_kind, bool detect_conflicts,
-                              bool retry_on_conflict);
+    Result<int32_t> TryCommit(
+        const std::vector<ManifestEntry>& delta_files,
+        const std::vector<ManifestEntry>& changelog_files,
+        const std::vector<IndexManifestEntry>& index_entries, int64_t identifier,
+        std::optional<int64_t> watermark, const std::map<std::string, std::string>& properties,
+        const std::map<RealtimePartitionBucket, OffsetRange>& realtime_ranges,
+        Snapshot::CommitKind commit_kind, bool reset_all_realtime_progress,
+        const std::vector<std::map<std::string, std::string>>& removed_realtime_partitions,
+        bool detect_conflicts, bool retry_on_conflict);
+
+    Result<int32_t> TryCommit(
+        const std::shared_ptr<CommitChangesProvider>& changes_provider, int64_t identifier,
+        std::optional<int64_t> watermark, const std::map<std::string, std::string>& properties,
+        const std::map<RealtimePartitionBucket, OffsetRange>& realtime_ranges,
+        Snapshot::CommitKind commit_kind, bool reset_all_realtime_progress,
+        const std::vector<std::map<std::string, std::string>>& removed_realtime_partitions,
+        bool detect_conflicts, bool retry_on_conflict);
 
     Result<bool> TryCommitOnce(const std::vector<ManifestEntry>& delta_files,
                                const std::vector<ManifestEntry>& changelog_files,
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 940608c72..6807ae35e 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -120,6 +120,9 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     }
 
     bool ignore_previous_files = ctx->IgnorePreviousFiles();
+    if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) {
+        return Status::Invalid("real-time write requires realtime.enabled=true");
+    }
     if (schema->PrimaryKeys().empty()) {
         // append table
         bool need_dv_maintainer_factory = options.DeletionVectorsEnabled();
diff --git a/src/paimon/core/operation/orphan_files_cleaner_impl.cpp b/src/paimon/core/operation/orphan_files_cleaner_impl.cpp
index 6b8e5ba04..015d13408 100644
--- a/src/paimon/core/operation/orphan_files_cleaner_impl.cpp
+++ b/src/paimon/core/operation/orphan_files_cleaner_impl.cpp
@@ -35,6 +35,7 @@
 #include "paimon/core/manifest/manifest_file.h"
 #include "paimon/core/manifest/manifest_file_meta.h"
 #include "paimon/core/manifest/manifest_list.h"
+#include "paimon/core/operation/commit/realtime_commit_properties.h"
 #include "paimon/core/operation/metrics/clean_metrics.h"
 #include "paimon/core/snapshot.h"
 #include "paimon/core/utils/duration.h"
@@ -85,7 +86,7 @@ bool OrphanFilesCleanerImpl::SupportToClean(const std::string& file_name) {
             return true;
         }
     }
-    return false;
+    return StringUtils::EndsWith(file_name, ".offsets");
 }
 
 Result<std::set<std::string>> OrphanFilesCleanerImpl::Clean() {
@@ -156,6 +157,10 @@ Result<std::set<std::string>> OrphanFilesCleanerImpl::ListPaimonFileDirs() const
     std::set<std::string> paimon_file_dirs;
     paimon_file_dirs.insert(snapshot_manager_->SnapshotDirectory());
     paimon_file_dirs.insert(FileStorePathFactory::ManifestPath(root_path_));
+    if (options_.RealtimeEnabled()) {
+        paimon_file_dirs.insert(
+            RealtimeCommitProperties::OffsetsDirectory(root_path_, options_.GetBranch()));
+    }
     // TODO(jinli.zjw): support clean index, stats, changelog in the future
     // paimon_file_dirs.insert(FileStorePathFactory::IndexPath(root_path_));
     // paimon_file_dirs.insert(FileStorePathFactory::StatisticsPath(root_path_));
@@ -294,6 +299,13 @@ Result<std::set<std::string>> OrphanFilesCleanerImpl::GetUsedFilesBySnapshot(
     used_files.insert(SnapshotManager::SNAPSHOT_PREFIX + std::to_string(snapshot.Id()));
     used_files.insert(snapshot.BaseManifestList());
     used_files.insert(snapshot.DeltaManifestList());
+    if (options_.RealtimeEnabled()) {
+        std::optional<std::string> offsets_path =
+            RealtimeCommitProperties::GetOffsetsPath(snapshot);
+        if (offsets_path) {
+            used_files.insert(PathUtil::GetName(offsets_path.value()));
+        }
+    }
     std::vector<ManifestFileMeta> manifests;
     PAIMON_RETURN_NOT_OK(manifest_list_->ReadIfFileExist(snapshot.BaseManifestList(),
                                                          /*filter=*/nullptr, &manifests));
diff --git a/src/paimon/core/realtime/arrow_realtime_store.cpp b/src/paimon/core/realtime/arrow_realtime_store.cpp
index cf1e37aac..18087a29f 100644
--- a/src/paimon/core/realtime/arrow_realtime_store.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store.cpp
@@ -25,12 +25,18 @@
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
+#include "arrow/compute/api_aggregate.h"
+#include "paimon/common/data/columnar/columnar_array.h"
+#include "paimon/common/data/columnar/columnar_row.h"
 #include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/predicate/predicate_filter.h"
 #include "paimon/common/reader/complete_row_kind_batch_reader.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/row_kind.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/projected_array.h"
+#include "paimon/common/utils/projected_row.h"
 #include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
 
@@ -53,6 +59,26 @@ uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
     return result;
 }
 
+bool SupportsMinMax(const std::shared_ptr<arrow::DataType>& type) {
+    switch (type->id()) {
+        case arrow::Type::BOOL:
+        case arrow::Type::INT8:
+        case arrow::Type::INT16:
+        case arrow::Type::INT32:
+        case arrow::Type::INT64:
+        case arrow::Type::FLOAT:
+        case arrow::Type::DOUBLE:
+        case arrow::Type::STRING:
+        case arrow::Type::BINARY:
+        case arrow::Type::DATE32:
+        case arrow::Type::TIMESTAMP:
+        case arrow::Type::DECIMAL128:
+            return true;
+        default:
+            return false;
+    }
+}
+
 }  // namespace
 
 class ArrowRealtimeStore::Segment : public RealtimeSegmentHandle {
@@ -165,11 +191,17 @@ class ArrowRealtimeStore::QueryBatchReader : public BatchReader {
  public:
     QueryBatchReader(const ReadView* view, int64_t offset_begin,
                      const std::shared_ptr<arrow::Schema>& read_schema,
-                     const std::shared_ptr<arrow::MemoryPool>& arrow_pool)
+                     const std::shared_ptr<PredicateFilter>& predicate_filter,
+                     std::vector<int32_t>&& statistics_mapping,
+                     const std::shared_ptr<arrow::MemoryPool>& arrow_pool,
+                     const std::shared_ptr<MemoryPool>& memory_pool)
         : view_(view),
           offset_begin_(offset_begin),
           read_schema_(read_schema),
           arrow_pool_(arrow_pool),
+          memory_pool_(memory_pool),
+          predicate_filter_(predicate_filter),
+          statistics_mapping_(std::move(statistics_mapping)),
           metrics_(std::make_shared<MetricsImpl>()) {}
 
     Result<ReadBatch> NextBatch() override {
@@ -189,6 +221,10 @@ class ArrowRealtimeStore::QueryBatchReader : public BatchReader {
             if (stored.offset_range.end <= offset_begin_) {
                 continue;
             }
+            PAIMON_ASSIGN_OR_RAISE(bool may_match, MayMatch(stored));
+            if (!may_match) {
+                continue;
+            }
             int64_t begin = std::max<int64_t>(0, offset_begin_ - stored.offset_range.begin);
             PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::StructArray> output, BuildOutput(stored));
             RoaringBitmap32 candidate_rows;
@@ -213,6 +249,25 @@ class ArrowRealtimeStore::QueryBatchReader : public BatchReader {
     }
 
  private:
+    Result<bool> MayMatch(const StoredBatch& stored) const {
+        if (!predicate_filter_ || !stored.statistics) {
+            return true;
+        }
+        const BatchStatistics& statistics = stored.statistics.value();
+        std::shared_ptr<InternalRow> min_row = std::make_shared<ColumnarRow>(
+            statistics.min_values, statistics.min_values->fields(), memory_pool_, /*row_id=*/0);
+        std::shared_ptr<InternalRow> max_row = std::make_shared<ColumnarRow>(
+            statistics.max_values, statistics.max_values->fields(), memory_pool_, /*row_id=*/0);
+        ProjectedRow projected_min(min_row, statistics_mapping_);
+        ProjectedRow projected_max(max_row, statistics_mapping_);
+        std::shared_ptr<InternalArray> null_counts =
+            std::make_shared<ColumnarArray>(statistics.null_counts.get(), memory_pool_,
+                                            /*offset=*/0, statistics.null_counts->length());
+        ProjectedArray projected_null_counts(null_counts, statistics_mapping_);
+        return predicate_filter_->Test(read_schema_, stored.data->length(), projected_min,
+                                       projected_max, projected_null_counts);
+    }
+
     Result<std::shared_ptr<arrow::StructArray>> BuildOutput(const StoredBatch& stored) {
         PAIMON_ASSIGN_OR_RAISE(
             std::shared_ptr<arrow::Array> projected,
@@ -231,14 +286,81 @@ class ArrowRealtimeStore::QueryBatchReader : public BatchReader {
     int64_t offset_begin_;
     std::shared_ptr<arrow::Schema> read_schema_;
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    std::shared_ptr<MemoryPool> memory_pool_;
+    std::shared_ptr<PredicateFilter> predicate_filter_;
+    std::vector<int32_t> statistics_mapping_;
     std::shared_ptr<Metrics> metrics_;
     size_t next_batch_ = 0;
 };
 
 ArrowRealtimeStore::ArrowRealtimeStore(const std::shared_ptr<arrow::Schema>& write_schema,
+                                       StatisticsMode statistics_mode,
                                        const std::shared_ptr<MemoryPool>& memory_pool,
                                        const std::shared_ptr<arrow::MemoryPool>& arrow_pool)
-    : write_schema_(write_schema), memory_pool_(memory_pool), arrow_pool_(arrow_pool) {}
+    : write_schema_(write_schema),
+      memory_pool_(memory_pool),
+      arrow_pool_(arrow_pool),
+      statistics_mode_(statistics_mode) {}
+
+Result<std::optional<ArrowRealtimeStore::BatchStatistics>> ArrowRealtimeStore::CollectStatistics(
+    const std::shared_ptr<arrow::StructArray>& data) const {
+    if (statistics_mode_ == StatisticsMode::NONE) {
+        return std::optional<BatchStatistics>();
+    }
+
+    arrow::ArrayVector min_values;
+    arrow::ArrayVector max_values;
+    min_values.reserve(data->num_fields());
+    max_values.reserve(data->num_fields());
+    arrow::Int64Builder null_count_builder(arrow_pool_.get());
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(null_count_builder.Reserve(data->num_fields()));
+    arrow::compute::ScalarAggregateOptions aggregate_options;
+    aggregate_options.skip_nulls = true;
+    aggregate_options.min_count = 1;
+    arrow::compute::ExecContext exec_context(arrow_pool_.get());
+
+    for (const std::shared_ptr<arrow::Array>& field : data->fields()) {
+        null_count_builder.UnsafeAppend(field->null_count());
+        if (!SupportsMinMax(field->type())) {
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> min_value,
+                arrow::MakeArrayOfNull(field->type(), /*length=*/1, arrow_pool_.get()));
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> max_value,
+                arrow::MakeArrayOfNull(field->type(), /*length=*/1, arrow_pool_.get()));
+            min_values.push_back(std::move(min_value));
+            max_values.push_back(std::move(max_value));
+            continue;
+        }
+
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            arrow::Datum min_max, arrow::compute::MinMax(field, aggregate_options, &exec_context));
+        std::shared_ptr<arrow::StructScalar> min_max_scalar =
+            std::dynamic_pointer_cast<arrow::StructScalar>(min_max.scalar());
+        if (!min_max_scalar || min_max_scalar->value.size() != 2) {
+            return Status::Invalid("Arrow min_max did not produce min and max scalars");
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> min_value,
+            arrow::MakeArrayFromScalar(*min_max_scalar->value[0], /*length=*/1, arrow_pool_.get()));
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> max_value,
+            arrow::MakeArrayFromScalar(*min_max_scalar->value[1], /*length=*/1, arrow_pool_.get()));
+        min_values.push_back(std::move(min_value));
+        max_values.push_back(std::move(max_value));
+    }
+
+    std::shared_ptr<arrow::Array> null_counts_array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(null_count_builder.Finish(&null_counts_array));
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::StructArray> min_values_struct,
+        arrow::StructArray::Make(min_values, write_schema_->fields()));
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::StructArray> max_values_struct,
+        arrow::StructArray::Make(max_values, write_schema_->fields()));
+    return std::optional<BatchStatistics>(BatchStatistics{
+        std::move(min_values_struct), std::move(max_values_struct), std::move(null_counts_array)});
+}
 
 Status ArrowRealtimeStore::Write(RealtimeWriteBatch&& write_batch) {
     if (!write_batch.batch) {
@@ -264,16 +386,23 @@ Status ArrowRealtimeStore::Write(RealtimeWriteBatch&& write_batch) {
     }
     std::shared_ptr<arrow::StructArray> struct_array =
         checked_pointer_cast<arrow::StructArray>(data);
+    PAIMON_ASSIGN_OR_RAISE(std::optional<BatchStatistics> statistics,
+                           CollectStatistics(struct_array));
 
     std::lock_guard<std::mutex> lock(mutex_);
     if (building_range_ && write_batch.offset_range.begin != building_range_->end) {
         return Status::Invalid("real-time offset ranges must be contiguous");
     }
     uint64_t memory_usage = GetArrayMemoryUsage(struct_array->data());
+    if (statistics) {
+        memory_usage += GetArrayMemoryUsage(statistics->min_values->data()) +
+                        GetArrayMemoryUsage(statistics->max_values->data()) +
+                        GetArrayMemoryUsage(statistics->null_counts->data());
+    }
     building_memory_usage_ += memory_usage;
-    building_batches_.push_back(StoredBatch{std::move(struct_array),
-                                            write_batch.batch->GetRowKind(),
-                                            write_batch.offset_range, memory_usage});
+    building_batches_.push_back(
+        StoredBatch{std::move(struct_array), write_batch.batch->GetRowKind(),
+                    write_batch.offset_range, std::move(statistics), memory_usage});
     if (!building_range_) {
         building_range_ = write_batch.offset_range;
     } else {
@@ -329,13 +458,20 @@ Result<std::vector<std::unique_ptr<BatchReader>>> ArrowRealtimeStore::CreateQuer
     }
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> read_schema,
                                       arrow::ImportSchema(context.read_schema));
-    // TODO(xinyu.lxy): Support predicate pushdown after adding batch statistics or index metadata.
-    // The default Arrow store currently ignores context.predicate and
-    // context.enable_predicate_pushdown, and returns all offset-matching rows as candidates.
+    std::shared_ptr<PredicateFilter> predicate_filter;
+    if (context.enable_predicate_pushdown && context.predicate) {
+        predicate_filter = std::dynamic_pointer_cast<PredicateFilter>(context.predicate);
+    }
+    std::vector<int32_t> statistics_mapping;
+    statistics_mapping.reserve(read_schema->num_fields());
+    for (const std::shared_ptr<arrow::Field>& field : read_schema->fields()) {
+        statistics_mapping.push_back(write_schema_->GetFieldIndex(field->name()));
+    }
     std::vector<std::unique_ptr<BatchReader>> readers;
     if (arrow_view->GetOffsetRange() && arrow_view->GetOffsetRange()->end > offset_begin) {
         std::unique_ptr<BatchReader> reader = std::make_unique<QueryBatchReader>(
-            arrow_view.get(), offset_begin, read_schema, arrow_pool_);
+            arrow_view.get(), offset_begin, read_schema, predicate_filter,
+            std::move(statistics_mapping), arrow_pool_, memory_pool_);
         reader = std::make_unique<CompleteRowKindBatchReader>(std::move(reader), memory_pool_);
         readers.push_back(std::move(reader));
     }
diff --git a/src/paimon/core/realtime/arrow_realtime_store.h b/src/paimon/core/realtime/arrow_realtime_store.h
index bd9ecfbf7..97339852f 100644
--- a/src/paimon/core/realtime/arrow_realtime_store.h
+++ b/src/paimon/core/realtime/arrow_realtime_store.h
@@ -28,6 +28,7 @@
 #include "paimon/realtime/realtime_store.h"
 
 namespace arrow {
+class Array;
 class MemoryPool;
 class Schema;
 class StructArray;
@@ -40,6 +41,7 @@ class MemoryPool;
 class ArrowRealtimeStore : public RealtimeStore {
  public:
     ArrowRealtimeStore(const std::shared_ptr<arrow::Schema>& write_schema,
+                       StatisticsMode statistics_mode,
                        const std::shared_ptr<MemoryPool>& memory_pool,
                        const std::shared_ptr<arrow::MemoryPool>& arrow_pool);
 
@@ -61,10 +63,17 @@ class ArrowRealtimeStore : public RealtimeStore {
     uint64_t GetMemoryUsage() const override;
 
  private:
+    struct BatchStatistics {
+        std::shared_ptr<arrow::StructArray> min_values;
+        std::shared_ptr<arrow::StructArray> max_values;
+        std::shared_ptr<arrow::Array> null_counts;
+    };
+
     struct StoredBatch {
         std::shared_ptr<arrow::StructArray> data;
         std::vector<RecordBatch::RowKind> row_kinds;
         OffsetRange offset_range;
+        std::optional<BatchStatistics> statistics;
         uint64_t memory_usage;
     };
 
@@ -73,9 +82,13 @@ class ArrowRealtimeStore : public RealtimeStore {
     class CommitBatchReader;
     class QueryBatchReader;
 
+    Result<std::optional<BatchStatistics>> CollectStatistics(
+        const std::shared_ptr<arrow::StructArray>& data) const;
+
     std::shared_ptr<arrow::Schema> write_schema_;
     std::shared_ptr<MemoryPool> memory_pool_;
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    StatisticsMode statistics_mode_;
     mutable std::mutex mutex_;
     std::vector<StoredBatch> building_batches_;
     std::vector<std::shared_ptr<Segment>> sealed_segments_;
diff --git a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
index 0eecb5d02..1d7219c41 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
@@ -30,8 +30,8 @@
 namespace paimon {
 
 Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
-    std::unique_ptr<ArrowSchema> write_schema, const std::map<std::string, std::string>&,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
+    std::unique_ptr<ArrowSchema> write_schema, StatisticsMode statistics_mode,
+    const std::map<std::string, std::string>&, const std::shared_ptr<MemoryPool>& memory_pool) {
     if (!write_schema || !write_schema->release) {
         return Status::Invalid("real-time store write schema is null");
     }
@@ -42,7 +42,8 @@ Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> imported_schema,
                                       arrow::ImportSchema(write_schema.get()));
     std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
-    return std::make_shared<ArrowRealtimeStore>(imported_schema, memory_pool, arrow_pool);
+    return std::make_shared<ArrowRealtimeStore>(imported_schema, statistics_mode, memory_pool,
+                                                arrow_pool);
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
index cc08cc8e1..9aae99332 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
@@ -19,6 +19,7 @@
 
 #include "paimon/core/realtime/arrow_realtime_store.h"
 
+#include <map>
 #include <memory>
 #include <optional>
 #include <string>
@@ -28,7 +29,11 @@
 #include "arrow/c/bridge.h"
 #include "arrow/ipc/json_simple.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/checked_cast.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/realtime/arrow_realtime_store_factory.h"
 #include "paimon/record_batch.h"
 #include "paimon/testing/utils/testharness.h"
 
@@ -56,7 +61,11 @@ class ArrowRealtimeStoreTest : public testing::Test {
             {arrow::field("id", arrow::int64()), arrow::field("value", arrow::utf8())});
         pool_ = GetDefaultPool();
         arrow_pool_ = GetArrowPool(pool_);
-        store_ = std::make_shared<ArrowRealtimeStore>(schema_, pool_, arrow_pool_);
+        store_ = CreateStore(StatisticsMode::NONE);
+    }
+
+    std::shared_ptr<ArrowRealtimeStore> CreateStore(StatisticsMode statistics_mode) const {
+        return std::make_shared<ArrowRealtimeStore>(schema_, statistics_mode, pool_, arrow_pool_);
     }
 
     std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) const {
@@ -86,6 +95,21 @@ class ArrowRealtimeStoreTest : public testing::Test {
         return c_schema;
     }
 
+    std::vector<int64_t> ReadIds(const BatchReader::ReadBatchWithBitmap& batch) const {
+        std::shared_ptr<arrow::Array> array =
+            arrow::ImportArray(batch.first.first.get(), batch.first.second.get()).ValueOrDie();
+        std::shared_ptr<arrow::StructArray> struct_array =
+            checked_pointer_cast<arrow::StructArray>(array);
+        std::shared_ptr<arrow::Int64Array> ids =
+            checked_pointer_cast<arrow::Int64Array>(struct_array->field(/*pos=*/1));
+        std::vector<int64_t> result;
+        for (RoaringBitmap32::Iterator iter = batch.second.Begin(); iter != batch.second.End();
+             ++iter) {
+            result.push_back(ids->Value(*iter));
+        }
+        return result;
+    }
+
  protected:
     std::shared_ptr<arrow::Schema> schema_;
     std::shared_ptr<MemoryPool> pool_;
@@ -205,6 +229,66 @@ TEST_F(ArrowRealtimeStoreTest, TestCommitReaderPreservesSlicedBatch) {
         << "expected: " << expected_array->ToString() << ", actual: " << actual_array->ToString();
 }
 
+TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) {
+    ArrowRealtimeStoreFactory factory;
+    std::unique_ptr<ArrowSchema> write_schema = MakeReadSchema(schema_);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> realtime_store,
+                         factory.Create(std::move(write_schema), StatisticsMode::FULL, {}, pool_));
+    std::shared_ptr<ArrowRealtimeStore> store =
+        std::dynamic_pointer_cast<ArrowRealtimeStore>(realtime_store);
+    ASSERT_NE(nullptr, store);
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, "a"], [1, "b"]])"), OffsetRange(0, 2)}));
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[10, "c"], [11, "d"]])"), OffsetRange(2, 4)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(schema_);
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::GreaterThan(
+        /*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT, Literal(int64_t{5}));
+    RealtimeQueryContext context{read_schema.get(), predicate, /*enable_predicate_pushdown=*/true};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(1, readers.size());
+
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch, readers[0]->NextBatchWithBitmap());
+    ASSERT_FALSE(BatchReader::IsEofBatch(batch));
+    ASSERT_EQ(std::vector<int64_t>({10, 11}), ReadIds(batch));
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap eof, readers[0]->NextBatchWithBitmap());
+    ASSERT_TRUE(BatchReader::IsEofBatch(eof));
+
+    std::unique_ptr<ArrowSchema> unfiltered_read_schema = MakeReadSchema(schema_);
+    RealtimeQueryContext unfiltered_context{unfiltered_read_schema.get(), predicate,
+                                            /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> unfiltered_readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, unfiltered_context));
+    ASSERT_EQ(1, unfiltered_readers.size());
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap unfiltered_batch,
+                         unfiltered_readers[0]->NextBatchWithBitmap());
+    ASSERT_EQ(std::vector<int64_t>({0, 1}), ReadIds(unfiltered_batch));
+}
+
+TEST_F(ArrowRealtimeStoreTest, TestMissingStatisticsRetainsNonMatchingBatch) {
+    ASSERT_OK(
+        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[0, "a"], [1, "b"]])"), OffsetRange(0, 2)}));
+    ASSERT_OK(store_->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[10, "c"], [11, "d"]])"), OffsetRange(2, 4)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
+
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(schema_);
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::GreaterThan(
+        /*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT, Literal(int64_t{5}));
+    RealtimeQueryContext context{read_schema.get(), predicate,
+                                 /*enable_predicate_pushdown=*/true};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(1, readers.size());
+
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch, readers[0]->NextBatchWithBitmap());
+    ASSERT_FALSE(BatchReader::IsEofBatch(batch));
+    ASSERT_EQ(std::vector<int64_t>({0, 1}), ReadIds(batch));
+}
+
 TEST_F(ArrowRealtimeStoreTest, TestRejectsHandlesFromAnotherStoreImplementation) {
     ASSERT_NOK_WITH_MSG(store_->CreateCommitReaders(std::make_shared<ForeignSegment>()),
                         "segment was not created by the Arrow real-time store");
diff --git a/src/paimon/core/realtime/realtime_append_only_writer.cpp b/src/paimon/core/realtime/realtime_append_only_writer.cpp
index af48b289b..9d519d791 100644
--- a/src/paimon/core/realtime/realtime_append_only_writer.cpp
+++ b/src/paimon/core/realtime/realtime_append_only_writer.cpp
@@ -47,7 +47,7 @@ Result<std::shared_ptr<RealtimeAppendOnlyWriter>> RealtimeAppendOnlyWriter::Crea
     std::unique_ptr<::ArrowSchema> write_schema,
     const std::shared_ptr<RealtimeContext>& realtime_context,
     const std::shared_ptr<AppendOnlyWriter>& file_writer,
-    const std::shared_ptr<arrow::Schema>& input_schema,
+    const std::shared_ptr<arrow::Schema>& input_schema, StatisticsMode statistics_mode,
     const std::map<std::string, std::string>& options,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     if (!realtime_context) {
@@ -55,9 +55,10 @@ Result<std::shared_ptr<RealtimeAppendOnlyWriter>> RealtimeAppendOnlyWriter::Crea
     }
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
                            RealtimeContextImpl::Cast(realtime_context));
-    PAIMON_ASSIGN_OR_RAISE(RealtimeStoreState store_state,
-                           realtime_context_impl->GetOrCreateRealtimeStore(
-                               partition, bucket, std::move(write_schema), options, memory_pool));
+    PAIMON_ASSIGN_OR_RAISE(
+        RealtimeStoreState store_state,
+        realtime_context_impl->GetOrCreateRealtimeStore(partition, bucket, std::move(write_schema),
+                                                        statistics_mode, options, memory_pool));
     return std::shared_ptr<RealtimeAppendOnlyWriter>(new RealtimeAppendOnlyWriter(
         store_state.store, file_writer, input_schema, store_state.initial_offset, memory_pool));
 }
diff --git a/src/paimon/core/realtime/realtime_append_only_writer.h b/src/paimon/core/realtime/realtime_append_only_writer.h
index 29c198c64..a6190d3b0 100644
--- a/src/paimon/core/realtime/realtime_append_only_writer.h
+++ b/src/paimon/core/realtime/realtime_append_only_writer.h
@@ -47,7 +47,7 @@ class RealtimeAppendOnlyWriter : public BatchWriter {
         std::unique_ptr<::ArrowSchema> write_schema,
         const std::shared_ptr<RealtimeContext>& realtime_context,
         const std::shared_ptr<AppendOnlyWriter>& file_writer,
-        const std::shared_ptr<arrow::Schema>& input_schema,
+        const std::shared_ptr<arrow::Schema>& input_schema, StatisticsMode statistics_mode,
         const std::map<std::string, std::string>& options,
         const std::shared_ptr<MemoryPool>& memory_pool);
 
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 1c7a7cf55..f6bad5cf1 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -79,7 +79,8 @@ Status RealtimeContextImpl::Start() {
 
 Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     const std::map<std::string, std::string>& partition, int32_t bucket,
-    std::unique_ptr<ArrowSchema> write_schema, const std::map<std::string, std::string>& options,
+    std::unique_ptr<ArrowSchema> write_schema, StatisticsMode statistics_mode,
+    const std::map<std::string, std::string>& options,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     std::lock_guard<std::mutex> progress_lock(progress_mutex_);
     std::lock_guard<std::mutex> registry_lock(mutex_);
@@ -118,9 +119,9 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
         }
         return RealtimeStoreState{iter->second, initial_offset};
     }
-    Result<std::shared_ptr<RealtimeStore>> store_result =
-        factory_->Create(std::move(write_schema), options, memory_pool);
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<RealtimeStore> store,
+        factory_->Create(std::move(write_schema), statistics_mode, options, memory_pool));
     stores_.emplace(key, store);
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
@@ -229,12 +230,28 @@ Status RealtimeContextImpl::AdvanceCommittedProgress(int64_t snapshot_id,
             if (partition_bucket.bucket < 0 || committed_end_offset < 0) {
                 return Status::Invalid("invalid partition-bucket committed offset");
             }
+        }
+        // Only stores created by this context can contain state which cannot be restored in
+        // place. Offsets for other partition-buckets are reference state for lazy store creation
+        // and may be removed or rolled back without rebuilding the context.
+        std::lock_guard<std::mutex> registry_lock(mutex_);
+        for (const auto& store_entry : stores_) {
+            const RealtimePartitionBucket& partition_bucket = store_entry.first;
             auto previous_iter = committed_offsets_.find(partition_bucket);
-            if (previous_iter != committed_offsets_.end()) {
-                if (committed_end_offset < previous_iter->second) {
-                    return Status::Invalid(
-                        "real-time partition-bucket committed offset cannot move backwards");
-                }
+            if (previous_iter == committed_offsets_.end()) {
+                continue;
+            }
+
+            auto current_iter = committed_offsets.find(partition_bucket);
+            if (current_iter == committed_offsets.end()) {
+                return Status::Invalid(
+                    "real-time committed progress removed an active partition-bucket; recreate "
+                    "RealtimeContext");
+            }
+            if (current_iter->second < previous_iter->second) {
+                return Status::Invalid(
+                    "real-time committed offset moved backwards for an active partition-bucket; "
+                    "recreate RealtimeContext");
             }
         }
         committed_offsets_ = committed_offsets;
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index 13fec77ab..66c324cab 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -33,6 +33,7 @@
 
 #include "paimon/realtime/realtime_context.h"
 #include "paimon/result.h"
+#include "paimon/statistics_mode.h"
 #include "paimon/visibility.h"
 
 struct ArrowSchema;
@@ -66,7 +67,7 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
 
     Result<RealtimeStoreState> GetOrCreateRealtimeStore(
         const std::map<std::string, std::string>& partition, int32_t bucket,
-        std::unique_ptr<::ArrowSchema> write_schema,
+        std::unique_ptr<::ArrowSchema> write_schema, StatisticsMode statistics_mode,
         const std::map<std::string, std::string>& options,
         const std::shared_ptr<MemoryPool>& memory_pool);
 
@@ -78,6 +79,9 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
 
     Status ReleaseReadView(const std::string& opaque_ticket);
 
+    // Returns an error requiring a new context if a newer snapshot removes or moves committed
+    // progress backwards for a store created by this context. Progress for inactive stores is
+    // only reference state and can be replaced in place.
     Status AdvanceCommittedProgress(int64_t snapshot_id,
                                     const RealtimeOffsetMap& committed_offsets);
 
@@ -99,7 +103,9 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     std::mutex mutex_;
     std::mutex progress_mutex_;
     std::map<RealtimePartitionBucket, std::shared_ptr<RealtimeStore>> stores_;
+    // Full-table progress used as the initial offset when a store is created lazily.
     RealtimeOffsetMap committed_offsets_;
+    // Progress already reflected in stores owned by this context.
     RealtimeOffsetMap reclaimed_offsets_;
     std::optional<int64_t> last_refreshed_snapshot_id_;
     std::mutex read_views_mutex_;
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index ea050094f..017820fd4 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -92,6 +92,7 @@ class TestingRealtimeStore : public RealtimeStore {
 class TestingRealtimeStoreFactory : public RealtimeStoreFactory {
  public:
     Result<std::shared_ptr<RealtimeStore>> Create(std::unique_ptr<ArrowSchema> write_schema,
+                                                  StatisticsMode,
                                                   const std::map<std::string, std::string>&,
                                                   const std::shared_ptr<MemoryPool>&) override {
         if (!write_schema || !write_schema->release) {
@@ -121,18 +122,20 @@ Result<std::shared_ptr<RealtimeContextImpl>> CreateContext(
     return RealtimeContextImpl::Cast(context);
 }
 
-TEST(RealtimeContextTest, TestReusesIndexerAndCapturesRegisteredViews) {
+TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
     std::shared_ptr<MemoryPool> pool = GetDefaultPool();
 
-    ASSERT_OK_AND_ASSIGN(RealtimeStoreState first_state,
-                         context->GetOrCreateRealtimeStore({{"dt", "2026-08-02"}}, 0,
-                                                           MakeWriteSchema(), {{"k", "v"}}, pool));
+    ASSERT_OK_AND_ASSIGN(
+        RealtimeStoreState first_state,
+        context->GetOrCreateRealtimeStore({{"dt", "2026-08-02"}}, 0, MakeWriteSchema(),
+                                          StatisticsMode::NONE, {{"k", "v"}}, pool));
     ASSERT_EQ(0, first_state.initial_offset);
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState first_again_state,
-        context->GetOrCreateRealtimeStore({{"dt", "2026-08-02"}}, 0, MakeWriteSchema(), {}, pool));
+        context->GetOrCreateRealtimeStore({{"dt", "2026-08-02"}}, 0, MakeWriteSchema(),
+                                          StatisticsMode::NONE, {}, pool));
     ASSERT_EQ(first_state.store, first_again_state.store);
     ASSERT_EQ(0, first_again_state.initial_offset);
     ASSERT_EQ(1, factory->stores.size());
@@ -140,10 +143,12 @@ TEST(RealtimeContextTest, TestReusesIndexerAndCapturesRegisteredViews) {
 
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState second_state,
-        context->GetOrCreateRealtimeStore({{"dt", "2026-08-02"}}, 1, MakeWriteSchema(), {}, pool));
+        context->GetOrCreateRealtimeStore({{"dt", "2026-08-02"}}, 1, MakeWriteSchema(),
+                                          StatisticsMode::NONE, {}, pool));
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState third_state,
-        context->GetOrCreateRealtimeStore({{"dt", "2026-08-03"}}, 0, MakeWriteSchema(), {}, pool));
+        context->GetOrCreateRealtimeStore({{"dt", "2026-08-03"}}, 0, MakeWriteSchema(),
+                                          StatisticsMode::NONE, {}, pool));
     ASSERT_NE(first_state.store, second_state.store);
     ASSERT_NE(first_state.store, third_state.store);
     ASSERT_EQ(3, factory->stores.size());
@@ -166,8 +171,10 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     std::shared_ptr<MemoryPool> pool = GetDefaultPool();
     const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
 
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 0, MakeWriteSchema(), {}, pool));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 1, MakeWriteSchema(), {}, pool));
+    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 0, MakeWriteSchema(),
+                                                StatisticsMode::NONE, {}, pool));
+    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 1, MakeWriteSchema(),
+                                                StatisticsMode::NONE, {}, pool));
     ASSERT_EQ(2, factory->stores.size());
 
     ASSERT_NOK_WITH_MSG(context->AdvanceCommittedProgress(-1, {}),
@@ -186,7 +193,8 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
 
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState restored_state,
-        context->GetOrCreateRealtimeStore({{"dt", "unknown"}}, 0, MakeWriteSchema(), {}, pool));
+        context->GetOrCreateRealtimeStore({{"dt", "unknown"}}, 0, MakeWriteSchema(),
+                                          StatisticsMode::NONE, {}, pool));
     ASSERT_EQ(9, restored_state.initial_offset);
 
     ASSERT_OK(context->AdvanceCommittedProgress(
@@ -203,15 +211,41 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     ASSERT_EQ(std::vector<int64_t>({8}), factory->stores[1]->committed_offsets);
 }
 
+TEST(RealtimeContextTest, TestRemovedInactivePartitionDoesNotRequireReopen) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
+    const std::map<std::string, std::string> active_partition = {{"dt", "2026-08-02"}};
+    const std::map<std::string, std::string> inactive_partition = {{"dt", "2026-08-03"}};
+    const RealtimePartitionBucket active_partition_bucket(active_partition, /*bucket=*/0);
+    const RealtimePartitionBucket inactive_partition_bucket(inactive_partition, /*bucket=*/0);
+
+    ASSERT_OK(context->AdvanceCommittedProgress(
+        5, {{active_partition_bucket, /*offset=*/7}, {inactive_partition_bucket, /*offset=*/9}}));
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState active_state,
+                         context->GetOrCreateRealtimeStore(active_partition, 0, MakeWriteSchema(),
+                                                           StatisticsMode::NONE, {}, pool));
+    ASSERT_EQ(7, active_state.initial_offset);
+
+    ASSERT_OK(context->AdvanceCommittedProgress(6, {{active_partition_bucket, /*offset=*/7}}));
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState inactive_state,
+                         context->GetOrCreateRealtimeStore(inactive_partition, 0, MakeWriteSchema(),
+                                                           StatisticsMode::NONE, {}, pool));
+    ASSERT_EQ(0, inactive_state.initial_offset);
+}
+
 TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
     std::shared_ptr<MemoryPool> pool = GetDefaultPool();
     const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
 
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 0, MakeWriteSchema(), {}, pool));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 1, MakeWriteSchema(), {}, pool));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 2, MakeWriteSchema(), {}, pool));
+    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 0, MakeWriteSchema(),
+                                                StatisticsMode::NONE, {}, pool));
+    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 1, MakeWriteSchema(),
+                                                StatisticsMode::NONE, {}, pool));
+    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 2, MakeWriteSchema(),
+                                                StatisticsMode::NONE, {}, pool));
     ASSERT_EQ(3, factory->stores.size());
     factory->stores[1]->fail_next_advance = true;
 
@@ -225,9 +259,9 @@ TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     ASSERT_TRUE(factory->stores[1]->committed_offsets.empty());
     ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[2]->committed_offsets);
 
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState failed_store_state,
-        context->GetOrCreateRealtimeStore(partition, 1, MakeWriteSchema(), {}, pool));
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState failed_store_state,
+                         context->GetOrCreateRealtimeStore(partition, 1, MakeWriteSchema(),
+                                                           StatisticsMode::NONE, {}, pool));
     ASSERT_EQ(8, failed_store_state.initial_offset);
 
     ASSERT_OK(context->AdvanceCommittedProgress(5, committed_offsets));
@@ -237,11 +271,45 @@ TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     ASSERT_EQ(std::vector<int64_t>({8}), factory->stores[1]->committed_offsets);
 }
 
+TEST(RealtimeContextTest, TestRequiresReopenWhenCommittedProgressMovesBackwards) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
+    const std::map<std::string, std::string> first_partition = {{"dt", "2026-08-02"}};
+    const std::map<std::string, std::string> second_partition = {{"dt", "2026-08-03"}};
+    const RealtimePartitionBucket first_partition_bucket(first_partition, /*bucket=*/0);
+    const RealtimePartitionBucket second_partition_bucket(second_partition, /*bucket=*/0);
+
+    ASSERT_OK(context->GetOrCreateRealtimeStore(first_partition, 0, MakeWriteSchema(),
+                                                StatisticsMode::NONE, {}, pool));
+    ASSERT_OK(context->GetOrCreateRealtimeStore(second_partition, 0, MakeWriteSchema(),
+                                                StatisticsMode::NONE, {}, pool));
+    ASSERT_OK(context->AdvanceCommittedProgress(
+        5, {{first_partition_bucket, /*offset=*/7}, {second_partition_bucket, /*offset=*/9}}));
+    ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
+    ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[1]->committed_offsets);
+
+    ASSERT_NOK_WITH_MSG(
+        context->AdvanceCommittedProgress(
+            6, {{first_partition_bucket, /*offset=*/6}, {second_partition_bucket, /*offset=*/10}}),
+        "recreate RealtimeContext");
+    ASSERT_NOK_WITH_MSG(
+        context->AdvanceCommittedProgress(6, {{first_partition_bucket, /*offset=*/10}}),
+        "recreate RealtimeContext");
+    ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
+    ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[1]->committed_offsets);
+
+    ASSERT_OK(context->AdvanceCommittedProgress(
+        6, {{first_partition_bucket, /*offset=*/10}, {second_partition_bucket, /*offset=*/11}}));
+    ASSERT_EQ(std::vector<int64_t>({7, 10}), factory->stores[0]->committed_offsets);
+    ASSERT_EQ(std::vector<int64_t>({9, 11}), factory->stores[1]->committed_offsets);
+}
+
 TEST(RealtimeContextTest, TestPinsResolvesAndReleasesReadViewTicket) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
     ASSERT_OK(context->GetOrCreateRealtimeStore(/*partition=*/{}, /*bucket=*/0, MakeWriteSchema(),
-                                                {}, GetDefaultPool()));
+                                                StatisticsMode::NONE, {}, GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimePartitionBucketView> views,
                          context->AcquireReadViews());
     ASSERT_EQ(1, views.size());
@@ -265,7 +333,7 @@ TEST(RealtimeContextTest, TestExpiresAbandonedReadViewTicket) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
     ASSERT_OK(context->GetOrCreateRealtimeStore(/*partition=*/{}, /*bucket=*/0, MakeWriteSchema(),
-                                                {}, GetDefaultPool()));
+                                                StatisticsMode::NONE, {}, GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimePartitionBucketView> views,
                          context->AcquireReadViews());
     ASSERT_EQ(1, views.size());
diff --git a/src/paimon/core/table/source/append_count_reader.cpp b/src/paimon/core/table/source/append_count_reader.cpp
index 8f9af90ac..5d684af67 100644
--- a/src/paimon/core/table/source/append_count_reader.cpp
+++ b/src/paimon/core/table/source/append_count_reader.cpp
@@ -21,6 +21,7 @@
 
 #include "paimon/core/deletionvectors/deletion_vector.h"
 #include "paimon/core/table/source/data_split_impl.h"
+#include "paimon/core/table/source/realtime_split.h"
 #include "paimon/status.h"
 
 namespace paimon {
@@ -35,6 +36,16 @@ Result<int64_t> AppendCountReader::CountRows() {
 }
 
 Result<int64_t> AppendCountReader::CountSingleSplit(const std::shared_ptr<Split>& split) const {
+    std::shared_ptr<RealtimeSplit> realtime_split = std::dynamic_pointer_cast<RealtimeSplit>(split);
+    if (realtime_split) {
+        int64_t total = realtime_split->MemoryEndOffset() - realtime_split->CommittedEndOffset();
+        for (const std::shared_ptr<Split>& disk_split : realtime_split->DiskSplits()) {
+            PAIMON_ASSIGN_OR_RAISE(int64_t disk_count, CountSingleSplit(disk_split));
+            total += disk_count;
+        }
+        return total;
+    }
+
     auto data_split = std::dynamic_pointer_cast<DataSplitImpl>(split);
     if (!data_split) {
         return Status::Invalid("split cannot be cast to DataSplitImpl");
diff --git a/src/paimon/core/table/source/append_only_table_read.cpp b/src/paimon/core/table/source/append_only_table_read.cpp
index 20786c9f0..6885dc374 100644
--- a/src/paimon/core/table/source/append_only_table_read.cpp
+++ b/src/paimon/core/table/source/append_only_table_read.cpp
@@ -183,17 +183,54 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateDiskReader(
 
 Result<std::unique_ptr<CountReader>> AppendOnlyTableRead::CreateCountReader(
     const std::vector<std::shared_ptr<Split>>& splits) {
-    for (const std::shared_ptr<Split>& split : splits) {
-        if (std::dynamic_pointer_cast<RealtimeSplit>(split)) {
-            return Status::NotImplemented(
-                "CreateCountReader does not support process-local real-time splits");
-        }
-    }
     if (context_->GetPredicate() != nullptr) {
         return Status::NotImplemented(
             "CreateCountReader with predicate pushdown is not supported yet");
     }
 
+    std::vector<std::shared_ptr<RealtimeSplit>> realtime_splits;
+    for (const std::shared_ptr<Split>& split : splits) {
+        std::shared_ptr<RealtimeSplit> realtime_split =
+            std::dynamic_pointer_cast<RealtimeSplit>(split);
+        if (realtime_split) {
+            realtime_splits.push_back(std::move(realtime_split));
+        }
+    }
+    if (!realtime_splits.empty()) {
+        const std::shared_ptr<RealtimeContext> realtime_context = context_->GetRealtimeContext();
+        if (!realtime_context) {
+            return Status::Invalid("reading a real-time split requires a real-time context");
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                               RealtimeContextImpl::Cast(realtime_context));
+        for (const std::shared_ptr<RealtimeSplit>& realtime_split : realtime_splits) {
+            if (realtime_split->Version() != RealtimeSplit::kCurrentVersion) {
+                return Status::Invalid("unsupported real-time split version");
+            }
+            if (realtime_split->MemoryEndOffset() < realtime_split->CommittedEndOffset()) {
+                return Status::Invalid("real-time memory upper offset is behind committed offset");
+            }
+            PAIMON_ASSIGN_OR_RAISE(
+                RealtimePartitionBucketView memory,
+                realtime_context_impl->ResolveReadView(realtime_split->OpaqueTicket()));
+            const RealtimePartitionBucket expected_partition_bucket(realtime_split->Partition(),
+                                                                    realtime_split->Bucket());
+            if (memory.partition_bucket != expected_partition_bucket) {
+                return Status::Invalid(
+                    "real-time read-view ticket belongs to another partition-bucket");
+            }
+            const std::optional<OffsetRange> memory_range = memory.read_view->GetOffsetRange();
+            if (!memory_range || memory_range->end != realtime_split->MemoryEndOffset()) {
+                return Status::Invalid(
+                    "real-time read-view ticket does not match the split offset range");
+            }
+        }
+        for (const std::shared_ptr<RealtimeSplit>& realtime_split : realtime_splits) {
+            PAIMON_RETURN_NOT_OK(
+                realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
+        }
+    }
+
     return std::make_unique<AppendCountReader>(splits, context_->GetCoreOptions().GetFileSystem(),
                                                GetMemoryPool());
 }
diff --git a/src/paimon/core/table/source/realtime_split.h b/src/paimon/core/table/source/realtime_split.h
index 2e264e827..7f5714534 100644
--- a/src/paimon/core/table/source/realtime_split.h
+++ b/src/paimon/core/table/source/realtime_split.h
@@ -31,7 +31,10 @@
 
 namespace paimon {
 
-/// Split combining committed disk splits and a ticket for one immutable memory view.
+/// Split combining disk splits and one immutable memory view.
+///
+/// Append scans keep earlier disk splits independently schedulable and place only the tail disk
+/// split in this wrapper. Other table semantics may choose a different disk grouping policy.
 ///
 /// `committed_end_offset` and `memory_end_offset` are exclusive bounds. Disk covers the committed
 /// prefix and memory readers return the remaining `[committed_end_offset, memory_end_offset)`
diff --git a/src/paimon/core/table/source/realtime_table_scan.cpp b/src/paimon/core/table/source/realtime_table_scan.cpp
index 5a76e81ff..c275208c5 100644
--- a/src/paimon/core/table/source/realtime_table_scan.cpp
+++ b/src/paimon/core/table/source/realtime_table_scan.cpp
@@ -19,6 +19,7 @@
 
 #include "paimon/core/table/source/realtime_table_scan.h"
 
+#include <iterator>
 #include <map>
 #include <optional>
 #include <utility>
@@ -119,7 +120,6 @@ Result<std::vector<std::shared_ptr<Split>>> RealtimeTableScan::CreateRealtimeSpl
             .push_back(split);
     }
 
-    // TODO(xinyu.lxy): Support splitting one partition-bucket into multiple real-time splits.
     std::vector<std::shared_ptr<Split>> result;
     std::vector<std::string> pinned_tickets;
     ScopeGuard ticket_guard([this, &pinned_tickets]() {
@@ -151,9 +151,17 @@ Result<std::vector<std::shared_ptr<Split>>> RealtimeTableScan::CreateRealtimeSpl
             result.insert(result.end(), grouped_disk_splits.begin(), grouped_disk_splits.end());
             continue;
         }
+
+        // Append tables can schedule all but the tail disk split independently. The tail split
+        // carries the immutable memory view so disk and memory are still concatenated by one
+        // RealtimeSplit without collapsing the whole partition-bucket into one scheduling unit.
+        auto tail_disk_split = std::prev(grouped_disk_splits.end());
+        result.insert(result.end(), grouped_disk_splits.begin(), tail_disk_split);
+        std::vector<std::shared_ptr<Split>> realtime_disk_splits;
+        realtime_disk_splits.push_back(std::move(*tail_disk_split));
         RealtimePartitionBucketView& memory = memory_iter->second;
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Split> realtime_split,
-                               create_realtime_split(key, std::move(grouped_disk_splits), memory));
+                               create_realtime_split(key, std::move(realtime_disk_splits), memory));
         result.push_back(std::move(realtime_split));
         active_memory.erase(memory_iter);
     }
diff --git a/src/paimon/core/table/source/table_read.cpp b/src/paimon/core/table/source/table_read.cpp
index e7e3f02f9..3d6a36766 100644
--- a/src/paimon/core/table/source/table_read.cpp
+++ b/src/paimon/core/table/source/table_read.cpp
@@ -90,6 +90,9 @@ Result<std::unique_ptr<TableRead>> CreateTableRead(
     const std::shared_ptr<MemoryPool>& memory_pool, const std::shared_ptr<Executor>& executor) {
     const auto& core_options = internal_context->GetCoreOptions();
     const auto& table_schema = internal_context->GetTableSchema();
+    if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) {
+        return Status::Invalid("real-time read requires realtime.enabled=true");
+    }
     auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> external_paths,
                            core_options.CreateExternalPaths());
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index a543a051c..2dda955ac 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -222,6 +222,9 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
     if (!context.GetRealtimeContext()) {
         return Status::OK();
     }
+    if (!core_options.RealtimeEnabled()) {
+        return Status::Invalid("real-time scan requires realtime.enabled=true");
+    }
     if (!table_schema.PrimaryKeys().empty()) {
         return Status::Invalid("real-time union read currently supports append tables only");
     }
diff --git a/src/paimon/core/utils/partition_utils.h b/src/paimon/core/utils/partition_utils.h
new file mode 100644
index 000000000..b576d674f
--- /dev/null
+++ b/src/paimon/core/utils/partition_utils.h
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <string>
+
+#include "paimon/common/utils/binary_row_partition_computer.h"
+#include "paimon/result.h"
+
+namespace paimon {
+
+class PartitionUtils {
+ public:
+    PartitionUtils() = delete;
+    ~PartitionUtils() = delete;
+
+    static Result<bool> MatchPartitionSpec(const std::map<std::string, std::string>& partition,
+                                           const std::map<std::string, std::string>& partition_spec,
+                                           const BinaryRowPartitionComputer& partition_computer) {
+        for (const auto& entry : partition_spec) {
+            if (partition.find(entry.first) == partition.end()) {
+                return false;
+            }
+        }
+        // Dynamic overwrite already supplies canonical partition values. Avoid trying to parse
+        // legacy DATE names such as "19723" as user-facing DATE literals again.
+        if (MatchNormalizedPartitionSpec(partition, partition_spec)) {
+            return true;
+        }
+        std::map<std::string, std::string> normalized_partition_spec;
+        PAIMON_ASSIGN_OR_RAISE(normalized_partition_spec,
+                               partition_computer.NormalizePartitionSpec(partition_spec));
+        return MatchNormalizedPartitionSpec(partition, normalized_partition_spec);
+    }
+
+    static bool MatchNormalizedPartitionSpec(
+        const std::map<std::string, std::string>& partition,
+        const std::map<std::string, std::string>& normalized_partition_spec) {
+        for (const auto& [key, value] : normalized_partition_spec) {
+            auto iter = partition.find(key);
+            if (iter == partition.end() || iter->second != value) {
+                return false;
+            }
+        }
+        return true;
+    }
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/utils/partition_utils_test.cpp b/src/paimon/core/utils/partition_utils_test.cpp
new file mode 100644
index 000000000..2ac62b92e
--- /dev/null
+++ b/src/paimon/core/utils/partition_utils_test.cpp
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/utils/partition_utils.h"
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include "arrow/type.h"
+#include "gtest/gtest.h"
+#include "paimon/common/utils/binary_row_partition_computer.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+TEST(PartitionUtilsTest, MatchNormalizedPartitionSpec) {
+    const std::map<std::string, std::string> partition = {{"dt", "2026-08-21"}, {"region", "cn"}};
+
+    ASSERT_TRUE(
+        PartitionUtils::MatchNormalizedPartitionSpec(partition, /*normalized_partition_spec=*/{}));
+    ASSERT_TRUE(PartitionUtils::MatchNormalizedPartitionSpec(partition, {{"dt", "2026-08-21"}}));
+    ASSERT_TRUE(PartitionUtils::MatchNormalizedPartitionSpec(
+        partition, {{"dt", "2026-08-21"}, {"region", "cn"}}));
+    ASSERT_FALSE(PartitionUtils::MatchNormalizedPartitionSpec(partition, {{"dt", "2026-08-22"}}));
+    ASSERT_FALSE(PartitionUtils::MatchNormalizedPartitionSpec(partition, {{"hour", "12"}}));
+}
+
+TEST(PartitionUtilsTest, MatchPartitionSpecNormalizesPartialSpec) {
+    std::shared_ptr<arrow::Schema> schema =
+        arrow::schema({arrow::field("dt", arrow::date32()), arrow::field("region", arrow::utf8())});
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BinaryRowPartitionComputer> partition_computer,
+                         BinaryRowPartitionComputer::Create(
+                             /*partition_keys=*/{"dt", "region"}, schema,
+                             /*default_part_value=*/"__DEFAULT_PARTITION__",
+                             /*legacy_partition_name_enabled=*/true, GetDefaultPool()));
+    const std::map<std::string, std::string> partition = {{"dt", "19723"}, {"region", "cn"}};
+    ASSERT_OK_AND_ASSIGN(
+        bool raw_spec_matches,
+        PartitionUtils::MatchPartitionSpec(partition, {{"dt", "2024-01-01"}}, *partition_computer));
+    ASSERT_TRUE(raw_spec_matches);
+
+    ASSERT_OK_AND_ASSIGN(
+        bool normalized_spec_matches,
+        PartitionUtils::MatchPartitionSpec(partition, {{"dt", "19723"}}, *partition_computer));
+    ASSERT_TRUE(normalized_spec_matches);
+
+    ASSERT_OK_AND_ASSIGN(bool unknown_key_matches,
+                         PartitionUtils::MatchPartitionSpec(
+                             partition, {{"unknown_partition_key", "value"}}, *partition_computer));
+    ASSERT_FALSE(unknown_key_matches);
+}
+
+}  // namespace paimon::test
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 67cc21d01..6298137ea 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -24,9 +24,11 @@
 #include <cstdint>
 #include <deque>
 #include <iterator>
+#include <limits>
 #include <map>
 #include <memory>
 #include <mutex>
+#include <set>
 #include <string>
 #include <thread>
 #include <tuple>
@@ -51,10 +53,12 @@
 #include "paimon/file_store_commit.h"
 #include "paimon/file_store_write.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/orphan_files_cleaner.h"
 #include "paimon/predicate/function.h"
 #include "paimon/predicate/predicate.h"
 #include "paimon/predicate/predicate_builder.h"
 #include "paimon/read_context.h"
+#include "paimon/reader/count_reader.h"
 #include "paimon/realtime/realtime_context.h"
 #include "paimon/realtime/realtime_store.h"
 #include "paimon/record_batch.h"
@@ -62,11 +66,20 @@
 #include "paimon/table/source/table_read.h"
 #include "paimon/table/source/table_scan.h"
 #include "paimon/testing/utils/read_result_collector.h"
+#include "paimon/testing/utils/test_helper.h"
 #include "paimon/testing/utils/testharness.h"
 #include "paimon/write_context.h"
 
 namespace paimon::test {
 
+namespace {
+
+constexpr char kDropPartitionCommitUser[] = "drop_partition_commit_user";
+constexpr char kRollbackCommitUser[] = "rollback_commit_user";
+constexpr char kTruncateCommitUser[] = "truncate_commit_user";
+
+}  // namespace
+
 class UnsupportedFunction : public Function {
  public:
     Type GetType() const override {
@@ -184,9 +197,10 @@ class RealtimeWriteInteTest : public ::testing::Test {
                    arrow::field("pt", arrow::utf8())};
         schema_ = arrow::schema(fields_);
         options_ = {
-            {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"},
-            {Options::FILE_SYSTEM, "local"},   {Options::BUCKET, "1"},
-            {Options::BUCKET_KEY, "id"},       {Options::TARGET_FILE_SIZE, "1048576"},
+            {Options::MANIFEST_FORMAT, "orc"},   {Options::FILE_FORMAT, "orc"},
+            {Options::FILE_SYSTEM, "local"},     {Options::BUCKET, "1"},
+            {Options::BUCKET_KEY, "id"},         {Options::TARGET_FILE_SIZE, "1048576"},
+            {Options::REALTIME_ENABLED, "true"},
         };
     }
 
@@ -255,6 +269,43 @@ class RealtimeWriteInteTest : public ::testing::Test {
         return builder.SetBucket(bucket).Finish();
     }
 
+    Result<std::unique_ptr<RecordBatch>> MakeDatePartitionBatch(
+        int64_t first_id, int64_t count, int32_t date, const std::string& partition) const {
+        if (count <= 0) {
+            return Status::Invalid("cannot create an empty test batch");
+        }
+        std::string json = "[";
+        for (int64_t i = 0; i < count; ++i) {
+            if (i > 0) {
+                json += ",";
+            }
+            int64_t id = first_id + i;
+            json += "[" + std::to_string(id) + ",\"value-" + std::to_string(id) + "\"," +
+                    std::to_string(date) + "]";
+        }
+        json += "]";
+
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> array,
+            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), json));
+        ArrowArray c_array;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
+        return RecordBatchBuilder(&c_array)
+            .SetPartition({{"pt", partition}})
+            .SetBucket(/*bucket=*/0)
+            .Finish();
+    }
+
+    Result<std::unique_ptr<RecordBatch>> MakeUnpartitionedBatchFromJson(
+        const std::string& json) const {
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> array,
+            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), json));
+        ArrowArray c_array;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
+        return RecordBatchBuilder(&c_array).SetBucket(/*bucket=*/0).Finish();
+    }
+
     static std::vector<Row> MakeRows(int64_t first_id, int64_t count,
                                      const std::string& partition) {
         std::vector<Row> rows;
@@ -277,6 +328,104 @@ class RealtimeWriteInteTest : public ::testing::Test {
                                           /*watermark=*/std::nullopt);
     }
 
+    Result<int64_t> DropPartition(const std::map<std::string, std::string>& partition,
+                                  int64_t commit_identifier) const {
+        CommitContextBuilder builder(table_path_, kDropPartitionCommitUser);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
+                               builder.SetOptions(options_).Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
+                               FileStoreCommit::Create(std::move(context)));
+        PAIMON_RETURN_NOT_OK(commit->DropPartition({partition}, commit_identifier));
+        PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_));
+        SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_);
+        PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
+                               snapshot_manager.LatestSnapshot());
+        if (!latest_snapshot) {
+            return Status::Invalid("drop partition did not produce a snapshot");
+        }
+        return latest_snapshot->Id();
+    }
+
+    Result<int64_t> TruncateTable(int64_t commit_identifier) const {
+        CommitContextBuilder builder(table_path_, kTruncateCommitUser);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
+                               builder.SetOptions(options_).Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
+                               FileStoreCommit::Create(std::move(context)));
+        PAIMON_RETURN_NOT_OK(commit->TruncateTable(commit_identifier));
+        PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_));
+        SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_);
+        PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
+                               snapshot_manager.LatestSnapshot());
+        if (!latest_snapshot) {
+            return Status::Invalid("truncate did not produce a snapshot");
+        }
+        return latest_snapshot->Id();
+    }
+
+    Result<int64_t> RollbackToAsLatest(int64_t target_snapshot_id) const {
+        CommitContextBuilder builder(table_path_, kRollbackCommitUser);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
+                               builder.SetOptions(options_).Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
+                               FileStoreCommit::Create(std::move(context)));
+        PAIMON_ASSIGN_OR_RAISE(bool rolled_back, commit->RollbackToAsLatest(target_snapshot_id));
+        if (!rolled_back) {
+            return Status::Invalid("failed to commit rollback snapshot");
+        }
+        PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_));
+        SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_);
+        PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
+                               snapshot_manager.LatestSnapshot());
+        if (!latest_snapshot) {
+            return Status::Invalid("rollback did not produce a snapshot");
+        }
+        return latest_snapshot->Id();
+    }
+
+    Result<Snapshot> CompactAndCommit(const std::map<std::string, std::string>& partition,
+                                      int32_t bucket, int64_t commit_identifier) const {
+        WriteContextBuilder write_builder(table_path_, commit_user_);
+        write_builder.SetOptions(options_).WithStreamingMode(true);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<WriteContext> write_context, write_builder.Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreWrite> compaction_writer,
+                               FileStoreWrite::Create(std::move(write_context)));
+        PAIMON_RETURN_NOT_OK(compaction_writer->Compact(partition, bucket,
+                                                        /*full_compaction=*/true));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::vector<std::shared_ptr<CommitMessage>> compaction_messages,
+            compaction_writer->PrepareCommit(/*wait_compaction=*/true, commit_identifier));
+        if (compaction_messages.empty()) {
+            return Status::Invalid("compaction did not produce a commit message");
+        }
+
+        CommitContextBuilder commit_builder(table_path_, commit_user_);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> commit_context,
+                               commit_builder.SetOptions(options_).Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> compaction_commit,
+                               FileStoreCommit::Create(std::move(commit_context)));
+        PAIMON_RETURN_NOT_OK(compaction_commit->Commit(compaction_messages, commit_identifier));
+        PAIMON_RETURN_NOT_OK(compaction_writer->Close());
+
+        PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options_));
+        SnapshotManager snapshot_manager(core_options.GetFileSystem(), table_path_);
+        PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> compact_snapshot,
+                               snapshot_manager.LatestSnapshot());
+        if (!compact_snapshot) {
+            return Status::Invalid("compaction did not produce a snapshot");
+        }
+        return compact_snapshot.value();
+    }
+
+    Result<int32_t> ExpireSnapshots() const {
+        CommitContextBuilder builder(table_path_, commit_user_);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
+                               builder.SetOptions(options_).Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
+                               FileStoreCommit::Create(std::move(context)));
+        return commit->Expire();
+    }
+
     Result<std::shared_ptr<Plan>> CreatePlan(
         const std::shared_ptr<RealtimeContext>& realtime_context,
         const std::shared_ptr<Predicate>& predicate) const {
@@ -314,6 +463,36 @@ class RealtimeWriteInteTest : public ::testing::Test {
         return CollectedReadResult{std::move(reader), std::move(result)};
     }
 
+    void ReadPlanWithSchemaAndCheck(const std::shared_ptr<Plan>& plan,
+                                    const std::shared_ptr<RealtimeContext>& realtime_context,
+                                    const std::shared_ptr<arrow::Schema>& read_schema,
+                                    const std::string& expected_json) const {
+        std::unique_ptr<ArrowSchema> c_read_schema = std::make_unique<ArrowSchema>();
+        ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_read_schema.get()).ok());
+        ReadContextBuilder read_builder(table_path_);
+        read_builder.SetOptions(options_)
+            .SetReadSchema(std::move(c_read_schema))
+            .WithRealtimeContext(realtime_context)
+            .WithMemoryPool(pool_);
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+                             TableRead::Create(std::move(read_context)));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader,
+                             table_read->CreateReader(plan->Splits()));
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result,
+                             ReadResultCollector::CollectResult(reader.get()));
+
+        arrow::FieldVector result_fields = {arrow::field("_VALUE_KIND", arrow::int8())};
+        result_fields.insert(result_fields.end(), read_schema->fields().begin(),
+                             read_schema->fields().end());
+        std::shared_ptr<arrow::Array> expected =
+            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), expected_json)
+                .ValueOrDie();
+        ASSERT_NE(nullptr, result);
+        ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*result))
+            << result->ToString();
+    }
+
     Result<std::vector<Row>> ReadRows(
         const std::shared_ptr<RealtimeContext>& realtime_context) const {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
@@ -367,6 +546,20 @@ class RealtimeWriteInteTest : public ::testing::Test {
         return ReadRows(/*realtime_context=*/nullptr);
     }
 
+    Result<int64_t> CountRows(const std::shared_ptr<Plan>& plan,
+                              const std::shared_ptr<RealtimeContext>& realtime_context) const {
+        ReadContextBuilder read_builder(table_path_);
+        read_builder.SetOptions(options_)
+            .WithRealtimeContext(realtime_context)
+            .WithMemoryPool(pool_);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
+                               TableRead::Create(std::move(read_context)));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CountReader> count_reader,
+                               table_read->CreateCountReader(plan->Splits()));
+        return count_reader->CountRows();
+    }
+
     Result<uint64_t> GetRealtimeMemoryUsage(
         const std::shared_ptr<RealtimeContext>& realtime_context) const {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
@@ -427,6 +620,52 @@ class RealtimeWriteInteTest : public ::testing::Test {
         ASSERT_EQ(expected_rows, actual_rows);
     }
 
+    void CheckDropDatePartitionRemovesOffset(bool legacy_partition_name_enabled) {
+        fields_ = {arrow::field("id", arrow::int64()), arrow::field("payload", arrow::utf8()),
+                   arrow::field("pt", arrow::date32())};
+        schema_ = arrow::schema(fields_);
+        options_[Options::PARTITION_GENERATE_LEGACY_NAME] =
+            legacy_partition_name_enabled ? "true" : "false";
+        CreateTable(/*partition_keys=*/{"pt"});
+
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                             RealtimeContext::Create());
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                             CreateRealtimeWriter(realtime_context));
+        constexpr int32_t kDate = 19723;
+        constexpr int64_t kRowCount = 3;
+        const std::string partition = "2024-01-01";
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeDatePartitionBatch(/*first_id=*/0, kRowCount, kDate, partition));
+        ASSERT_OK(writer->Write(std::move(batch)));
+        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
+                             writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+        ASSERT_EQ(1, commits.size());
+        ASSERT_OK(Commit(commits, /*commit_identifier=*/0));
+
+        const std::string normalized_partition =
+            legacy_partition_name_enabled ? std::to_string(kDate) : partition;
+        RealtimePartitionBucket partition_bucket({{"pt", normalized_partition}}, /*bucket=*/0);
+        ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets_before_drop, ReadCommittedOffsets());
+        ASSERT_EQ(1, offsets_before_drop.size());
+        ASSERT_EQ(kRowCount, offsets_before_drop.at(partition_bucket));
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan_before_drop,
+                             CreatePlan(/*realtime_context=*/nullptr, /*predicate=*/nullptr));
+        ASSERT_OK_AND_ASSIGN(int64_t rows_before_drop,
+                             CountRows(plan_before_drop, /*realtime_context=*/nullptr));
+        ASSERT_EQ(kRowCount, rows_before_drop);
+
+        ASSERT_OK(DropPartition({{"pt", partition}}, /*commit_identifier=*/1));
+        ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets_after_drop, ReadCommittedOffsets());
+        ASSERT_TRUE(offsets_after_drop.empty());
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan_after_drop,
+                             CreatePlan(/*realtime_context=*/nullptr, /*predicate=*/nullptr));
+        ASSERT_OK_AND_ASSIGN(int64_t rows_after_drop,
+                             CountRows(plan_after_drop, /*realtime_context=*/nullptr));
+        ASSERT_EQ(0, rows_after_drop);
+        ASSERT_OK(writer->Close());
+    }
+
     std::unique_ptr<UniqueTestDirectory> dir_;
     std::string table_path_;
     std::string commit_user_ = "realtime_commit_user";
@@ -436,6 +675,44 @@ class RealtimeWriteInteTest : public ::testing::Test {
     std::shared_ptr<MemoryPool> pool_;
 };
 
+TEST_F(RealtimeWriteInteTest, TestRealtimeOperationsRequireEnabledOption) {
+    CreateTable(/*partition_keys=*/{});
+    std::map<std::string, std::string> disabled_options = options_;
+    disabled_options[Options::REALTIME_ENABLED] = "false";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+
+    WriteContextBuilder write_builder(table_path_, commit_user_);
+    write_builder.SetOptions(disabled_options)
+        .WithStreamingMode(true)
+        .WithRealtimeContext(realtime_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, write_builder.Finish());
+    ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)),
+                        "real-time write requires realtime.enabled=true");
+
+    ScanContextBuilder scan_builder(table_path_);
+    scan_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ScanContext> scan_context, scan_builder.Finish());
+    ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)),
+                        "real-time scan requires realtime.enabled=true");
+
+    ReadContextBuilder read_builder(table_path_);
+    read_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+    ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)),
+                        "real-time read requires realtime.enabled=true");
+
+    CommitContextBuilder commit_builder(table_path_, commit_user_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
+                         commit_builder.SetOptions(disabled_options).Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
+                         FileStoreCommit::Create(std::move(commit_context)));
+    ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(/*realtime_commits=*/{},
+                                                   /*commit_identifier=*/0,
+                                                   /*watermark=*/std::nullopt),
+                        "CommitWithProgress requires realtime.enabled=true");
+}
+
 TEST_F(RealtimeWriteInteTest, TestAppendCommitAndRead) {
     CreateTable(/*partition_keys=*/{});
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());
@@ -478,6 +755,52 @@ TEST_F(RealtimeWriteInteTest, TestRollingFilesPreserveProgress) {
     ASSERT_EQ(expected_rows, actual_rows);
 }
 
+TEST_F(RealtimeWriteInteTest, TestAppendScanKeepsDiskSplitsIndependent) {
+    options_[Options::TARGET_FILE_ROW_NUM] = "2";
+    options_[Options::SOURCE_SPLIT_OPEN_FILE_COST] = "1";
+    options_[Options::SOURCE_SPLIT_TARGET_SIZE] = "1";
+    CreateTable(/*partition_keys=*/{});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    std::vector<Row> expected_rows;
+    for (int64_t first_id = 0; first_id < 6; first_id += 2) {
+        std::vector<Row> rows = MakeRows(first_id, /*count=*/2, /*partition=*/"p0");
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch(rows, /*partitioned=*/false));
+        ASSERT_OK(writer->Write(std::move(batch)));
+        expected_rows.insert(expected_rows.end(), rows.begin(), rows.end());
+    }
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(commits, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+
+    std::vector<Row> memory_rows = MakeRows(/*first_id=*/6, /*count=*/2, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
+                         MakeBatch(memory_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(memory_batch)));
+    expected_rows.insert(expected_rows.end(), memory_rows.begin(), memory_rows.end());
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_EQ(3, plan->Splits().size());
+    ASSERT_EQ(nullptr, std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]));
+    ASSERT_EQ(nullptr, std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[1]));
+    std::shared_ptr<RealtimeSplit> realtime_split =
+        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[2]);
+    ASSERT_NE(nullptr, realtime_split);
+    ASSERT_EQ(1, realtime_split->DiskSplits().size());
+    ASSERT_EQ(6, realtime_split->CommittedEndOffset());
+    ASSERT_EQ(8, realtime_split->MemoryEndOffset());
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
+    ASSERT_EQ(expected_rows, actual_rows);
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestCommitOrdersPreparedOffsetRanges) {
     CreateTable(/*partition_keys=*/{});
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());
@@ -513,6 +836,247 @@ TEST_F(RealtimeWriteInteTest, TestCommitOrdersPreparedOffsetRanges) {
     ASSERT_EQ(expected_rows, actual_rows);
 }
 
+TEST_F(RealtimeWriteInteTest, TestCommitWithProgressRetryReturnsLatestSnapshot) {
+    CreateTable(/*partition_keys=*/{});
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());
+    std::vector<Row> expected_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch(expected_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+
+    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id, Commit(commits, /*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t retry_snapshot_id, Commit(commits, /*commit_identifier=*/0));
+    ASSERT_EQ(first_snapshot_id, retry_snapshot_id);
+
+    ASSERT_OK(writer->RefreshCommittedSnapshot(first_snapshot_id));
+    std::vector<Row> second_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+                         MakeBatch(second_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(second_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
+                         Commit(second_commits, /*commit_identifier=*/1));
+
+    ASSERT_OK_AND_ASSIGN(retry_snapshot_id, Commit(commits, /*commit_identifier=*/0));
+    ASSERT_EQ(second_snapshot_id, retry_snapshot_id);
+    ASSERT_NE(first_snapshot_id, retry_snapshot_id);
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
+    ASSERT_EQ(5, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+
+    expected_rows.insert(expected_rows.end(), second_rows.begin(), second_rows.end());
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows());
+    ASSERT_EQ(expected_rows, actual_rows);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestCommitWithProgressRejectsCoveredRangesFromAnotherUser) {
+    CreateTable(/*partition_keys=*/{});
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());
+    std::vector<Row> expected_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch(expected_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK(Commit(commits, /*commit_identifier=*/0));
+
+    CommitContextBuilder builder(table_path_, "another_realtime_commit_user");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> context,
+                         builder.SetOptions(options_).Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
+                         FileStoreCommit::Create(std::move(context)));
+    ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(commits, /*commit_identifier=*/0,
+                                                   /*watermark=*/std::nullopt),
+                        "another commit user or identifier");
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows());
+    ASSERT_EQ(expected_rows, actual_rows);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeWriteAcrossAppendCompaction) {
+    options_[Options::TARGET_FILE_ROW_NUM] = "2";
+    options_[Options::COMPACTION_MIN_FILE_NUM] = "2";
+    CreateTable(/*partition_keys=*/{});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    std::vector<Row> first_rows;
+    for (int64_t first_id = 0; first_id < 5; first_id += 2) {
+        std::vector<Row> rows =
+            MakeRows(first_id, std::min<int64_t>(2, 5 - first_id), /*partition=*/"p0");
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch(rows, /*partitioned=*/false));
+        ASSERT_OK(writer->Write(std::move(batch)));
+        first_rows.insert(first_rows.end(), rows.begin(), rows.end());
+    }
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, first_commits.size());
+    ASSERT_EQ(OffsetRange(0, 5), first_commits[0].offset_range);
+    std::shared_ptr<CommitMessageImpl> first_commit_message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(first_commits[0].commit_message);
+    ASSERT_NE(nullptr, first_commit_message);
+    ASSERT_EQ(3, first_commit_message->GetNewFilesIncrement().NewFiles().size());
+    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id, Commit(first_commits, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(first_snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(Snapshot compact_snapshot, CompactAndCommit(/*partition=*/{}, /*bucket=*/0,
+                                                                     /*commit_identifier=*/1));
+    ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot.GetCommitKind());
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap compacted_offsets, ReadCommittedOffsets());
+    ASSERT_EQ(5, compacted_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+
+    ASSERT_OK(writer->RefreshCommittedSnapshot(compact_snapshot.Id()));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_compaction, ReadRows(realtime_context));
+    ASSERT_EQ(first_rows, rows_after_compaction);
+
+    std::vector<Row> second_rows = MakeRows(/*first_id=*/5, /*count=*/2, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+                         MakeBatch(second_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(second_batch)));
+    std::vector<Row> expected_rows = first_rows;
+    expected_rows.insert(expected_rows.end(), second_rows.begin(), second_rows.end());
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_with_building_memory, ReadRows(realtime_context));
+    ASSERT_EQ(expected_rows, rows_with_building_memory);
+
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_EQ(1, second_commits.size());
+    ASSERT_EQ(OffsetRange(5, 7), second_commits[0].offset_range);
+    ASSERT_OK_AND_ASSIGN(int64_t final_snapshot_id,
+                         Commit(second_commits, /*commit_identifier=*/2));
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap final_offsets, ReadCommittedOffsets());
+    ASSERT_EQ(7, final_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows_before_refresh, ReadRows(realtime_context));
+    ASSERT_EQ(expected_rows, final_rows_before_refresh);
+    ASSERT_OK(writer->RefreshCommittedSnapshot(final_snapshot_id));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows_after_refresh, ReadRows(realtime_context));
+    ASSERT_EQ(expected_rows, final_rows_after_refresh);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeOffsetFileLifecycle) {
+    options_[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1";
+    options_[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1";
+    options_[Options::SNAPSHOT_TIME_RETAINED] = "1ms";
+    CreateTable(/*partition_keys=*/{});
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());
+
+    std::vector<Row> first_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+                         MakeBatch(first_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(first_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id, Commit(first_commits, /*commit_identifier=*/0));
+
+    std::vector<Row> second_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+                         MakeBatch(second_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(second_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
+                         Commit(second_commits, /*commit_identifier=*/1));
+
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options_));
+    std::shared_ptr<FileSystem> file_system = core_options.GetFileSystem();
+    SnapshotManager snapshot_manager(file_system, table_path_);
+    ASSERT_OK_AND_ASSIGN(Snapshot first_snapshot, snapshot_manager.LoadSnapshot(first_snapshot_id));
+    ASSERT_OK_AND_ASSIGN(Snapshot second_snapshot,
+                         snapshot_manager.LoadSnapshot(second_snapshot_id));
+    std::optional<std::string> first_offsets_path =
+        RealtimeCommitProperties::GetOffsetsPath(first_snapshot);
+    std::optional<std::string> second_offsets_path =
+        RealtimeCommitProperties::GetOffsetsPath(second_snapshot);
+    ASSERT_TRUE(first_offsets_path);
+    ASSERT_TRUE(second_offsets_path);
+    ASSERT_NE(first_offsets_path, second_offsets_path);
+
+    std::string orphan_offsets_path = PathUtil::JoinPath(
+        RealtimeCommitProperties::OffsetsDirectory(table_path_, core_options.GetBranch()),
+        "orphan.offsets");
+    ASSERT_OK(file_system->WriteFile(orphan_offsets_path, "orphan", /*overwrite=*/false));
+    CleanContextBuilder clean_builder(table_path_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CleanContext> clean_context,
+                         clean_builder.WithFileSystem(file_system)
+                             .WithOlderThanMs(std::numeric_limits<int64_t>::max())
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<OrphanFilesCleaner> cleaner,
+                         OrphanFilesCleaner::Create(std::move(clean_context)));
+    ASSERT_OK_AND_ASSIGN(std::set<std::string> cleaned_paths, cleaner->Clean());
+    ASSERT_EQ(std::set<std::string>({orphan_offsets_path}), cleaned_paths);
+    ASSERT_OK_AND_ASSIGN(bool first_offsets_exist, file_system->Exists(first_offsets_path.value()));
+    ASSERT_TRUE(first_offsets_exist);
+    ASSERT_OK_AND_ASSIGN(bool second_offsets_exist,
+                         file_system->Exists(second_offsets_path.value()));
+    ASSERT_TRUE(second_offsets_exist);
+
+    ASSERT_OK_AND_ASSIGN(int32_t expired_snapshots, ExpireSnapshots());
+    ASSERT_EQ(1, expired_snapshots);
+    ASSERT_OK_AND_ASSIGN(first_offsets_exist, file_system->Exists(first_offsets_path.value()));
+    ASSERT_FALSE(first_offsets_exist);
+    ASSERT_OK_AND_ASSIGN(second_offsets_exist, file_system->Exists(second_offsets_path.value()));
+    ASSERT_TRUE(second_offsets_exist);
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets,
+                         RealtimeCommitProperties::ReadOffsets(second_snapshot, file_system));
+    ASSERT_EQ(5, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestCompactionSnapshotRetainsSharedOffsetFile) {
+    options_[Options::TARGET_FILE_ROW_NUM] = "2";
+    options_[Options::COMPACTION_MIN_FILE_NUM] = "2";
+    options_[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1";
+    options_[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1";
+    options_[Options::SNAPSHOT_TIME_RETAINED] = "1ms";
+    CreateTable(/*partition_keys=*/{});
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());
+
+    for (int64_t first_id = 0; first_id < 5; first_id += 2) {
+        std::vector<Row> rows =
+            MakeRows(first_id, std::min<int64_t>(2, 5 - first_id), /*partition=*/"p0");
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch(rows, /*partitioned=*/false));
+        ASSERT_OK(writer->Write(std::move(batch)));
+    }
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t realtime_snapshot_id, Commit(commits, /*commit_identifier=*/0));
+
+    ASSERT_OK_AND_ASSIGN(Snapshot compact_snapshot, CompactAndCommit(/*partition=*/{}, /*bucket=*/0,
+                                                                     /*commit_identifier=*/1));
+    ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot.GetCommitKind());
+
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options_));
+    std::shared_ptr<FileSystem> file_system = core_options.GetFileSystem();
+    SnapshotManager snapshot_manager(file_system, table_path_);
+    ASSERT_OK_AND_ASSIGN(Snapshot realtime_snapshot,
+                         snapshot_manager.LoadSnapshot(realtime_snapshot_id));
+    std::optional<std::string> realtime_offsets_path =
+        RealtimeCommitProperties::GetOffsetsPath(realtime_snapshot);
+    std::optional<std::string> compact_offsets_path =
+        RealtimeCommitProperties::GetOffsetsPath(compact_snapshot);
+    ASSERT_TRUE(realtime_offsets_path);
+    ASSERT_TRUE(compact_offsets_path);
+    ASSERT_EQ(realtime_offsets_path, compact_offsets_path);
+
+    ASSERT_OK_AND_ASSIGN(int32_t expired_snapshots, ExpireSnapshots());
+    ASSERT_EQ(1, expired_snapshots);
+    ASSERT_OK_AND_ASSIGN(bool offsets_exist, file_system->Exists(compact_offsets_path.value()));
+    ASSERT_TRUE(offsets_exist);
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets,
+                         RealtimeCommitProperties::ReadOffsets(compact_snapshot, file_system));
+    ASSERT_EQ(5, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestReadMemoryBeforePrepareCommit) {
     CreateTable(/*partition_keys=*/{});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
@@ -816,46 +1380,84 @@ TEST_F(RealtimeWriteInteTest, TestReadCommittedDiskAndBuildingMemory) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestProjectionAndPredicateForMemoryAndDisk) {
+TEST_F(RealtimeWriteInteTest, TestCountMemoryAndDiskAcrossRefresh) {
     CreateTable(/*partition_keys=*/{});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create());
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                          CreateRealtimeWriter(realtime_context));
-    std::shared_ptr<Predicate> scan_predicate =
-        PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT,
-                                      Literal(static_cast<int64_t>(1)));
-    std::shared_ptr<Predicate> read_predicate =
-        PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"id", FieldType::BIGINT,
-                                      Literal(static_cast<int64_t>(1)));
-    const std::vector<std::string> read_fields = {"payload", "id"};
-    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
-        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("payload", arrow::utf8()),
-         arrow::field("id", arrow::int64())});
 
     std::vector<Row> disk_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                          MakeBatch(disk_rows, /*partitioned=*/false));
     ASSERT_OK(writer->Write(std::move(disk_batch)));
-
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> memory_plan,
-                         CreatePlan(realtime_context, scan_predicate));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_result,
-                         ReadPlan(memory_plan, realtime_context, read_fields, read_predicate,
-                                  /*enable_predicate_filter=*/true));
-    std::shared_ptr<arrow::Array> expected_memory =
-        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
-            [0, "value-2", 2]
-        ])")
-            .ValueOrDie();
-    ASSERT_NE(nullptr, memory_result.data);
-    ASSERT_TRUE(
-        std::make_shared<arrow::ChunkedArray>(expected_memory)->Equals(*memory_result.data));
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_OK_AND_ASSIGN(int64_t memory_count, CountRows(memory_plan, realtime_context));
+    ASSERT_EQ(3, memory_count);
 
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
                          writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
-    ASSERT_OK(Commit(disk_commits, /*commit_identifier=*/0));
-    std::vector<Row> memory_rows = MakeRows(/*first_id=*/3, /*count=*/3, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(int64_t committed_snapshot_id,
+                         Commit(disk_commits, /*commit_identifier=*/0));
+    std::vector<Row> memory_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
+                         MakeBatch(memory_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(memory_batch)));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> union_plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_OK_AND_ASSIGN(int64_t union_count, CountRows(union_plan, realtime_context));
+    ASSERT_EQ(5, union_count);
+
+    ASSERT_OK(writer->RefreshCommittedSnapshot(committed_snapshot_id));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> refreshed_plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_OK_AND_ASSIGN(int64_t refreshed_count, CountRows(refreshed_plan, realtime_context));
+    ASSERT_EQ(union_count, refreshed_count);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestProjectionAndPredicateForMemoryAndDisk) {
+    CreateTable(/*partition_keys=*/{});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    std::shared_ptr<Predicate> scan_predicate =
+        PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT,
+                                      Literal(static_cast<int64_t>(1)));
+    std::shared_ptr<Predicate> read_predicate =
+        PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"id", FieldType::BIGINT,
+                                      Literal(static_cast<int64_t>(1)));
+    const std::vector<std::string> read_fields = {"payload", "id"};
+    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
+        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("payload", arrow::utf8()),
+         arrow::field("id", arrow::int64())});
+
+    std::vector<Row> disk_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
+                         MakeBatch(disk_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(disk_batch)));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> memory_plan,
+                         CreatePlan(realtime_context, scan_predicate));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_result,
+                         ReadPlan(memory_plan, realtime_context, read_fields, read_predicate,
+                                  /*enable_predicate_filter=*/true));
+    std::shared_ptr<arrow::Array> expected_memory =
+        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
+            [0, "value-2", 2]
+        ])")
+            .ValueOrDie();
+    ASSERT_NE(nullptr, memory_result.data);
+    ASSERT_TRUE(
+        std::make_shared<arrow::ChunkedArray>(expected_memory)->Equals(*memory_result.data));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK(Commit(disk_commits, /*commit_identifier=*/0));
+    std::vector<Row> memory_rows = MakeRows(/*first_id=*/3, /*count=*/3, /*partition=*/"p0");
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                          MakeBatch(memory_rows, /*partitioned=*/false));
     ASSERT_OK(writer->Write(std::move(memory_batch)));
@@ -928,6 +1530,310 @@ TEST_F(RealtimeWriteInteTest, TestDiskPredicatePushdownWithoutMemoryFiltering) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestMemoryBatchStatisticsPredicatePushdown) {
+    CreateTable(/*partition_keys=*/{});
+    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
+        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("id", arrow::int64()),
+         arrow::field("payload", arrow::utf8()), arrow::field("pt", arrow::utf8())});
+    auto make_expected = [&](const std::string& json) {
+        std::shared_ptr<arrow::Array> array =
+            arrow::ipc::internal::json::ArrayFromJSON(result_type, json).ValueOrDie();
+        return std::make_shared<arrow::ChunkedArray>(array);
+    };
+    std::vector<std::shared_ptr<Predicate>> predicates = {
+        PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT,
+                                      Literal(static_cast<int64_t>(100))),
+        PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT,
+                                      Literal(static_cast<int64_t>(5))),
+        PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT,
+                                Literal(static_cast<int64_t>(10))),
+    };
+
+    auto check_candidates =
+        [&](const std::string& statistics_mode,
+            const std::vector<std::shared_ptr<arrow::ChunkedArray>>& expected) -> Status {
+        if (expected.size() != predicates.size()) {
+            return Status::Invalid("unexpected real-time candidate result count");
+        }
+        options_[Options::REALTIME_STORE_STATS_MODE] = statistics_mode;
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContext> realtime_context,
+                               RealtimeContext::Create());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreWrite> writer,
+                               CreateRealtimeWriter(realtime_context));
+        for (int64_t first_id : {0, 10, 20}) {
+            std::vector<Row> rows = MakeRows(first_id, /*count=*/3, /*partition=*/"p0");
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> batch,
+                                   MakeBatch(rows, /*partitioned=*/false));
+            PAIMON_RETURN_NOT_OK(writer->Write(std::move(batch)));
+        }
+
+        for (size_t i = 0; i < predicates.size(); ++i) {
+            const std::shared_ptr<Predicate>& predicate = predicates[i];
+            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
+                                   CreatePlan(realtime_context, predicate));
+            PAIMON_ASSIGN_OR_RAISE(
+                CollectedReadResult result,
+                ReadPlan(plan, realtime_context, {"id", "payload", "pt"}, predicate,
+                         /*enable_predicate_filter=*/false));
+            std::shared_ptr<arrow::ChunkedArray> actual =
+                result.data ? result.data : make_expected("[]");
+            if (!expected[i]->Equals(*actual)) {
+                return Status::Invalid("unexpected real-time candidate rows: " +
+                                       actual->ToString());
+            }
+        }
+        PAIMON_RETURN_NOT_OK(writer->Close());
+        return Status::OK();
+    };
+
+    std::shared_ptr<arrow::ChunkedArray> all_rows = make_expected(R"([
+        [0, 0, "value-0", "p0"],
+        [0, 1, "value-1", "p0"],
+        [0, 2, "value-2", "p0"],
+        [0, 10, "value-10", "p0"],
+        [0, 11, "value-11", "p0"],
+        [0, 12, "value-12", "p0"],
+        [0, 20, "value-20", "p0"],
+        [0, 21, "value-21", "p0"],
+        [0, 22, "value-22", "p0"]
+    ])");
+    ASSERT_OK(check_candidates("none", {all_rows, all_rows, all_rows}));
+
+    std::shared_ptr<arrow::ChunkedArray> partially_filtered = make_expected(R"([
+        [0, 10, "value-10", "p0"],
+        [0, 11, "value-11", "p0"],
+        [0, 12, "value-12", "p0"],
+        [0, 20, "value-20", "p0"],
+        [0, 21, "value-21", "p0"],
+        [0, 22, "value-22", "p0"]
+    ])");
+    std::shared_ptr<arrow::ChunkedArray> matching_batch = make_expected(R"([
+        [0, 10, "value-10", "p0"],
+        [0, 11, "value-11", "p0"],
+        [0, 12, "value-12", "p0"]
+    ])");
+    ASSERT_OK(check_candidates("full", {make_expected("[]"), partially_filtered, matching_batch}));
+}
+
+TEST_F(RealtimeWriteInteTest, TestMemoryBatchStatisticsPredicatePushdownWithDisk) {
+    options_[Options::FILE_FORMAT] = "parquet";
+    options_[Options::WRITE_BATCH_SIZE] = "1";
+    options_[Options::REALTIME_STORE_STATS_MODE] = "full";
+    options_["parquet.page.size"] = "1";
+    options_["parquet.enable-dictionary"] = "false";
+    options_["parquet.write.enable-page-index"] = "true";
+    options_["parquet.read.enable-page-index-filter"] = "true";
+    CreateTable(/*partition_keys=*/{});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    std::vector<Row> disk_rows = MakeRows(/*first_id=*/0, /*count=*/6, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
+                         MakeBatch(disk_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(disk_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK(Commit(disk_commits, /*commit_identifier=*/0));
+
+    for (int64_t first_id : {0, 10}) {
+        std::vector<Row> rows = MakeRows(first_id, /*count=*/3, /*partition=*/"p0");
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch(rows, /*partitioned=*/false));
+        ASSERT_OK(writer->Write(std::move(batch)));
+    }
+    std::shared_ptr<Predicate> predicate =
+        PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT,
+                                      Literal(static_cast<int64_t>(3)));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, CreatePlan(realtime_context, predicate));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult result,
+                         ReadPlan(plan, realtime_context, {"id", "payload", "pt"}, predicate,
+                                  /*enable_predicate_filter=*/false));
+
+    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
+        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("id", arrow::int64()),
+         arrow::field("payload", arrow::utf8()), arrow::field("pt", arrow::utf8())});
+    std::shared_ptr<arrow::Array> expected =
+        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
+            [0, 4, "value-4", "p0"],
+            [0, 5, "value-5", "p0"],
+            [0, 10, "value-10", "p0"],
+            [0, 11, "value-11", "p0"],
+            [0, 12, "value-12", "p0"]
+        ])")
+            .ValueOrDie();
+    ASSERT_NE(nullptr, result.data);
+    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*result.data))
+        << result.data->ToString();
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestNullPredicateForMemoryAndDisk) {
+    options_[Options::FILE_FORMAT] = "parquet";
+    options_[Options::WRITE_BATCH_SIZE] = "1";
+    options_[Options::REALTIME_STORE_STATS_MODE] = "full";
+    options_["parquet.page.size"] = "1";
+    options_["parquet.enable-dictionary"] = "false";
+    options_["parquet.write.enable-page-index"] = "true";
+    options_["parquet.write.max-row-group-length"] = "1";
+    options_["parquet.read.enable-page-index-filter"] = "true";
+    CreateTable(/*partition_keys=*/{});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
+                         MakeUnpartitionedBatchFromJson(R"([
+                             [0, null, "p0"],
+                             [1, "disk-value", "p0"]
+                         ])"));
+    ASSERT_OK(writer->Write(std::move(disk_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK(Commit(disk_commits, /*commit_identifier=*/0));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> non_null_memory_batch,
+                         MakeUnpartitionedBatchFromJson(R"([
+                             [2, "memory-value-2", "p0"],
+                             [3, "memory-value-3", "p0"]
+                         ])"));
+    ASSERT_OK(writer->Write(std::move(non_null_memory_batch)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> nullable_memory_batch,
+                         MakeUnpartitionedBatchFromJson(R"([
+                             [4, null, "p0"],
+                             [5, "memory-value-5", "p0"]
+                         ])"));
+    ASSERT_OK(writer->Write(std::move(nullable_memory_batch)));
+
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::IsNull(
+        /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, CreatePlan(realtime_context, predicate));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult result,
+                         ReadPlan(plan, realtime_context, {"id", "payload", "pt"}, predicate,
+                                  /*enable_predicate_filter=*/false));
+
+    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
+        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("id", arrow::int64()),
+         arrow::field("payload", arrow::utf8()), arrow::field("pt", arrow::utf8())});
+    std::shared_ptr<arrow::Array> expected =
+        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
+            [0, 0, null, "p0"],
+            [0, 4, null, "p0"],
+            [0, 5, "memory-value-5", "p0"]
+        ])")
+            .ValueOrDie();
+    ASSERT_NE(nullptr, result.data);
+    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*result.data))
+        << result.data->ToString();
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestUnionReadAfterColumnRename) {
+    CreateTable(/*partition_keys=*/{});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> first_context, RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
+                         CreateRealtimeWriter(first_context));
+    std::vector<Row> disk_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
+                         MakeBatch(disk_rows, /*partitioned=*/false));
+    ASSERT_OK(first_writer->Write(std::move(disk_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
+                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK(Commit(disk_commits, /*commit_identifier=*/0));
+    ASSERT_OK(first_writer->Close());
+
+    std::shared_ptr<arrow::Field> renamed_payload = arrow::field("renamed_payload", arrow::utf8());
+    ASSERT_OK(TestHelper::WriteNextSchema(
+        dir_->GetFileSystem(), table_path_,
+        {DataField(0, fields_[0]), DataField(1, renamed_payload), DataField(2, fields_[2])},
+        /*highest_field_id=*/2, options_));
+    fields_[1] = renamed_payload;
+    schema_ = arrow::schema(fields_);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> second_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
+                         CreateRealtimeWriter(second_context));
+    std::vector<Row> memory_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
+                         MakeBatch(memory_rows, /*partitioned=*/false));
+    ASSERT_OK(second_writer->Write(std::move(memory_batch)));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(second_context, /*predicate=*/nullptr));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult result,
+                         ReadPlan(plan, second_context, {"id", "renamed_payload", "pt"},
+                                  /*predicate=*/nullptr, /*enable_predicate_filter=*/false));
+    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
+        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("id", arrow::int64()),
+         renamed_payload, arrow::field("pt", arrow::utf8())});
+    std::shared_ptr<arrow::Array> expected =
+        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
+            [0, 0, "value-0", "p0"],
+            [0, 1, "value-1", "p0"],
+            [0, 2, "value-2", "p0"],
+            [0, 3, "value-3", "p0"],
+            [0, 4, "value-4", "p0"]
+        ])")
+            .ValueOrDie();
+    ASSERT_NE(nullptr, result.data);
+    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*result.data))
+        << result.data->ToString();
+    ASSERT_OK(second_writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestUnionReadWithNestedStructProjection) {
+    std::shared_ptr<arrow::DataType> address_type =
+        arrow::struct_({arrow::field("city", arrow::utf8()), arrow::field("zip", arrow::int64())});
+    std::shared_ptr<arrow::DataType> profile_type = arrow::struct_(
+        {arrow::field("name", arrow::utf8()), arrow::field("address", address_type)});
+    fields_ = {arrow::field("id", arrow::int64()), arrow::field("profile", profile_type),
+               arrow::field("pt", arrow::utf8())};
+    schema_ = arrow::schema(fields_);
+    CreateTable(/*partition_keys=*/{});
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
+                         MakeUnpartitionedBatchFromJson(R"([
+                             [0, ["disk-0", ["hangzhou", 310000]], "p0"],
+                             [1, ["disk-1", ["shanghai", 200000]], "p0"]
+                         ])"));
+    ASSERT_OK(writer->Write(std::move(disk_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t disk_snapshot_id, Commit(disk_commits, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(disk_snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
+                         MakeUnpartitionedBatchFromJson(R"([
+                             [2, ["memory-2", ["beijing", 100000]], "p0"],
+                             [3, ["memory-3", ["shenzhen", 518000]], "p0"]
+                         ])"));
+    ASSERT_OK(writer->Write(std::move(memory_batch)));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    std::shared_ptr<arrow::DataType> projected_address_type =
+        arrow::struct_({arrow::field("city", arrow::utf8())});
+    std::shared_ptr<arrow::DataType> projected_profile_type =
+        arrow::struct_({arrow::field("address", projected_address_type)});
+    std::shared_ptr<arrow::Schema> projected_schema = arrow::schema(
+        {arrow::field("id", arrow::int64()), arrow::field("profile", projected_profile_type),
+         arrow::field("pt", arrow::utf8())});
+    ReadPlanWithSchemaAndCheck(plan, realtime_context, projected_schema, R"([
+        [0, 0, [["hangzhou"]], "p0"],
+        [0, 1, [["shanghai"]], "p0"],
+        [0, 2, [["beijing"]], "p0"],
+        [0, 3, [["shenzhen"]], "p0"]
+    ])");
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestRefreshCommittedSnapshotReclaimsMemory) {
     CreateTable(/*partition_keys=*/{});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
@@ -1109,6 +2015,161 @@ TEST_F(RealtimeWriteInteTest, TestRepeatedCommitReadAndRefresh) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestRefreshLatestSnapshotReclaimsMultipleCommittedSegments) {
+    CreateTable(/*partition_keys=*/{});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    constexpr int64_t kSnapshotCount = 3;
+    constexpr int64_t kRowsPerSnapshot = 2;
+    std::vector<Row> expected_rows;
+    int64_t latest_snapshot_id = -1;
+    for (int64_t snapshot_index = 0; snapshot_index < kSnapshotCount; ++snapshot_index) {
+        std::vector<Row> rows =
+            MakeRows(snapshot_index * kRowsPerSnapshot, kRowsPerSnapshot, /*partition=*/"p0");
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch(rows, /*partitioned=*/false));
+        ASSERT_OK(writer->Write(std::move(batch)));
+        ASSERT_OK_AND_ASSIGN(
+            std::vector<RealtimeCommitProgress> commits,
+            writer->PrepareCommitWithProgress(/*commit_identifier=*/snapshot_index));
+        ASSERT_EQ(1, commits.size());
+        ASSERT_OK_AND_ASSIGN(latest_snapshot_id,
+                             Commit(commits, /*commit_identifier=*/snapshot_index));
+        expected_rows.insert(expected_rows.end(), rows.begin(), rows.end());
+    }
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
+    ASSERT_EQ(expected_rows, rows_before_refresh);
+    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_before_refresh,
+                         GetRealtimeMemoryUsage(realtime_context));
+    ASSERT_GT(memory_usage_before_refresh, 0);
+
+    ASSERT_OK(writer->RefreshCommittedSnapshot(latest_snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
+    ASSERT_EQ(rows_before_refresh, rows_after_refresh);
+    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_after_refresh,
+                         GetRealtimeMemoryUsage(realtime_context));
+    ASSERT_EQ(0, memory_usage_after_refresh);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestOverwriteRequiresReopenRealtimeContext) {
+    CreateTable(/*partition_keys=*/{});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    std::vector<Row> committed_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> committed_batch,
+                         MakeBatch(committed_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(committed_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t committed_snapshot_id, Commit(commits, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(committed_snapshot_id));
+
+    std::vector<Row> building_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> building_batch,
+                         MakeBatch(building_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(building_batch)));
+    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_before_overwrite,
+                         GetRealtimeMemoryUsage(realtime_context));
+    ASSERT_GT(memory_usage_before_overwrite, 0);
+
+    ASSERT_OK_AND_ASSIGN(int64_t overwrite_snapshot_id, TruncateTable(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options_));
+    SnapshotManager snapshot_manager(core_options.GetFileSystem(), table_path_);
+    ASSERT_OK_AND_ASSIGN(Snapshot overwrite_snapshot,
+                         snapshot_manager.LoadSnapshot(overwrite_snapshot_id));
+    ASSERT_EQ(Snapshot::CommitKind::Overwrite(), overwrite_snapshot.GetCommitKind());
+    ASSERT_FALSE(RealtimeCommitProperties::GetOffsetsPath(overwrite_snapshot));
+
+    ASSERT_NOK_WITH_MSG(writer->RefreshCommittedSnapshot(overwrite_snapshot_id),
+                        "recreate RealtimeContext");
+    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_after_failed_refresh,
+                         GetRealtimeMemoryUsage(realtime_context));
+    ASSERT_EQ(memory_usage_before_overwrite, memory_usage_after_failed_refresh);
+    ASSERT_OK(writer->Close());
+    writer.reset();
+    realtime_context.reset();
+
+    ASSERT_OK_AND_ASSIGN(realtime_context, RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(writer, CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> replay_batch,
+                         MakeBatch(building_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(replay_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> replay_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_EQ(1, replay_commits.size());
+    ASSERT_EQ(OffsetRange(0, 2), replay_commits[0].offset_range);
+    ASSERT_OK_AND_ASSIGN(int64_t replay_snapshot_id,
+                         Commit(replay_commits, /*commit_identifier=*/2));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(replay_snapshot_id));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> replayed_rows, ReadRows(realtime_context));
+    ASSERT_EQ(building_rows, replayed_rows);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestReopenRealtimeContextAfterRollback) {
+    CreateTable(/*partition_keys=*/{});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    std::vector<Row> first_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+                         MakeBatch(first_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(first_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id, Commit(first_commits, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(first_snapshot_id));
+
+    std::vector<Row> second_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+                         MakeBatch(second_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(second_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
+                         Commit(second_commits, /*commit_identifier=*/1));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(second_snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(int64_t rollback_snapshot_id, RollbackToAsLatest(first_snapshot_id));
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap rollback_offsets, ReadCommittedOffsets());
+    ASSERT_EQ(1, rollback_offsets.size());
+    ASSERT_EQ(3, rollback_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+    ASSERT_NOK_WITH_MSG(writer->RefreshCommittedSnapshot(rollback_snapshot_id),
+                        "recreate RealtimeContext");
+    ASSERT_OK(writer->Close());
+    writer.reset();
+    realtime_context.reset();
+
+    // Reopen the same real-time writer identity from the target snapshot's progress and replay
+    // input after that restored boundary.
+    ASSERT_OK_AND_ASSIGN(realtime_context, RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(writer, CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> replay_batch,
+                         MakeBatch(second_rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(replay_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> replay_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_EQ(1, replay_commits.size());
+    ASSERT_EQ(OffsetRange(3, 5), replay_commits[0].offset_range);
+    ASSERT_OK(Commit(replay_commits, /*commit_identifier=*/2));
+
+    std::vector<Row> expected_rows = first_rows;
+    expected_rows.insert(expected_rows.end(), second_rows.begin(), second_rows.end());
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows());
+    ASSERT_EQ(expected_rows, actual_rows);
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestConcurrentWritePrepareCommitReadAndRefresh) {
     CreateTable(/*partition_keys=*/{});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
@@ -1392,7 +2453,166 @@ TEST_F(RealtimeWriteInteTest, TestMultiplePartitions) {
     }
     ASSERT_EQ(committed_offsets.end(),
               committed_offsets.find(RealtimePartitionBucket({{"pt", "p2"}}, /*bucket=*/0)));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> final_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(2, final_commits.size());
+    ASSERT_OK_AND_ASSIGN(int64_t final_snapshot_id, Commit(final_commits, /*commit_identifier=*/1));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(final_snapshot_id));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> committed_rows, ReadRows(realtime_context));
+    ASSERT_EQ(expected_rows, committed_rows);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestDropPartitionRequiresReopenRealtimeContext) {
+    CreateTable(/*partition_keys=*/{"pt"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    std::vector<Row> disk_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
+                         MakeBatch(disk_rows, /*partitioned=*/true));
+    ASSERT_OK(writer->Write(std::move(disk_batch)));
+    std::vector<Row> retained_disk_rows =
+        MakeRows(/*first_id=*/10, /*count=*/3, /*partition=*/"p1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> retained_disk_batch,
+                         MakeBatch(retained_disk_rows, /*partitioned=*/true));
+    ASSERT_OK(writer->Write(std::move(retained_disk_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t disk_snapshot_id, Commit(disk_commits, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(disk_snapshot_id));
+
+    std::vector<Row> memory_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
+                         MakeBatch(memory_rows, /*partitioned=*/true));
+    ASSERT_OK(writer->Write(std::move(memory_batch)));
+    std::vector<Row> rows_before_drop = disk_rows;
+    rows_before_drop.insert(rows_before_drop.end(), memory_rows.begin(), memory_rows.end());
+    rows_before_drop.insert(rows_before_drop.end(), retained_disk_rows.begin(),
+                            retained_disk_rows.end());
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows_before_drop, ReadRows(realtime_context));
+    ASSERT_EQ(rows_before_drop, actual_rows_before_drop);
+
+    ASSERT_OK_AND_ASSIGN(int64_t drop_snapshot_id,
+                         DropPartition({{"pt", "p0"}}, /*commit_identifier=*/1));
+    RealtimePartitionBucket partition_bucket({{"pt", "p0"}}, /*bucket=*/0);
+    RealtimePartitionBucket retained_partition_bucket({{"pt", "p1"}}, /*bucket=*/0);
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets_after_drop, ReadCommittedOffsets());
+    ASSERT_EQ(1, offsets_after_drop.size());
+    ASSERT_EQ(3, offsets_after_drop.at(retained_partition_bucket));
+    ASSERT_EQ(offsets_after_drop.end(), offsets_after_drop.find(partition_bucket));
+    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_before_refresh,
+                         GetRealtimeMemoryUsage(realtime_context));
+    ASSERT_NOK_WITH_MSG(writer->RefreshCommittedSnapshot(drop_snapshot_id),
+                        "recreate RealtimeContext");
+    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_after_refresh,
+                         GetRealtimeMemoryUsage(realtime_context));
+    ASSERT_EQ(memory_usage_before_refresh, memory_usage_after_refresh);
     ASSERT_OK(writer->Close());
+    writer.reset();
+    realtime_context.reset();
+
+    // Reopen with p1's retained progress and replay p0 input that existed only in the old context.
+    ASSERT_OK_AND_ASSIGN(realtime_context, RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(writer, CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> replay_batch,
+                         MakeBatch(memory_rows, /*partitioned=*/true));
+    ASSERT_OK(writer->Write(std::move(replay_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> replayed_rows, ReadRows(realtime_context));
+    // The retained p1 disk split is read before the tail real-time split containing p0.
+    std::vector<Row> expected_replayed_rows = retained_disk_rows;
+    expected_replayed_rows.insert(expected_replayed_rows.end(), memory_rows.begin(),
+                                  memory_rows.end());
+    ASSERT_EQ(expected_replayed_rows, replayed_rows);
+
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> memory_commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_EQ(1, memory_commits.size());
+    ASSERT_EQ(OffsetRange(0, 2), memory_commits[0].offset_range);
+    ASSERT_OK_AND_ASSIGN(int64_t memory_snapshot_id,
+                         Commit(memory_commits, /*commit_identifier=*/2));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(memory_snapshot_id));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_memory_commit, ReadRows(realtime_context));
+    std::vector<Row> expected_committed_rows = memory_rows;
+    expected_committed_rows.insert(expected_committed_rows.end(), retained_disk_rows.begin(),
+                                   retained_disk_rows.end());
+    ASSERT_EQ(expected_committed_rows, rows_after_memory_commit);
+    ASSERT_OK_AND_ASSIGN(uint64_t final_memory_usage, GetRealtimeMemoryUsage(realtime_context));
+    ASSERT_EQ(0, final_memory_usage);
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap final_offsets, ReadCommittedOffsets());
+    ASSERT_EQ(2, final_offsets.size());
+    ASSERT_EQ(2, final_offsets.at(partition_bucket));
+    ASSERT_EQ(3, final_offsets.at(retained_partition_bucket));
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestDropInactivePartitionDoesNotRequireReopenRealtimeContext) {
+    CreateTable(/*partition_keys=*/{"pt"});
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> seed_context, RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> seed_writer,
+                         CreateRealtimeWriter(seed_context));
+    for (int64_t partition_index = 0; partition_index < 2; ++partition_index) {
+        std::string partition = "p" + std::to_string(partition_index);
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch(MakeRows(partition_index * 10, /*count=*/3, partition),
+                                       /*partitioned=*/true));
+        ASSERT_OK(seed_writer->Write(std::move(batch)));
+    }
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> seed_commits,
+                         seed_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(2, seed_commits.size());
+    ASSERT_OK(Commit(seed_commits, /*commit_identifier=*/0));
+    ASSERT_OK(seed_writer->Close());
+    seed_writer.reset();
+    seed_context.reset();
+
+    // The new context loads offsets for both partitions, but creates a store only for p0.
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0_batch,
+                         MakeBatch(MakeRows(/*first_id=*/20, /*count=*/1, /*partition=*/"p0"),
+                                   /*partitioned=*/true));
+    ASSERT_OK(writer->Write(std::move(p0_batch)));
+
+    ASSERT_OK_AND_ASSIGN(int64_t drop_snapshot_id,
+                         DropPartition({{"pt", "p1"}}, /*commit_identifier=*/1));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(drop_snapshot_id));
+    const RealtimePartitionBucket p0_partition_bucket({{"pt", "p0"}}, /*bucket=*/0);
+    const RealtimePartitionBucket p1_partition_bucket({{"pt", "p1"}}, /*bucket=*/0);
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets_after_drop, ReadCommittedOffsets());
+    ASSERT_EQ(1, offsets_after_drop.size());
+    ASSERT_EQ(3, offsets_after_drop.at(p0_partition_bucket));
+    ASSERT_EQ(offsets_after_drop.end(), offsets_after_drop.find(p1_partition_bucket));
+
+    // Since p1 was never active in this context, writing it after the drop starts from zero.
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p1_batch,
+                         MakeBatch(MakeRows(/*first_id=*/30, /*count=*/2, /*partition=*/"p1"),
+                                   /*partitioned=*/true));
+    ASSERT_OK(writer->Write(std::move(p1_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_EQ(2, commits.size());
+    auto p1_commit =
+        std::find_if(commits.begin(), commits.end(), [&](const RealtimeCommitProgress& commit) {
+            return commit.partition_bucket == p1_partition_bucket;
+        });
+    ASSERT_NE(commits.end(), p1_commit);
+    ASSERT_EQ(OffsetRange(0, 2), p1_commit->offset_range);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestDropDatePartitionRemovesOffsetWithLegacyPartitionName) {
+    CheckDropDatePartitionRemovesOffset(/*legacy_partition_name_enabled=*/true);
+}
+
+TEST_F(RealtimeWriteInteTest, TestDropDatePartitionRemovesOffsetWithoutLegacyPartitionName) {
+    CheckDropDatePartitionRemovesOffset(/*legacy_partition_name_enabled=*/false);
 }
 
 TEST_F(RealtimeWriteInteTest, TestMultipleBucketsRestoreIndependentOffsets) {

From 6650b42b036efd7fcb64df678bc56be32a928ed4 Mon Sep 17 00:00:00 2001
From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com>
Date: Mon, 24 Aug 2026 15:29:08 +0800
Subject: [PATCH 15/93] chore: remove required status checks in .asf.yaml
 (#241)

---
 .asf.yaml | 34 +---------------------------------
 1 file changed, 1 insertion(+), 33 deletions(-)

diff --git a/.asf.yaml b/.asf.yaml
index 0cc39ed40..e4846c164 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -54,39 +54,7 @@ github:
         dismiss_stale_reviews: true
         require_last_push_approval: true
         required_approving_review_count: 1
-      required_status_checks:
-        - name: "pre-commit"
-          app_slug: -1
-        - name: "rat-license-check"
-          app_slug: -1
-        - name: "script-tests"
-          app_slug: -1
-        - name: "asan-ubsan-x86_64"
-          app_slug: -1
-        - name: "tsan-x86_64"
-          app_slug: -1
-        - name: "clang-debug-x86_64"
-          app_slug: -1
-        - name: "clang-release-x86_64"
-          app_slug: -1
-        - name: "gcc-debug-x86_64"
-          app_slug: -1
-        - name: "gcc-release-x86_64"
-          app_slug: -1
-        - name: "gcc-debug-aarch64"
-          app_slug: -1
-        - name: "gcc-release-aarch64"
-          app_slug: -1
-        - name: "clang-debug-aarch64"
-          app_slug: -1
-        - name: "asan-ubsan-aarch64"
-          app_slug: -1
-        - name: "clang-release-aarch64"
-          app_slug: -1
-        - name: "tsan-aarch64"
-          app_slug: -1
-        - name: "gcc8-test"
-          app_slug: -1
+
   pull_requests:
     allow_auto_merge: false
     allow_update_branch: true

From 6a2c90f01a9029d037f29a82950811e30563859b Mon Sep 17 00:00:00 2001
From: Jingsong Lee <jingsonglee0@gmail.com>
Date: Tue, 25 Aug 2026 13:14:48 +0800
Subject: [PATCH 16/93] feat(rest): support DLF authentication (#244)

---
 CMakeLists.txt                                |   6 +-
 cmake_modules/arrow.diff                      |   7 +
 cmake_modules/orc.diff                        |   9 +
 docs/source/building.rst                      |   3 +-
 docs/source/user_guide/catalog.rst            |  42 +-
 include/paimon/catalog_options.h              |  31 +-
 src/paimon/CMakeLists.txt                     |   8 +
 src/paimon/common/catalog_options.cpp         |   9 +
 src/paimon/common/utils/http_client.cpp       |   5 +
 src/paimon/common/utils/http_client.h         |   2 +
 src/paimon/common/utils/options_utils.h       |  13 +
 .../common/utils/options_utils_test.cpp       |  20 +
 src/paimon/rest/dlf_auth.cpp                  | 811 ++++++++++++++++++
 src/paimon/rest/dlf_auth.h                    | 208 +++++
 src/paimon/rest/dlf_auth_test.cpp             | 468 ++++++++++
 src/paimon/rest/rest_api.cpp                  |  31 +-
 src/paimon/rest/rest_api.h                    |   8 +-
 src/paimon/rest/rest_auth.cpp                 |  12 +-
 src/paimon/rest/rest_auth.h                   |   5 +
 src/paimon/rest/rest_catalog_test.cpp         |   8 +-
 src/paimon/rest/rest_http_client.cpp          |  13 +-
 src/paimon/rest/rest_http_client.h            |  12 +-
 src/paimon/rest/rest_http_client_test.cpp     |  19 +
 23 files changed, 1716 insertions(+), 34 deletions(-)
 create mode 100644 src/paimon/rest/dlf_auth.cpp
 create mode 100644 src/paimon/rest/dlf_auth.h
 create mode 100644 src/paimon/rest/dlf_auth_test.cpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e99034dcc..084e6bf03 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -68,7 +68,8 @@ option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF)
 option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF)
 option(PAIMON_ENABLE_TANTIVY
        "Whether to enable tantivy-fulltext global index (Rust FFI, experimental)" OFF)
-option(PAIMON_ENABLE_REST "Whether to enable the rest catalog (requires libcurl)" OFF)
+option(PAIMON_ENABLE_REST
+       "Whether to enable the rest catalog (requires libcurl and OpenSSL)" OFF)
 if(PAIMON_ENABLE_ORC)
     add_definitions(-DPAIMON_ENABLE_ORC)
 endif()
@@ -79,6 +80,9 @@ endif()
 if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST)
     find_package(CURL REQUIRED)
 endif()
+if(PAIMON_ENABLE_REST)
+    find_package(OpenSSL 1.1.0 REQUIRED)
+endif()
 if(PAIMON_ENABLE_AVRO)
     add_definitions(-DPAIMON_ENABLE_AVRO)
 endif()
diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff
index b8b83517b..bb71e9c39 100644
--- a/cmake_modules/arrow.diff
+++ b/cmake_modules/arrow.diff
@@ -15,6 +15,13 @@ diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/Thi
 index 8cb3ec83f5..0765df8fa8 100644
 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
 +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+@@ -814,5 +814,6 @@ if(DEFINED ENV{ARROW_THRIFT_URL})
+   set(THRIFT_SOURCE_URL "$ENV{ARROW_THRIFT_URL}")
+ else()
+   set_urls(THRIFT_SOURCE_URL
++           "https://archive.apache.org/dist/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+            "https://www.apache.org/dyn/closer.cgi?action=download&filename=/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+            "https://downloads.apache.org/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
 @@ -983,6 +983,11 @@ if(CMAKE_TOOLCHAIN_FILE)
    list(APPEND EP_COMMON_CMAKE_ARGS -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE})
  endif()
diff --git a/cmake_modules/orc.diff b/cmake_modules/orc.diff
index e4ca4e299..88742dbdc 100644
--- a/cmake_modules/orc.diff
+++ b/cmake_modules/orc.diff
@@ -435,3 +435,12 @@ index 9b2c829c7..434841224 100644
  set(ORC_FORMAT_VERSION "1.0.0")
  set(LZ4_VERSION "1.10.0")
  set(SNAPPY_VERSION "1.2.1")
+@@ -140,7 +142,7 @@ if(DEFINED ENV{ORC_FORMAT_URL})
+   set(ORC_FORMAT_SOURCE_URL "$ENV{ORC_FORMAT_URL}")
+   message(STATUS "Using ORC_FORMAT_URL: ${ORC_FORMAT_SOURCE_URL}")
+ else()
+-  set(ORC_FORMAT_SOURCE_URL "https://www.apache.org/dyn/closer.lua/orc/orc-format-${ORC_FORMAT_VERSION}/orc-format-${ORC_FORMAT_VERSION}.tar.gz?action=download" )
++  set(ORC_FORMAT_SOURCE_URL "https://archive.apache.org/dist/orc/orc-format-${ORC_FORMAT_VERSION}/orc-format-${ORC_FORMAT_VERSION}.tar.gz" )
+   message(STATUS "Using DEFAULT URL: ${ORC_FORMAT_SOURCE_URL}")
+ endif()
+ ExternalProject_Add (orc-format_ep
diff --git a/docs/source/building.rst b/docs/source/building.rst
index 466d461b4..32c7e67cd 100644
--- a/docs/source/building.rst
+++ b/docs/source/building.rst
@@ -182,7 +182,8 @@ boolean flags to ``cmake``.
   Linux ``x86_64``; see :ref:`cpp-building-platforms`.
 * ``-DPAIMON_ENABLE_LUCENE=ON``: Support for Lucene full-text search indexes
 * ``-DPAIMON_ENABLE_TANTIVY=ON``: Enable the experimental Tantivy full-text index Rust FFI.
-* ``-DPAIMON_ENABLE_REST=ON``: Support for the REST catalog (``metastore=rest``), requires the libcurl development package.
+* ``-DPAIMON_ENABLE_REST=ON``: Support for the REST catalog
+  (``metastore=rest``), requires the libcurl and OpenSSL development packages.
 
 Third-party dependency source
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst
index c57c50c12..b695044dd 100644
--- a/docs/source/user_guide/catalog.rst
+++ b/docs/source/user_guide/catalog.rst
@@ -50,9 +50,25 @@ registered on the REST server. The catalog is configured through the
 
 * ``metastore``: must be ``rest`` to select the REST catalog.
 * ``uri``: server url of the REST catalog server.
-* ``token.provider``: authentication provider of the REST catalog; currently only
-  ``bear`` is supported (the protocol's historical spelling of "bearer").
+* ``token.provider``: authentication provider of the REST catalog. ``bear``
+  (the protocol's historical spelling of "bearer") and ``dlf`` are supported.
 * ``token``: token of the ``bear`` token provider.
+* ``dlf.region``: region used by DLF request signing. It is inferred from the
+  endpoint URI when omitted.
+* ``dlf.access-key-id`` and ``dlf.access-key-secret``: static DLF access key.
+* ``dlf.security-token``: optional STS security token used with a static access
+  key.
+* ``dlf.token-path``: path to a JSON file containing refreshable DLF credentials.
+* ``dlf.token-loader``: refreshable credential loader. ``local_file`` reads
+  ``dlf.token-path`` and ``ecs`` obtains an STS token from an ECS RAM role.
+* ``dlf.token-ecs-metadata-url``: ECS RAM role metadata endpoint. It defaults to
+  ``http://100.100.100.200/latest/meta-data/Ram/security-credentials/``.
+* ``dlf.token-ecs-role-name``: optional ECS RAM role name. The loader discovers
+  the role from the metadata endpoint when it is omitted.
+* ``dlf.signing-algorithm``: ``default`` selects DLF4-HMAC-SHA256 for DLF VPC
+  endpoints and ``openapi`` selects ROA HMAC-SHA1 for DlfNext OpenAPI endpoints.
+  When omitted, an endpoint containing ``dlfnext`` selects ``openapi`` and other
+  endpoints select ``default``.
 * ``table-default.<key>``: table option defaults applied when a created table
   left ``<key>`` unset.
 * ``header.<name>``: sent as the ``<name>`` http header on every request to the
@@ -70,6 +86,25 @@ registered on the REST server. The catalog is configured through the
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::Catalog> catalog,
                           paimon::Catalog::Create(/*root_path=*/"my_instance", options));
 
+For DLF, configure one credential source. Static AK/SK credentials, an optional
+STS token, a refreshable local token file, and ECS RAM role credentials are
+supported. A local or ECS token has the Java-compatible JSON fields
+``AccessKeyId``, ``AccessKeySecret``, ``SecurityToken`` and ``Expiration``. The
+last field uses UTC ``yyyy-MM-dd'T'HH:mm:ss'Z'`` format. Refreshable credentials
+are reloaded when less than one hour of validity remains.
+
+.. code-block:: cpp
+
+   std::map<std::string, std::string> options = {
+       {"metastore", "rest"},
+       {"uri", "https://dlfnext.cn-hangzhou.aliyuncs.com"},
+       {"token.provider", "dlf"},
+       {"dlf.access-key-id", "<access-key-id>"},
+       {"dlf.access-key-secret", "<access-key-secret>"},
+       // Optional for temporary credentials:
+       {"dlf.security-token", "<security-token>"},
+   };
+
 On creation the catalog queries the server's ``/v1/config`` endpoint and merges
 its response with the options above: the server's overrides win over the client
 options, which in turn win over the server's defaults.
@@ -81,5 +116,4 @@ through the regular ``Catalog`` API, and table snapshots can be listed through
 The C++ REST catalog covers the database, table and snapshot operations of the
 ``Catalog`` API. The parts of the Java REST catalog that have no C++ counterpart
 yet — altering a database or a table, views, functions, partitions, tags, branch
-management and consumers — are not supported, and neither is the ``dlf`` token
-provider.
+management and consumers — are not supported.
diff --git a/include/paimon/catalog_options.h b/include/paimon/catalog_options.h
index f58a876c3..e959483dd 100644
--- a/include/paimon/catalog_options.h
+++ b/include/paimon/catalog_options.h
@@ -32,10 +32,37 @@ struct PAIMON_EXPORT CatalogOptions {
     /// "token" - Token of the "bear" token provider of the REST catalog.
     static const char TOKEN[];
 
-    /// "token.provider" - Authentication provider of the REST catalog. Only "bear" is
-    /// supported ("bear" is the protocol's historical spelling of "bearer", do not "fix" it).
+    /// "token.provider" - Authentication provider of the REST catalog. Supported values are
+    /// "bear" (the protocol's historical spelling of "bearer") and "dlf".
     static const char TOKEN_PROVIDER[];
 
+    /// "dlf.region" - Region used by DLF request signing. Inferred from URI when absent.
+    static const char DLF_REGION[];
+
+    /// "dlf.token-path" - Path of a JSON file containing refreshable DLF credentials.
+    static const char DLF_TOKEN_PATH[];
+
+    /// "dlf.access-key-id" - DLF access key id.
+    static const char DLF_ACCESS_KEY_ID[];
+
+    /// "dlf.access-key-secret" - DLF access key secret.
+    static const char DLF_ACCESS_KEY_SECRET[];
+
+    /// "dlf.security-token" - Optional STS security token used with a DLF access key.
+    static const char DLF_SECURITY_TOKEN[];
+
+    /// "dlf.token-loader" - Refreshable DLF token loader ("ecs" or "local_file").
+    static const char DLF_TOKEN_LOADER[];
+
+    /// "dlf.token-ecs-metadata-url" - ECS RAM role metadata endpoint.
+    static const char DLF_TOKEN_ECS_METADATA_URL[];
+
+    /// "dlf.token-ecs-role-name" - Optional ECS RAM role name.
+    static const char DLF_TOKEN_ECS_ROLE_NAME[];
+
+    /// "dlf.signing-algorithm" - DLF signer ("default" or "openapi").
+    static const char DLF_SIGNING_ALGORITHM[];
+
     /// "table-default." - Prefix of the catalog options that provide table option
     /// defaults: "table-default.<key>=<value>" applies "<key>=<value>" to a created
     /// table when the caller left "<key>" unset.
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index a9810424a..8b3a29536 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -196,6 +196,10 @@ if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST)
     list(APPEND PAIMON_COMMON_SRCS common/utils/http_client.cpp)
     set(PAIMON_CURL_LINK_LIBS CURL::libcurl)
 endif()
+set(PAIMON_REST_LINK_LIBS)
+if(PAIMON_ENABLE_REST)
+    set(PAIMON_REST_LINK_LIBS OpenSSL::Crypto)
+endif()
 if(PAIMON_ENABLE_S3)
     list(APPEND PAIMON_COMMON_SRCS common/fs/object_store_file_system.cpp)
 endif()
@@ -449,6 +453,7 @@ if(PAIMON_ENABLE_REST)
          rest/resource_paths.cpp
          rest/rest_api.cpp
          rest/rest_auth.cpp
+         rest/dlf_auth.cpp
          rest/rest_catalog.cpp
          rest/rest_messages.cpp
          rest/rest_util.cpp)
@@ -468,6 +473,7 @@ add_paimon_lib(paimon
                Threads::Threads
                RapidJSON
                ${PAIMON_CURL_LINK_LIBS}
+               ${PAIMON_REST_LINK_LIBS}
                DataSketches
                STATIC_LINK_LIBS
                arrow
@@ -480,6 +486,7 @@ add_paimon_lib(paimon
                RapidJSON
                DataSketches
                ${PAIMON_CURL_LINK_LIBS}
+               ${PAIMON_REST_LINK_LIBS}
                SHARED_LINK_FLAGS
                ${PAIMON_VERSION_SCRIPT_FLAGS})
 
@@ -965,6 +972,7 @@ if(PAIMON_BUILD_TESTS)
                         rest/rest_http_client_test.cpp
                         rest/mock_rest_server.cpp
                         rest/resource_paths_test.cpp
+                        rest/dlf_auth_test.cpp
                         rest/rest_catalog_test.cpp
                         rest/rest_messages_test.cpp
                         rest/rest_util_test.cpp
diff --git a/src/paimon/common/catalog_options.cpp b/src/paimon/common/catalog_options.cpp
index 6e89e80a8..2722f06bf 100644
--- a/src/paimon/common/catalog_options.cpp
+++ b/src/paimon/common/catalog_options.cpp
@@ -22,6 +22,15 @@ const char CatalogOptions::METASTORE[] = "metastore";
 const char CatalogOptions::URI[] = "uri";
 const char CatalogOptions::TOKEN[] = "token";
 const char CatalogOptions::TOKEN_PROVIDER[] = "token.provider";
+const char CatalogOptions::DLF_REGION[] = "dlf.region";
+const char CatalogOptions::DLF_TOKEN_PATH[] = "dlf.token-path";
+const char CatalogOptions::DLF_ACCESS_KEY_ID[] = "dlf.access-key-id";
+const char CatalogOptions::DLF_ACCESS_KEY_SECRET[] = "dlf.access-key-secret";
+const char CatalogOptions::DLF_SECURITY_TOKEN[] = "dlf.security-token";
+const char CatalogOptions::DLF_TOKEN_LOADER[] = "dlf.token-loader";
+const char CatalogOptions::DLF_TOKEN_ECS_METADATA_URL[] = "dlf.token-ecs-metadata-url";
+const char CatalogOptions::DLF_TOKEN_ECS_ROLE_NAME[] = "dlf.token-ecs-role-name";
+const char CatalogOptions::DLF_SIGNING_ALGORITHM[] = "dlf.signing-algorithm";
 const char CatalogOptions::TABLE_DEFAULT_OPTION_PREFIX[] = "table-default.";
 
 }  // namespace paimon
diff --git a/src/paimon/common/utils/http_client.cpp b/src/paimon/common/utils/http_client.cpp
index e61c7faa8..1542f58a1 100644
--- a/src/paimon/common/utils/http_client.cpp
+++ b/src/paimon/common/utils/http_client.cpp
@@ -164,6 +164,9 @@ CurlHttpClient::~CurlHttpClient() = default;
 
 Result<HttpResponse> CurlHttpClient::Execute(const HttpRequest& request,
                                              const HttpBodyConsumer& consumer) const {
+    if (request.request_timeout_ms < 0) {
+        return Status::Invalid("HTTP request timeout must not be negative");
+    }
     for (int32_t attempt = 0; attempt < kMaxAttempts; ++attempt) {
         CURL* handle = impl_->Acquire();
         if (handle == nullptr) {
@@ -184,6 +187,8 @@ Result<HttpResponse> CurlHttpClient::Execute(const HttpRequest& request,
         curl_easy_setopt(handle, CURLOPT_HTTPHEADER, headers);
         curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1L);
         curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, 30000L);
+        curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS,
+                         static_cast<long>(request.request_timeout_ms));  // NOLINT(runtime/int)
         curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, WriteCallback);
         curl_easy_setopt(handle, CURLOPT_WRITEDATA, &context);
         curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, HeaderCallback);
diff --git a/src/paimon/common/utils/http_client.h b/src/paimon/common/utils/http_client.h
index 0dd3e742c..8b04a7003 100644
--- a/src/paimon/common/utils/http_client.h
+++ b/src/paimon/common/utils/http_client.h
@@ -46,6 +46,8 @@ struct HttpRequest {
     HttpMethod method = HttpMethod::GET;
     std::string url;
     HttpHeaders headers;
+    /// Overall request timeout in milliseconds; zero keeps libcurl's no-timeout default.
+    int32_t request_timeout_ms = 0;
 };
 
 struct HttpResponse {
diff --git a/src/paimon/common/utils/options_utils.h b/src/paimon/common/utils/options_utils.h
index 90b30b54b..c20140071 100644
--- a/src/paimon/common/utils/options_utils.h
+++ b/src/paimon/common/utils/options_utils.h
@@ -76,6 +76,19 @@ class OptionsUtils {
         return value.value();
     }
 
+    template <typename T>
+    static Result<std::optional<T>> GetOptionalValueFromMap(
+        const std::map<std::string, std::string>& key_value_map, const std::string& key) {
+        Result<T> value = GetValueFromMap<T>(key_value_map, key);
+        if (value.ok()) {
+            return std::optional<T>(value.value());
+        }
+        if (value.status().IsNotExist()) {
+            return std::optional<T>();
+        }
+        return value.status();
+    }
+
     /// Fetch options with specific prefix and remove prefix for key.
     static std::map<std::string, std::string> FetchOptionsWithPrefix(
         const std::string& prefix, const std::map<std::string, std::string>& options) {
diff --git a/src/paimon/common/utils/options_utils_test.cpp b/src/paimon/common/utils/options_utils_test.cpp
index 7a09a9850..d4641184f 100644
--- a/src/paimon/common/utils/options_utils_test.cpp
+++ b/src/paimon/common/utils/options_utils_test.cpp
@@ -63,6 +63,26 @@ TEST(OptionsUtilsTest, TestGetValueFromMap) {
     ASSERT_EQ(999, empty);
 }
 
+TEST(OptionsUtilsTest, TestGetOptionalValueFromMap) {
+    const std::map<std::string, std::string> key_value_map = {
+        {"key_int", "10"}, {"key_empty", ""}, {"key_invalid", "ab"}};
+
+    ASSERT_OK_AND_ASSIGN(std::optional<int32_t> optional_value,
+                         OptionsUtils::GetOptionalValueFromMap<int32_t>(key_value_map, "key_int"));
+    ASSERT_EQ(std::optional<int32_t>(10), optional_value);
+    ASSERT_OK_AND_ASSIGN(
+        std::optional<int32_t> optional_missing,
+        OptionsUtils::GetOptionalValueFromMap<int32_t>(key_value_map, "key_nonexist"));
+    ASSERT_EQ(std::nullopt, optional_missing);
+    ASSERT_OK_AND_ASSIGN(
+        std::optional<std::string> optional_empty,
+        OptionsUtils::GetOptionalValueFromMap<std::string>(key_value_map, "key_empty"));
+    ASSERT_EQ(std::optional<std::string>(""), optional_empty);
+    ASSERT_TRUE(OptionsUtils::GetOptionalValueFromMap<int64_t>(key_value_map, "key_invalid")
+                    .status()
+                    .IsInvalid());
+}
+
 TEST(OptionsUtilsTest, TestFetchOptionsWithPrefix) {
     std::map<std::string, std::string> options = {{"key1", "value1"}, {"test.key2", "value2"}};
     auto new_options = OptionsUtils::FetchOptionsWithPrefix("test.", options);
diff --git a/src/paimon/rest/dlf_auth.cpp b/src/paimon/rest/dlf_auth.cpp
new file mode 100644
index 000000000..4c592f31f
--- /dev/null
+++ b/src/paimon/rest/dlf_auth.cpp
@@ -0,0 +1,811 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/rest/dlf_auth.h"
+
+#include <openssl/evp.h>
+
+#include <array>
+#include <cctype>
+#include <climits>
+#include <ctime>
+#include <fstream>
+#include <iomanip>
+#include <limits>
+#include <regex>
+#include <set>
+#include <sstream>
+#include <string_view>
+#include <thread>
+#include <utility>
+
+#include "fmt/format.h"
+#include "paimon/catalog_options.h"
+#include "paimon/common/utils/options_utils.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/common/utils/url_utils.h"
+#include "paimon/common/utils/uuid.h"
+#include "paimon/rest/rest_http_client.h"
+#include "rapidjson/document.h"
+
+namespace paimon {
+
+namespace {
+
+constexpr int32_t kEcsMetadataRequestTimeoutMillis = 3 * 60 * 1000;
+
+constexpr int64_t kTokenExpirationSafeTimeMillis = 60 * 60 * 1000;
+constexpr size_t kMaxTokenResponseBytes = 1024 * 1024;
+constexpr const char kDefaultEcsMetadataUrl[] =
+    "http://100.100.100.200/latest/meta-data/Ram/security-credentials/";
+
+constexpr const char kAuthorizationHeader[] = "Authorization";
+constexpr const char kContentMd5Header[] = "Content-MD5";
+constexpr const char kContentTypeHeader[] = "Content-Type";
+constexpr const char kDlfDateHeader[] = "x-dlf-date";
+constexpr const char kDlfSecurityTokenHeader[] = "x-dlf-security-token";
+constexpr const char kDlfVersionHeader[] = "x-dlf-version";
+constexpr const char kDlfContentSha256Header[] = "x-dlf-content-sha256";
+constexpr const char kUnsignedPayload[] = "UNSIGNED-PAYLOAD";
+constexpr const char kJsonMediaType[] = "application/json";
+
+constexpr const char kOpenApiDateHeader[] = "Date";
+constexpr const char kOpenApiAcceptHeader[] = "Accept";
+constexpr const char kOpenApiHostHeader[] = "Host";
+constexpr const char kAcsSignatureMethodHeader[] = "x-acs-signature-method";
+constexpr const char kAcsSignatureNonceHeader[] = "x-acs-signature-nonce";
+constexpr const char kAcsSignatureVersionHeader[] = "x-acs-signature-version";
+constexpr const char kAcsVersionHeader[] = "x-acs-version";
+constexpr const char kAcsSecurityTokenHeader[] = "x-acs-security-token";
+
+void TrimWhitespace(std::string* value) {
+    size_t begin = 0;
+    while (begin < value->size() && std::isspace(static_cast<unsigned char>((*value)[begin]))) {
+        ++begin;
+    }
+    size_t end = value->size();
+    while (end > begin && std::isspace(static_cast<unsigned char>((*value)[end - 1]))) {
+        --end;
+    }
+    *value = value->substr(begin, end - begin);
+}
+
+Result<std::string> RequiredNonEmptyOption(const std::map<std::string, std::string>& options,
+                                           const std::string& key) {
+    Result<std::string> value = OptionsUtils::GetValueFromMap<std::string>(options, key);
+    if (!value.ok()) {
+        if (!value.status().IsNotExist()) {
+            return value.status();
+        }
+        return Status::Invalid(fmt::format("option '{}' must be configured for DLF auth", key));
+    }
+    if (value.value().empty()) {
+        return Status::Invalid(fmt::format("option '{}' must be configured for DLF auth", key));
+    }
+    return value.value();
+}
+
+Result<std::string> RequiredJsonString(const rapidjson::Value& object, const char* key) {
+    if (!object.HasMember(key) || !object[key].IsString() || object[key].GetStringLength() == 0) {
+        return Status::Invalid(fmt::format("DLF token field '{}' must be a non-empty string", key));
+    }
+    return std::string(object[key].GetString(), object[key].GetStringLength());
+}
+
+Result<std::optional<std::string>> OptionalJsonString(const rapidjson::Value& object,
+                                                      const char* key) {
+    if (!object.HasMember(key) || object[key].IsNull()) {
+        return std::optional<std::string>();
+    }
+    if (!object[key].IsString()) {
+        return Status::Invalid(fmt::format("DLF token field '{}' must be a string", key));
+    }
+    return std::optional<std::string>(
+        std::string(object[key].GetString(), object[key].GetStringLength()));
+}
+
+Result<std::tm> ToUtc(std::chrono::system_clock::time_point time) {
+    std::time_t seconds = std::chrono::system_clock::to_time_t(time);
+    std::tm utc{};
+    if (gmtime_r(&seconds, &utc) == nullptr) {
+        return Status::Invalid("failed to convert DLF signing time to UTC");
+    }
+    return utc;
+}
+
+Result<std::string> FormatDlfTime(std::chrono::system_clock::time_point time) {
+    PAIMON_ASSIGN_OR_RAISE(std::tm utc, ToUtc(time));
+    std::array<char, 32> buffer{};
+    if (std::strftime(buffer.data(), buffer.size(), "%Y%m%dT%H%M%SZ", &utc) == 0) {
+        return Status::Invalid("failed to format DLF signing time");
+    }
+    return std::string(buffer.data());
+}
+
+Result<std::string> FormatRfc1123Time(std::chrono::system_clock::time_point time) {
+    PAIMON_ASSIGN_OR_RAISE(std::tm utc, ToUtc(time));
+    static constexpr std::array<const char*, 7> kWeekdays = {"Sun", "Mon", "Tue", "Wed",
+                                                             "Thu", "Fri", "Sat"};
+    static constexpr std::array<const char*, 12> kMonths = {
+        "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"};
+    if (utc.tm_wday < 0 || utc.tm_wday >= static_cast<int32_t>(kWeekdays.size()) ||
+        utc.tm_mon < 0 || utc.tm_mon >= static_cast<int32_t>(kMonths.size())) {
+        return Status::Invalid("failed to format DLF OpenAPI signing time");
+    }
+    return fmt::format("{}, {:02d} {} {:04d} {:02d}:{:02d}:{:02d} GMT", kWeekdays[utc.tm_wday],
+                       utc.tm_mday, kMonths[utc.tm_mon], utc.tm_year + 1900, utc.tm_hour,
+                       utc.tm_min, utc.tm_sec);
+}
+
+Result<int64_t> ParseExpiration(const std::string& expiration) {
+    static const std::regex kExpirationPattern(
+        "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$");
+    if (!std::regex_match(expiration, kExpirationPattern)) {
+        return Status::Invalid("invalid DLF token expiration");
+    }
+    std::tm utc{};
+    std::istringstream stream(expiration);
+    stream >> std::get_time(&utc, "%Y-%m-%dT%H:%M:%SZ");
+    if (stream.fail() || stream.peek() != std::char_traits<char>::eof()) {
+        return Status::Invalid("invalid DLF token expiration");
+    }
+    int32_t year = utc.tm_year;
+    int32_t month = utc.tm_mon;
+    int32_t day = utc.tm_mday;
+    int32_t hour = utc.tm_hour;
+    int32_t minute = utc.tm_min;
+    int32_t second = utc.tm_sec;
+    std::time_t timestamp = timegm(&utc);
+    std::tm verified{};
+    if (timestamp == static_cast<std::time_t>(-1) || gmtime_r(&timestamp, &verified) == nullptr) {
+        return Status::Invalid("invalid DLF token expiration");
+    }
+    if (verified.tm_year != year || verified.tm_mon != month || verified.tm_mday != day ||
+        verified.tm_hour != hour || verified.tm_min != minute || verified.tm_sec != second) {
+        return Status::Invalid("invalid DLF token expiration");
+    }
+    if (timestamp > std::numeric_limits<int64_t>::max() / 1000) {
+        return Status::Invalid("DLF token expiration is out of range");
+    }
+    return static_cast<int64_t>(timestamp) * 1000;
+}
+
+using Bytes = std::vector<uint8_t>;
+using EvpMdContext = std::unique_ptr<EVP_MD_CTX, decltype(&EVP_MD_CTX_free)>;
+using EvpPkey = std::unique_ptr<EVP_PKEY, decltype(&EVP_PKEY_free)>;
+
+Result<Bytes> Digest(const EVP_MD* digest, std::string_view data) {
+    EvpMdContext context(EVP_MD_CTX_new(), EVP_MD_CTX_free);
+    if (!context || EVP_DigestInit_ex(context.get(), digest, nullptr) != 1 ||
+        EVP_DigestUpdate(context.get(), data.data(), data.size()) != 1) {
+        return Status::IOError("failed to calculate DLF request digest");
+    }
+    Bytes output(EVP_MAX_MD_SIZE);
+    unsigned int output_size = 0;
+    if (EVP_DigestFinal_ex(context.get(), output.data(), &output_size) != 1) {
+        return Status::IOError("failed to calculate DLF request digest");
+    }
+    output.resize(output_size);
+    return output;
+}
+
+Result<Bytes> Hmac(const EVP_MD* digest, const Bytes& key, std::string_view data) {
+    if (key.size() > static_cast<size_t>(INT_MAX)) {
+        return Status::Invalid("DLF signing key is too large");
+    }
+    EvpPkey signing_key(
+        EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, nullptr, key.data(), static_cast<int32_t>(key.size())),
+        EVP_PKEY_free);
+    EvpMdContext context(EVP_MD_CTX_new(), EVP_MD_CTX_free);
+    if (!signing_key || !context ||
+        EVP_DigestSignInit(context.get(), nullptr, digest, nullptr, signing_key.get()) != 1 ||
+        EVP_DigestSignUpdate(context.get(), data.data(), data.size()) != 1) {
+        return Status::IOError("failed to calculate DLF request signature");
+    }
+    size_t output_size = 0;
+    if (EVP_DigestSignFinal(context.get(), nullptr, &output_size) != 1) {
+        return Status::IOError("failed to calculate DLF request signature");
+    }
+    Bytes output(output_size);
+    if (EVP_DigestSignFinal(context.get(), output.data(), &output_size) != 1) {
+        return Status::IOError("failed to calculate DLF request signature");
+    }
+    output.resize(output_size);
+    return output;
+}
+
+Bytes ToBytes(const std::string& value) {
+    return Bytes(value.begin(), value.end());
+}
+
+std::string HexEncode(const Bytes& value) {
+    static constexpr char kHex[] = "0123456789abcdef";
+    std::string encoded;
+    encoded.reserve(value.size() * 2);
+    for (uint8_t byte : value) {
+        encoded.push_back(kHex[byte >> 4]);
+        encoded.push_back(kHex[byte & 0x0f]);
+    }
+    return encoded;
+}
+
+Result<std::string> Base64Encode(const Bytes& value) {
+    if (value.size() > static_cast<size_t>(INT_MAX)) {
+        return Status::Invalid("DLF digest is too large to encode");
+    }
+    size_t capacity = 4 * ((value.size() + 2) / 3) + 1;
+    std::string encoded(capacity, '\0');
+    int32_t size = EVP_EncodeBlock(reinterpret_cast<unsigned char*>(encoded.data()), value.data(),
+                                   static_cast<int32_t>(value.size()));
+    if (size < 0) {
+        return Status::IOError("failed to encode DLF request digest");
+    }
+    encoded.resize(static_cast<size_t>(size));
+    return encoded;
+}
+
+Result<std::string> Md5Base64(const std::string& value) {
+    PAIMON_ASSIGN_OR_RAISE(Bytes digest, Digest(EVP_md5(), value));
+    return Base64Encode(digest);
+}
+
+std::string Trimmed(const std::string& value) {
+    std::string trimmed = value;
+    TrimWhitespace(&trimmed);
+    return trimmed;
+}
+
+std::string DefaultCanonicalRequest(const RestAuthParameter& parameter,
+                                    const DlfRequestSigner::Headers& headers) {
+    std::string canonical = parameter.method + "\n" + parameter.resource_path + "\n";
+    bool first = true;
+    for (const auto& [key, value] : parameter.parameters) {
+        if (!first) {
+            canonical += "&";
+        }
+        canonical += Trimmed(key);
+        if (!value.empty()) {
+            canonical += "=" + Trimmed(value);
+        }
+        first = false;
+    }
+
+    static const std::set<std::string> kSignedHeaders = {
+        "content-md5", "content-type",  "x-dlf-content-sha256",
+        "x-dlf-date",  "x-dlf-version", "x-dlf-security-token"};
+    std::map<std::string, std::string> sorted_headers;
+    for (const auto& [key, value] : headers) {
+        std::string lower_key = StringUtils::ToLowerCase(key);
+        if (kSignedHeaders.count(lower_key) > 0) {
+            sorted_headers[lower_key] = Trimmed(value);
+        }
+    }
+    for (const auto& [key, value] : sorted_headers) {
+        canonical += "\n" + key + ":" + value;
+    }
+    auto content_iter = headers.find(kDlfContentSha256Header);
+    std::string content_sha =
+        content_iter == headers.end() ? std::string(kUnsignedPayload) : content_iter->second;
+    return canonical + "\n" + content_sha;
+}
+
+Result<std::string> RequiredHeader(const DlfRequestSigner::Headers& headers,
+                                   const std::string& name) {
+    auto iter = headers.find(name);
+    if (iter == headers.end() || iter->second.empty()) {
+        return Status::Invalid(fmt::format("DLF signing header '{}' is missing", name));
+    }
+    return iter->second;
+}
+
+std::string OpenApiCanonicalizedHeaders(const DlfRequestSigner::Headers& headers) {
+    std::map<std::string, std::string> sorted;
+    for (const auto& [key, value] : headers) {
+        std::string lower_key = StringUtils::ToLowerCase(key);
+        if (StringUtils::StartsWith(lower_key, "x-acs-")) {
+            sorted[lower_key] = Trimmed(value);
+        }
+    }
+    std::string canonical;
+    for (const auto& [key, value] : sorted) {
+        canonical += key + ":" + value + "\n";
+    }
+    return canonical;
+}
+
+std::string OpenApiCanonicalizedResource(const RestAuthParameter& parameter) {
+    std::string resource = UrlUtils::DecodeString(parameter.resource_path);
+    if (parameter.parameters.empty()) {
+        return resource;
+    }
+    resource += "?";
+    bool first = true;
+    for (const auto& [key, value] : parameter.parameters) {
+        if (!first) {
+            resource += "&";
+        }
+        resource += key;
+        std::string decoded = UrlUtils::DecodeString(value);
+        if (!decoded.empty()) {
+            resource += "=" + decoded;
+        }
+        first = false;
+    }
+    return resource;
+}
+
+Result<std::string> GenerateNonce(std::chrono::system_clock::time_point now) {
+    std::string uuid;
+    if (!UUID::Generate(&uuid)) {
+        return Status::IOError("failed to generate DLF OpenAPI signing nonce");
+    }
+    int64_t millis =
+        std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
+    std::ostringstream thread_id;
+    thread_id << std::this_thread::get_id();
+    return fmt::format("{}{}{}", uuid, millis, thread_id.str());
+}
+
+Result<std::unique_ptr<DlfRequestSigner>> CreateSigner(const std::string& algorithm,
+                                                       const std::string& region) {
+    if (algorithm == DlfDefaultSigner::kIdentifier) {
+        return std::make_unique<DlfDefaultSigner>(region);
+    }
+    if (algorithm == DlfOpenApiSigner::kIdentifier) {
+        return std::make_unique<DlfOpenApiSigner>();
+    }
+    return Status::Invalid(fmt::format(
+        "unsupported DLF signing algorithm '{}', supported values are 'default' and 'openapi'",
+        algorithm));
+}
+
+}  // namespace
+
+DlfToken::DlfToken(const std::string& access_key_id, const std::string& access_key_secret,
+                   const std::optional<std::string>& security_token,
+                   const std::optional<int64_t>& expiration_at_millis)
+    : access_key_id_(access_key_id),
+      access_key_secret_(access_key_secret),
+      security_token_(security_token),
+      expiration_at_millis_(expiration_at_millis) {}
+
+Result<DlfToken> DlfToken::FromJson(const std::string& json) {
+    rapidjson::Document document;
+    document.Parse(json.data(), json.size());
+    if (document.HasParseError() || !document.IsObject()) {
+        return Status::Invalid("failed to parse DLF token JSON");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::string access_key_id, RequiredJsonString(document, "AccessKeyId"));
+    PAIMON_ASSIGN_OR_RAISE(std::string access_key_secret,
+                           RequiredJsonString(document, "AccessKeySecret"));
+    PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> security_token,
+                           OptionalJsonString(document, "SecurityToken"));
+    PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> expiration,
+                           OptionalJsonString(document, "Expiration"));
+    std::optional<int64_t> expiration_at_millis;
+    if (expiration) {
+        PAIMON_ASSIGN_OR_RAISE(int64_t parsed_expiration, ParseExpiration(expiration.value()));
+        expiration_at_millis = parsed_expiration;
+    }
+    return DlfToken(access_key_id, access_key_secret, security_token, expiration_at_millis);
+}
+
+bool DlfToken::ShouldRefresh(std::chrono::system_clock::time_point now) const {
+    if (!expiration_at_millis_) {
+        return false;
+    }
+    int64_t now_millis =
+        std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
+    return expiration_at_millis_.value() - now_millis < kTokenExpirationSafeTimeMillis;
+}
+
+DlfLocalFileTokenLoader::DlfLocalFileTokenLoader(const std::string& token_file_path,
+                                                 int32_t max_attempts,
+                                                 std::chrono::milliseconds retry_delay)
+    : token_file_path_(token_file_path), max_attempts_(max_attempts), retry_delay_(retry_delay) {}
+
+Result<DlfToken> DlfLocalFileTokenLoader::LoadToken() {
+    if (token_file_path_.empty()) {
+        return Status::Invalid("DLF token file path is empty");
+    }
+    if (max_attempts_ <= 0 || retry_delay_.count() < 0) {
+        return Status::Invalid("invalid DLF token file retry configuration");
+    }
+    Status last_status = Status::Invalid("failed to load DLF token file");
+    for (int32_t attempt = 1; attempt <= max_attempts_; ++attempt) {
+        std::ifstream file(token_file_path_, std::ios::binary);
+        if (!file.is_open()) {
+            last_status = Status::IOError(
+                fmt::format("failed to read DLF token file '{}'", token_file_path_));
+        } else {
+            std::string contents(kMaxTokenResponseBytes + 1, '\0');
+            file.read(contents.data(), static_cast<std::streamsize>(contents.size()));
+            std::streamsize size = file.gcount();
+            if (file.bad()) {
+                last_status = Status::IOError(
+                    fmt::format("failed to read DLF token file '{}'", token_file_path_));
+            } else if (size > static_cast<std::streamsize>(kMaxTokenResponseBytes)) {
+                last_status = Status::Invalid("DLF token file is too large");
+            } else {
+                contents.resize(static_cast<size_t>(size));
+                Result<DlfToken> token = DlfToken::FromJson(contents);
+                if (token.ok()) {
+                    return token;
+                }
+                last_status = Status::Invalid("failed to parse DLF token file");
+            }
+        }
+        if (attempt < max_attempts_) {
+            std::this_thread::sleep_for(retry_delay_ * attempt);
+        }
+    }
+    return last_status;
+}
+
+std::string DlfLocalFileTokenLoader::Description() const {
+    return token_file_path_;
+}
+
+DlfEcsTokenLoader::DlfEcsTokenLoader(const std::string& metadata_url,
+                                     const std::optional<std::string>& role_name,
+                                     std::unique_ptr<HttpClient> http_client)
+    : metadata_url_(metadata_url), role_name_(role_name), http_client_(std::move(http_client)) {}
+
+std::unique_ptr<DlfEcsTokenLoader> DlfEcsTokenLoader::Create(
+    const std::string& metadata_url, const std::optional<std::string>& role_name) {
+    return std::make_unique<DlfEcsTokenLoader>(metadata_url, role_name,
+                                               std::make_unique<CurlHttpClient>());
+}
+
+Result<std::string> DlfEcsTokenLoader::Get(const std::string& url) const {
+    if (!http_client_) {
+        return Status::Invalid("DLF ECS metadata HTTP client is not configured");
+    }
+    HttpRequest request;
+    request.url = url;
+    request.request_timeout_ms = kEcsMetadataRequestTimeoutMillis;
+    std::string body;
+    Result<HttpResponse> response =
+        http_client_->Execute(request, [&body](const char* data, int64_t size) {
+            if (size < 0 || body.size() + static_cast<size_t>(size) > kMaxTokenResponseBytes) {
+                return Status::Invalid("DLF ECS metadata response is too large");
+            }
+            body.append(data, static_cast<size_t>(size));
+            return Status::OK();
+        });
+    if (!response.ok()) {
+        return Status::IOError("failed to request DLF credentials from ECS metadata service: ",
+                               response.status().message());
+    }
+    HttpResponse http_response = std::move(response).value();
+    if (http_response.status_code < 200 || http_response.status_code >= 300) {
+        return Status::IOError(fmt::format("DLF ECS metadata service returned HTTP status {}",
+                                           http_response.status_code));
+    }
+    if (StringUtils::IsNullOrWhitespaceOnly(body)) {
+        return Status::Invalid("DLF ECS metadata service returned an empty response");
+    }
+    return body;
+}
+
+Result<DlfToken> DlfEcsTokenLoader::LoadToken() {
+    if (metadata_url_.empty()) {
+        return Status::Invalid("DLF ECS metadata URL is empty");
+    }
+    if (!role_name_) {
+        PAIMON_ASSIGN_OR_RAISE(std::string role, Get(metadata_url_));
+        TrimWhitespace(&role);
+        if (role.empty()) {
+            return Status::Invalid("DLF ECS metadata service returned an empty role name");
+        }
+        role_name_ = role;
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::string token_json, Get(metadata_url_ + role_name_.value()));
+    Result<DlfToken> token = DlfToken::FromJson(token_json);
+    if (!token.ok()) {
+        return Status::Invalid("failed to parse DLF ECS token response");
+    }
+    return token;
+}
+
+std::string DlfEcsTokenLoader::Description() const {
+    return metadata_url_;
+}
+
+DlfDefaultSigner::DlfDefaultSigner(const std::string& region) : region_(region) {}
+
+Result<DlfRequestSigner::Headers> DlfDefaultSigner::SignHeaders(
+    const std::string& body, std::chrono::system_clock::time_point now,
+    const std::optional<std::string>& security_token, const std::string& host) const {
+    PAIMON_ASSIGN_OR_RAISE(std::string date_time, FormatDlfTime(now));
+    Headers headers = {{kDlfDateHeader, date_time},
+                       {kDlfContentSha256Header, kUnsignedPayload},
+                       {kDlfVersionHeader, "v1"}};
+    if (!body.empty()) {
+        PAIMON_ASSIGN_OR_RAISE(std::string content_md5, Md5Base64(body));
+        headers[kContentTypeHeader] = kJsonMediaType;
+        headers[kContentMd5Header] = content_md5;
+    }
+    if (security_token) {
+        headers[kDlfSecurityTokenHeader] = security_token.value();
+    }
+    return headers;
+}
+
+Result<std::string> DlfDefaultSigner::Authorization(const RestAuthParameter& parameter,
+                                                    const DlfToken& token, const std::string& host,
+                                                    const Headers& sign_headers) const {
+    PAIMON_ASSIGN_OR_RAISE(std::string date_time, RequiredHeader(sign_headers, kDlfDateHeader));
+    if (date_time.size() < 8) {
+        return Status::Invalid("DLF signing date is invalid");
+    }
+    std::string date = date_time.substr(0, 8);
+    std::string scope = fmt::format("{}/{}/DlfNext/aliyun_v4_request", date, region_);
+    std::string canonical_request = DefaultCanonicalRequest(parameter, sign_headers);
+    PAIMON_ASSIGN_OR_RAISE(Bytes canonical_hash, Digest(EVP_sha256(), canonical_request));
+    std::string string_to_sign =
+        fmt::format("DLF4-HMAC-SHA256\n{}\n{}\n{}", date_time, scope, HexEncode(canonical_hash));
+
+    PAIMON_ASSIGN_OR_RAISE(
+        Bytes date_key,
+        Hmac(EVP_sha256(), ToBytes("aliyun_v4" + token.GetAccessKeySecret()), date));
+    PAIMON_ASSIGN_OR_RAISE(Bytes region_key, Hmac(EVP_sha256(), date_key, region_));
+    PAIMON_ASSIGN_OR_RAISE(Bytes service_key, Hmac(EVP_sha256(), region_key, "DlfNext"));
+    PAIMON_ASSIGN_OR_RAISE(Bytes signing_key, Hmac(EVP_sha256(), service_key, "aliyun_v4_request"));
+    PAIMON_ASSIGN_OR_RAISE(Bytes signature, Hmac(EVP_sha256(), signing_key, string_to_sign));
+    return fmt::format("DLF4-HMAC-SHA256 Credential={}/{},Signature={}", token.GetAccessKeyId(),
+                       scope, HexEncode(signature));
+}
+
+Result<DlfRequestSigner::Headers> DlfOpenApiSigner::SignHeaders(
+    const std::string& body, std::chrono::system_clock::time_point now,
+    const std::optional<std::string>& security_token, const std::string& host) const {
+    if (host.empty()) {
+        return Status::Invalid("DLF OpenAPI signing host is empty");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::string date, FormatRfc1123Time(now));
+    PAIMON_ASSIGN_OR_RAISE(std::string nonce, GenerateNonce(now));
+    Headers headers = {{kOpenApiDateHeader, date},        {kOpenApiAcceptHeader, kJsonMediaType},
+                       {kOpenApiHostHeader, host},        {kAcsSignatureMethodHeader, "HMAC-SHA1"},
+                       {kAcsSignatureNonceHeader, nonce}, {kAcsSignatureVersionHeader, "1.0"},
+                       {kAcsVersionHeader, "2026-01-18"}};
+    if (!body.empty()) {
+        PAIMON_ASSIGN_OR_RAISE(std::string content_md5, Md5Base64(body));
+        headers[kContentMd5Header] = content_md5;
+        headers[kContentTypeHeader] = kJsonMediaType;
+    }
+    if (security_token) {
+        headers[kAcsSecurityTokenHeader] = security_token.value();
+    }
+    return headers;
+}
+
+Result<std::string> DlfOpenApiSigner::Authorization(const RestAuthParameter& parameter,
+                                                    const DlfToken& token, const std::string& host,
+                                                    const Headers& sign_headers) const {
+    PAIMON_ASSIGN_OR_RAISE(std::string accept, RequiredHeader(sign_headers, kOpenApiAcceptHeader));
+    PAIMON_ASSIGN_OR_RAISE(std::string date, RequiredHeader(sign_headers, kOpenApiDateHeader));
+    std::string content_md5;
+    auto md5_iter = sign_headers.find(kContentMd5Header);
+    if (md5_iter != sign_headers.end()) {
+        content_md5 = md5_iter->second;
+    }
+    std::string content_type;
+    auto type_iter = sign_headers.find(kContentTypeHeader);
+    if (type_iter != sign_headers.end()) {
+        content_type = type_iter->second;
+    }
+    std::string string_to_sign =
+        parameter.method + "\n" + accept + "\n" + content_md5 + "\n" + content_type + "\n" + date +
+        "\n" + OpenApiCanonicalizedHeaders(sign_headers) + OpenApiCanonicalizedResource(parameter);
+    PAIMON_ASSIGN_OR_RAISE(Bytes signature,
+                           Hmac(EVP_sha1(), ToBytes(token.GetAccessKeySecret()), string_to_sign));
+    PAIMON_ASSIGN_OR_RAISE(std::string encoded_signature, Base64Encode(signature));
+    return fmt::format("acs {}:{}", token.GetAccessKeyId(), encoded_signature);
+}
+
+DlfAuthProvider::DlfAuthProvider(std::unique_ptr<DlfTokenLoader> token_loader,
+                                 const std::optional<DlfToken>& token, const std::string& host,
+                                 std::unique_ptr<DlfRequestSigner> signer, Clock clock)
+    : token_loader_(std::move(token_loader)),
+      token_(token),
+      host_(host),
+      signer_(std::move(signer)),
+      clock_(std::move(clock)) {}
+
+Result<std::unique_ptr<DlfAuthProvider>> DlfAuthProvider::Create(
+    const std::map<std::string, std::string>& options) {
+    PAIMON_ASSIGN_OR_RAISE(std::string uri, RequiredNonEmptyOption(options, CatalogOptions::URI));
+    std::string region;
+    PAIMON_ASSIGN_OR_RAISE(
+        std::optional<std::string> configured_region,
+        OptionsUtils::GetOptionalValueFromMap<std::string>(options, CatalogOptions::DLF_REGION));
+    if (configured_region) {
+        if (configured_region->empty()) {
+            return Status::Invalid("option 'dlf.region' must not be empty");
+        }
+        region = configured_region.value();
+    } else {
+        PAIMON_ASSIGN_OR_RAISE(region, ParseRegionFromUri(uri));
+    }
+
+    std::string algorithm;
+    PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> configured_algorithm,
+                           OptionsUtils::GetOptionalValueFromMap<std::string>(
+                               options, CatalogOptions::DLF_SIGNING_ALGORITHM));
+    if (configured_algorithm) {
+        algorithm = configured_algorithm.value();
+    } else {
+        algorithm = ParseSigningAlgorithmFromUri(uri);
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> loader_name,
+                           OptionsUtils::GetOptionalValueFromMap<std::string>(
+                               options, CatalogOptions::DLF_TOKEN_LOADER));
+    PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> token_path,
+                           OptionsUtils::GetOptionalValueFromMap<std::string>(
+                               options, CatalogOptions::DLF_TOKEN_PATH));
+    if (loader_name) {
+        if (loader_name.value() == "ecs") {
+            PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> configured_metadata_url,
+                                   OptionsUtils::GetOptionalValueFromMap<std::string>(
+                                       options, CatalogOptions::DLF_TOKEN_ECS_METADATA_URL));
+            std::string metadata_url = configured_metadata_url.value_or(kDefaultEcsMetadataUrl);
+            PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> role_name,
+                                   OptionsUtils::GetOptionalValueFromMap<std::string>(
+                                       options, CatalogOptions::DLF_TOKEN_ECS_ROLE_NAME));
+            return FromTokenLoader(DlfEcsTokenLoader::Create(metadata_url, role_name), uri, region,
+                                   algorithm, std::chrono::system_clock::now);
+        }
+        if (loader_name.value() == "local_file") {
+            PAIMON_ASSIGN_OR_RAISE(std::string path,
+                                   RequiredNonEmptyOption(options, CatalogOptions::DLF_TOKEN_PATH));
+            return FromTokenLoader(
+                std::make_unique<DlfLocalFileTokenLoader>(path, 5, std::chrono::seconds(1)), uri,
+                region, algorithm, std::chrono::system_clock::now);
+        }
+        return Status::NotImplemented(
+            fmt::format("unsupported DLF token loader '{}', supported values are 'ecs' and "
+                        "'local_file'",
+                        loader_name.value()));
+    }
+    if (token_path) {
+        if (token_path->empty()) {
+            return Status::Invalid("option 'dlf.token-path' must not be empty");
+        }
+        return FromTokenLoader(std::make_unique<DlfLocalFileTokenLoader>(token_path.value(), 5,
+                                                                         std::chrono::seconds(1)),
+                               uri, region, algorithm, std::chrono::system_clock::now);
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> access_key_id,
+                           OptionsUtils::GetOptionalValueFromMap<std::string>(
+                               options, CatalogOptions::DLF_ACCESS_KEY_ID));
+    PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> access_key_secret,
+                           OptionsUtils::GetOptionalValueFromMap<std::string>(
+                               options, CatalogOptions::DLF_ACCESS_KEY_SECRET));
+    if (access_key_id && !access_key_id->empty() && access_key_secret &&
+        !access_key_secret->empty()) {
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> security_token,
+                               OptionsUtils::GetOptionalValueFromMap<std::string>(
+                                   options, CatalogOptions::DLF_SECURITY_TOKEN));
+        DlfToken token(access_key_id.value(), access_key_secret.value(), security_token,
+                       std::nullopt);
+        return FromAccessKey(token, uri, region, algorithm, std::chrono::system_clock::now);
+    }
+    return Status::Invalid("DLF token path or access key must be configured for DLF auth");
+}
+
+Result<std::unique_ptr<DlfAuthProvider>> DlfAuthProvider::FromAccessKey(
+    const DlfToken& token, const std::string& uri, const std::string& region,
+    const std::string& signing_algorithm, Clock clock) {
+    if (token.GetAccessKeyId().empty() || token.GetAccessKeySecret().empty()) {
+        return Status::Invalid("DLF access key id and secret must not be empty");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::string host, ExtractHost(uri));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<DlfRequestSigner> signer,
+                           CreateSigner(signing_algorithm, region));
+    return std::unique_ptr<DlfAuthProvider>(
+        new DlfAuthProvider(nullptr, token, host, std::move(signer), std::move(clock)));
+}
+
+Result<std::unique_ptr<DlfAuthProvider>> DlfAuthProvider::FromTokenLoader(
+    std::unique_ptr<DlfTokenLoader> token_loader, const std::string& uri, const std::string& region,
+    const std::string& signing_algorithm, Clock clock) {
+    if (!token_loader) {
+        return Status::Invalid("DLF token loader must not be null");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::string host, ExtractHost(uri));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<DlfRequestSigner> signer,
+                           CreateSigner(signing_algorithm, region));
+    return std::unique_ptr<DlfAuthProvider>(new DlfAuthProvider(
+        std::move(token_loader), std::nullopt, host, std::move(signer), std::move(clock)));
+}
+
+Result<DlfToken> DlfAuthProvider::GetFreshToken(std::chrono::system_clock::time_point now) const {
+    std::scoped_lock lock(token_mutex_);
+    if (token_ && !token_->ShouldRefresh(now)) {
+        return token_.value();
+    }
+    if (!token_loader_) {
+        return Status::Invalid("DLF credentials expired and no token loader is configured");
+    }
+    PAIMON_ASSIGN_OR_RAISE(DlfToken loaded, token_loader_->LoadToken());
+    if (loaded.GetAccessKeyId().empty() || loaded.GetAccessKeySecret().empty()) {
+        return Status::Invalid("DLF token loader returned empty access key credentials");
+    }
+    token_ = loaded;
+    return loaded;
+}
+
+Result<std::map<std::string, std::string>> DlfAuthProvider::MergeAuthHeader(
+    const std::map<std::string, std::string>& base_header,
+    const RestAuthParameter& parameter) const {
+    PAIMON_ASSIGN_OR_RAISE(DlfToken token, GetFreshToken(clock_()));
+    std::chrono::system_clock::time_point signing_time = clock_();
+    PAIMON_ASSIGN_OR_RAISE(
+        DlfRequestSigner::Headers sign_headers,
+        signer_->SignHeaders(parameter.data, signing_time, token.GetSecurityToken(), host_));
+    PAIMON_ASSIGN_OR_RAISE(std::string authorization,
+                           signer_->Authorization(parameter, token, host_, sign_headers));
+    std::map<std::string, std::string> headers = base_header;
+    for (const auto& [key, value] : sign_headers) {
+        headers[key] = value;
+    }
+    headers[kAuthorizationHeader] = authorization;
+    return headers;
+}
+
+Result<std::string> DlfAuthProvider::ParseRegionFromUri(const std::string& uri) {
+    static const std::regex kRegionPattern("(?:pre-)?([a-z]+-[a-z]+(?:-[0-9]+)?)");
+    std::smatch match;
+    if (std::regex_search(uri, match, kRegionPattern) && match.size() > 1 &&
+        !match.str(1).empty()) {
+        return match.str(1);
+    }
+    return Status::Invalid(
+        "could not determine DLF region from option 'dlf.region' or REST catalog URI");
+}
+
+std::string DlfAuthProvider::ParseSigningAlgorithmFromUri(const std::string& uri) {
+    std::string lower_uri = StringUtils::ToLowerCase(uri);
+    return lower_uri.find("dlfnext") == std::string::npos ? DlfDefaultSigner::kIdentifier
+                                                          : DlfOpenApiSigner::kIdentifier;
+}
+
+Result<std::string> DlfAuthProvider::ExtractHost(const std::string& uri) {
+    std::string host = RestHttpClient::NormalizeUri(uri);
+    std::string lower_uri = StringUtils::ToLowerCase(host);
+    if (StringUtils::StartsWith(lower_uri, "http://")) {
+        host.erase(0, 7);
+    } else if (StringUtils::StartsWith(lower_uri, "https://")) {
+        host.erase(0, 8);
+    }
+    size_t path = host.find('/');
+    if (path != std::string::npos) {
+        host.resize(path);
+    }
+    if (host.empty() || host.find('?') != std::string::npos ||
+        host.find('#') != std::string::npos || host.find('@') != std::string::npos) {
+        return Status::Invalid("could not determine DLF signing host from REST catalog URI");
+    }
+    return host;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/rest/dlf_auth.h b/src/paimon/rest/dlf_auth.h
new file mode 100644
index 000000000..c10b655b7
--- /dev/null
+++ b/src/paimon/rest/dlf_auth.h
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <chrono>
+#include <cstdint>
+#include <functional>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "paimon/common/utils/http_client.h"
+#include "paimon/rest/rest_auth.h"
+
+namespace paimon {
+
+/// Access key credentials used to sign DLF REST requests.
+class DlfToken {
+ public:
+    DlfToken(const std::string& access_key_id, const std::string& access_key_secret,
+             const std::optional<std::string>& security_token,
+             const std::optional<int64_t>& expiration_at_millis);
+
+    static Result<DlfToken> FromJson(const std::string& json);
+
+    const std::string& GetAccessKeyId() const {
+        return access_key_id_;
+    }
+
+    const std::string& GetAccessKeySecret() const {
+        return access_key_secret_;
+    }
+
+    const std::optional<std::string>& GetSecurityToken() const {
+        return security_token_;
+    }
+
+    const std::optional<int64_t>& GetExpirationAtMillis() const {
+        return expiration_at_millis_;
+    }
+
+    bool ShouldRefresh(std::chrono::system_clock::time_point now) const;
+
+ private:
+    std::string access_key_id_;
+    std::string access_key_secret_;
+    std::optional<std::string> security_token_;
+    std::optional<int64_t> expiration_at_millis_;
+};
+
+/// Loads refreshable DLF credentials.
+class DlfTokenLoader {
+ public:
+    virtual ~DlfTokenLoader() = default;
+
+    virtual Result<DlfToken> LoadToken() = 0;
+    virtual std::string Description() const = 0;
+};
+
+/// Loads a DLF STS token from a JSON file.
+class DlfLocalFileTokenLoader : public DlfTokenLoader {
+ public:
+    DlfLocalFileTokenLoader(const std::string& token_file_path, int32_t max_attempts,
+                            std::chrono::milliseconds retry_delay);
+
+    Result<DlfToken> LoadToken() override;
+    std::string Description() const override;
+
+ private:
+    std::string token_file_path_;
+    int32_t max_attempts_;
+    std::chrono::milliseconds retry_delay_;
+};
+
+/// Loads a DLF STS token from the Alibaba Cloud ECS metadata service.
+class DlfEcsTokenLoader : public DlfTokenLoader {
+ public:
+    DlfEcsTokenLoader(const std::string& metadata_url, const std::optional<std::string>& role_name,
+                      std::unique_ptr<HttpClient> http_client);
+
+    static std::unique_ptr<DlfEcsTokenLoader> Create(const std::string& metadata_url,
+                                                     const std::optional<std::string>& role_name);
+
+    Result<DlfToken> LoadToken() override;
+    std::string Description() const override;
+
+ private:
+    Result<std::string> Get(const std::string& url) const;
+
+    std::string metadata_url_;
+    std::optional<std::string> role_name_;
+    std::unique_ptr<HttpClient> http_client_;
+};
+
+/// Signs a DLF REST request using one of the endpoint-specific algorithms.
+class DlfRequestSigner {
+ public:
+    using Headers = std::map<std::string, std::string>;
+
+    virtual ~DlfRequestSigner() = default;
+
+    virtual Result<Headers> SignHeaders(const std::string& body,
+                                        std::chrono::system_clock::time_point now,
+                                        const std::optional<std::string>& security_token,
+                                        const std::string& host) const = 0;
+
+    virtual Result<std::string> Authorization(const RestAuthParameter& parameter,
+                                              const DlfToken& token, const std::string& host,
+                                              const Headers& sign_headers) const = 0;
+};
+
+/// DLF4-HMAC-SHA256 signer used by the default DLF VPC endpoint.
+class DlfDefaultSigner : public DlfRequestSigner {
+ public:
+    static constexpr const char* kIdentifier = "default";
+
+    explicit DlfDefaultSigner(const std::string& region);
+
+    Result<Headers> SignHeaders(const std::string& body, std::chrono::system_clock::time_point now,
+                                const std::optional<std::string>& security_token,
+                                const std::string& host) const override;
+
+    Result<std::string> Authorization(const RestAuthParameter& parameter, const DlfToken& token,
+                                      const std::string& host,
+                                      const Headers& sign_headers) const override;
+
+ private:
+    std::string region_;
+};
+
+/// ROA HMAC-SHA1 signer used by DlfNext OpenAPI endpoints.
+class DlfOpenApiSigner : public DlfRequestSigner {
+ public:
+    static constexpr const char* kIdentifier = "openapi";
+
+    Result<Headers> SignHeaders(const std::string& body, std::chrono::system_clock::time_point now,
+                                const std::optional<std::string>& security_token,
+                                const std::string& host) const override;
+
+    Result<std::string> Authorization(const RestAuthParameter& parameter, const DlfToken& token,
+                                      const std::string& host,
+                                      const Headers& sign_headers) const override;
+};
+
+/// Generates DLF authentication headers and refreshes expiring credentials.
+class DlfAuthProvider : public AuthProvider {
+ public:
+    using Clock = std::function<std::chrono::system_clock::time_point()>;
+
+    static Result<std::unique_ptr<DlfAuthProvider>> Create(
+        const std::map<std::string, std::string>& options);
+
+    static Result<std::unique_ptr<DlfAuthProvider>> FromAccessKey(
+        const DlfToken& token, const std::string& uri, const std::string& region,
+        const std::string& signing_algorithm, Clock clock);
+
+    static Result<std::unique_ptr<DlfAuthProvider>> FromTokenLoader(
+        std::unique_ptr<DlfTokenLoader> token_loader, const std::string& uri,
+        const std::string& region, const std::string& signing_algorithm, Clock clock);
+
+    Result<std::map<std::string, std::string>> MergeAuthHeader(
+        const std::map<std::string, std::string>& base_header,
+        const RestAuthParameter& parameter) const override;
+
+    bool AllowsRedirects() const override {
+        return false;
+    }
+
+    static Result<std::string> ParseRegionFromUri(const std::string& uri);
+    static std::string ParseSigningAlgorithmFromUri(const std::string& uri);
+    static Result<std::string> ExtractHost(const std::string& uri);
+
+ private:
+    DlfAuthProvider(std::unique_ptr<DlfTokenLoader> token_loader,
+                    const std::optional<DlfToken>& token, const std::string& host,
+                    std::unique_ptr<DlfRequestSigner> signer, Clock clock);
+
+    Result<DlfToken> GetFreshToken(std::chrono::system_clock::time_point now) const;
+
+    std::unique_ptr<DlfTokenLoader> token_loader_;
+    mutable std::optional<DlfToken> token_;
+    std::string host_;
+    std::unique_ptr<DlfRequestSigner> signer_;
+    Clock clock_;
+    mutable std::mutex token_mutex_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/rest/dlf_auth_test.cpp b/src/paimon/rest/dlf_auth_test.cpp
new file mode 100644
index 000000000..afb9cec50
--- /dev/null
+++ b/src/paimon/rest/dlf_auth_test.cpp
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/rest/dlf_auth.h"
+
+#include <atomic>
+#include <chrono>
+#include <fstream>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "paimon/catalog_options.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+namespace {
+
+using StringMap = std::map<std::string, std::string>;
+
+std::chrono::system_clock::time_point FixedTime() {
+    return std::chrono::system_clock::from_time_t(1744775086);
+}
+
+class SequenceTokenLoader : public DlfTokenLoader {
+ public:
+    explicit SequenceTokenLoader(
+        std::vector<DlfToken> tokens,
+        std::chrono::milliseconds load_delay = std::chrono::milliseconds(0))
+        : tokens_(std::move(tokens)), load_delay_(load_delay) {}
+
+    Result<DlfToken> LoadToken() override {
+        std::this_thread::sleep_for(load_delay_);
+        int32_t index = load_count_.fetch_add(1);
+        if (index >= static_cast<int32_t>(tokens_.size())) {
+            return Status::Invalid("test token loader exhausted");
+        }
+        return tokens_[index];
+    }
+
+    std::string Description() const override {
+        return "test sequence";
+    }
+
+    int32_t GetLoadCount() const {
+        return load_count_.load();
+    }
+
+ private:
+    std::vector<DlfToken> tokens_;
+    std::chrono::milliseconds load_delay_;
+    std::atomic<int32_t> load_count_{0};
+};
+
+class MockEcsHttpClient : public HttpClient {
+ public:
+    explicit MockEcsHttpClient(const std::string& metadata_url, bool direct_token = false)
+        : metadata_url_(metadata_url), direct_token_(direct_token) {}
+
+    Result<HttpResponse> Execute(const HttpRequest& request,
+                                 const HttpBodyConsumer& consumer) const override {
+        last_request_timeout_ms_.store(request.request_timeout_ms);
+        HttpResponse response;
+        response.status_code = 200;
+        std::string body;
+        if (request.url == metadata_url_) {
+            if (direct_token_) {
+                token_requests_.fetch_add(1);
+                body = R"({"AccessKeyId":"ecs-ak","AccessKeySecret":"ecs-sk",)"
+                       R"("SecurityToken":"ecs-sts","Expiration":"2027-04-16T05:44:46Z"})";
+            } else {
+                role_requests_.fetch_add(1);
+                body = " test-role\n";
+            }
+        } else if (request.url == metadata_url_ + "test-role") {
+            token_requests_.fetch_add(1);
+            body = R"({"AccessKeyId":"ecs-ak","AccessKeySecret":"ecs-sk",)"
+                   R"("SecurityToken":"ecs-sts","Expiration":"2027-04-16T05:44:46Z"})";
+        } else {
+            response.status_code = 404;
+        }
+        if (!body.empty()) {
+            PAIMON_RETURN_NOT_OK(consumer(body.data(), static_cast<int64_t>(body.size())));
+            response.body_size = static_cast<int64_t>(body.size());
+        }
+        return response;
+    }
+
+    int32_t GetRoleRequestCount() const {
+        return role_requests_.load();
+    }
+
+    int32_t GetTokenRequestCount() const {
+        return token_requests_.load();
+    }
+
+    int64_t GetLastRequestTimeoutMillis() const {
+        return last_request_timeout_ms_.load();
+    }
+
+ private:
+    std::string metadata_url_;
+    bool direct_token_;
+    mutable std::atomic<int32_t> role_requests_{0};
+    mutable std::atomic<int32_t> token_requests_{0};
+    mutable std::atomic<int64_t> last_request_timeout_ms_{-1};
+};
+
+class FailingEcsHttpClient : public HttpClient {
+ public:
+    Result<HttpResponse> Execute(const HttpRequest&, const HttpBodyConsumer&) const override {
+        return Status::IOError("connection refused");
+    }
+};
+
+}  // namespace
+
+TEST(DlfDefaultSignerTest, SignsJavaCompatibleRequest) {
+    DlfDefaultSigner signer("cn-beijing");
+    const std::string body = R"({"name":"t1"})";
+    DlfToken token("YourAccessKeyId", "YourAccessKeySecret", "securityToken", std::nullopt);
+    RestAuthParameter parameter =
+        RestAuthParameter::Create("POST", "/v1/wh/databases/db/tables",
+                                  {{"warehouse", "my instance"}, {"branch", "main"}}, body);
+
+    ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers,
+                         signer.SignHeaders(body, FixedTime(), token.GetSecurityToken(), "unused"));
+    ASSERT_EQ("20250416T034446Z", headers.at("x-dlf-date"));
+    ASSERT_EQ("Od9T1x3c2+JusJPFMpXe9Q==", headers.at("Content-MD5"));
+    ASSERT_EQ("application/json", headers.at("Content-Type"));
+    ASSERT_EQ("UNSIGNED-PAYLOAD", headers.at("x-dlf-content-sha256"));
+    ASSERT_EQ("v1", headers.at("x-dlf-version"));
+    ASSERT_EQ("securityToken", headers.at("x-dlf-security-token"));
+
+    ASSERT_OK_AND_ASSIGN(std::string authorization,
+                         signer.Authorization(parameter, token, "unused", headers));
+    ASSERT_EQ(
+        "DLF4-HMAC-SHA256 Credential=YourAccessKeyId/20250416/cn-beijing/"
+        "DlfNext/aliyun_v4_request,Signature="
+        "22594f8bbb8bb0ec296ced6003b7ffdf7022a8ca3815da5b53090daa11a06558",
+        authorization);
+}
+
+TEST(DlfDefaultSignerTest, MatchesJavaGoldenAuthorization) {
+    // Mirrors Java DLFAuthSignatureTest#testGetAuthorization.
+    DlfDefaultSigner signer("cn-hangzhou");
+    const std::string body = R"({"name":"database","options":{"a":"b"}})";
+    DlfToken token("access-key-id", "access-key-secret", "securityToken", std::nullopt);
+    RestAuthParameter parameter = RestAuthParameter::Create("POST", "/v1/paimon/databases",
+                                                            {{"k1", "v1"}, {"k2", "v2"}}, body);
+    const std::chrono::system_clock::time_point signing_time =
+        std::chrono::system_clock::from_time_t(1701605532);
+
+    ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers,
+                         signer.SignHeaders(body, signing_time, token.GetSecurityToken(), "host"));
+    ASSERT_OK_AND_ASSIGN(std::string authorization,
+                         signer.Authorization(parameter, token, "host", headers));
+    ASSERT_EQ(
+        "DLF4-HMAC-SHA256 Credential=access-key-id/20231203/cn-hangzhou/"
+        "DlfNext/aliyun_v4_request,Signature="
+        "c72caf1d40b55b1905d891ee3e3de48a2f8bebefa7e39e4f277acc93c269c5e3",
+        authorization);
+}
+
+TEST(DlfDefaultSignerTest, OmitsBodyAndSecurityTokenHeadersWhenAbsent) {
+    DlfDefaultSigner signer("cn-hangzhou");
+    ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers,
+                         signer.SignHeaders("", FixedTime(), std::nullopt, "unused"));
+    ASSERT_EQ(3, headers.size());
+    ASSERT_EQ(0, headers.count("Content-MD5"));
+    ASSERT_EQ(0, headers.count("Content-Type"));
+    ASSERT_EQ(0, headers.count("x-dlf-security-token"));
+}
+
+TEST(DlfOpenApiSignerTest, SignsJavaCompatibleRequest) {
+    DlfOpenApiSigner signer;
+    const std::string body = R"({"CategoryName":"test","CategoryType":"UNSTRUCTURED"})";
+    const std::string host = "dlfnext.cn-beijing.aliyuncs.com";
+    DlfToken token("YourAccessKeyId", "YourAccessKeySecret", "securityToken", std::nullopt);
+    RestAuthParameter parameter =
+        RestAuthParameter::Create("POST", "/llm-p2e4XXXXXXXXsvtn/datacenter/category", {}, body);
+
+    ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers,
+                         signer.SignHeaders(body, FixedTime(), token.GetSecurityToken(), host));
+    headers["x-acs-signature-nonce"] = "ef34aae7-7bd2-413d-a541-680cd2c48538";
+    ASSERT_EQ("Wed, 16 Apr 2025 03:44:46 GMT", headers.at("Date"));
+    ASSERT_EQ("q2qaEcR4P47+Z7CUzHRTBw==", headers.at("Content-MD5"));
+    ASSERT_EQ("application/json", headers.at("Accept"));
+    ASSERT_EQ("application/json", headers.at("Content-Type"));
+    ASSERT_EQ(host, headers.at("Host"));
+    ASSERT_EQ("HMAC-SHA1", headers.at("x-acs-signature-method"));
+    ASSERT_EQ("1.0", headers.at("x-acs-signature-version"));
+    ASSERT_EQ("2026-01-18", headers.at("x-acs-version"));
+    ASSERT_EQ("securityToken", headers.at("x-acs-security-token"));
+
+    ASSERT_OK_AND_ASSIGN(std::string authorization,
+                         signer.Authorization(parameter, token, host, headers));
+    ASSERT_EQ("acs YourAccessKeyId:wX4CDPSCtfgYkxdK9tJIO3ez5VI=", authorization);
+}
+
+TEST(DlfOpenApiSignerTest, DecodesPathAndQueryValuesBeforeSigning) {
+    DlfOpenApiSigner signer;
+    DlfToken token("ak", "sk", std::nullopt, std::nullopt);
+    RestAuthParameter parameter = RestAuthParameter::Create(
+        "GET", "/v1/%24snapshots", {{"z", ""}, {"name", "hello world"}}, "");
+    ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers,
+                         signer.SignHeaders("", FixedTime(), std::nullopt, "host"));
+    headers["x-acs-signature-nonce"] = "fixed-nonce";
+
+    ASSERT_OK_AND_ASSIGN(std::string authorization,
+                         signer.Authorization(parameter, token, "host", headers));
+    ASSERT_EQ("acs ak:vD+M7291KKoOTvt2gQuD6jPDlw8=", authorization);
+    ASSERT_EQ(0, headers.count("Content-MD5"));
+    ASSERT_EQ(0, headers.count("Content-Type"));
+}
+
+TEST(DlfTokenTest, ParsesExpirationAndRefreshBoundary) {
+    ASSERT_OK_AND_ASSIGN(
+        DlfToken token,
+        DlfToken::FromJson(R"({"AccessKeyId":"ak","AccessKeySecret":"sk","SecurityToken":"sts",)"
+                           R"("Expiration":"2025-04-16T05:44:46Z","Ignored":"value"})"));
+    ASSERT_EQ("ak", token.GetAccessKeyId());
+    ASSERT_EQ("sk", token.GetAccessKeySecret());
+    ASSERT_EQ(std::optional<std::string>("sts"), token.GetSecurityToken());
+    ASSERT_EQ(std::optional<int64_t>(1744782286000), token.GetExpirationAtMillis());
+    ASSERT_FALSE(token.ShouldRefresh(FixedTime() + std::chrono::hours(1)));
+    ASSERT_TRUE(
+        token.ShouldRefresh(FixedTime() + std::chrono::hours(1) + std::chrono::milliseconds(1)));
+
+    DlfToken permanent("ak", "sk", std::nullopt, std::nullopt);
+    ASSERT_FALSE(permanent.ShouldRefresh(FixedTime() + std::chrono::hours(100000)));
+}
+
+TEST(DlfTokenTest, ParseFailureDoesNotLeakCredentials) {
+    const std::string secret = "STSSECRET_AKID_9999";
+    Status status =
+        DlfToken::FromJson(R"({"AccessKeyId":"ak","AccessKeySecret":")" + secret).status();
+    ASSERT_FALSE(status.ok());
+    ASSERT_EQ(std::string::npos, status.ToString().find(secret));
+}
+
+TEST(DlfLocalFileTokenLoaderTest, LoadsTokenAndRedactsMalformedContent) {
+    std::unique_ptr<UniqueTestDirectory> test_dir = UniqueTestDirectory::Create();
+    ASSERT_NE(nullptr, test_dir);
+    const std::string path = test_dir->Str() + "/dlf-token.json";
+    {
+        std::ofstream file(path);
+        file << R"({"AccessKeyId":"file-ak","AccessKeySecret":"file-sk",)"
+                R"("SecurityToken":"file-sts","Expiration":"2027-04-16T05:44:46Z"})";
+    }
+    DlfLocalFileTokenLoader loader(path, 1, std::chrono::milliseconds(0));
+    ASSERT_OK_AND_ASSIGN(DlfToken token, loader.LoadToken());
+    ASSERT_EQ("file-ak", token.GetAccessKeyId());
+    ASSERT_EQ(std::optional<std::string>("file-sts"), token.GetSecurityToken());
+
+    std::map<std::string, std::string> options = {
+        {CatalogOptions::URI, "https://cn-hangzhou-vpc.dlf.aliyuncs.com"},
+        {CatalogOptions::TOKEN_PROVIDER, "dlf"},
+        {CatalogOptions::DLF_TOKEN_PATH, path},
+        {CatalogOptions::DLF_ACCESS_KEY_ID, "ignored-ak"},
+        {CatalogOptions::DLF_ACCESS_KEY_SECRET, "ignored-sk"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<AuthProvider> provider, AuthProvider::Create(options));
+    RestAuthParameter parameter = RestAuthParameter::Create("GET", "/v1/config", {}, "");
+    ASSERT_OK_AND_ASSIGN(StringMap headers, provider->MergeAuthHeader({}, parameter));
+    ASSERT_NE(std::string::npos, headers.at("Authorization").find("Credential=file-ak/"));
+
+    options[CatalogOptions::DLF_TOKEN_LOADER] = "local_file";
+    ASSERT_OK_AND_ASSIGN(provider, AuthProvider::Create(options));
+    ASSERT_OK(provider->MergeAuthHeader({}, parameter));
+
+    const std::string secret = "FILE_SECRET_9999";
+    {
+        std::ofstream file(path);
+        file << R"({"AccessKeyId":"ak","AccessKeySecret":")" << secret;
+    }
+    Status status = loader.LoadToken().status();
+    ASSERT_FALSE(status.ok());
+    ASSERT_EQ(std::string::npos, status.ToString().find(secret));
+}
+
+TEST(DlfEcsTokenLoaderTest, DiscoversRoleOnceAndRefreshesToken) {
+    const std::string metadata_url = "http://100.100.100.200/metadata/";
+    auto http_client = std::make_unique<MockEcsHttpClient>(metadata_url);
+    MockEcsHttpClient* http_client_ptr = http_client.get();
+    DlfEcsTokenLoader loader(metadata_url, std::nullopt, std::move(http_client));
+
+    ASSERT_OK_AND_ASSIGN(DlfToken first, loader.LoadToken());
+    ASSERT_OK_AND_ASSIGN(DlfToken second, loader.LoadToken());
+    ASSERT_EQ("ecs-ak", first.GetAccessKeyId());
+    ASSERT_EQ("ecs-ak", second.GetAccessKeyId());
+    ASSERT_EQ(1, http_client_ptr->GetRoleRequestCount());
+    ASSERT_EQ(2, http_client_ptr->GetTokenRequestCount());
+    ASSERT_EQ(180000, http_client_ptr->GetLastRequestTimeoutMillis());
+}
+
+TEST(DlfEcsTokenLoaderTest, ExplicitEmptyRoleUsesMetadataUrlAsTokenEndpoint) {
+    const std::string metadata_url = "http://100.100.100.200/metadata/token";
+    auto http_client = std::make_unique<MockEcsHttpClient>(metadata_url, /*direct_token=*/true);
+    MockEcsHttpClient* http_client_ptr = http_client.get();
+    DlfEcsTokenLoader loader(metadata_url, std::string(""), std::move(http_client));
+
+    ASSERT_OK_AND_ASSIGN(DlfToken token, loader.LoadToken());
+    ASSERT_EQ("ecs-ak", token.GetAccessKeyId());
+    ASSERT_EQ(0, http_client_ptr->GetRoleRequestCount());
+    ASSERT_EQ(1, http_client_ptr->GetTokenRequestCount());
+    ASSERT_EQ(180000, http_client_ptr->GetLastRequestTimeoutMillis());
+}
+
+TEST(DlfEcsTokenLoaderTest, PreservesTransportFailureDetail) {
+    DlfEcsTokenLoader loader("http://100.100.100.200/metadata/token", std::string(""),
+                             std::make_unique<FailingEcsHttpClient>());
+    Status status = loader.LoadToken().status();
+    ASSERT_NOK_WITH_MSG(status, "failed to request DLF credentials from ECS metadata service");
+    ASSERT_NOK_WITH_MSG(status, "connection refused");
+}
+
+TEST(DlfAuthProviderTest, RefreshesWithinSafeWindow) {
+    std::atomic<int64_t> now_seconds{1744775086};
+    std::vector<DlfToken> tokens;
+    tokens.emplace_back("ak-1", "sk-1", std::nullopt, 1744782286000);
+    tokens.emplace_back("ak-2", "sk-2", std::nullopt, std::nullopt);
+    auto loader = std::make_unique<SequenceTokenLoader>(tokens);
+    SequenceTokenLoader* loader_ptr = loader.get();
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<DlfAuthProvider> provider,
+        DlfAuthProvider::FromTokenLoader(
+            std::move(loader), "https://cn-beijing-vpc.dlf.aliyuncs.com", "cn-beijing", "default",
+            [&] { return std::chrono::system_clock::from_time_t(now_seconds.load()); }));
+    RestAuthParameter parameter = RestAuthParameter::Create("GET", "/v1/config", {}, "");
+
+    ASSERT_OK_AND_ASSIGN(StringMap first, provider->MergeAuthHeader({}, parameter));
+    ASSERT_NE(std::string::npos, first.at("Authorization").find("Credential=ak-1/"));
+    ASSERT_OK(provider->MergeAuthHeader({}, parameter));
+    ASSERT_EQ(1, loader_ptr->GetLoadCount());
+
+    now_seconds.fetch_add(3601);
+    ASSERT_OK_AND_ASSIGN(StringMap refreshed, provider->MergeAuthHeader({}, parameter));
+    ASSERT_NE(std::string::npos, refreshed.at("Authorization").find("Credential=ak-2/"));
+    ASSERT_EQ(2, loader_ptr->GetLoadCount());
+}
+
+TEST(DlfAuthProviderTest, ConcurrentFirstUseLoadsTokenOnce) {
+    auto loader = std::make_unique<SequenceTokenLoader>(
+        std::vector<DlfToken>{
+            DlfToken("concurrent-ak", "concurrent-sk", std::nullopt, std::nullopt)},
+        std::chrono::milliseconds(10));
+    SequenceTokenLoader* loader_ptr = loader.get();
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<DlfAuthProvider> provider,
+                         DlfAuthProvider::FromTokenLoader(std::move(loader),
+                                                          "https://cn-beijing-vpc.dlf.aliyuncs.com",
+                                                          "cn-beijing", "default", FixedTime));
+    RestAuthParameter parameter = RestAuthParameter::Create("GET", "/v1/config", {}, "");
+    std::vector<std::thread> threads;
+    std::vector<Status> statuses;
+    std::mutex statuses_mutex;
+    for (int32_t i = 0; i < 16; ++i) {
+        threads.emplace_back([&] {
+            Status status = provider->MergeAuthHeader({}, parameter).status();
+            std::scoped_lock lock(statuses_mutex);
+            statuses.push_back(status);
+        });
+    }
+    for (std::thread& thread : threads) {
+        thread.join();
+    }
+    ASSERT_EQ(16, statuses.size());
+    for (const Status& status : statuses) {
+        ASSERT_OK(status);
+    }
+    ASSERT_EQ(1, loader_ptr->GetLoadCount());
+}
+
+TEST(DlfAuthProviderTest, SelectsEndpointSignerAndCredentialSource) {
+    std::map<std::string, std::string> options = {
+        {CatalogOptions::URI, "https://dlfnext.cn-hangzhou.aliyuncs.com"},
+        {CatalogOptions::TOKEN_PROVIDER, "dlf"},
+        {CatalogOptions::DLF_ACCESS_KEY_ID, "ak"},
+        {CatalogOptions::DLF_ACCESS_KEY_SECRET, "sk"},
+        {CatalogOptions::DLF_SECURITY_TOKEN, "sts"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<AuthProvider> provider, AuthProvider::Create(options));
+    RestAuthParameter parameter = RestAuthParameter::Create("GET", "/v1/config", {}, "");
+    ASSERT_OK_AND_ASSIGN(StringMap headers, provider->MergeAuthHeader({{"Authorization", "old"},
+                                                                       {"x-acs-version", "old"},
+                                                                       {"custom-header", "kept"}},
+                                                                      parameter));
+    ASSERT_NE(std::string::npos, headers.at("Authorization").find("acs ak:"));
+    ASSERT_EQ("2026-01-18", headers.at("x-acs-version"));
+    ASSERT_EQ("sts", headers.at("x-acs-security-token"));
+    ASSERT_EQ("kept", headers.at("custom-header"));
+    ASSERT_FALSE(provider->AllowsRedirects());
+
+    BearTokenAuthProvider bear_provider("token");
+    ASSERT_TRUE(bear_provider.AllowsRedirects());
+
+    ASSERT_EQ("openapi", DlfAuthProvider::ParseSigningAlgorithmFromUri(options.at("uri")));
+    ASSERT_EQ("default", DlfAuthProvider::ParseSigningAlgorithmFromUri(
+                             "https://cn-hangzhou-vpc.dlf.aliyuncs.com"));
+    ASSERT_OK_AND_ASSIGN(std::string region,
+                         DlfAuthProvider::ParseRegionFromUri(options.at("uri")));
+    ASSERT_EQ("cn-hangzhou", region);
+    ASSERT_OK_AND_ASSIGN(std::string host,
+                         DlfAuthProvider::ExtractHost("https://example.com:8443/prefix"));
+    ASSERT_EQ("example.com:8443", host);
+}
+
+TEST(DlfAuthProviderTest, NormalizesSigningHostLikeTransport) {
+    std::map<std::string, std::string> options = {
+        {CatalogOptions::URI, " https://dlfnext.cn-hangzhou.aliyuncs.com "},
+        {CatalogOptions::TOKEN_PROVIDER, "dlf"},
+        {CatalogOptions::DLF_ACCESS_KEY_ID, "ak"},
+        {CatalogOptions::DLF_ACCESS_KEY_SECRET, "sk"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<AuthProvider> provider, AuthProvider::Create(options));
+    RestAuthParameter parameter = RestAuthParameter::Create("GET", "/v1/config", {}, "");
+    ASSERT_OK_AND_ASSIGN(StringMap headers, provider->MergeAuthHeader({}, parameter));
+    ASSERT_EQ("dlfnext.cn-hangzhou.aliyuncs.com", headers.at("Host"));
+}
+
+TEST(DlfAuthProviderTest, RejectsIncompleteOrUnknownConfiguration) {
+    const std::map<std::string, std::string> base = {
+        {CatalogOptions::URI, "https://dlfnext.cn-hangzhou.aliyuncs.com"},
+        {CatalogOptions::TOKEN_PROVIDER, "dlf"}};
+    ASSERT_NOK_WITH_MSG(AuthProvider::Create(base).status(), "token path or access key");
+
+    std::map<std::string, std::string> options = base;
+    options[CatalogOptions::DLF_TOKEN_LOADER] = "ecs";
+    options[CatalogOptions::DLF_TOKEN_ECS_METADATA_URL] = "http://metadata/";
+    options[CatalogOptions::DLF_TOKEN_ECS_ROLE_NAME] = "role";
+    ASSERT_OK(AuthProvider::Create(options));
+
+    options = base;
+    options[CatalogOptions::DLF_TOKEN_LOADER] = "unknown";
+    ASSERT_NOK_WITH_MSG(AuthProvider::Create(options).status(), "unsupported DLF token loader");
+
+    options = base;
+    options[CatalogOptions::DLF_ACCESS_KEY_ID] = "ak";
+    options[CatalogOptions::DLF_ACCESS_KEY_SECRET] = "sk";
+    options[CatalogOptions::DLF_SIGNING_ALGORITHM] = "unknown";
+    ASSERT_NOK_WITH_MSG(AuthProvider::Create(options).status(),
+                        "unsupported DLF signing algorithm");
+
+    options[CatalogOptions::DLF_SIGNING_ALGORITHM] = "default";
+    options[CatalogOptions::URI] = "http://127.0.0.1:8080";
+    ASSERT_NOK_WITH_MSG(AuthProvider::Create(options).status(), "DLF region");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/rest/rest_api.cpp b/src/paimon/rest/rest_api.cpp
index 393a0938a..437484ef0 100644
--- a/src/paimon/rest/rest_api.cpp
+++ b/src/paimon/rest/rest_api.cpp
@@ -81,11 +81,12 @@ Result<std::unique_ptr<RestApi>> RestApi::Create(const std::map<std::string, std
             RestAuthParameter::Create("GET", ResourcePaths::Config(), query_params, "");
         PAIMON_ASSIGN_OR_RAISE(StringMap headers,
                                auth_provider->MergeAuthHeader(base_headers, auth_parameter));
-        PAIMON_ASSIGN_OR_RAISE(
-            RestHttpClient::Response response,
-            client->Execute("GET", ResourcePaths::Config(), query_params, headers, ""));
+        bool follow_redirects = auth_provider->AllowsRedirects();
+        PAIMON_ASSIGN_OR_RAISE(RestHttpClient::Response response,
+                               client->Execute("GET", ResourcePaths::Config(), query_params,
+                                               headers, "", follow_redirects));
         if (!response.IsSuccessful()) {
-            return ErrorToStatus(response);
+            return ErrorToStatus(response, follow_redirects);
         }
         ConfigResponse config;
         PAIMON_RETURN_NOT_OK(ParseResponseBody(response.body, ResourcePaths::Config(), &config));
@@ -109,7 +110,19 @@ Result<std::unique_ptr<RestApi>> RestApi::Create(const std::map<std::string, std
                                                 ResourcePaths(prefix)));
 }
 
-Status RestApi::ErrorToStatus(const RestHttpClient::Response& response) {
+Status RestApi::ErrorToStatus(const RestHttpClient::Response& response, bool follow_redirects) {
+    if (!follow_redirects && response.code >= 300 && response.code < HttpStatus::kBadRequest) {
+        std::string message = fmt::format(
+            "rest endpoint returned redirect status {}, which is not followed for "
+            "signed requests",
+            response.code);
+        std::string request_id = RestUtil::ExtractRequestId(response.headers);
+        if (request_id != RestUtil::kUnknownRequestId) {
+            message += fmt::format(" requestId:{}", request_id);
+        }
+        return Status::IOError(message).WithDetail(
+            std::make_shared<RestErrorDetail>(response.code));
+    }
     // The code of the parsed error body takes precedence over the http status, which
     // a gateway may have rewritten.
     int64_t code = response.code;
@@ -188,10 +201,12 @@ Result<RestHttpClient::Response> RestApi::Execute(
     }
     PAIMON_ASSIGN_OR_RAISE(StringMap headers,
                            auth_provider_->MergeAuthHeader(request_headers, auth_parameter));
-    PAIMON_ASSIGN_OR_RAISE(RestHttpClient::Response response,
-                           client_->Execute(method, path, query_params, headers, body));
+    bool follow_redirects = auth_provider_->AllowsRedirects();
+    PAIMON_ASSIGN_OR_RAISE(
+        RestHttpClient::Response response,
+        client_->Execute(method, path, query_params, headers, body, follow_redirects));
     if (!response.IsSuccessful()) {
-        return ErrorToStatus(response);
+        return ErrorToStatus(response, follow_redirects);
     }
     return response;
 }
diff --git a/src/paimon/rest/rest_api.h b/src/paimon/rest/rest_api.h
index c2394cedc..f23fe1350 100644
--- a/src/paimon/rest/rest_api.h
+++ b/src/paimon/rest/rest_api.h
@@ -105,9 +105,11 @@ class RestApi {
 
     /// Maps a non-successful http response to a status: 404 becomes `NotExist`, 409
     /// becomes `Exist`, 400 becomes `Invalid`, 501 becomes `NotImplemented` and the
-    /// other codes become `IOError`. The status carries a `RestErrorDetail` with the
-    /// mapped code.
-    static Status ErrorToStatus(const RestHttpClient::Response& response);
+    /// other codes become `IOError`. A redirect returned while `follow_redirects` is
+    /// false is reported as deliberately rejected for a signed request. The status
+    /// carries a `RestErrorDetail` with the mapped code.
+    static Status ErrorToStatus(const RestHttpClient::Response& response,
+                                bool follow_redirects = true);
 
  private:
     RestApi(std::unique_ptr<RestHttpClient> client, std::unique_ptr<AuthProvider> auth_provider,
diff --git a/src/paimon/rest/rest_auth.cpp b/src/paimon/rest/rest_auth.cpp
index 65d6d7551..30a1869af 100644
--- a/src/paimon/rest/rest_auth.cpp
+++ b/src/paimon/rest/rest_auth.cpp
@@ -20,6 +20,7 @@
 #include "paimon/catalog_options.h"
 #include "paimon/common/utils/string_utils.h"
 #include "paimon/common/utils/url_utils.h"
+#include "paimon/rest/dlf_auth.h"
 
 namespace paimon {
 
@@ -52,8 +53,8 @@ Result<std::unique_ptr<AuthProvider>> AuthProvider::Create(
         return Status::Invalid(fmt::format("option '{}' must be configured for the rest catalog",
                                            CatalogOptions::TOKEN_PROVIDER));
     }
-    // Matched leniently in lower case; other clients may match the provider name
-    // case-sensitively, so only the exact "bear" spelling is portable.
+    // Matched leniently in lower case; other clients may match provider names
+    // case-sensitively, so the exact "bear" and "dlf" spellings are portable.
     std::string provider = StringUtils::ToLowerCase(provider_iter->second);
     if (provider == "bear") {
         auto token_iter = options.find(CatalogOptions::TOKEN);
@@ -64,8 +65,11 @@ Result<std::unique_ptr<AuthProvider>> AuthProvider::Create(
         }
         return std::make_unique<BearTokenAuthProvider>(token_iter->second);
     }
-    return Status::NotImplemented(
-        fmt::format("unsupported token provider: {}, only 'bear' is supported for now", provider));
+    if (provider == "dlf") {
+        return DlfAuthProvider::Create(options);
+    }
+    return Status::NotImplemented(fmt::format(
+        "unsupported token provider: {}, supported providers are 'bear' and 'dlf'", provider));
 }
 
 }  // namespace paimon
diff --git a/src/paimon/rest/rest_auth.h b/src/paimon/rest/rest_auth.h
index 22da08d99..72120d241 100644
--- a/src/paimon/rest/rest_auth.h
+++ b/src/paimon/rest/rest_auth.h
@@ -52,6 +52,11 @@ class AuthProvider {
         const std::map<std::string, std::string>& base_header,
         const RestAuthParameter& parameter) const = 0;
 
+    /// Whether the transport may follow a redirect without regenerating auth headers.
+    virtual bool AllowsRedirects() const {
+        return true;
+    }
+
     /// Creates the provider configured by `CatalogOptions::TOKEN_PROVIDER`.
     static Result<std::unique_ptr<AuthProvider>> Create(
         const std::map<std::string, std::string>& options);
diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp
index fda03cc4a..f1bb93aab 100644
--- a/src/paimon/rest/rest_catalog_test.cpp
+++ b/src/paimon/rest/rest_catalog_test.cpp
@@ -414,7 +414,7 @@ TEST_F(RestCatalogTest, CreateRejectsInvalidOptions) {
     ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "'token.provider' must be configured");
 
     options_ = valid_options;
-    options_[CatalogOptions::TOKEN_PROVIDER] = "dlf";
+    options_[CatalogOptions::TOKEN_PROVIDER] = "unsupported";
     Status unsupported_provider = CreateRestCatalog().status();
     ASSERT_TRUE(unsupported_provider.IsNotImplemented()) << unsupported_provider.ToString();
     ASSERT_NOK_WITH_MSG(unsupported_provider, "unsupported token provider");
@@ -991,6 +991,12 @@ TEST(RestApiErrorTest, ErrorToStatus) {
     ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "rest request failed with code 429");
     response.code = 418;
     ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "rest request failed with code 418");
+
+    response.code = 302;
+    Status signed_redirect = RestApi::ErrorToStatus(response, /*follow_redirects=*/false);
+    ASSERT_NOK_WITH_MSG(signed_redirect, "redirect status 302");
+    ASSERT_NOK_WITH_MSG(signed_redirect, "not followed for signed requests");
+    ASSERT_EQ(302, checked_pointer_cast<RestErrorDetail>(signed_redirect.detail())->GetCode());
 }
 
 TEST(RestApiErrorTest, MalformedSuccessBodyFails) {
diff --git a/src/paimon/rest/rest_http_client.cpp b/src/paimon/rest/rest_http_client.cpp
index 52e6417e7..10695818b 100644
--- a/src/paimon/rest/rest_http_client.cpp
+++ b/src/paimon/rest/rest_http_client.cpp
@@ -249,7 +249,7 @@ std::string RestHttpClient::BuildQueryString(
 Result<RestHttpClient::Response> RestHttpClient::ExecuteOnce(
     const std::string& method, const std::string& url,
     const std::map<std::string, std::string>& headers, const std::string& body,
-    bool* transport_retriable) const {
+    bool follow_redirects, bool* transport_retriable) const {
     CURL* curl = handle_pool_->Acquire();
     if (curl == nullptr) {
         return Status::IOError("failed to create curl handle");
@@ -271,10 +271,11 @@ Result<RestHttpClient::Response> RestHttpClient::ExecuteOnce(
     curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response.body);
     curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, WriteHeaderCallback);
     curl_easy_setopt(curl, CURLOPT_HEADERDATA, &response.headers);
-    // Follow redirects transparently, restricted to http(s) targets. Without
+    // Redirects can be disabled for auth headers whose signatures are bound to the
+    // original request. Otherwise they are restricted to http(s) targets. Without
     // CURLOPT_POSTREDIR a 301/302 would replay a body-carrying request as a bodyless
     // GET. A 303 is left to become a GET, which is what it is defined to mean.
-    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, follow_redirects ? 1L : 0L);
     curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 50L);
     curl_easy_setopt(curl, CURLOPT_POSTREDIR,
                      static_cast<long>(CURL_REDIR_POST_301 |  // NOLINT(runtime/int)
@@ -399,7 +400,8 @@ std::optional<int64_t> RestHttpClient::GetRetryDelayMs(int32_t execution_count,
 Result<RestHttpClient::Response> RestHttpClient::Execute(
     const std::string& method, const std::string& path,
     const std::map<std::string, std::string>& query_params,
-    const std::map<std::string, std::string>& headers, const std::string& body) const {
+    const std::map<std::string, std::string>& headers, const std::string& body,
+    bool follow_redirects) const {
     if (method != "GET" && method != "POST" && method != "DELETE") {
         return Status::Invalid(fmt::format("unsupported http method: {}", method));
     }
@@ -423,7 +425,8 @@ Result<RestHttpClient::Response> RestHttpClient::Execute(
     while (true) {
         execution_count++;
         bool transport_retriable = false;
-        Result<Response> result = ExecuteOnce(method, url, headers, body, &transport_retriable);
+        Result<Response> result =
+            ExecuteOnce(method, url, headers, body, follow_redirects, &transport_retriable);
         bool retriable;
         if (result.ok()) {
             retriable = IsRetriableCode(result.value().code);
diff --git a/src/paimon/rest/rest_http_client.h b/src/paimon/rest/rest_http_client.h
index 077127957..412e98536 100644
--- a/src/paimon/rest/rest_http_client.h
+++ b/src/paimon/rest/rest_http_client.h
@@ -50,8 +50,8 @@ struct HttpStatus {
 /// response header (delta-seconds or HTTP-date form). A backoff sleep is bounded by
 /// `retry_max_delay_ms` and the whole request by `retry_timeout_ms`; a `Retry-After`
 /// beyond the remaining budget stops retrying rather than shortening the sleep.
-/// Redirects to http(s) targets are followed transparently, keeping the method and
-/// body of POST/DELETE requests.
+/// Redirects to http(s) targets are followed by default, keeping the method and body
+/// of POST/DELETE requests; callers can disable them for request-bound signatures.
 class RestHttpClient {
  public:
     struct Config {
@@ -96,11 +96,12 @@ class RestHttpClient {
     /// final response, which may carry a non-2xx code, or an error status when the
     /// request could not be transported at all. Only transient transport errors (an
     /// established connection breaking mid-request or a truncated response body) are
-    /// retried; every other transport failure fails immediately.
+    /// retried; every other transport failure fails immediately. `follow_redirects`
+    /// must be false when authentication headers are bound to the original request.
     Result<Response> Execute(const std::string& method, const std::string& path,
                              const std::map<std::string, std::string>& query_params,
                              const std::map<std::string, std::string>& headers,
-                             const std::string& body) const;
+                             const std::string& body, bool follow_redirects = true) const;
 
     const std::string& GetBaseUri() const {
         return base_uri_;
@@ -135,7 +136,8 @@ class RestHttpClient {
     /// may be retried; see `Execute` for which kinds are not.
     Result<Response> ExecuteOnce(const std::string& method, const std::string& url,
                                  const std::map<std::string, std::string>& headers,
-                                 const std::string& body, bool* transport_retriable) const;
+                                 const std::string& body, bool follow_redirects,
+                                 bool* transport_retriable) const;
 
     std::optional<int64_t> GetRetryDelayMs(int32_t execution_count, const Response* response,
                                            int64_t remaining_budget_ms) const;
diff --git a/src/paimon/rest/rest_http_client_test.cpp b/src/paimon/rest/rest_http_client_test.cpp
index 34d07fe66..82ea9e2a6 100644
--- a/src/paimon/rest/rest_http_client_test.cpp
+++ b/src/paimon/rest/rest_http_client_test.cpp
@@ -489,6 +489,25 @@ TEST(RestHttpClientTest, RedirectIsFollowed) {
     ASSERT_EQ(0, response.headers.count("location"));
 }
 
+TEST(RestHttpClientTest, RedirectCanBeDisabledForSignedRequests) {
+    std::atomic<int32_t> request_count{0};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<MockRestServer> server,
+                         MockRestServer::Start([&](const MockRestServer::Request& request) {
+                             request_count++;
+                             MockRestServer::Response response;
+                             response.code = 302;
+                             response.headers["Location"] = "/v1/config";
+                             return response;
+                         }));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RestHttpClient> client,
+                         RestHttpClient::Create(server->GetBaseUri()));
+    ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response,
+                         client->Execute("GET", "/old", {}, {{"x-acs-security-token", "secret"}},
+                                         "", /*follow_redirects=*/false));
+    ASSERT_EQ(302, response.code);
+    ASSERT_EQ(1, request_count.load());
+}
+
 TEST(RestHttpClientTest, PostRedirectKeepsMethodAndBody) {
     std::mutex mutex;
     MockRestServer::Request last_request;

From 83773eb31a7e8784dabae43930cb8eefa6e63053 Mon Sep 17 00:00:00 2001
From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com>
Date: Tue, 25 Aug 2026 14:27:22 +0800
Subject: [PATCH 17/93] chore: fix REST Catalog license headers (#246)

---
 include/paimon/catalog_options.h                   | 14 ++++++++------
 src/paimon/common/catalog_options.cpp              | 14 ++++++++------
 src/paimon/common/utils/http_client_test.cpp       | 14 ++++++++------
 src/paimon/common/utils/sensitive_config_utils.cpp | 14 ++++++++------
 src/paimon/common/utils/sensitive_config_utils.h   | 14 ++++++++------
 .../common/utils/sensitive_config_utils_test.cpp   | 14 ++++++++------
 src/paimon/common/utils/url_utils.cpp              | 14 ++++++++------
 src/paimon/common/utils/url_utils.h                | 14 ++++++++------
 src/paimon/common/utils/url_utils_test.cpp         | 14 ++++++++------
 src/paimon/core/catalog/catalog_utils.cpp          | 14 ++++++++------
 src/paimon/core/catalog/catalog_utils.h            | 14 ++++++++------
 src/paimon/rest/mock_rest_server.cpp               | 14 ++++++++------
 src/paimon/rest/mock_rest_server.h                 | 14 ++++++++------
 src/paimon/rest/resource_paths.cpp                 | 14 ++++++++------
 src/paimon/rest/resource_paths.h                   | 14 ++++++++------
 src/paimon/rest/resource_paths_test.cpp            | 14 ++++++++------
 src/paimon/rest/rest_api.cpp                       | 14 ++++++++------
 src/paimon/rest/rest_api.h                         | 14 ++++++++------
 src/paimon/rest/rest_auth.cpp                      | 14 ++++++++------
 src/paimon/rest/rest_auth.h                        | 14 ++++++++------
 src/paimon/rest/rest_catalog.cpp                   | 14 ++++++++------
 src/paimon/rest/rest_catalog.h                     | 14 ++++++++------
 src/paimon/rest/rest_catalog_test.cpp              | 14 ++++++++------
 src/paimon/rest/rest_http_client.cpp               | 14 ++++++++------
 src/paimon/rest/rest_http_client.h                 | 14 ++++++++------
 src/paimon/rest/rest_http_client_test.cpp          | 14 ++++++++------
 src/paimon/rest/rest_messages.cpp                  | 14 ++++++++------
 src/paimon/rest/rest_messages.h                    | 14 ++++++++------
 src/paimon/rest/rest_messages_test.cpp             | 14 ++++++++------
 src/paimon/rest/rest_util.cpp                      | 14 ++++++++------
 src/paimon/rest/rest_util.h                        | 14 ++++++++------
 src/paimon/rest/rest_util_test.cpp                 | 14 ++++++++------
 32 files changed, 256 insertions(+), 192 deletions(-)

diff --git a/include/paimon/catalog_options.h b/include/paimon/catalog_options.h
index e959483dd..05b1af278 100644
--- a/include/paimon/catalog_options.h
+++ b/include/paimon/catalog_options.h
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/common/catalog_options.cpp b/src/paimon/common/catalog_options.cpp
index 2722f06bf..1a856dad7 100644
--- a/src/paimon/common/catalog_options.cpp
+++ b/src/paimon/common/catalog_options.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/common/utils/http_client_test.cpp b/src/paimon/common/utils/http_client_test.cpp
index 5bf3ce232..f9cf82d96 100644
--- a/src/paimon/common/utils/http_client_test.cpp
+++ b/src/paimon/common/utils/http_client_test.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/common/utils/sensitive_config_utils.cpp b/src/paimon/common/utils/sensitive_config_utils.cpp
index 28740412c..a3cff3c66 100644
--- a/src/paimon/common/utils/sensitive_config_utils.cpp
+++ b/src/paimon/common/utils/sensitive_config_utils.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/common/utils/sensitive_config_utils.h b/src/paimon/common/utils/sensitive_config_utils.h
index 717156098..05cc8ac63 100644
--- a/src/paimon/common/utils/sensitive_config_utils.h
+++ b/src/paimon/common/utils/sensitive_config_utils.h
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/common/utils/sensitive_config_utils_test.cpp b/src/paimon/common/utils/sensitive_config_utils_test.cpp
index 5e8b5350b..9e8989f76 100644
--- a/src/paimon/common/utils/sensitive_config_utils_test.cpp
+++ b/src/paimon/common/utils/sensitive_config_utils_test.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/common/utils/url_utils.cpp b/src/paimon/common/utils/url_utils.cpp
index 883a4b77f..8d9bfd536 100644
--- a/src/paimon/common/utils/url_utils.cpp
+++ b/src/paimon/common/utils/url_utils.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/common/utils/url_utils.h b/src/paimon/common/utils/url_utils.h
index a045dc067..2fedbe73c 100644
--- a/src/paimon/common/utils/url_utils.h
+++ b/src/paimon/common/utils/url_utils.h
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/common/utils/url_utils_test.cpp b/src/paimon/common/utils/url_utils_test.cpp
index ebdc2bdc7..56b0620dc 100644
--- a/src/paimon/common/utils/url_utils_test.cpp
+++ b/src/paimon/common/utils/url_utils_test.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp
index 7a6768ab3..eae23ad8e 100644
--- a/src/paimon/core/catalog/catalog_utils.cpp
+++ b/src/paimon/core/catalog/catalog_utils.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h
index b9a54ac11..e92cd8443 100644
--- a/src/paimon/core/catalog/catalog_utils.h
+++ b/src/paimon/core/catalog/catalog_utils.h
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/mock_rest_server.cpp b/src/paimon/rest/mock_rest_server.cpp
index 0bf4ec012..e9f0b36b7 100644
--- a/src/paimon/rest/mock_rest_server.cpp
+++ b/src/paimon/rest/mock_rest_server.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/mock_rest_server.h b/src/paimon/rest/mock_rest_server.h
index 240f661e6..c688ee398 100644
--- a/src/paimon/rest/mock_rest_server.h
+++ b/src/paimon/rest/mock_rest_server.h
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/resource_paths.cpp b/src/paimon/rest/resource_paths.cpp
index 484482378..d4b6adba4 100644
--- a/src/paimon/rest/resource_paths.cpp
+++ b/src/paimon/rest/resource_paths.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/resource_paths.h b/src/paimon/rest/resource_paths.h
index ea36bf480..009e6e073 100644
--- a/src/paimon/rest/resource_paths.h
+++ b/src/paimon/rest/resource_paths.h
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/resource_paths_test.cpp b/src/paimon/rest/resource_paths_test.cpp
index 782f3f690..a0ea565a4 100644
--- a/src/paimon/rest/resource_paths_test.cpp
+++ b/src/paimon/rest/resource_paths_test.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_api.cpp b/src/paimon/rest/rest_api.cpp
index 437484ef0..c249ede05 100644
--- a/src/paimon/rest/rest_api.cpp
+++ b/src/paimon/rest/rest_api.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_api.h b/src/paimon/rest/rest_api.h
index f23fe1350..5d642a194 100644
--- a/src/paimon/rest/rest_api.h
+++ b/src/paimon/rest/rest_api.h
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_auth.cpp b/src/paimon/rest/rest_auth.cpp
index 30a1869af..1af3b06aa 100644
--- a/src/paimon/rest/rest_auth.cpp
+++ b/src/paimon/rest/rest_auth.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_auth.h b/src/paimon/rest/rest_auth.h
index 72120d241..639273f74 100644
--- a/src/paimon/rest/rest_auth.h
+++ b/src/paimon/rest/rest_auth.h
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp
index 5d2bb8238..c928c6a25 100644
--- a/src/paimon/rest/rest_catalog.cpp
+++ b/src/paimon/rest/rest_catalog.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h
index 984aad708..e952263aa 100644
--- a/src/paimon/rest/rest_catalog.h
+++ b/src/paimon/rest/rest_catalog.h
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp
index f1bb93aab..140f3f4c1 100644
--- a/src/paimon/rest/rest_catalog_test.cpp
+++ b/src/paimon/rest/rest_catalog_test.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_http_client.cpp b/src/paimon/rest/rest_http_client.cpp
index 10695818b..99b233f1a 100644
--- a/src/paimon/rest/rest_http_client.cpp
+++ b/src/paimon/rest/rest_http_client.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_http_client.h b/src/paimon/rest/rest_http_client.h
index 412e98536..3ebd89472 100644
--- a/src/paimon/rest/rest_http_client.h
+++ b/src/paimon/rest/rest_http_client.h
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_http_client_test.cpp b/src/paimon/rest/rest_http_client_test.cpp
index 82ea9e2a6..9b93705a2 100644
--- a/src/paimon/rest/rest_http_client_test.cpp
+++ b/src/paimon/rest/rest_http_client_test.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_messages.cpp b/src/paimon/rest/rest_messages.cpp
index 324a8408b..df6911429 100644
--- a/src/paimon/rest/rest_messages.cpp
+++ b/src/paimon/rest/rest_messages.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_messages.h b/src/paimon/rest/rest_messages.h
index b943e6e11..0244fa362 100644
--- a/src/paimon/rest/rest_messages.h
+++ b/src/paimon/rest/rest_messages.h
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_messages_test.cpp b/src/paimon/rest/rest_messages_test.cpp
index 7a29b77ca..86fbfbd3d 100644
--- a/src/paimon/rest/rest_messages_test.cpp
+++ b/src/paimon/rest/rest_messages_test.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_util.cpp b/src/paimon/rest/rest_util.cpp
index b887d2567..fc8e6f698 100644
--- a/src/paimon/rest/rest_util.cpp
+++ b/src/paimon/rest/rest_util.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_util.h b/src/paimon/rest/rest_util.h
index 407217c67..1f5708c3a 100644
--- a/src/paimon/rest/rest_util.h
+++ b/src/paimon/rest/rest_util.h
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/paimon/rest/rest_util_test.cpp b/src/paimon/rest/rest_util_test.cpp
index e76baf156..c994e4462 100644
--- a/src/paimon/rest/rest_util_test.cpp
+++ b/src/paimon/rest/rest_util_test.cpp
@@ -1,11 +1,13 @@
 /*
- * Copyright 2026-present Alibaba Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,

From ff38c0d2188cef349c96e57b97bccaa264052a03 Mon Sep 17 00:00:00 2001
From: lxy <38709059+lxy-9602@users.noreply.github.com>
Date: Tue, 25 Aug 2026 17:16:20 +0800
Subject: [PATCH 18/93] refactor(shredding): unify shredding read plans for map
 shredding and variant (#235)

---
 src/paimon/CMakeLists.txt                     |   4 +-
 .../map_shared_shredding_file_reader.h        | 112 ---------
 ...ap_shared_shredding_read_plan_factory.cpp} | 227 ++++--------------
 .../map_shared_shredding_read_plan_factory.h  |  49 ++++
 ...ared_shredding_read_plan_factory_test.cpp} | 111 ++++-----
 .../data/shredding/shredding_file_reader.cpp  |   2 +-
 .../core/operation/abstract_split_read.cpp    |  76 +++---
 .../core/operation/abstract_split_read.h      |  14 +-
 .../operation/data_evolution_split_read.h     |   1 -
 .../core/operation/merge_file_split_read.h    |   1 -
 .../core/operation/raw_file_split_read.h      |   2 +-
 11 files changed, 183 insertions(+), 416 deletions(-)
 delete mode 100644 src/paimon/common/data/shredding/map_shared_shredding_file_reader.h
 rename src/paimon/common/data/shredding/{map_shared_shredding_file_reader.cpp => map_shared_shredding_read_plan_factory.cpp} (77%)
 create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory.h
 rename src/paimon/common/data/shredding/{map_shared_shredding_file_reader_test.cpp => map_shared_shredding_read_plan_factory_test.cpp} (90%)

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 8b3a29536..61fd7c96d 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -173,7 +173,7 @@ set(PAIMON_COMMON_SRCS
     common/data/shredding/map_shared_shredding_batch_converter.cpp
     common/data/shredding/map_shared_shredding_column_allocator.cpp
     common/data/shredding/lru_map_shared_shredding_column_allocator.cpp
-    common/data/shredding/map_shared_shredding_file_reader.cpp
+    common/data/shredding/map_shared_shredding_read_plan_factory.cpp
     common/data/shredding/shredding_file_reader.cpp
     common/utils/delta_varint_compressor.cpp
     common/utils/fields_comparator.cpp
@@ -682,7 +682,7 @@ if(PAIMON_BUILD_TESTS)
                     common/data/shredding/sequential_map_shared_shredding_column_allocator_test.cpp
                     common/data/shredding/map_shared_shredding_field_dict_test.cpp
                     common/data/shredding/map_shared_shredding_context_test.cpp
-                    common/data/shredding/map_shared_shredding_file_reader_test.cpp
+                    common/data/shredding/map_shared_shredding_read_plan_factory_test.cpp
                     STATIC_LINK_LIBS
                     paimon_shared
                     test_utils_static
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_file_reader.h b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.h
deleted file mode 100644
index 744d09fd1..000000000
--- a/src/paimon/common/data/shredding/map_shared_shredding_file_reader.h
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <map>
-#include <memory>
-#include <optional>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "arrow/api.h"
-#include "paimon/common/data/shredding/map_shared_shredding_utils.h"
-#include "paimon/memory/memory_pool.h"
-#include "paimon/reader/file_batch_reader.h"
-
-namespace paimon {
-
-class MapFieldReadPlan {
- public:
-    virtual ~MapFieldReadPlan() = default;
-
-    MapFieldReadPlan(const std::shared_ptr<arrow::Field>& logical_field,
-                     const std::shared_ptr<arrow::Field>& physical_read_field)
-        : logical_field_(logical_field), physical_read_field_(physical_read_field) {}
-
-    const std::shared_ptr<arrow::Field>& LogicalField() const {
-        return logical_field_;
-    }
-
-    const std::shared_ptr<arrow::Field>& PhysicalReadField() const {
-        return physical_read_field_;
-    }
-
-    virtual Result<std::shared_ptr<arrow::Array>> Materialize(
-        const std::shared_ptr<arrow::Array>& physical_array,
-        arrow::MemoryPool* arrow_pool) const = 0;
-
- private:
-    std::shared_ptr<arrow::Field> logical_field_;
-    std::shared_ptr<arrow::Field> physical_read_field_;
-};
-
-class MapFieldReadPlanFactory {
- public:
-    static Result<std::unique_ptr<MapFieldReadPlan>> CreateMapReadPlan(
-        const std::shared_ptr<arrow::Field>& logical_map_field,
-        const MapSharedShreddingFieldMeta& meta);
-
-    static Result<std::unique_ptr<MapFieldReadPlan>> CreateSharedSelectedKeysReadPlan(
-        const std::shared_ptr<arrow::Field>& selected_keys_field,
-        const MapSharedShreddingFieldMeta& meta);
-
-    static Result<std::unique_ptr<MapFieldReadPlan>> CreateDefaultSelectedKeysReadPlan(
-        const std::shared_ptr<arrow::Field>& file_map_field,
-        const std::shared_ptr<arrow::Field>& selected_keys_field);
-};
-
-class MapSharedShreddingFileReader : public FileBatchReader {
- public:
-    MapSharedShreddingFileReader(
-        std::unique_ptr<FileBatchReader>&& reader,
-        std::map<std::string, std::unique_ptr<MapFieldReadPlan>>&& field_read_plans,
-        const std::shared_ptr<MemoryPool>& pool);
-
-    Result<std::unique_ptr<::ArrowSchema>> GetFileSchema() const override;
-
-    Status SetReadSchema(::ArrowSchema* read_schema, const std::shared_ptr<Predicate>& predicate,
-                         const std::optional<RoaringBitmap32>& selection_bitmap) override;
-
-    Result<ReadBatch> NextBatch() override;
-
-    Result<ReadBatchWithBitmap> NextBatchWithBitmap() override;
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override;
-
-    void Close() override;
-
-    Result<uint64_t> GetPreviousBatchFileRowId(uint64_t batch_row_id) const override;
-
-    Result<uint64_t> GetNumberOfRows() const override;
-
-    bool SupportPreciseBitmapSelection() const override;
-
- private:
-    static Result<std::shared_ptr<arrow::Field>> ToLogicalMapField(
-        const std::shared_ptr<arrow::Field>& physical_field);
-
- private:
-    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
-    std::unique_ptr<FileBatchReader> reader_;
-    std::map<std::string, std::unique_ptr<MapFieldReadPlan>> field_read_plans_;
-};
-
-}  // namespace paimon
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp b/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory.cpp
similarity index 77%
rename from src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp
rename to src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory.cpp
index b1cdf963b..4cb28f694 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp
+++ b/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory.cpp
@@ -17,19 +17,14 @@
  * under the License.
  */
 
-#include "paimon/common/data/shredding/map_shared_shredding_file_reader.h"
+#include "paimon/common/data/shredding/map_shared_shredding_read_plan_factory.h"
 
-#include <optional>
 #include <set>
 #include <string_view>
 #include <utility>
 #include <vector>
 
-#include "arrow/c/bridge.h"
-#include "arrow/util/key_value_metadata.h"
 #include "fmt/format.h"
-#include "paimon/common/reader/reader_utils.h"
-#include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/core/utils/nested_projection_utils.h"
@@ -69,16 +64,35 @@ void CollectPhysicalColumns(
     }
 }
 
-class FullMapReadPlan : public MapFieldReadPlan {
+class MapShreddingColumnReadPlan : public ShreddingColumnReadPlan {
+ public:
+    MapShreddingColumnReadPlan(std::shared_ptr<arrow::Field> logical_field,
+                               std::shared_ptr<arrow::Field> physical_field)
+        : logical_field_(std::move(logical_field)), physical_field_(std::move(physical_field)) {}
+
+    const std::shared_ptr<arrow::Field>& LogicalField() const override {
+        return logical_field_;
+    }
+
+    const std::shared_ptr<arrow::Field>& PhysicalField() const override {
+        return physical_field_;
+    }
+
+ private:
+    std::shared_ptr<arrow::Field> logical_field_;
+    std::shared_ptr<arrow::Field> physical_field_;
+};
+
+class FullMapReadPlan : public MapShreddingColumnReadPlan {
  public:
     FullMapReadPlan(const std::shared_ptr<arrow::Field>& logical_field,
                     const std::shared_ptr<arrow::Field>& physical_read_field,
                     std::vector<std::pair<std::string, int32_t>>&& selected_key_ids)
-        : MapFieldReadPlan(logical_field, physical_read_field),
+        : MapShreddingColumnReadPlan(logical_field, physical_read_field),
           selected_key_ids_(std::move(selected_key_ids)),
           logical_map_type_(checked_pointer_cast<arrow::MapType>(logical_field->type())) {}
 
-    Result<std::shared_ptr<arrow::Array>> Materialize(
+    Result<std::shared_ptr<arrow::Array>> Assemble(
         const std::shared_ptr<arrow::Array>& physical_array,
         arrow::MemoryPool* arrow_pool) const override;
 
@@ -87,7 +101,7 @@ class FullMapReadPlan : public MapFieldReadPlan {
     std::shared_ptr<arrow::MapType> logical_map_type_;
 };
 
-class SharedSelectedKeysReadPlan : public MapFieldReadPlan {
+class SharedSelectedKeysReadPlan : public MapShreddingColumnReadPlan {
  public:
     struct SelectedKey {
         int32_t field_id = -1;
@@ -98,10 +112,10 @@ class SharedSelectedKeysReadPlan : public MapFieldReadPlan {
     SharedSelectedKeysReadPlan(const std::shared_ptr<arrow::Field>& logical_field,
                                const std::shared_ptr<arrow::Field>& physical_read_field,
                                std::vector<SelectedKey>&& selected_keys)
-        : MapFieldReadPlan(logical_field, physical_read_field),
+        : MapShreddingColumnReadPlan(logical_field, physical_read_field),
           selected_keys_(std::move(selected_keys)) {}
 
-    Result<std::shared_ptr<arrow::Array>> Materialize(
+    Result<std::shared_ptr<arrow::Array>> Assemble(
         const std::shared_ptr<arrow::Array>& physical_array,
         arrow::MemoryPool* arrow_pool) const override;
 
@@ -159,14 +173,15 @@ Result<std::shared_ptr<arrow::Array>> MaskSinglePhysicalColumn(
     return arrow::MakeArray(std::move(result_data));
 }
 
-class DefaultSelectedKeysReadPlan : public MapFieldReadPlan {
+class DefaultSelectedKeysReadPlan : public MapShreddingColumnReadPlan {
  public:
     DefaultSelectedKeysReadPlan(const std::shared_ptr<arrow::Field>& logical_field,
                                 const std::shared_ptr<arrow::Field>& physical_read_field,
                                 const std::vector<std::string>& selected_keys)
-        : MapFieldReadPlan(logical_field, physical_read_field), selected_keys_(selected_keys) {}
+        : MapShreddingColumnReadPlan(logical_field, physical_read_field),
+          selected_keys_(selected_keys) {}
 
-    Result<std::shared_ptr<arrow::Array>> Materialize(
+    Result<std::shared_ptr<arrow::Array>> Assemble(
         const std::shared_ptr<arrow::Array>& physical_array,
         arrow::MemoryPool* arrow_pool) const override;
 
@@ -176,7 +191,8 @@ class DefaultSelectedKeysReadPlan : public MapFieldReadPlan {
 
 }  // namespace
 
-Result<std::unique_ptr<MapFieldReadPlan>> MapFieldReadPlanFactory::CreateMapReadPlan(
+Result<std::shared_ptr<ShreddingColumnReadPlan>>
+MapSharedShreddingReadPlanFactory::CreateMapReadPlan(
     const std::shared_ptr<arrow::Field>& logical_map_field,
     const MapSharedShreddingFieldMeta& meta) {
     if (logical_map_field->type()->id() != arrow::Type::MAP) {
@@ -213,12 +229,13 @@ Result<std::unique_ptr<MapFieldReadPlan>> MapFieldReadPlanFactory::CreateMapRead
             logical_map_type->item_type(), selected_physical_column_ids,
             logical_map_type->item_field()->nullable(), include_overflow);
     auto physical_read_field = logical_map_field->WithType(physical_type);
-    std::unique_ptr<MapFieldReadPlan> read_plan = std::make_unique<FullMapReadPlan>(
+    std::shared_ptr<ShreddingColumnReadPlan> read_plan = std::make_shared<FullMapReadPlan>(
         logical_map_field, physical_read_field, ResolveSelectedKeyIds(meta, selected_keys));
     return read_plan;
 }
 
-Result<std::unique_ptr<MapFieldReadPlan>> MapFieldReadPlanFactory::CreateSharedSelectedKeysReadPlan(
+Result<std::shared_ptr<ShreddingColumnReadPlan>>
+MapSharedShreddingReadPlanFactory::CreateSharedSelectedKeysReadPlan(
     const std::shared_ptr<arrow::Field>& selected_keys_field,
     const MapSharedShreddingFieldMeta& meta) {
     PAIMON_ASSIGN_OR_RAISE(
@@ -253,13 +270,14 @@ Result<std::unique_ptr<MapFieldReadPlan>> MapFieldReadPlanFactory::CreateSharedS
             value_field->type(), selected_physical_column_ids, value_field->nullable(),
             include_overflow);
     auto physical_read_field = selected_keys_field->WithType(physical_type);
-    std::unique_ptr<MapFieldReadPlan> read_plan = std::make_unique<SharedSelectedKeysReadPlan>(
-        selected_keys_field, physical_read_field, std::move(selected_key_plans));
+    std::shared_ptr<ShreddingColumnReadPlan> read_plan =
+        std::make_shared<SharedSelectedKeysReadPlan>(selected_keys_field, physical_read_field,
+                                                     std::move(selected_key_plans));
     return read_plan;
 }
 
-Result<std::unique_ptr<MapFieldReadPlan>>
-MapFieldReadPlanFactory::CreateDefaultSelectedKeysReadPlan(
+Result<std::shared_ptr<ShreddingColumnReadPlan>>
+MapSharedShreddingReadPlanFactory::CreateDefaultSelectedKeysReadPlan(
     const std::shared_ptr<arrow::Field>& file_map_field,
     const std::shared_ptr<arrow::Field>& selected_keys_field) {
     if (file_map_field->type()->id() != arrow::Type::MAP) {
@@ -271,145 +289,13 @@ MapFieldReadPlanFactory::CreateDefaultSelectedKeysReadPlan(
         std::vector<std::string> selected_keys,
         NestedProjectionUtils::ValidateMapSharedShreddingAccessField(selected_keys_field));
     auto physical_read_field = selected_keys_field->WithType(file_map_field->type());
-    std::unique_ptr<MapFieldReadPlan> read_plan = std::make_unique<DefaultSelectedKeysReadPlan>(
-        selected_keys_field, physical_read_field, selected_keys);
+    std::shared_ptr<ShreddingColumnReadPlan> read_plan =
+        std::make_shared<DefaultSelectedKeysReadPlan>(selected_keys_field, physical_read_field,
+                                                      selected_keys);
     return read_plan;
 }
 
-MapSharedShreddingFileReader::MapSharedShreddingFileReader(
-    std::unique_ptr<FileBatchReader>&& reader,
-    std::map<std::string, std::unique_ptr<MapFieldReadPlan>>&& field_read_plans,
-    const std::shared_ptr<MemoryPool>& pool)
-    : arrow_pool_(GetArrowPool(pool)),
-      reader_(std::move(reader)),
-      field_read_plans_(std::move(field_read_plans)) {}
-
-Result<std::unique_ptr<::ArrowSchema>> MapSharedShreddingFileReader::GetFileSchema() const {
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> physical_schema,
-                           reader_->GetFileSchema());
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> physical_arrow_schema,
-                                      arrow::ImportSchema(physical_schema.get()));
-
-    arrow::FieldVector logical_fields = physical_arrow_schema->fields();
-    for (int32_t i = 0; i < physical_arrow_schema->num_fields(); ++i) {
-        const auto& field = physical_arrow_schema->field(i);
-        std::shared_ptr<arrow::KeyValueMetadata> metadata =
-            std::const_pointer_cast<arrow::KeyValueMetadata>(field->metadata());
-        if (!MapSharedShreddingUtils::HasShreddingMetadata(metadata)) {
-            continue;
-        }
-        PAIMON_ASSIGN_OR_RAISE(logical_fields[i], ToLogicalMapField(field));
-    }
-
-    auto logical_schema = arrow::schema(std::move(logical_fields));
-    auto c_logical_schema = std::make_unique<ArrowSchema>();
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*logical_schema, c_logical_schema.get()));
-    return c_logical_schema;
-}
-
-Result<std::shared_ptr<arrow::Field>> MapSharedShreddingFileReader::ToLogicalMapField(
-    const std::shared_ptr<arrow::Field>& physical_field) {
-    if (!physical_field || !physical_field->type() ||
-        physical_field->type()->id() != arrow::Type::STRUCT) {
-        return Status::Invalid(fmt::format("shared-shredding field {} is not a physical struct",
-                                           physical_field ? physical_field->name() : "<null>"));
-    }
-    auto physical_type = checked_pointer_cast<arrow::StructType>(physical_field->type());
-    std::shared_ptr<arrow::DataType> value_type;
-    bool value_nullable = true;
-    for (const auto& child : physical_type->fields()) {
-        if (child->name() == MapSharedShreddingDefine::kFieldMapping ||
-            child->name() == MapSharedShreddingDefine::kOverflow) {
-            continue;
-        }
-        value_type = child->type();
-        value_nullable = child->nullable();
-        break;
-    }
-    if (!value_type) {
-        return Status::Invalid(fmt::format("cannot infer shared-shredding value type for field {}",
-                                           physical_field->name()));
-    }
-    return arrow::field(
-        physical_field->name(),
-        arrow::map(arrow::utf8(), arrow::field("value", value_type, value_nullable)),
-        physical_field->nullable());
-}
-
-Status MapSharedShreddingFileReader::SetReadSchema(
-    ::ArrowSchema* read_schema, const std::shared_ptr<Predicate>& predicate,
-    const std::optional<RoaringBitmap32>& selection_bitmap) {
-    if (!read_schema) {
-        return Status::Invalid(
-            "invalid read schema in MapSharedShreddingFileReader, cannot be null");
-    }
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> logical_read_schema,
-                                      arrow::ImportSchema(read_schema));
-    bool converted = false;
-    arrow::FieldVector physical_read_fields = logical_read_schema->fields();
-    for (size_t i = 0; i < logical_read_schema->fields().size(); ++i) {
-        const auto& field = logical_read_schema->field(i);
-        auto plan_iter = field_read_plans_.find(field->name());
-        if (plan_iter != field_read_plans_.end()) {
-            physical_read_fields[i] = plan_iter->second->PhysicalReadField();
-            converted = true;
-        }
-    }
-    if (!converted) {
-        return Status::Invalid("suppose not fall into MapSharedShreddingFileReader");
-    }
-    auto physical_read_schema = arrow::schema(std::move(physical_read_fields));
-    std::unique_ptr<ArrowSchema> c_physical_read_schema = std::make_unique<ArrowSchema>();
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(
-        arrow::ExportSchema(*physical_read_schema, c_physical_read_schema.get()));
-    return reader_->SetReadSchema(c_physical_read_schema.get(), predicate, selection_bitmap);
-}
-
-Result<BatchReader::ReadBatch> MapSharedShreddingFileReader::NextBatch() {
-    return Status::Invalid(
-        "paimon inner reader MapSharedShreddingFileReader should use NextBatchWithBitmap");
-}
-
-Result<BatchReader::ReadBatchWithBitmap> MapSharedShreddingFileReader::NextBatchWithBitmap() {
-    PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap,
-                           reader_->NextBatchWithBitmap());
-    if (BatchReader::IsEofBatch(batch_with_bitmap)) {
-        return batch_with_bitmap;
-    }
-
-    auto& [batch, bitmap] = batch_with_bitmap;
-    auto& [c_array, c_schema] = batch;
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> arrow_array,
-                                      arrow::ImportArray(c_array.get(), c_schema.get()));
-    if (!arrow_array || arrow_array->type_id() != arrow::Type::STRUCT) {
-        return Status::Invalid("cannot cast batch to StructArray in MapSharedShreddingFileReader");
-    }
-    auto struct_array = checked_pointer_cast<arrow::StructArray>(arrow_array);
-
-    arrow::ArrayVector resolved_arrays = struct_array->fields();
-    arrow::FieldVector resolved_fields = struct_array->struct_type()->fields();
-    for (int32_t field_idx = 0; field_idx < struct_array->num_fields(); ++field_idx) {
-        const auto& physical_field = struct_array->struct_type()->field(field_idx);
-        auto plan_iter = field_read_plans_.find(physical_field->name());
-        if (plan_iter == field_read_plans_.end()) {
-            continue;
-        }
-        PAIMON_ASSIGN_OR_RAISE(
-            resolved_arrays[field_idx],
-            plan_iter->second->Materialize(struct_array->field(field_idx), arrow_pool_.get()));
-        resolved_fields[field_idx] = plan_iter->second->LogicalField();
-    }
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> new_struct_array,
-                                      arrow::StructArray::Make(resolved_arrays, resolved_fields));
-    auto new_c_array = std::make_unique<ArrowArray>();
-    auto new_c_schema = std::make_unique<ArrowSchema>();
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(
-        arrow::ExportArray(*new_struct_array, new_c_array.get(), new_c_schema.get()));
-    batch = std::make_pair(std::move(new_c_array), std::move(new_c_schema));
-    return batch_with_bitmap;
-}
-
-Result<std::shared_ptr<arrow::Array>> FullMapReadPlan::Materialize(
+Result<std::shared_ptr<arrow::Array>> FullMapReadPlan::Assemble(
     const std::shared_ptr<arrow::Array>& physical_array, arrow::MemoryPool* arrow_pool) const {
     if (!physical_array || physical_array->type_id() != arrow::Type::STRUCT) {
         return Status::Invalid(fmt::format("cannot cast physical shredding field {} to StructArray",
@@ -547,7 +433,7 @@ Result<std::shared_ptr<arrow::Array>> FullMapReadPlan::Materialize(
     return map_array;
 }
 
-Result<std::shared_ptr<arrow::Array>> SharedSelectedKeysReadPlan::Materialize(
+Result<std::shared_ptr<arrow::Array>> SharedSelectedKeysReadPlan::Assemble(
     const std::shared_ptr<arrow::Array>& physical_array, arrow::MemoryPool* arrow_pool) const {
     if (!physical_array || physical_array->type_id() != arrow::Type::STRUCT) {
         return Status::Invalid(fmt::format("cannot cast physical shredding field {} to StructArray",
@@ -716,7 +602,7 @@ Result<std::shared_ptr<arrow::Array>> SharedSelectedKeysReadPlan::Materialize(
     return result;
 }
 
-Result<std::shared_ptr<arrow::Array>> DefaultSelectedKeysReadPlan::Materialize(
+Result<std::shared_ptr<arrow::Array>> DefaultSelectedKeysReadPlan::Assemble(
     const std::shared_ptr<arrow::Array>& physical_array, arrow::MemoryPool* arrow_pool) const {
     if (!physical_array || physical_array->type_id() != arrow::Type::MAP) {
         return Status::Invalid(
@@ -774,25 +660,4 @@ Result<std::shared_ptr<arrow::Array>> DefaultSelectedKeysReadPlan::Materialize(
     return result;
 }
 
-std::shared_ptr<Metrics> MapSharedShreddingFileReader::GetReaderMetrics() const {
-    return reader_->GetReaderMetrics();
-}
-
-void MapSharedShreddingFileReader::Close() {
-    reader_->Close();
-}
-
-Result<uint64_t> MapSharedShreddingFileReader::GetPreviousBatchFileRowId(
-    uint64_t batch_row_id) const {
-    return reader_->GetPreviousBatchFileRowId(batch_row_id);
-}
-
-Result<uint64_t> MapSharedShreddingFileReader::GetNumberOfRows() const {
-    return reader_->GetNumberOfRows();
-}
-
-bool MapSharedShreddingFileReader::SupportPreciseBitmapSelection() const {
-    return reader_->SupportPreciseBitmapSelection();
-}
-
 }  // namespace paimon
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory.h b/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory.h
new file mode 100644
index 000000000..871c51c38
--- /dev/null
+++ b/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory.h
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "arrow/api.h"
+#include "paimon/common/data/shredding/map_shared_shredding_utils.h"
+#include "paimon/common/data/shredding/shredding_read_plan.h"
+
+namespace paimon {
+
+/// Builds per-column read plans for shared-shredding MAP columns and selected-key MAP access.
+class MapSharedShreddingReadPlanFactory {
+ public:
+    MapSharedShreddingReadPlanFactory() = delete;
+    ~MapSharedShreddingReadPlanFactory() = delete;
+
+    static Result<std::shared_ptr<ShreddingColumnReadPlan>> CreateMapReadPlan(
+        const std::shared_ptr<arrow::Field>& logical_map_field,
+        const MapSharedShreddingFieldMeta& meta);
+
+    static Result<std::shared_ptr<ShreddingColumnReadPlan>> CreateSharedSelectedKeysReadPlan(
+        const std::shared_ptr<arrow::Field>& selected_keys_field,
+        const MapSharedShreddingFieldMeta& meta);
+
+    static Result<std::shared_ptr<ShreddingColumnReadPlan>> CreateDefaultSelectedKeysReadPlan(
+        const std::shared_ptr<arrow::Field>& file_map_field,
+        const std::shared_ptr<arrow::Field>& selected_keys_field);
+};
+
+}  // namespace paimon
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory_test.cpp
similarity index 90%
rename from src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp
rename to src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory_test.cpp
index b2dd9374b..e02f0ffb0 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp
+++ b/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory_test.cpp
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-#include "paimon/common/data/shredding/map_shared_shredding_file_reader.h"
+#include "paimon/common/data/shredding/map_shared_shredding_read_plan_factory.h"
 
 #include <map>
 #include <memory>
@@ -33,6 +33,7 @@
 #include "gtest/gtest.h"
 #include "paimon/common/data/shredding/map_shared_shredding_utils.h"
 #include "paimon/common/data/shredding/map_shredding_defs.h"
+#include "paimon/common/data/shredding/shredding_file_reader.h"
 #include "paimon/common/fs/external_path_provider.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/core/append/append_only_writer.h"
@@ -51,7 +52,7 @@
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
-class MapSharedShreddingFileReaderTest : public ::testing::Test {
+class MapSharedShreddingReadPlanFactoryTest : public ::testing::Test {
  public:
     void SetUp() override {
         pool_ = GetDefaultPool();
@@ -100,12 +101,12 @@ class MapSharedShreddingFileReaderTest : public ::testing::Test {
             .ValueOrDie();
     }
 
-    std::unique_ptr<MapSharedShreddingFileReader> WrapReader(
+    std::unique_ptr<ShreddingFileReader> WrapReader(
         std::unique_ptr<FileBatchReader>&& reader,
         const std::optional<std::string>& selected_keys_str = std::nullopt) const {
         EXPECT_OK_AND_ASSIGN(auto c_file_schema, reader->GetFileSchema());
         auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie();
-        std::map<std::string, std::unique_ptr<MapFieldReadPlan>> field_read_plans;
+        std::map<std::string, std::shared_ptr<ShreddingColumnReadPlan>> field_read_plans;
         for (const auto& field : file_schema->fields()) {
             auto metadata = std::const_pointer_cast<arrow::KeyValueMetadata>(field->metadata());
             if (!MapSharedShreddingUtils::HasShreddingMetadata(metadata)) {
@@ -124,20 +125,22 @@ class MapSharedShreddingFileReaderTest : public ::testing::Test {
             EXPECT_TRUE(item_field);
             auto map_type = checked_pointer_cast<arrow::MapType>(arrow::map(
                 arrow::utf8(), arrow::field("value", item_field->type(), item_field->nullable())));
-            std::shared_ptr<arrow::Field> logical_map_field = field->WithType(map_type);
+            std::shared_ptr<arrow::Field> logical_map_field =
+                arrow::field(field->name(), map_type, field->nullable());
             if (selected_keys_str.has_value()) {
                 logical_map_field = logical_map_field->WithMetadata(arrow::KeyValueMetadata::Make(
                     {DataField::MAP_SELECTED_KEYS}, {selected_keys_str.value()}));
             }
-            EXPECT_OK_AND_ASSIGN(auto field_read_plan, MapFieldReadPlanFactory::CreateMapReadPlan(
-                                                           logical_map_field, meta));
+            EXPECT_OK_AND_ASSIGN(
+                auto field_read_plan,
+                MapSharedShreddingReadPlanFactory::CreateMapReadPlan(logical_map_field, meta));
             field_read_plans.emplace(field->name(), std::move(field_read_plan));
         }
-        return std::make_unique<MapSharedShreddingFileReader>(std::move(reader),
-                                                              std::move(field_read_plans), pool_);
+        return std::make_unique<ShreddingFileReader>(std::move(reader), std::move(field_read_plans),
+                                                     pool_);
     }
 
-    Result<std::unique_ptr<MapSharedShreddingFileReader>> CreateReader(
+    Result<std::unique_ptr<ShreddingFileReader>> CreateReader(
         std::shared_ptr<arrow::Array> physical_array = nullptr,
         std::shared_ptr<arrow::Schema> physical_schema = nullptr,
         const std::optional<std::string>& selected_keys = std::nullopt) const {
@@ -236,20 +239,7 @@ class MapSharedShreddingFileReaderTest : public ::testing::Test {
     };
 };
 
-TEST_F(MapSharedShreddingFileReaderTest, TestGetFileSchemaReturnsLogicalMapSchema) {
-    ASSERT_OK_AND_ASSIGN(auto reader, CreateReader());
-
-    ASSERT_OK_AND_ASSIGN(auto c_schema, reader->GetFileSchema());
-    auto schema = arrow::ImportSchema(c_schema.get()).ValueOrDie();
-
-    ASSERT_TRUE(schema->Equals(logical_schema_, /*check_metadata=*/false))
-        << "Expected:\n"
-        << logical_schema_->ToString() << "\nActual:\n"
-        << schema->ToString();
-    ASSERT_FALSE(schema->field(1)->HasMetadata());
-}
-
-TEST_F(MapSharedShreddingFileReaderTest, TestAllExistSelectedKeysWithoutOverflow) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestAllExistSelectedKeysWithoutOverflow) {
     ASSERT_OK_AND_ASSIGN(auto reader,
                          CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr,
                                       /*selected_keys=*/"b"));
@@ -271,7 +261,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestAllExistSelectedKeysWithoutOverflow
     AssertChunkedArrayEquals(expected, actual);
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestAllExistSelectedKeysWithOverflow) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestAllExistSelectedKeysWithOverflow) {
     ASSERT_OK_AND_ASSIGN(auto reader,
                          CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr,
                                       /*selected_keys=*/"a,c"));
@@ -293,7 +283,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestAllExistSelectedKeysWithOverflow) {
     AssertChunkedArrayEquals(expected, actual);
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjection) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestSelectedKeysStructProjection) {
     ASSERT_OK_AND_ASSIGN(auto physical_schema, PhysicalSchemaWithMetadata());
     ASSERT_OK_AND_ASSIGN(auto physical_array, PhysicalArray());
     auto mock_reader = std::make_unique<MockFileBatchReader>(
@@ -306,13 +296,13 @@ TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjection) {
     auto selected_field = arrow::field(
         "tags", selected_type, /*nullable=*/true,
         arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c,missing"}));
-    ASSERT_OK_AND_ASSIGN(
-        auto field_read_plan,
-        MapFieldReadPlanFactory::CreateSharedSelectedKeysReadPlan(selected_field, TagsMeta()));
-    std::map<std::string, std::unique_ptr<MapFieldReadPlan>> contexts;
+    ASSERT_OK_AND_ASSIGN(auto field_read_plan,
+                         MapSharedShreddingReadPlanFactory::CreateSharedSelectedKeysReadPlan(
+                             selected_field, TagsMeta()));
+    std::map<std::string, std::shared_ptr<ShreddingColumnReadPlan>> contexts;
     contexts.emplace("tags", std::move(field_read_plan));
-    auto reader = std::make_unique<MapSharedShreddingFileReader>(std::move(mock_reader),
-                                                                 std::move(contexts), pool_);
+    auto reader =
+        std::make_unique<ShreddingFileReader>(std::move(mock_reader), std::move(contexts), pool_);
 
     auto read_schema =
         ExportSchema(arrow::schema({arrow::field("id", arrow::int32()), selected_field}));
@@ -333,7 +323,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjection) {
     AssertChunkedArrayEquals(expected, actual);
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjectionSharesValueBuffers) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestSelectedKeysStructProjectionSharesValueBuffers) {
     ASSERT_OK_AND_ASSIGN(auto physical_array, PhysicalArray());
     auto physical_root = checked_pointer_cast<arrow::StructArray>(physical_array);
     auto physical_tags =
@@ -347,11 +337,11 @@ TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjectionSharesV
     auto selected_field = arrow::field(
         "tags", selected_type, /*nullable=*/true,
         arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,b,e,missing"}));
-    ASSERT_OK_AND_ASSIGN(
-        auto field_read_plan,
-        MapFieldReadPlanFactory::CreateSharedSelectedKeysReadPlan(selected_field, TagsMeta()));
+    ASSERT_OK_AND_ASSIGN(auto field_read_plan,
+                         MapSharedShreddingReadPlanFactory::CreateSharedSelectedKeysReadPlan(
+                             selected_field, TagsMeta()));
     ASSERT_OK_AND_ASSIGN(auto result,
-                         field_read_plan->Materialize(physical_tags, arrow::default_memory_pool()));
+                         field_read_plan->Assemble(physical_tags, arrow::default_memory_pool()));
     auto result_struct = checked_pointer_cast<arrow::StructArray>(result);
 
     auto expected = arrow::ipc::internal::json::ArrayFromJSON(selected_type, R"([
@@ -370,7 +360,8 @@ TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjectionSharesV
     ASSERT_EQ(physical_tags->data()->buffers[0], result_struct->data()->buffers[0]);
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjectionSharesNestedValueBuffers) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest,
+       TestSelectedKeysStructProjectionSharesNestedValueBuffers) {
     auto item_type = arrow::list(arrow::int64());
     auto logical_schema =
         arrow::schema({arrow::field("id", arrow::int32()),
@@ -402,9 +393,9 @@ TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjectionSharesN
                      arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"b"}));
     ASSERT_OK_AND_ASSIGN(
         auto field_read_plan,
-        MapFieldReadPlanFactory::CreateSharedSelectedKeysReadPlan(selected_field, meta));
+        MapSharedShreddingReadPlanFactory::CreateSharedSelectedKeysReadPlan(selected_field, meta));
     ASSERT_OK_AND_ASSIGN(auto result,
-                         field_read_plan->Materialize(physical_tags, arrow::default_memory_pool()));
+                         field_read_plan->Assemble(physical_tags, arrow::default_memory_pool()));
     auto result_struct = checked_pointer_cast<arrow::StructArray>(result);
     auto result_list = checked_pointer_cast<arrow::ListArray>(result_struct->field(0));
 
@@ -423,7 +414,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjectionSharesN
               result_list->data()->child_data[0]->buffers[1]);
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjectionFromDefaultMap) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestSelectedKeysStructProjectionFromDefaultMap) {
     auto map_type = checked_pointer_cast<arrow::MapType>(
         arrow::map(arrow::utf8(), arrow::field("value", arrow::int64())));
     auto file_schema =
@@ -446,12 +437,12 @@ TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjectionFromDef
         arrow::field("tags", selected_type, /*nullable=*/true,
                      arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,missing"}));
     ASSERT_OK_AND_ASSIGN(auto field_read_plan,
-                         MapFieldReadPlanFactory::CreateDefaultSelectedKeysReadPlan(
+                         MapSharedShreddingReadPlanFactory::CreateDefaultSelectedKeysReadPlan(
                              file_schema->field(1), selected_field));
-    std::map<std::string, std::unique_ptr<MapFieldReadPlan>> contexts;
+    std::map<std::string, std::shared_ptr<ShreddingColumnReadPlan>> contexts;
     contexts.emplace("tags", std::move(field_read_plan));
-    auto reader = std::make_unique<MapSharedShreddingFileReader>(std::move(mock_reader),
-                                                                 std::move(contexts), pool_);
+    auto reader =
+        std::make_unique<ShreddingFileReader>(std::move(mock_reader), std::move(contexts), pool_);
 
     auto read_schema =
         ExportSchema(arrow::schema({arrow::field("id", arrow::int32()), selected_field}));
@@ -471,15 +462,15 @@ TEST_F(MapSharedShreddingFileReaderTest, TestSelectedKeysStructProjectionFromDef
     AssertChunkedArrayEquals(expected, actual);
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestInvalidSelectedKeysStructProjection) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestInvalidSelectedKeysStructProjection) {
     auto file_map_field = arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64()));
     auto mismatched_count_field =
         arrow::field("tags", arrow::struct_({arrow::field("a", arrow::int64())}), /*nullable=*/true,
                      arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,b"}));
-    ASSERT_NOK_WITH_MSG(MapFieldReadPlanFactory::CreateSharedSelectedKeysReadPlan(
+    ASSERT_NOK_WITH_MSG(MapSharedShreddingReadPlanFactory::CreateSharedSelectedKeysReadPlan(
                             mismatched_count_field, TagsMeta()),
                         "metadata size 2 does not match STRUCT field count 1");
-    ASSERT_NOK_WITH_MSG(MapFieldReadPlanFactory::CreateDefaultSelectedKeysReadPlan(
+    ASSERT_NOK_WITH_MSG(MapSharedShreddingReadPlanFactory::CreateDefaultSelectedKeysReadPlan(
                             file_map_field, mismatched_count_field),
                         "metadata size 2 does not match STRUCT field count 1");
 
@@ -487,15 +478,15 @@ TEST_F(MapSharedShreddingFileReaderTest, TestInvalidSelectedKeysStructProjection
         "tags",
         arrow::struct_({arrow::field("a", arrow::int64()), arrow::field("b", arrow::utf8())}),
         /*nullable=*/true, arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,b"}));
-    ASSERT_NOK_WITH_MSG(MapFieldReadPlanFactory::CreateSharedSelectedKeysReadPlan(
+    ASSERT_NOK_WITH_MSG(MapSharedShreddingReadPlanFactory::CreateSharedSelectedKeysReadPlan(
                             mismatched_type_field, TagsMeta()),
                         "must have the same value type");
-    ASSERT_NOK_WITH_MSG(MapFieldReadPlanFactory::CreateDefaultSelectedKeysReadPlan(
+    ASSERT_NOK_WITH_MSG(MapSharedShreddingReadPlanFactory::CreateDefaultSelectedKeysReadPlan(
                             file_map_field, mismatched_type_field),
                         "must have the same value type");
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestPartialExistSelectedKeys) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestPartialExistSelectedKeys) {
     ASSERT_OK_AND_ASSIGN(auto reader,
                          CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr,
                                       /*selected_keys=*/"a,c,missing"));
@@ -518,7 +509,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestPartialExistSelectedKeys) {
     AssertChunkedArrayEquals(expected, actual);
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestMissingSelectedKeysReadsWholeMap) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestMissingSelectedKeysReadsWholeMap) {
     ASSERT_OK_AND_ASSIGN(auto reader, CreateReader());
     auto read_schema = ExportSchema(ReadSchema(std::nullopt));
     ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr,
@@ -538,7 +529,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestMissingSelectedKeysReadsWholeMap) {
     AssertChunkedArrayEquals(expected, actual);
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestSpecialSelectedKeys) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestSpecialSelectedKeys) {
     MapSharedShreddingFieldMeta meta;
     meta.name_to_id = {{"", 0}, {"     ", 1}, {".", 2}, {"a", 3}};
     meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0}}, {3, {1}}};
@@ -591,7 +582,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestSpecialSelectedKeys) {
     ])");
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestUnknownSelectedKeyReturnsEmptyMap) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestUnknownSelectedKeyReturnsEmptyMap) {
     ASSERT_OK_AND_ASSIGN(auto reader,
                          CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr,
                                       /*selected_keys=*/"missing"));
@@ -614,7 +605,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestUnknownSelectedKeyReturnsEmptyMap)
     AssertChunkedArrayEquals(expected, actual);
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestInvalidNullFieldMappingField) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestInvalidNullFieldMappingField) {
     ASSERT_OK_AND_ASSIGN(auto physical_schema, PhysicalSchemaWithMetadata());
     std::string json = R"([
         [1, [null, 10, null, null]]
@@ -631,7 +622,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestInvalidNullFieldMappingField) {
                         "__field_mapping cannot be null");
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestInvalidNullFieldMappingFieldElement) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestInvalidNullFieldMappingFieldElement) {
     ASSERT_OK_AND_ASSIGN(auto physical_schema, PhysicalSchemaWithMetadata());
     std::string json = R"([
         [1, [[0, null], 10, null, null]]
@@ -648,7 +639,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestInvalidNullFieldMappingFieldElement
                         "__field_mapping element cannot be null");
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestListValue) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestListValue) {
     std::shared_ptr<arrow::Schema> logical_schema = arrow::schema({
         arrow::field("id", arrow::int32()),
         arrow::field("tags", arrow::map(arrow::utf8(), arrow::list(arrow::int32()))),
@@ -704,7 +695,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestListValue) {
     AssertChunkedArrayEquals(expected, actual);
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestOrcDictionaryEncodedStringValue) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestOrcDictionaryEncodedStringValue) {
     std::shared_ptr<arrow::Schema> logical_schema = arrow::schema({
         arrow::field("id", arrow::int32()),
         arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())),
@@ -764,7 +755,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestOrcDictionaryEncodedStringValue) {
     AssertChunkedArrayEquals(expected, actual);
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestOrcDictionaryEncodedStringListValue) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestOrcDictionaryEncodedStringListValue) {
     std::shared_ptr<arrow::Schema> logical_schema = arrow::schema({
         arrow::field("id", arrow::int32()),
         arrow::field("tags", arrow::map(arrow::utf8(), arrow::list(arrow::utf8()))),
@@ -824,7 +815,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestOrcDictionaryEncodedStringListValue
     AssertChunkedArrayEquals(expected, actual);
 }
 
-TEST_F(MapSharedShreddingFileReaderTest, TestReadsRealFormatFile) {
+TEST_F(MapSharedShreddingReadPlanFactoryTest, TestReadsRealFormatFile) {
     // TODO(lisizhuo.lsz): support other format
     auto options = options_;
     std::string format = "orc";
diff --git a/src/paimon/common/data/shredding/shredding_file_reader.cpp b/src/paimon/common/data/shredding/shredding_file_reader.cpp
index 0f47350dd..1c2e34f57 100644
--- a/src/paimon/common/data/shredding/shredding_file_reader.cpp
+++ b/src/paimon/common/data/shredding/shredding_file_reader.cpp
@@ -84,7 +84,7 @@ Result<BatchReader::ReadBatchWithBitmap> ShreddingFileReader::NextBatchWithBitma
     auto& [c_array, c_schema] = batch;
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> arrow_array,
                                       arrow::ImportArray(c_array.get(), c_schema.get()));
-    if (arrow_array->type_id() != arrow::Type::STRUCT) {
+    if (!arrow_array || arrow_array->type_id() != arrow::Type::STRUCT) {
         return Status::Invalid("cannot cast batch to StructArray in ShreddingFileReader");
     }
     auto struct_array = checked_pointer_cast<arrow::StructArray>(arrow_array);
diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp
index bb82c5d82..d057e1d72 100644
--- a/src/paimon/core/operation/abstract_split_read.cpp
+++ b/src/paimon/core/operation/abstract_split_read.cpp
@@ -28,11 +28,10 @@
 #include "fmt/format.h"
 #include "paimon/common/data/blob_defs.h"
 #include "paimon/common/data/blob_utils.h"
-#include "paimon/common/data/shredding/map_shared_shredding_file_reader.h"
+#include "paimon/common/data/shredding/map_shared_shredding_read_plan_factory.h"
 #include "paimon/common/data/shredding/map_shared_shredding_utils.h"
 #include "paimon/common/data/shredding/shredding_file_reader.h"
 #include "paimon/common/data/variant/variant_shredding_read_plan_factory.h"
-#include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/reader/delegating_prefetch_reader.h"
 #include "paimon/common/reader/predicate_batch_reader.h"
 #include "paimon/common/reader/prefetch_file_batch_reader_impl.h"
@@ -221,13 +220,11 @@ Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::CreateFieldMappingRe
     }
     std::set<int32_t> skip_map_selected_keys_filter_field_ids;
     if (file_format_identifier != "blob") {
-        std::pair<std::unique_ptr<FileBatchReader>, std::set<int32_t>> shared_shredding_result;
-        PAIMON_ASSIGN_OR_RAISE(shared_shredding_result, ApplySharedShreddingReaderIfNeeded(
-                                                            std::move(file_reader), read_schema));
-        file_reader = std::move(shared_shredding_result.first);
-        skip_map_selected_keys_filter_field_ids = std::move(shared_shredding_result.second);
-        PAIMON_ASSIGN_OR_RAISE(
-            file_reader, ApplyVariantShreddingReaderIfNeeded(std::move(file_reader), read_schema));
+        std::pair<std::unique_ptr<FileBatchReader>, std::set<int32_t>> shredding_result;
+        PAIMON_ASSIGN_OR_RAISE(shredding_result,
+                               ApplyShreddingReaderIfNeeded(std::move(file_reader), read_schema));
+        file_reader = std::move(shredding_result.first);
+        skip_map_selected_keys_filter_field_ids = std::move(shredding_result.second);
     }
     if (NeedCompleteRowTrackingFields(options_.RowTrackingEnabled(), read_schema)) {
         // A blob file has no self-describing schema: its physical fields are declared by the
@@ -260,15 +257,16 @@ Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::CreateFieldMappingRe
 }
 
 Result<std::pair<std::unique_ptr<FileBatchReader>, std::set<int32_t>>>
-AbstractSplitRead::ApplySharedShreddingReaderIfNeeded(
+AbstractSplitRead::ApplyShreddingReaderIfNeeded(
     std::unique_ptr<FileBatchReader>&& file_reader,
     const std::shared_ptr<arrow::Schema>& read_schema) const {
-    std::set<int32_t> handled_shared_shredding_field_ids;
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> file_schema,
                            file_reader->GetFileSchema());
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> file_arrow_schema,
                                       arrow::ImportSchema(file_schema.get()));
-    std::map<std::string, std::unique_ptr<MapFieldReadPlan>> field_read_plans;
+
+    std::set<int32_t> handled_shared_shredding_field_ids;
+    std::map<std::string, std::shared_ptr<ShreddingColumnReadPlan>> plans;
     for (const auto& read_field : read_schema->fields()) {
         const auto& field_name = read_field->name();
         auto file_field = file_arrow_schema->GetFieldByName(field_name);
@@ -286,64 +284,46 @@ AbstractSplitRead::ApplySharedShreddingReaderIfNeeded(
             continue;
         }
 
-        std::unique_ptr<MapFieldReadPlan> field_read_plan;
+        std::shared_ptr<ShreddingColumnReadPlan> plan;
         if (is_shared_shredding_map_access) {
             if (is_shared_shredding_file) {
                 PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingFieldMeta meta,
                                        MapSharedShreddingUtils::DeserializeMetadata(metadata));
                 PAIMON_ASSIGN_OR_RAISE(
-                    field_read_plan,
-                    MapFieldReadPlanFactory::CreateSharedSelectedKeysReadPlan(read_field, meta));
+                    plan, MapSharedShreddingReadPlanFactory::CreateSharedSelectedKeysReadPlan(
+                              read_field, meta));
             } else {
-                PAIMON_ASSIGN_OR_RAISE(field_read_plan,
-                                       MapFieldReadPlanFactory::CreateDefaultSelectedKeysReadPlan(
-                                           file_field, read_field));
+                PAIMON_ASSIGN_OR_RAISE(
+                    plan, MapSharedShreddingReadPlanFactory::CreateDefaultSelectedKeysReadPlan(
+                              file_field, read_field));
             }
         } else {
             PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingFieldMeta meta,
                                    MapSharedShreddingUtils::DeserializeMetadata(metadata));
-            PAIMON_ASSIGN_OR_RAISE(field_read_plan,
-                                   MapFieldReadPlanFactory::CreateMapReadPlan(read_field, meta));
+            PAIMON_ASSIGN_OR_RAISE(
+                plan, MapSharedShreddingReadPlanFactory::CreateMapReadPlan(read_field, meta));
         }
-        field_read_plans.emplace(field_name, std::move(field_read_plan));
+        plans.emplace(field_name, std::move(plan));
         PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
                                NestedProjectionUtils::GetPaimonFieldId(read_field));
         handled_shared_shredding_field_ids.insert(field_id);
     }
-    if (!field_read_plans.empty()) {
-        file_reader = std::make_unique<MapSharedShreddingFileReader>(
-            std::move(file_reader), std::move(field_read_plans), pool_);
-    }
-    return std::make_pair(std::move(file_reader), std::move(handled_shared_shredding_field_ids));
-}
 
-Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::ApplyVariantShreddingReaderIfNeeded(
-    std::unique_ptr<FileBatchReader>&& file_reader,
-    const std::shared_ptr<arrow::Schema>& read_schema) const {
-    bool has_variant_field = false;
-    for (const auto& read_field : read_schema->fields()) {
-        // Variant columns may be nested inside struct columns; a variant-access projection also
-        // matches because it carries the variant extension marker itself.
-        if (VariantTypeUtils::ContainsVariantField(read_field)) {
-            has_variant_field = true;
-            break;
+    std::map<std::string, std::shared_ptr<ShreddingColumnReadPlan>> variant_plans;
+    PAIMON_ASSIGN_OR_RAISE(variant_plans, VariantShreddingReadPlanFactory::CreateReadPlans(
+                                              read_schema, file_arrow_schema, pool_));
+    for (auto& [field_name, plan] : variant_plans) {
+        if (!plans.emplace(field_name, std::move(plan)).second) {
+            return Status::Invalid(
+                fmt::format("multiple shredding read plans exist for field {}", field_name));
         }
     }
-    if (!has_variant_field) {
-        return std::move(file_reader);
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> file_schema,
-                           file_reader->GetFileSchema());
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> file_arrow_schema,
-                                      arrow::ImportSchema(file_schema.get()));
-    std::map<std::string, std::shared_ptr<ShreddingColumnReadPlan>> plans;
-    PAIMON_ASSIGN_OR_RAISE(plans, VariantShreddingReadPlanFactory::CreateReadPlans(
-                                      read_schema, file_arrow_schema, pool_));
+
     if (!plans.empty()) {
         file_reader =
             std::make_unique<ShreddingFileReader>(std::move(file_reader), std::move(plans), pool_);
     }
-    return std::move(file_reader);
+    return std::make_pair(std::move(file_reader), std::move(handled_shared_shredding_field_ids));
 }
 
 Result<std::vector<DataField>> AbstractSplitRead::ProjectFieldsForRowTrackingAndDataEvolution(
diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h
index 27349fec1..a56b48fdf 100644
--- a/src/paimon/core/operation/abstract_split_read.h
+++ b/src/paimon/core/operation/abstract_split_read.h
@@ -117,16 +117,12 @@ class AbstractSplitRead : public SplitRead {
         const std::optional<std::vector<Range>>& row_ranges,
         const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) const;
 
+    /// The returned field ID set contains MAP fields handled by shredding read plans. It tells
+    /// FieldMappingReader to skip its generic selected-key filtering because the plans have already
+    /// applied any requested key selection.
     Result<std::pair<std::unique_ptr<FileBatchReader>, std::set<int32_t>>>
-    ApplySharedShreddingReaderIfNeeded(std::unique_ptr<FileBatchReader>&& file_reader,
-                                       const std::shared_ptr<arrow::Schema>& read_schema) const;
-
-    /// Wraps the reader with a `ShreddingFileReader` when any read variant column needs
-    /// reassembly or path extraction; a plain read of an unshredded variant column is passed
-    /// through untouched.
-    Result<std::unique_ptr<FileBatchReader>> ApplyVariantShreddingReaderIfNeeded(
-        std::unique_ptr<FileBatchReader>&& file_reader,
-        const std::shared_ptr<arrow::Schema>& read_schema) const;
+    ApplyShreddingReaderIfNeeded(std::unique_ptr<FileBatchReader>&& file_reader,
+                                 const std::shared_ptr<arrow::Schema>& read_schema) const;
 
     static bool NeedCompleteRowTrackingFields(bool row_tracking_enabled,
                                               const std::shared_ptr<arrow::Schema>& read_schema);
diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h
index b4f80adb6..fad0e6742 100644
--- a/src/paimon/core/operation/data_evolution_split_read.h
+++ b/src/paimon/core/operation/data_evolution_split_read.h
@@ -64,7 +64,6 @@ struct DeletionFile;
 /// ->(ConcatBatchReader across blob files | BlobFallbackBatchReader across blob sequence layers)
 /// ->FieldMappingReader->(ApplyDeletionVectorBatchReader)->(ApplyBitmapIndexBatchReader)
 /// ->(CompleteRowTrackingFieldsBatchReader)->(ShreddingFileReader)
-/// ->(MapSharedShreddingFileReader)
 /// ->(VectorFileBatchReader)->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader
 ///
 ///
diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h
index d4bfa727c..5003cb55a 100644
--- a/src/paimon/core/operation/merge_file_split_read.h
+++ b/src/paimon/core/operation/merge_file_split_read.h
@@ -74,7 +74,6 @@ class MergeFunctionWrapper;
 /// files->KeyValueProjectionReader/AsyncKeyValueProjectionReader
 /// ->DropDeleteReader->SortMergeReader->ConcatKeyValueRecordReader->KeyValueDataFileRecordReader
 /// ->FieldMappingReader->(ApplyDeletionVectorBatchReader)->(ShreddingFileReader)
-/// ->(MapSharedShreddingFileReader)
 /// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader
 class MergeFileSplitRead : public AbstractSplitRead {
  public:
diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h
index 6a97b9b37..93eab5509 100644
--- a/src/paimon/core/operation/raw_file_split_read.h
+++ b/src/paimon/core/operation/raw_file_split_read.h
@@ -54,7 +54,7 @@ struct DeletionFile;
 /// splits)->CompleteRowKindBatchReader->(PredicateBatchReader)
 /// ->ConcatBatchReader across
 /// files->FieldMappingReader->(ApplyBitmapIndexBatchReader)->(CompleteRowTrackingFieldsBatchReader)
-/// ->(ShreddingFileReader)->(MapSharedShreddingFileReader)->(VectorFileBatchReader)
+/// ->(ShreddingFileReader)->(VectorFileBatchReader)
 /// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader
 
 class RawFileSplitRead : public AbstractSplitRead {

From 370582dc1e3ec414bd83b739681e02686db86464 Mon Sep 17 00:00:00 2001
From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com>
Date: Tue, 25 Aug 2026 17:59:46 +0800
Subject: [PATCH 19/93] feat(file-index): support BSI and Bloom filter writers
 (#234)

---
 .../file_index/bitmap/bitmap_file_index.cpp   |  24 +---
 .../file_index/bitmap/bitmap_file_index.h     |   2 +-
 .../bloomfilter/bloom_filter_file_index.cpp   |  88 +++++++++++--
 .../bloomfilter/bloom_filter_file_index.h     |  38 +++++-
 .../bloom_filter_file_index_test.cpp          |  55 +++++++-
 .../file_index/bloomfilter/fast_hash.cpp      |  16 ++-
 .../file_index/bloomfilter/fast_hash_test.cpp |  31 +++++
 .../bsi/bit_slice_index_bitmap_file_index.cpp | 105 ++++++++++++++++
 .../bsi/bit_slice_index_bitmap_file_index.h   |  34 +++--
 ...bit_slice_index_bitmap_file_index_test.cpp | 119 ++++++++++++------
 .../rangebitmap/range_bitmap_file_index.cpp   |  33 ++---
 .../rangebitmap/range_bitmap_file_index.h     |   6 +-
 .../range_bitmap_file_index_test.cpp          |  10 +-
 .../rangebitmap/range_bitmap_io_test.cpp      |   8 +-
 .../common/lookup/lookup_store_factory.cpp    |   3 +-
 src/paimon/common/sst/sst_file_io_test.cpp    |   4 +-
 src/paimon/common/utils/bloom_filter.cpp      |  24 ++--
 src/paimon/common/utils/bloom_filter.h        |   8 +-
 src/paimon/common/utils/bloom_filter64.cpp    |  52 ++++++--
 src/paimon/common/utils/bloom_filter64.h      |  11 +-
 .../common/utils/bloom_filter64_test.cpp      |  42 ++++++-
 src/paimon/common/utils/bloom_filter_test.cpp |  35 ++++--
 src/paimon/common/utils/math.h                |  27 ++++
 src/paimon/common/utils/math_test.cpp         |  18 +++
 .../core/io/data_file_index_writer_test.cpp   |  31 ++++-
 25 files changed, 658 insertions(+), 166 deletions(-)

diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp b/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp
index 6df161436..43bb71b0e 100644
--- a/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp
+++ b/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp
@@ -103,24 +103,18 @@ Result<std::shared_ptr<FileIndexWriter>> BitmapFileIndex::CreateWriter(
             "invalid schema for BitmapFileIndexWriter, supposed to have single "
             "field.");
     }
-    auto arrow_field = arrow_schema->field(0);
-    return BitmapFileIndexWriter::Create(arrow_schema, arrow_field->name(), options_, pool);
+    return BitmapFileIndexWriter::Create(arrow_schema->field(0), options_, pool);
 }
 
 Result<std::shared_ptr<BitmapFileIndexWriter>> BitmapFileIndexWriter::Create(
-    const std::shared_ptr<arrow::Schema>& arrow_schema, const std::string& field_name,
-    const std::map<std::string, std::string>& options, const std::shared_ptr<MemoryPool>& pool) {
+    const std::shared_ptr<arrow::Field>& field, const std::map<std::string, std::string>& options,
+    const std::shared_ptr<MemoryPool>& pool) {
     PAIMON_ASSIGN_OR_RAISE(int8_t version,
                            OptionsUtils::GetValueFromMap<int8_t>(options, BitmapFileIndex::VERSION,
                                                                  BitmapFileIndex::VERSION_2));
-    auto arrow_field = arrow_schema->GetFieldByName(field_name);
-    if (!arrow_field) {
-        return Status::Invalid(
-            fmt::format("field {} not in arrow_schema for BitmapFileIndexWriter", field_name));
-    }
-    auto struct_type = arrow::struct_({arrow_field});
+    std::shared_ptr<arrow::DataType> struct_type = arrow::struct_({field});
     return std::shared_ptr<BitmapFileIndexWriter>(
-        new BitmapFileIndexWriter(version, struct_type, arrow_field->type(), options, pool));
+        new BitmapFileIndexWriter(version, struct_type, field->type(), options, pool));
 }
 
 BitmapFileIndexWriter::BitmapFileIndexWriter(int8_t version,
@@ -137,15 +131,7 @@ BitmapFileIndexWriter::BitmapFileIndexWriter(int8_t version,
 Status BitmapFileIndexWriter::AddBatch(::ArrowArray* batch) {
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> arrow_array,
                                       arrow::ImportArray(batch, struct_type_));
-    if (!arrow_array || arrow_array->type_id() != arrow::Type::STRUCT) {
-        return Status::Invalid("invalid batch for BitmapFileIndexWriter, expected a struct array");
-    }
     auto struct_array = checked_pointer_cast<arrow::StructArray>(arrow_array);
-    if (struct_array->num_fields() != 1) {
-        return Status::Invalid(
-            "invalid batch for BitmapFileIndexWriter, expected a struct array with exactly one "
-            "field");
-    }
     PAIMON_ASSIGN_OR_RAISE(
         std::vector<Literal> array_values,
         LiteralConverter::ConvertLiteralsFromArray(*(struct_array->field(0)), /*own_data=*/true));
diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index.h b/src/paimon/common/file_index/bitmap/bitmap_file_index.h
index 9cf15ddc3..6aa0f4169 100644
--- a/src/paimon/common/file_index/bitmap/bitmap_file_index.h
+++ b/src/paimon/common/file_index/bitmap/bitmap_file_index.h
@@ -71,7 +71,7 @@ class PAIMON_EXPORT BitmapFileIndex : public FileIndexer {
 class BitmapFileIndexWriter : public FileIndexWriter {
  public:
     static Result<std::shared_ptr<BitmapFileIndexWriter>> Create(
-        const std::shared_ptr<arrow::Schema>& arrow_schema, const std::string& field_name,
+        const std::shared_ptr<arrow::Field>& field,
         const std::map<std::string, std::string>& options, const std::shared_ptr<MemoryPool>& pool);
 
     Status AddBatch(::ArrowArray* batch) override;
diff --git a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp
index ca33c696f..86d07d63b 100644
--- a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp
+++ b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp
@@ -19,10 +19,19 @@
 #include "paimon/common/file_index/bloomfilter/bloom_filter_file_index.h"
 
 #include <cstddef>
+#include <cstdint>
+#include <cstring>
 #include <functional>
 #include <utility>
+#include <vector>
 
+#include "arrow/c/bridge.h"
 #include "fmt/format.h"
+#include "paimon/common/predicate/literal_converter.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/math.h"
+#include "paimon/common/utils/options_utils.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/memory/bytes.h"
 #include "paimon/predicate/literal.h"
@@ -31,7 +40,8 @@
 namespace paimon {
 class MemoryPool;
 
-BloomFilterFileIndex::BloomFilterFileIndex(const std::map<std::string, std::string>& options) {}
+BloomFilterFileIndex::BloomFilterFileIndex(const std::map<std::string, std::string>& options)
+    : options_(options) {}
 Result<std::shared_ptr<FileIndexReader>> BloomFilterFileIndex::CreateReader(
     ::ArrowSchema* c_arrow_schema, int32_t start, int32_t length,
     const std::shared_ptr<InputStream>& input_stream,
@@ -58,15 +68,77 @@ Result<std::shared_ptr<FileIndexReader>> BloomFilterFileIndex::CreateReader(
     return BloomFilterFileIndexReader::Create(arrow_type, bytes);
 }
 
+Result<std::shared_ptr<FileIndexWriter>> BloomFilterFileIndex::CreateWriter(
+    ::ArrowSchema* c_arrow_schema, const std::shared_ptr<MemoryPool>& pool) const {
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> arrow_schema,
+                                      arrow::ImportSchema(c_arrow_schema));
+    if (arrow_schema->num_fields() != 1) {
+        return Status::Invalid(
+            "invalid schema for BloomFilterFileIndexWriter, supposed to have single field.");
+    }
+    return BloomFilterFileIndexWriter::Create(arrow_schema->field(0), options_, pool);
+}
+
+Result<std::shared_ptr<BloomFilterFileIndexWriter>> BloomFilterFileIndexWriter::Create(
+    const std::shared_ptr<arrow::Field>& field, const std::map<std::string, std::string>& options,
+    const std::shared_ptr<MemoryPool>& pool) {
+    PAIMON_ASSIGN_OR_RAISE(FastHash::HashFunction hash_function,
+                           FastHash::GetHashFunction(field->type()));
+    PAIMON_ASSIGN_OR_RAISE(
+        int32_t items, OptionsUtils::GetValueFromMap<int32_t>(options, BloomFilterFileIndex::kItems,
+                                                              BloomFilterFileIndex::kDefaultItems));
+    PAIMON_ASSIGN_OR_RAISE(
+        double fpp, OptionsUtils::GetValueFromMap<double>(options, BloomFilterFileIndex::kFpp,
+                                                          BloomFilterFileIndex::kDefaultFpp));
+    std::shared_ptr<arrow::DataType> struct_type = arrow::struct_({field});
+    PAIMON_ASSIGN_OR_RAISE(BloomFilter64 filter, BloomFilter64::Create(items, fpp, pool));
+    return std::shared_ptr<BloomFilterFileIndexWriter>(
+        new BloomFilterFileIndexWriter(struct_type, hash_function, std::move(filter), pool));
+}
+
+BloomFilterFileIndexWriter::BloomFilterFileIndexWriter(
+    const std::shared_ptr<arrow::DataType>& struct_type,
+    const FastHash::HashFunction& hash_function, BloomFilter64&& filter,
+    const std::shared_ptr<MemoryPool>& pool)
+    : struct_type_(struct_type),
+      hash_function_(hash_function),
+      filter_(std::move(filter)),
+      pool_(pool) {}
+
+Status BloomFilterFileIndexWriter::AddBatch(::ArrowArray* batch) {
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
+                                      arrow::ImportArray(batch, struct_type_));
+    std::shared_ptr<arrow::StructArray> struct_array =
+        checked_pointer_cast<arrow::StructArray>(array);
+    PAIMON_ASSIGN_OR_RAISE(
+        std::vector<Literal> values,
+        LiteralConverter::ConvertLiteralsFromArray(*struct_array->field(0), /*own_data=*/false));
+    for (const Literal& value : values) {
+        if (!value.IsNull()) {
+            filter_.AddHash(hash_function_(value));
+        }
+    }
+    return Status::OK();
+}
+
+Result<PAIMON_UNIQUE_PTR<Bytes>> BloomFilterFileIndexWriter::SerializedBytes() const {
+    constexpr int32_t kHeaderLength = sizeof(int32_t);
+    const int32_t bit_set_length = filter_.GetBitSet().ByteLength();
+    PAIMON_UNIQUE_PTR<Bytes> bytes =
+        Bytes::AllocateBytes(kHeaderLength + bit_set_length, pool_.get());
+    const int32_t num_hash_functions = ToBigEndian(filter_.GetNumHashFunctions());
+    std::memcpy(bytes->data(), &num_hash_functions, sizeof(num_hash_functions));
+    filter_.GetBitSet().ToByteArray(kHeaderLength, bit_set_length, bytes->data());
+    return bytes;
+}
+
 Result<std::shared_ptr<BloomFilterFileIndexReader>> BloomFilterFileIndexReader::Create(
     const std::shared_ptr<arrow::DataType>& arrow_type, const std::shared_ptr<Bytes>& bytes) {
-    // compatible with java, little endian
-    const char* data = bytes->data();
-    auto num_hash_functions =
-        static_cast<int32_t>((static_cast<uint32_t>(static_cast<uint8_t>(data[0])) << 24) |
-                             (static_cast<uint32_t>(static_cast<uint8_t>(data[1])) << 16) |
-                             (static_cast<uint32_t>(static_cast<uint8_t>(data[2])) << 8) |
-                             static_cast<uint32_t>(static_cast<uint8_t>(data[3])));
+    // Compatible with Java's big-endian numHashFunctions header.
+    int32_t big_endian_num_hash_functions;
+    std::memcpy(&big_endian_num_hash_functions, bytes->data(),
+                sizeof(big_endian_num_hash_functions));
+    const int32_t num_hash_functions = FromBigEndian(big_endian_num_hash_functions);
     PAIMON_ASSIGN_OR_RAISE(FastHash::HashFunction hash_function,
                            FastHash::GetHashFunction(arrow_type));
     auto bit_set = std::make_unique<BloomFilter64::BitSet>(bytes, /*offset=*/sizeof(int32_t));
diff --git a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.h b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.h
index 8152c2204..62d082c76 100644
--- a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.h
+++ b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.h
@@ -25,10 +25,10 @@
 
 #include "arrow/c/bridge.h"
 #include "paimon/common/file_index/bloomfilter/fast_hash.h"
-#include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/bloom_filter64.h"
 #include "paimon/file_index/file_index_reader.h"
 #include "paimon/file_index/file_index_result.h"
+#include "paimon/file_index/file_index_writer.h"
 #include "paimon/file_index/file_indexer.h"
 #include "paimon/result.h"
 namespace paimon {
@@ -55,11 +55,37 @@ class BloomFilterFileIndex : public FileIndexer {
         const std::shared_ptr<MemoryPool>& pool) const override;
 
     Result<std::shared_ptr<FileIndexWriter>> CreateWriter(
-        ::ArrowSchema* arrow_schema, const std::shared_ptr<MemoryPool>& pool) const override {
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::DataType> arrow_type,
-                                          arrow::ImportType(arrow_schema));
-        return Status::NotImplemented("do not support index writer in bloom filter");
-    }
+        ::ArrowSchema* arrow_schema, const std::shared_ptr<MemoryPool>& pool) const override;
+
+    static constexpr int32_t kDefaultItems = 1000000;
+    static constexpr double kDefaultFpp = 0.1;
+    static constexpr char kItems[] = "items";
+    static constexpr char kFpp[] = "fpp";
+
+ private:
+    std::map<std::string, std::string> options_;
+};
+
+class BloomFilterFileIndexWriter : public FileIndexWriter {
+ public:
+    static Result<std::shared_ptr<BloomFilterFileIndexWriter>> Create(
+        const std::shared_ptr<arrow::Field>& field,
+        const std::map<std::string, std::string>& options, const std::shared_ptr<MemoryPool>& pool);
+
+    Status AddBatch(::ArrowArray* batch) override;
+
+    Result<PAIMON_UNIQUE_PTR<Bytes>> SerializedBytes() const override;
+
+ private:
+    BloomFilterFileIndexWriter(const std::shared_ptr<arrow::DataType>& struct_type,
+                               const FastHash::HashFunction& hash_function, BloomFilter64&& filter,
+                               const std::shared_ptr<MemoryPool>& pool);
+
+ private:
+    std::shared_ptr<arrow::DataType> struct_type_;
+    FastHash::HashFunction hash_function_;
+    BloomFilter64 filter_;
+    std::shared_ptr<MemoryPool> pool_;
 };
 
 class BloomFilterFileIndexReader : public FileIndexReader {
diff --git a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index_test.cpp b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index_test.cpp
index 209f0b409..23fadce28 100644
--- a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index_test.cpp
+++ b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index_test.cpp
@@ -18,10 +18,16 @@
 
 #include "paimon/common/file_index/bloomfilter/bloom_filter_file_index.h"
 
+#include <cstring>
+#include <map>
+#include <string>
 #include <utility>
 #include <vector>
 
+#include "arrow/c/bridge.h"
+#include "arrow/ipc/json_simple.h"
 #include "gtest/gtest.h"
+#include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/field_type_utils.h"
 #include "paimon/data/timestamp.h"
 #include "paimon/defs.h"
@@ -49,21 +55,60 @@ class BloomFilterIndexReaderTest : public ::testing::Test {
         return c_schema;
     }
 
+    Result<PAIMON_UNIQUE_PTR<Bytes>> WriteIndex(
+        const std::shared_ptr<arrow::DataType>& data_type, const std::string& json,
+        const std::map<std::string, std::string>& options) const {
+        const std::shared_ptr<arrow::Schema> schema =
+            arrow::schema({arrow::field("f0", data_type)});
+        std::shared_ptr<arrow::Array> array =
+            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), json)
+                .ValueOrDie();
+        ::ArrowSchema c_schema;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema));
+        BloomFilterFileIndex file_index(options);
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileIndexWriter> writer,
+                               file_index.CreateWriter(&c_schema, pool_));
+        ::ArrowArray c_array;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
+        PAIMON_RETURN_NOT_OK(writer->AddBatch(&c_array));
+        return writer->SerializedBytes();
+    }
+
  private:
     std::shared_ptr<MemoryPool> pool_;
 };
 
+TEST_F(BloomFilterIndexReaderTest, TestWriterRejectsInvalidOptionsAndType) {
+    auto create_writer = [&](const std::shared_ptr<arrow::DataType>& type,
+                             const std::map<std::string, std::string>& options) {
+        BloomFilterFileIndex file_index(options);
+        return file_index.CreateWriter(CreateArrowSchema(type).get(), pool_);
+    };
+    ASSERT_NOK_WITH_MSG(create_writer(arrow::int32(), {{"items", "0"}}),
+                        "items must be greater than 0");
+    ASSERT_NOK_WITH_MSG(create_writer(arrow::int32(), {{"fpp", "1"}}),
+                        "fpp must be greater than 0 and less than 1");
+    ASSERT_NOK_WITH_MSG(create_writer(arrow::boolean(), {}),
+                        "bloom filter index does not support BOOLEAN");
+}
+
 TEST_F(BloomFilterIndexReaderTest, TestStringType) {
-    // data: "a", "b", ""
-    std::vector<uint8_t> index_bytes = {0, 0, 0, 6, 0, 32, 32, 3, 208, 32, 0, 64, 73, 16, 201};
-    auto input_stream = std::make_shared<ByteArrayInputStream>(
-        reinterpret_cast<char*>(index_bytes.data()), index_bytes.size());
+    // Java writer output for data: "a", "b", "" with items=10 and fpp=0.02.
+    const std::vector<uint8_t> expected = {0, 0, 0, 6, 0, 32, 32, 3, 208, 32, 0, 64, 73, 16, 201};
+    ASSERT_OK_AND_ASSIGN(
+        PAIMON_UNIQUE_PTR<Bytes> bytes,
+        WriteIndex(arrow::utf8(), R"([["a"], ["b"], [""]])", {{"items", "10"}, {"fpp", "0.02"}}));
+    ASSERT_EQ(expected.size(), bytes->size());
+    ASSERT_EQ(0, std::memcmp(expected.data(), bytes->data(), expected.size()));
+
+    std::shared_ptr<ByteArrayInputStream> input_stream =
+        std::make_shared<ByteArrayInputStream>(bytes->data(), bytes->size());
 
     BloomFilterFileIndex file_index({});
     ASSERT_OK_AND_ASSIGN(
         auto reader,
         file_index.CreateReader(CreateArrowSchema(arrow::utf8()).get(),
-                                /*start=*/0, /*length=*/index_bytes.size(), input_stream, pool_));
+                                /*start=*/0, /*length=*/bytes->size(), input_stream, pool_));
     ASSERT_TRUE(reader);
     ASSERT_TRUE(reader->VisitEqual(Literal(FieldType::STRING, "a", 1)).value()->IsRemain().value());
     ASSERT_TRUE(reader->VisitEqual(Literal(FieldType::STRING, "b", 1)).value()->IsRemain().value());
diff --git a/src/paimon/common/file_index/bloomfilter/fast_hash.cpp b/src/paimon/common/file_index/bloomfilter/fast_hash.cpp
index d98dc476e..b1d8784fc 100644
--- a/src/paimon/common/file_index/bloomfilter/fast_hash.cpp
+++ b/src/paimon/common/file_index/bloomfilter/fast_hash.cpp
@@ -19,6 +19,7 @@
 #include "paimon/common/file_index/bloomfilter/fast_hash.h"
 
 #include <cassert>
+#include <cmath>
 #include <cstring>
 #include <string>
 #include <utility>
@@ -34,6 +35,11 @@
 #include "xxhash.h"  // NOLINT(build/include_subdir)
 
 namespace paimon {
+namespace {
+constexpr int32_t kCanonicalFloatNaNBits = 0x7fc00000;
+constexpr int64_t kCanonicalDoubleNaNBits = 0x7ff8000000000000L;
+}  // namespace
+
 Result<FastHash::HashFunction> FastHash::GetHashFunction(
     const std::shared_ptr<arrow::DataType>& arrow_type) {
     PAIMON_ASSIGN_OR_RAISE(FieldType field_type,
@@ -58,14 +64,20 @@ Result<FastHash::HashFunction> FastHash::GetHashFunction(
             });
         case FieldType::FLOAT:
             return HashFunction([](const Literal& literal) -> int64_t {
-                auto raw_value = literal.GetValue<float>();
+                const auto raw_value = literal.GetValue<float>();
+                if (std::isnan(raw_value)) {
+                    return GetLongHash(kCanonicalFloatNaNBits);
+                }
                 int32_t bits = 0;
                 std::memcpy(&bits, &raw_value, sizeof(raw_value));
                 return GetLongHash(bits);
             });
         case FieldType::DOUBLE:
             return HashFunction([](const Literal& literal) -> int64_t {
-                auto raw_value = literal.GetValue<double>();
+                const auto raw_value = literal.GetValue<double>();
+                if (std::isnan(raw_value)) {
+                    return GetLongHash(kCanonicalDoubleNaNBits);
+                }
                 int64_t bits;
                 std::memcpy(&bits, &raw_value, sizeof(raw_value));
                 return GetLongHash(bits);
diff --git a/src/paimon/common/file_index/bloomfilter/fast_hash_test.cpp b/src/paimon/common/file_index/bloomfilter/fast_hash_test.cpp
index 680d591bd..8a528e443 100644
--- a/src/paimon/common/file_index/bloomfilter/fast_hash_test.cpp
+++ b/src/paimon/common/file_index/bloomfilter/fast_hash_test.cpp
@@ -18,6 +18,9 @@
 
 #include "paimon/common/file_index/bloomfilter/fast_hash.h"
 
+#include <cmath>
+#include <cstdint>
+#include <cstring>
 #include <limits>
 #include <string>
 #include <vector>
@@ -164,4 +167,32 @@ TEST_F(FastHashTest, TestCompatibleWithJava) {
     }
 }
 
+TEST_F(FastHashTest, TestNaNCompatibleWithJava) {
+    auto float_from_bits = [](uint32_t bits) {
+        float value;
+        std::memcpy(&value, &bits, sizeof(value));
+        return value;
+    };
+    const float float_nan = float_from_bits(0x7fc12345);
+    const float negative_float_nan = float_from_bits(0xffc54321);
+    ASSERT_TRUE(std::isnan(float_nan));
+    ASSERT_TRUE(std::isnan(negative_float_nan));
+    ASSERT_OK_AND_ASSIGN(auto float_hash_function, FastHash::GetHashFunction(arrow::float32()));
+    CheckResult(float_hash_function, {Literal(float_nan), Literal(negative_float_nan)},
+                {0x67c27c6d9936ae63, 0x67c27c6d9936ae63});
+
+    auto double_from_bits = [](uint64_t bits) {
+        double value;
+        std::memcpy(&value, &bits, sizeof(value));
+        return value;
+    };
+    const double double_nan = double_from_bits(0x7ff8123456789abc);
+    const double negative_double_nan = double_from_bits(0xfff8abcdef012345);
+    ASSERT_TRUE(std::isnan(double_nan));
+    ASSERT_TRUE(std::isnan(negative_double_nan));
+    ASSERT_OK_AND_ASSIGN(auto double_hash_function, FastHash::GetHashFunction(arrow::float64()));
+    CheckResult(double_hash_function, {Literal(double_nan), Literal(negative_double_nan)},
+                {0x13d2d3f2cc0e846e, 0x13d2d3f2cc0e846e});
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp
index 7ce53e9a6..4cb98cdca 100644
--- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp
+++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp
@@ -18,13 +18,23 @@
 
 #include "paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h"
 
+#include <algorithm>
 #include <cassert>
 #include <climits>
 #include <cstddef>
 #include <cstdint>
+#include <limits>
+#include <optional>
+#include <utility>
+#include <vector>
 
+#include "arrow/c/bridge.h"
 #include "fmt/format.h"
 #include "paimon/common/file_index/bsi/bit_slice_index_roaring_bitmap.h"
+#include "paimon/common/io/memory_segment_output_stream.h"
+#include "paimon/common/memory/memory_segment_utils.h"
+#include "paimon/common/predicate/literal_converter.h"
+#include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/common/utils/field_type_utils.h"
@@ -53,6 +63,101 @@ class MemoryPool;
 BitSliceIndexBitmapFileIndex::BitSliceIndexBitmapFileIndex(
     const std::map<std::string, std::string>& options) {}
 
+Result<std::shared_ptr<FileIndexWriter>> BitSliceIndexBitmapFileIndex::CreateWriter(
+    ::ArrowSchema* c_arrow_schema, const std::shared_ptr<MemoryPool>& pool) const {
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> arrow_schema,
+                                      arrow::ImportSchema(c_arrow_schema));
+    if (arrow_schema->num_fields() != 1) {
+        return Status::Invalid(
+            "invalid schema for BitSliceIndexBitmapFileIndexWriter, supposed to have single "
+            "field.");
+    }
+    const std::shared_ptr<arrow::Field> field = arrow_schema->field(0);
+    PAIMON_ASSIGN_OR_RAISE(ValueMapperType value_mapper, GetValueMapper(field->type()));
+    return std::make_shared<BitSliceIndexBitmapFileIndexWriter>(field, value_mapper, pool);
+}
+
+BitSliceIndexBitmapFileIndexWriter::BitSliceIndexBitmapFileIndexWriter(
+    const std::shared_ptr<arrow::Field>& field,
+    const BitSliceIndexBitmapFileIndex::ValueMapperType& value_mapper,
+    const std::shared_ptr<MemoryPool>& pool)
+    : struct_type_(arrow::struct_({field})),
+      field_name_(field->name()),
+      value_mapper_(value_mapper),
+      pool_(pool) {}
+
+Status BitSliceIndexBitmapFileIndexWriter::AddBatch(::ArrowArray* batch) {
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
+                                      arrow::ImportArray(batch, struct_type_));
+    auto struct_array = checked_pointer_cast<arrow::StructArray>(array);
+    if (struct_array->length() > static_cast<int64_t>(std::numeric_limits<int32_t>::max()) -
+                                     static_cast<int64_t>(values_.size())) {
+        return Status::Invalid("bsi index row count exceeds the supported int32 range");
+    }
+    PAIMON_ASSIGN_OR_RAISE(
+        std::vector<Literal> literals,
+        LiteralConverter::ConvertLiteralsFromArray(*struct_array->field(0), /*own_data=*/false));
+    values_.reserve(values_.size() + literals.size());
+    for (const Literal& literal : literals) {
+        if (literal.IsNull()) {
+            values_.emplace_back(std::nullopt);
+            continue;
+        }
+        PAIMON_ASSIGN_OR_RAISE(int64_t value, value_mapper_(literal));
+        if (value == std::numeric_limits<int64_t>::min()) {
+            return Status::Invalid(
+                fmt::format("bsi index does not support INT64_MIN for field '{}'", field_name_));
+        }
+        values_.emplace_back(value);
+        if (value < 0) {
+            const int64_t absolute_value = SafeAbs(value);
+            negative_min_ = std::min(negative_min_, absolute_value);
+            negative_max_ = std::max(negative_max_, absolute_value);
+        } else {
+            positive_min_ = std::min(positive_min_, value);
+            positive_max_ = std::max(positive_max_, value);
+        }
+    }
+    return Status::OK();
+}
+
+Result<PAIMON_UNIQUE_PTR<Bytes>> BitSliceIndexBitmapFileIndexWriter::SerializedBytes() const {
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<BitSliceIndexRoaringBitmap::Appender> positive,
+        BitSliceIndexRoaringBitmap::Appender::Create(positive_min_, positive_max_));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<BitSliceIndexRoaringBitmap::Appender> negative,
+        BitSliceIndexRoaringBitmap::Appender::Create(negative_min_, negative_max_));
+    for (size_t i = 0; i < values_.size(); ++i) {
+        if (!values_[i]) {
+            continue;
+        }
+        const int64_t value = values_[i].value();
+        if (value < 0) {
+            PAIMON_RETURN_NOT_OK(negative->Append(static_cast<int32_t>(i), SafeAbs(value)));
+        } else {
+            PAIMON_RETURN_NOT_OK(positive->Append(static_cast<int32_t>(i), value));
+        }
+    }
+
+    MemorySegmentOutputStream output_stream(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_);
+    output_stream.SetOrder(ByteOrder::PAIMON_BIG_ENDIAN);
+    output_stream.WriteValue<int8_t>(BitSliceIndexBitmapFileIndex::VERSION_1);
+    output_stream.WriteValue<int32_t>(static_cast<int32_t>(values_.size()));
+    const bool has_positive = positive->IsNotEmpty();
+    output_stream.WriteValue<bool>(has_positive);
+    if (has_positive) {
+        output_stream.WriteBytes(positive->Serialize(pool_));
+    }
+    const bool has_negative = negative->IsNotEmpty();
+    output_stream.WriteValue<bool>(has_negative);
+    if (has_negative) {
+        output_stream.WriteBytes(negative->Serialize(pool_));
+    }
+    return MemorySegmentUtils::CopyToBytes(output_stream.Segments(), /*offset=*/0,
+                                           /*num_bytes=*/output_stream.CurrentSize(), pool_.get());
+}
+
 Result<std::shared_ptr<FileIndexReader>> BitSliceIndexBitmapFileIndex::CreateReader(
     ::ArrowSchema* c_arrow_schema, int32_t start, int32_t length,
     const std::shared_ptr<InputStream>& input_stream,
diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h
index e661ea2a3..d7dfe092f 100644
--- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h
+++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h
@@ -22,15 +22,15 @@
 #include <functional>
 #include <map>
 #include <memory>
+#include <optional>
 #include <string>
 #include <utility>
 #include <vector>
 
-#include "arrow/c/bridge.h"
 #include "arrow/type.h"
-#include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/file_index/file_index_reader.h"
 #include "paimon/file_index/file_index_result.h"
+#include "paimon/file_index/file_index_writer.h"
 #include "paimon/file_index/file_indexer.h"
 #include "paimon/predicate/literal.h"
 #include "paimon/result.h"
@@ -54,14 +54,12 @@ class BitSliceIndexBitmapFileIndex : public FileIndexer {
         const std::shared_ptr<MemoryPool>& pool) const override;
 
     Result<std::shared_ptr<FileIndexWriter>> CreateWriter(
-        ::ArrowSchema* arrow_schema, const std::shared_ptr<MemoryPool>& pool) const override {
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::DataType> arrow_type,
-                                          arrow::ImportType(arrow_schema));
-        return Status::NotImplemented("do not support index writer in bsi");
-    }
+        ::ArrowSchema* arrow_schema, const std::shared_ptr<MemoryPool>& pool) const override;
 
     using ValueMapperType = std::function<Result<int64_t>(const Literal& literal)>;
 
+    static constexpr int8_t VERSION_1 = 1;
+
  private:
     static Result<ValueMapperType> GetValueMapper(
         const std::shared_ptr<arrow::DataType>& arrow_type);
@@ -74,9 +72,29 @@ class BitSliceIndexBitmapFileIndex : public FileIndexer {
         }
         return static_cast<int64_t>(literal.GetValue<T>());
     }
+};
+
+class BitSliceIndexBitmapFileIndexWriter : public FileIndexWriter {
+ public:
+    BitSliceIndexBitmapFileIndexWriter(
+        const std::shared_ptr<arrow::Field>& field,
+        const BitSliceIndexBitmapFileIndex::ValueMapperType& value_mapper,
+        const std::shared_ptr<MemoryPool>& pool);
+
+    Status AddBatch(::ArrowArray* batch) override;
+
+    Result<PAIMON_UNIQUE_PTR<Bytes>> SerializedBytes() const override;
 
  private:
-    static constexpr int8_t VERSION_1 = 1;
+    std::shared_ptr<arrow::DataType> struct_type_;
+    std::string field_name_;
+    BitSliceIndexBitmapFileIndex::ValueMapperType value_mapper_;
+    std::vector<std::optional<int64_t>> values_;
+    int64_t positive_min_ = 0;
+    int64_t positive_max_ = 0;
+    int64_t negative_min_ = 0;
+    int64_t negative_max_ = 0;
+    std::shared_ptr<MemoryPool> pool_;
 };
 
 class BitSliceIndexBitmapFileIndexReader
diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp
index 128c1a69c..760434e19 100644
--- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp
+++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp
@@ -19,9 +19,13 @@
 #include "paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h"
 
 #include <cstdint>
+#include <string>
 #include <utility>
 
+#include "arrow/c/bridge.h"
+#include "arrow/ipc/json_simple.h"
 #include "gtest/gtest.h"
+#include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/field_type_utils.h"
 #include "paimon/data/timestamp.h"
 #include "paimon/defs.h"
@@ -60,6 +64,24 @@ class BitSliceIndexBitmapIndexReaderTest : public ::testing::Test {
             << ", expected=" << RoaringBitmap32::From(expected).ToString();
     }
 
+    Result<PAIMON_UNIQUE_PTR<Bytes>> WriteIndex(const std::shared_ptr<arrow::DataType>& data_type,
+                                                const std::string& json) const {
+        const std::shared_ptr<arrow::Schema> schema =
+            arrow::schema({arrow::field("f0", data_type)});
+        std::shared_ptr<arrow::Array> array =
+            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), json)
+                .ValueOrDie();
+        ::ArrowSchema c_schema;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema));
+        BitSliceIndexBitmapFileIndex file_index({});
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileIndexWriter> writer,
+                               file_index.CreateWriter(&c_schema, pool_));
+        ::ArrowArray c_array;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
+        PAIMON_RETURN_NOT_OK(writer->AddBatch(&c_array));
+        return writer->SerializedBytes();
+    }
+
  private:
     std::shared_ptr<MemoryPool> pool_;
 };
@@ -75,48 +97,75 @@ TEST_F(BitSliceIndexBitmapIndexReaderTest, TestMix) {
         0, 0, 0, 2,  58, 48, 0,  0, 1, 0, 0,  0, 0, 0, 2, 0,  16, 0,  0,  0,  3, 0, 4, 0,  5,
         0, 0, 0, 0,  2,  58, 48, 0, 0, 1, 0,  0, 0, 0, 0, 0,  0,  16, 0,  0,  0, 5, 0, 58, 48,
         0, 0, 1, 0,  0,  0,  0,  0, 1, 0, 16, 0, 0, 0, 3, 0,  4,  0};
+    const auto check_mix_reader = [this](const std::shared_ptr<FileIndexReader>& reader) {
+        // test equal
+        CheckResult(reader->VisitEqual(Literal(2)).value(), {1, 7});
+        CheckResult(reader->VisitEqual(Literal(-2)).value(), {3, 4});
+        CheckResult(reader->VisitEqual(Literal(100)).value(), {});
+
+        // test not equal
+        CheckResult(reader->VisitNotEqual(Literal(2)).value(), {0, 3, 4, 5, 8, 9});
+        CheckResult(reader->VisitNotEqual(Literal(-2)).value(), {0, 1, 5, 7, 8, 9});
+        CheckResult(reader->VisitNotEqual(Literal(100)).value(), {0, 1, 3, 4, 5, 7, 8, 9});
+
+        // test in
+        CheckResult(reader->VisitIn({Literal(-1), Literal(1), Literal(2), Literal(3)}).value(),
+                    {0, 1, 5, 7});
+
+        // test not in
+        CheckResult(reader->VisitNotIn({Literal(-1), Literal(1), Literal(2), Literal(3)}).value(),
+                    {3, 4, 8, 9});
+
+        // test null
+        CheckResult(reader->VisitIsNull().value(), {2, 6, 10});
+
+        // test not null
+        CheckResult(reader->VisitIsNotNull().value(), {0, 1, 3, 4, 5, 7, 8, 9});
+
+        // test less than
+        CheckResult(reader->VisitLessThan(Literal(2)).value(), {0, 3, 4, 5, 8});
+        CheckResult(reader->VisitLessOrEqual(Literal(2)).value(), {0, 1, 3, 4, 5, 7, 8});
+        CheckResult(reader->VisitLessThan(Literal(-1)).value(), {3, 4});
+        CheckResult(reader->VisitLessOrEqual(Literal(-1)).value(), {3, 4, 5});
+
+        // test greater than
+        CheckResult(reader->VisitGreaterThan(Literal(-2)).value(), {0, 1, 5, 7, 8, 9});
+        CheckResult(reader->VisitGreaterOrEqual(Literal(-2)).value(), {0, 1, 3, 4, 5, 7, 8, 9});
+        CheckResult(reader->VisitGreaterThan(Literal(2)).value(), {9});
+        CheckResult(reader->VisitGreaterOrEqual(Literal(2)).value(), {1, 7, 9});
+    };
+
+    ASSERT_OK_AND_ASSIGN(
+        PAIMON_UNIQUE_PTR<Bytes> written_bytes,
+        WriteIndex(arrow::int32(),
+                   R"([[1], [2], [null], [-2], [-2], [-1], [null], [2], [0], [5], [null]])"));
+    auto written_stream =
+        std::make_shared<ByteArrayInputStream>(written_bytes->data(), written_bytes->size());
+    BitSliceIndexBitmapFileIndex file_index({});
+    ASSERT_OK_AND_ASSIGN(auto written_reader,
+                         file_index.CreateReader(CreateArrowSchema(arrow::int32()).get(),
+                                                 /*start=*/0, /*length=*/written_bytes->size(),
+                                                 written_stream, pool_));
+    check_mix_reader(written_reader);
+
+    // Reading the Java-produced fixture below. C++ and Java may choose different valid portable
+    // roaring containers for the same bitmap, so their complete byte streams need not be identical.
     auto input_stream =
         std::make_shared<ByteArrayInputStream>(index_bytes.data(), index_bytes.size());
-    BitSliceIndexBitmapFileIndex file_index({});
     ASSERT_OK_AND_ASSIGN(
-        auto reader,
+        auto java_bytes_reader,
         file_index.CreateReader(CreateArrowSchema(arrow::int32()).get(),
                                 /*start=*/0, /*length=*/index_bytes.size(), input_stream, pool_));
-    // test equal
-    CheckResult(reader->VisitEqual(Literal(2)).value(), {1, 7});
-    CheckResult(reader->VisitEqual(Literal(-2)).value(), {3, 4});
-    CheckResult(reader->VisitEqual(Literal(100)).value(), {});
-
-    // test not equal
-    CheckResult(reader->VisitNotEqual(Literal(2)).value(), {0, 3, 4, 5, 8, 9});
-    CheckResult(reader->VisitNotEqual(Literal(-2)).value(), {0, 1, 5, 7, 8, 9});
-    CheckResult(reader->VisitNotEqual(Literal(100)).value(), {0, 1, 3, 4, 5, 7, 8, 9});
-
-    // test in
-    CheckResult(reader->VisitIn({Literal(-1), Literal(1), Literal(2), Literal(3)}).value(),
-                {0, 1, 5, 7});
-
-    // test not in
-    CheckResult(reader->VisitNotIn({Literal(-1), Literal(1), Literal(2), Literal(3)}).value(),
-                {3, 4, 8, 9});
-
-    // test null
-    CheckResult(reader->VisitIsNull().value(), {2, 6, 10});
-
-    // test not null
-    CheckResult(reader->VisitIsNotNull().value(), {0, 1, 3, 4, 5, 7, 8, 9});
+    check_mix_reader(java_bytes_reader);
+}
 
-    // test less than
-    CheckResult(reader->VisitLessThan(Literal(2)).value(), {0, 3, 4, 5, 8});
-    CheckResult(reader->VisitLessOrEqual(Literal(2)).value(), {0, 1, 3, 4, 5, 7, 8});
-    CheckResult(reader->VisitLessThan(Literal(-1)).value(), {3, 4});
-    CheckResult(reader->VisitLessOrEqual(Literal(-1)).value(), {3, 4, 5});
+TEST_F(BitSliceIndexBitmapIndexReaderTest, TestWriterRejectsInt64MinAndUnsupportedType) {
+    ASSERT_NOK_WITH_MSG(WriteIndex(arrow::int64(), R"([[-9223372036854775808]])"),
+                        "bsi index does not support INT64_MIN for field 'f0'");
 
-    // test greater than
-    CheckResult(reader->VisitGreaterThan(Literal(-2)).value(), {0, 1, 5, 7, 8, 9});
-    CheckResult(reader->VisitGreaterOrEqual(Literal(-2)).value(), {0, 1, 3, 4, 5, 7, 8, 9});
-    CheckResult(reader->VisitGreaterThan(Literal(2)).value(), {9});
-    CheckResult(reader->VisitGreaterOrEqual(Literal(2)).value(), {1, 7, 9});
+    BitSliceIndexBitmapFileIndex file_index({});
+    ASSERT_NOK_WITH_MSG(file_index.CreateWriter(CreateArrowSchema(arrow::boolean()).get(), pool_),
+                        "BitSliceIndexBitmapFileIndex only support");
 }
 
 TEST_F(BitSliceIndexBitmapIndexReaderTest, TestPositiveOnly) {
diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp
index 5d0543bef..88e081bd1 100644
--- a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp
+++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp
@@ -60,18 +60,12 @@ Result<std::shared_ptr<FileIndexWriter>> RangeBitmapFileIndex::CreateWriter(
         return Status::Invalid(
             "invalid schema for RangeBitmapFileIndexWriter, supposed to have single field.");
     }
-    const auto arrow_field = arrow_schema_ptr->field(0);
-    return RangeBitmapFileIndexWriter::Create(arrow_schema_ptr, arrow_field->name(), options_,
-                                              pool);
+    return RangeBitmapFileIndexWriter::Create(arrow_schema_ptr->field(0), options_, pool);
 }
 
 Result<std::shared_ptr<RangeBitmapFileIndexWriter>> RangeBitmapFileIndexWriter::Create(
-    const std::shared_ptr<arrow::Schema>& arrow_schema, const std::string& field_name,
-    const std::map<std::string, std::string>& options, const std::shared_ptr<MemoryPool>& pool) {
-    const auto field = arrow_schema->GetFieldByName(field_name);
-    if (!field) {
-        return Status::Invalid(fmt::format("Field not found in schema: {}", field_name));
-    }
+    const std::shared_ptr<arrow::Field>& field, const std::map<std::string, std::string>& options,
+    const std::shared_ptr<MemoryPool>& pool) {
     PAIMON_ASSIGN_OR_RAISE(FieldType field_type,
                            FieldTypeUtils::ConvertToFieldType(field->type()->id()));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<KeyFactory> shared_key_factory,
@@ -82,27 +76,18 @@ Result<std::shared_ptr<RangeBitmapFileIndexWriter>> RangeBitmapFileIndexWriter::
         chunk_size_it != options.end()) {
         PAIMON_ASSIGN_OR_RAISE(parsed_chunk_size, MemorySize::ParseBytes(chunk_size_it->second));
     }
-    auto struct_type = arrow::struct_({field});
+    std::shared_ptr<arrow::DataType> struct_type = arrow::struct_({field});
     PAIMON_ASSIGN_OR_RAISE(
         std::unique_ptr<RangeBitmap::Appender> appender_ptr,
         RangeBitmap::Appender::Create(shared_key_factory, parsed_chunk_size, pool));
-    return std::make_shared<RangeBitmapFileIndexWriter>(
-        struct_type, field->type(), options, pool, shared_key_factory, std::move(appender_ptr));
+    return std::make_shared<RangeBitmapFileIndexWriter>(struct_type, pool, shared_key_factory,
+                                                        std::move(appender_ptr));
 }
 
 Status RangeBitmapFileIndexWriter::AddBatch(::ArrowArray* batch) {
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
                                       arrow::ImportArray(batch, struct_type_));
-    if (!array || array->type_id() != arrow::Type::STRUCT) {
-        return Status::Invalid(
-            "invalid batch for RangeBitmapFileIndexWriter, expected a struct array");
-    }
     auto struct_array = checked_pointer_cast<arrow::StructArray>(array);
-    if (struct_array->num_fields() != 1) {
-        return Status::Invalid(
-            "invalid batch for RangeBitmapFileIndexWriter, expected a struct array with exactly "
-            "one field");
-    }
     PAIMON_ASSIGN_OR_RAISE(std::vector<Literal> array_values,
                            LiteralConverter::ConvertLiteralsFromArray(*(struct_array->field(0)),
                                                                       /*own_data=*/true));
@@ -117,13 +102,9 @@ Result<PAIMON_UNIQUE_PTR<Bytes>> RangeBitmapFileIndexWriter::SerializedBytes() c
 }
 
 RangeBitmapFileIndexWriter::RangeBitmapFileIndexWriter(
-    const std::shared_ptr<arrow::DataType>& struct_type,
-    const std::shared_ptr<arrow::DataType>& arrow_type,
-    const std::map<std::string, std::string>& options, const std::shared_ptr<MemoryPool>& pool,
+    const std::shared_ptr<arrow::DataType>& struct_type, const std::shared_ptr<MemoryPool>& pool,
     const std::shared_ptr<KeyFactory>& key_factory, std::unique_ptr<RangeBitmap::Appender> appender)
     : struct_type_(struct_type),
-      arrow_type_(arrow_type),
-      options_(options),
       pool_(pool),
       key_factory_(key_factory),
       appender_(std::move(appender)) {}
diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h
index fc99cac33..64289a536 100644
--- a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h
+++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h
@@ -61,15 +61,13 @@ class PAIMON_EXPORT RangeBitmapFileIndex final : public FileIndexer {
 class RangeBitmapFileIndexWriter final : public FileIndexWriter {
  public:
     static Result<std::shared_ptr<RangeBitmapFileIndexWriter>> Create(
-        const std::shared_ptr<arrow::Schema>& arrow_schema, const std::string& field_name,
+        const std::shared_ptr<arrow::Field>& field,
         const std::map<std::string, std::string>& options, const std::shared_ptr<MemoryPool>& pool);
 
     Status AddBatch(::ArrowArray* batch) override;
     Result<PAIMON_UNIQUE_PTR<Bytes>> SerializedBytes() const override;
 
     RangeBitmapFileIndexWriter(const std::shared_ptr<arrow::DataType>& struct_type,
-                               const std::shared_ptr<arrow::DataType>& arrow_type,
-                               const std::map<std::string, std::string>& options,
                                const std::shared_ptr<MemoryPool>& pool,
                                const std::shared_ptr<KeyFactory>& key_factory,
                                std::unique_ptr<RangeBitmap::Appender> appender);
@@ -78,8 +76,6 @@ class RangeBitmapFileIndexWriter final : public FileIndexWriter {
     /// @note struct_type_ contains only one field with arrow_type_, used for import from C
     /// interface.
     std::shared_ptr<arrow::DataType> struct_type_;
-    std::shared_ptr<arrow::DataType> arrow_type_;
-    std::map<std::string, std::string> options_;
     std::shared_ptr<MemoryPool> pool_;
     std::shared_ptr<KeyFactory> key_factory_;
     std::unique_ptr<RangeBitmap::Appender> appender_;
diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp
index e14797849..6157ee020 100644
--- a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp
+++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp
@@ -123,17 +123,15 @@ Result<std::shared_ptr<RangeBitmapFileIndexReader>> RangeBitmapFileIndexTest::Cr
     std::shared_ptr<arrow::Array> arrow_array;
     PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&arrow_array));
     // Wrap in StructArray (single field) as required by RangeBitmapFileIndexWriter
-    arrow::FieldVector fields = {arrow::field("test_field", arrow_type)};
+    auto field = arrow::field("test_field", arrow_type);
+    arrow::FieldVector fields = {field};
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> struct_array,
                                       arrow::StructArray::Make({arrow_array}, fields));
     auto c_array = std::make_unique<::ArrowArray>();
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, c_array.get()));
-    // Create schema for the field
-    const auto schema = arrow::schema({arrow::field("test_field", arrow_type)});
     // Create writer
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<RangeBitmapFileIndexWriter> writer,
-        RangeBitmapFileIndexWriter::Create(schema, "test_field", options, pool_));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RangeBitmapFileIndexWriter> writer,
+                           RangeBitmapFileIndexWriter::Create(field, options, pool_));
     // Add the batch
     PAIMON_RETURN_NOT_OK(writer->AddBatch(c_array.get()));
     // Get serialized payload
diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp b/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp
index e2bb867ed..fd8e0c888 100644
--- a/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp
+++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp
@@ -65,18 +65,16 @@ class RangeBitmapIoTest : public ::testing::Test {
         PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&arrow_array));
 
         // Wrap in StructArray
-        arrow::FieldVector fields = {arrow::field("test_field", arrow_type)};
+        const std::shared_ptr<arrow::Field> field = arrow::field("test_field", arrow_type);
+        arrow::FieldVector fields = {field};
         PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> struct_array,
                                           arrow::StructArray::Make({arrow_array}, fields));
         auto c_array = std::make_unique<::ArrowArray>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, c_array.get()));
 
-        // Create schema
-        const auto schema = arrow::schema({arrow::field("test_field", arrow_type)});
-
         // Create writer and write data
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RangeBitmapFileIndexWriter> writer,
-                               RangeBitmapFileIndexWriter::Create(schema, "test_field", {}, pool_));
+                               RangeBitmapFileIndexWriter::Create(field, {}, pool_));
 
         PAIMON_RETURN_NOT_OK(writer->AddBatch(c_array.get()));
         return writer->SerializedBytes();
diff --git a/src/paimon/common/lookup/lookup_store_factory.cpp b/src/paimon/common/lookup/lookup_store_factory.cpp
index c5742e12a..d6cb8d400 100644
--- a/src/paimon/common/lookup/lookup_store_factory.cpp
+++ b/src/paimon/common/lookup/lookup_store_factory.cpp
@@ -36,7 +36,8 @@ Result<std::shared_ptr<BloomFilter>> LookupStoreFactory::BfGenerator(int64_t row
     if (row_count <= 0 || !options.LookupCacheBloomFilterEnabled()) {
         return std::shared_ptr<BloomFilter>();
     }
-    auto bloom_filter = BloomFilter::Create(row_count, options.GetLookupCacheBloomFilterFpp());
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<BloomFilter> bloom_filter,
+                           BloomFilter::Create(row_count, options.GetLookupCacheBloomFilterFpp()));
     MemorySegment memory_segment =
         MemorySegment::AllocateHeapMemory(bloom_filter->ByteLength(), pool);
     PAIMON_RETURN_NOT_OK(bloom_filter->SetMemorySegment(memory_segment));
diff --git a/src/paimon/common/sst/sst_file_io_test.cpp b/src/paimon/common/sst/sst_file_io_test.cpp
index eac9a8a89..54d4b1b13 100644
--- a/src/paimon/common/sst/sst_file_io_test.cpp
+++ b/src/paimon/common/sst/sst_file_io_test.cpp
@@ -93,7 +93,7 @@ TEST_P(SstFileIOTest, TestSimple) {
                          fs_->Create(index_path, /*overwrite=*/false));
 
     // write data
-    auto bf = BloomFilter::Create(30, 0.01);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<BloomFilter> bf, BloomFilter::Create(30, 0.01));
     auto seg_for_bf = MemorySegment::AllocateHeapMemory(bf->ByteLength(), pool_.get());
     ASSERT_OK(bf->SetMemorySegment(seg_for_bf));
     auto writer = std::make_shared<SstFileWriter>(out, bf, 50, factory, pool_);
@@ -234,7 +234,7 @@ TEST_F(SstFileIOTest, TestIOException) {
         CHECK_HOOK_STATUS(out_result.status(), i);
         std::shared_ptr<OutputStream> out = std::move(out_result).value();
 
-        auto bf = BloomFilter::Create(30, 0.01);
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<BloomFilter> bf, BloomFilter::Create(30, 0.01));
         MemorySegment seg_for_bf = MemorySegment::AllocateHeapMemory(bf->ByteLength(), pool_.get());
         ASSERT_OK(bf->SetMemorySegment(seg_for_bf));
         auto writer = std::make_shared<SstFileWriter>(out, bf, 50, factory, pool_);
diff --git a/src/paimon/common/utils/bloom_filter.cpp b/src/paimon/common/utils/bloom_filter.cpp
index d66420f47..ba9fa4b44 100644
--- a/src/paimon/common/utils/bloom_filter.cpp
+++ b/src/paimon/common/utils/bloom_filter.cpp
@@ -25,29 +25,35 @@
 
 namespace paimon {
 
-int32_t BloomFilter::OptimalNumOfBits(int64_t expect_entries, double fpp) {
-    if (expect_entries <= 0 || fpp <= 0.0 || fpp >= 1.0) {
+int32_t BloomFilter::OptimalNumOfBits(int64_t expected_entries, double fpp) {
+    if (expected_entries <= 0 || fpp <= 0.0 || fpp >= 1.0) {
         return 0;
     }
-    double result = -static_cast<double>(expect_entries) * log(fpp) / (log(2) * log(2));
+    double result = -static_cast<double>(expected_entries) * log(fpp) / (log(2) * log(2));
     if (result > INT32_MAX) return INT32_MAX;
     if (result < 0) return 0;
     return static_cast<int32_t>(result);
 }
 
-int32_t BloomFilter::OptimalNumOfHashFunctions(int64_t expect_entries, int64_t bit_size) {
-    if (expect_entries <= 0) {
+int32_t BloomFilter::OptimalNumOfHashFunctions(int64_t expected_entries, int64_t bit_size) {
+    if (expected_entries <= 0) {
         return 1;
     }
-    double ratio = static_cast<double>(bit_size) / static_cast<double>(expect_entries);
+    double ratio = static_cast<double>(bit_size) / static_cast<double>(expected_entries);
     double result = ratio * std::log(2.0);
     return std::max(1, static_cast<int32_t>(std::round(result)));
 }
 
-std::shared_ptr<BloomFilter> BloomFilter::Create(int64_t expect_entries, double fpp) {
+Result<std::shared_ptr<BloomFilter>> BloomFilter::Create(int64_t expected_entries, double fpp) {
+    if (expected_entries <= 0) {
+        return Status::Invalid("expected entries must be greater than 0 for bloom filter");
+    }
+    if (!std::isfinite(fpp) || fpp <= 0.0 || fpp >= 1.0) {
+        return Status::Invalid("fpp must be greater than 0 and less than 1 for bloom filter");
+    }
     auto bytes =
-        static_cast<int32_t>(ceil(BloomFilter::OptimalNumOfBits(expect_entries, fpp) / 8.0));
-    return std::make_shared<BloomFilter>(expect_entries, bytes);
+        static_cast<int32_t>(ceil(BloomFilter::OptimalNumOfBits(expected_entries, fpp) / 8.0));
+    return std::make_shared<BloomFilter>(expected_entries, bytes);
 }
 
 BloomFilter::BloomFilter(int64_t expected_entries, int32_t byte_length)
diff --git a/src/paimon/common/utils/bloom_filter.h b/src/paimon/common/utils/bloom_filter.h
index 8f1b32711..61e27af59 100644
--- a/src/paimon/common/utils/bloom_filter.h
+++ b/src/paimon/common/utils/bloom_filter.h
@@ -22,7 +22,7 @@
 #include <memory>
 
 #include "paimon/common/utils/bit_set.h"
-#include "paimon/memory/bytes.h"
+#include "paimon/result.h"
 #include "paimon/visibility.h"
 
 namespace paimon {
@@ -30,9 +30,9 @@ namespace paimon {
 /// Bloom filter based on MemorySegment.
 class PAIMON_EXPORT BloomFilter {
  public:
-    static int32_t OptimalNumOfBits(int64_t expect_entries, double fpp);
-    static int32_t OptimalNumOfHashFunctions(int64_t expect_entries, int64_t bit_size);
-    static std::shared_ptr<BloomFilter> Create(int64_t expect_entries, double fpp);
+    static int32_t OptimalNumOfBits(int64_t expected_entries, double fpp);
+    static int32_t OptimalNumOfHashFunctions(int64_t expected_entries, int64_t bit_size);
+    static Result<std::shared_ptr<BloomFilter>> Create(int64_t expected_entries, double fpp);
 
  public:
     BloomFilter(int64_t expected_entries, int32_t byte_length);
diff --git a/src/paimon/common/utils/bloom_filter64.cpp b/src/paimon/common/utils/bloom_filter64.cpp
index f2685a13f..7a975161c 100644
--- a/src/paimon/common/utils/bloom_filter64.cpp
+++ b/src/paimon/common/utils/bloom_filter64.cpp
@@ -21,6 +21,8 @@
 #include <algorithm>
 #include <cassert>
 #include <cmath>
+#include <cstring>
+#include <limits>
 #include <utility>
 
 #include "paimon/memory/bytes.h"
@@ -47,17 +49,42 @@ bool BloomFilter64::BitSet::Get(int32_t index) const {
 }
 
 int32_t BloomFilter64::BitSet::BitSize() const {
-    return (bytes_->size() - offset_) * BloomFilter64::BYTE_SIZE;
+    return ByteLength() * BloomFilter64::BYTE_SIZE;
 }
 
-BloomFilter64::BloomFilter64(int64_t items, double fpp, const std::shared_ptr<MemoryPool>& pool)
-    : pool_(pool) {
-    auto nb = static_cast<int32_t>(-items * std::log(fpp) / (std::log(2) * std::log(2)));
-    num_bits_ = nb + (BloomFilter64::BYTE_SIZE - (nb % BloomFilter64::BYTE_SIZE));
-    num_hash_functions_ = std::max(
-        1, static_cast<int32_t>(std::round(static_cast<double>(num_bits_) / items * std::log(2))));
-    auto bytes = std::make_shared<Bytes>(num_bits_ / BloomFilter64::BYTE_SIZE, pool_.get());
-    bit_set_ = std::make_unique<BitSet>(bytes, /*offset=*/0);
+int32_t BloomFilter64::BitSet::ByteLength() const {
+    return static_cast<int32_t>(bytes_->size() - offset_);
+}
+
+void BloomFilter64::BitSet::ToByteArray(int32_t offset, int32_t length, char* bytes) const {
+    assert(bytes);
+    assert(offset >= 0);
+    assert(length >= 0);
+    assert(static_cast<size_t>(offset_ + length) <= bytes_->size());
+    std::memcpy(bytes + offset, bytes_->data() + offset_, length);
+}
+
+Result<BloomFilter64> BloomFilter64::Create(int64_t items, double fpp,
+                                            const std::shared_ptr<MemoryPool>& pool) {
+    if (items <= 0) {
+        return Status::Invalid("items must be greater than 0 for bloom filter");
+    }
+    if (!std::isfinite(fpp) || fpp <= 0.0 || fpp >= 1.0) {
+        return Status::Invalid("fpp must be greater than 0 and less than 1 for bloom filter");
+    }
+    const double log_two = std::log(2);
+    const double estimated_bits = -static_cast<double>(items) * std::log(fpp) / (log_two * log_two);
+    if (estimated_bits > std::numeric_limits<int32_t>::max() - BYTE_SIZE) {
+        return Status::Invalid("bloom filter size exceeds the supported range");
+    }
+    const auto num_bits_without_padding = static_cast<int32_t>(estimated_bits);
+    const int32_t num_bits =
+        num_bits_without_padding + (BYTE_SIZE - (num_bits_without_padding % BYTE_SIZE));
+    const int32_t num_hash_functions = std::max(
+        1, static_cast<int32_t>(std::round(static_cast<double>(num_bits) / items * log_two)));
+    auto bytes = std::make_shared<Bytes>(num_bits / BYTE_SIZE, pool.get());
+    auto bit_set = std::make_unique<BitSet>(bytes, /*offset=*/0);
+    return BloomFilter64(num_hash_functions, std::move(bit_set), pool);
 }
 
 BloomFilter64::BloomFilter64(int32_t num_hash_functions, std::unique_ptr<BitSet>&& bit_set)
@@ -65,6 +92,13 @@ BloomFilter64::BloomFilter64(int32_t num_hash_functions, std::unique_ptr<BitSet>
       num_hash_functions_(num_hash_functions),
       bit_set_(std::move(bit_set)) {}
 
+BloomFilter64::BloomFilter64(int32_t num_hash_functions, std::unique_ptr<BitSet>&& bit_set,
+                             const std::shared_ptr<MemoryPool>& pool)
+    : num_bits_(bit_set->BitSize()),
+      num_hash_functions_(num_hash_functions),
+      pool_(pool),
+      bit_set_(std::move(bit_set)) {}
+
 void BloomFilter64::AddHash(int64_t hash64) {
     auto hash1 = static_cast<int32_t>(hash64);
     auto hash2 = static_cast<int32_t>(static_cast<uint64_t>(hash64) >> 32);
diff --git a/src/paimon/common/utils/bloom_filter64.h b/src/paimon/common/utils/bloom_filter64.h
index 3ba4e86f9..2440b3033 100644
--- a/src/paimon/common/utils/bloom_filter64.h
+++ b/src/paimon/common/utils/bloom_filter64.h
@@ -22,6 +22,7 @@
 #include <memory>
 
 #include "paimon/memory/bytes.h"
+#include "paimon/result.h"
 #include "paimon/visibility.h"
 
 namespace paimon {
@@ -31,7 +32,9 @@ class MemoryPool;
 /// Bloom filter 64 handle 64 bits hash.
 class PAIMON_EXPORT BloomFilter64 {
  public:
-    BloomFilter64(int64_t items, double fpp, const std::shared_ptr<MemoryPool>& pool);
+    static Result<BloomFilter64> Create(int64_t items, double fpp,
+                                        const std::shared_ptr<MemoryPool>& pool);
+
     class BitSet;
 
     BloomFilter64(int32_t num_hash_functions, std::unique_ptr<BitSet>&& bit_set);
@@ -54,6 +57,8 @@ class PAIMON_EXPORT BloomFilter64 {
         void Set(int32_t index);
         bool Get(int32_t index) const;
         int32_t BitSize() const;
+        int32_t ByteLength() const;
+        void ToByteArray(int32_t offset, int32_t length, char* bytes) const;
 
      private:
         static constexpr int8_t MASK = 0x07;
@@ -64,9 +69,11 @@ class PAIMON_EXPORT BloomFilter64 {
     };
 
  private:
+    BloomFilter64(int32_t num_hash_functions, std::unique_ptr<BitSet>&& bit_set,
+                  const std::shared_ptr<MemoryPool>& pool);
+
     static constexpr int32_t BYTE_SIZE = 8;
 
- private:
     int32_t num_bits_ = -1;
     int32_t num_hash_functions_ = -1;
     std::shared_ptr<MemoryPool> pool_;
diff --git a/src/paimon/common/utils/bloom_filter64_test.cpp b/src/paimon/common/utils/bloom_filter64_test.cpp
index aa548fb7d..6f60fd018 100644
--- a/src/paimon/common/utils/bloom_filter64_test.cpp
+++ b/src/paimon/common/utils/bloom_filter64_test.cpp
@@ -28,13 +28,14 @@
 #include "gtest/gtest.h"
 #include "paimon/memory/bytes.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
 
 TEST(BloomFilter64Test, TestSimple) {
     int32_t items = 10000;
     auto pool = GetDefaultPool();
-    BloomFilter64 bloom_filter(items, 0.02, pool);
+    ASSERT_OK_AND_ASSIGN(BloomFilter64 bloom_filter, BloomFilter64::Create(items, 0.02, pool));
     std::mt19937_64 engine(std::random_device{}());  // NOLINT(whitespace/braces)
     std::uniform_int_distribution<int64_t> distribution(std::numeric_limits<int64_t>::min(),
                                                         std::numeric_limits<int64_t>::max());
@@ -61,6 +62,39 @@ TEST(BloomFilter64Test, TestSimple) {
     ASSERT_TRUE(static_cast<double>(false_positives) / num < 0.03);
 }
 
+TEST(BloomFilter64Test, TestInvalidItemsAndFpp) {
+    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
+
+    ASSERT_NOK_WITH_MSG(BloomFilter64::Create(/*items=*/0, /*fpp=*/0.1, pool),
+                        "items must be greater than 0");
+    ASSERT_NOK_WITH_MSG(BloomFilter64::Create(/*items=*/-1, /*fpp=*/0.1, pool),
+                        "items must be greater than 0");
+    ASSERT_NOK_WITH_MSG(BloomFilter64::Create(/*items=*/100, /*fpp=*/0.0, pool),
+                        "fpp must be greater than 0 and less than 1");
+    ASSERT_NOK_WITH_MSG(BloomFilter64::Create(/*items=*/100, /*fpp=*/-0.1, pool),
+                        "fpp must be greater than 0 and less than 1");
+    ASSERT_NOK_WITH_MSG(BloomFilter64::Create(/*items=*/100, /*fpp=*/1.0, pool),
+                        "fpp must be greater than 0 and less than 1");
+    ASSERT_NOK_WITH_MSG(BloomFilter64::Create(/*items=*/100, /*fpp=*/1.1, pool),
+                        "fpp must be greater than 0 and less than 1");
+    ASSERT_NOK_WITH_MSG(
+        BloomFilter64::Create(/*items=*/std::numeric_limits<int64_t>::max(), /*fpp=*/0.1, pool),
+        "bloom filter size exceeds the supported range");
+}
+
+TEST(BloomFilter64Test, TestKeepMemoryPoolAlive) {
+    std::weak_ptr<MemoryPool> weak_pool;
+    {
+        std::shared_ptr<MemoryPool> pool(GetMemoryPool());
+        weak_pool = pool;
+        ASSERT_OK_AND_ASSIGN(BloomFilter64 bloom_filter,
+                             BloomFilter64::Create(/*items=*/100, /*fpp=*/0.1, pool));
+        pool.reset();
+        ASSERT_FALSE(weak_pool.expired());
+    }
+    ASSERT_TRUE(weak_pool.expired());
+}
+
 TEST(BloomFilter64Test, TestCompatibleWithJava) {
     // data: -10, -5, 0, 13, 100, 200, 500
     std::vector<uint8_t> se_bytes = {241, 255, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0};
@@ -74,10 +108,10 @@ TEST(BloomFilter64Test, TestCompatibleWithJava) {
         ASSERT_TRUE(bloom_filter.TestHash(value));
     }
 
-    BloomFilter64 bloom_filter2(10, 0.01, pool);
+    ASSERT_OK_AND_ASSIGN(BloomFilter64 bloom_filter2, BloomFilter64::Create(10, 0.01, pool));
     ASSERT_EQ(7, bloom_filter2.GetNumHashFunctions());
-    ASSERT_EQ(se_bytes.size() * BloomFilter64::BYTE_SIZE, bloom_filter2.num_bits_);
-    ASSERT_EQ(se_bytes.size(), bloom_filter2.GetBitSet().bytes_->size());
+    ASSERT_EQ(se_bytes.size() * 8, bloom_filter2.GetBitSet().BitSize());
+    ASSERT_EQ(se_bytes.size(), bloom_filter2.GetBitSet().ByteLength());
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/common/utils/bloom_filter_test.cpp b/src/paimon/common/utils/bloom_filter_test.cpp
index fdd6abb34..0832fdf5c 100644
--- a/src/paimon/common/utils/bloom_filter_test.cpp
+++ b/src/paimon/common/utils/bloom_filter_test.cpp
@@ -35,7 +35,8 @@ namespace paimon::test {
 TEST(BloomFilterTest, TestOneSegmentBuilder) {
     int32_t items = 100;
     auto pool = GetDefaultPool();
-    auto bloom_filter = BloomFilter::Create(items, 0.01);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<BloomFilter> bloom_filter,
+                         BloomFilter::Create(items, 0.01));
     auto seg = MemorySegment::AllocateHeapMemory(1024, pool.get());
     ASSERT_OK(bloom_filter->SetMemorySegment(seg));
 
@@ -54,12 +55,32 @@ TEST(BloomFilterTest, TestOneSegmentBuilder) {
 }
 
 TEST(BloomFilterTest, TestEstimatedHashFunctions) {
-    ASSERT_EQ(7, BloomFilter::Create(1000, 0.01)->GetNumHashFunctions());
-    ASSERT_EQ(7, BloomFilter::Create(10000, 0.01)->GetNumHashFunctions());
-    ASSERT_EQ(7, BloomFilter::Create(100000, 0.01)->GetNumHashFunctions());
-    ASSERT_EQ(4, BloomFilter::Create(100000, 0.05)->GetNumHashFunctions());
-    ASSERT_EQ(7, BloomFilter::Create(1000000, 0.01)->GetNumHashFunctions());
-    ASSERT_EQ(4, BloomFilter::Create(1000000, 0.05)->GetNumHashFunctions());
+    auto get_num_hash_functions = [](int64_t expected_entries, double fpp) {
+        EXPECT_OK_AND_ASSIGN(std::shared_ptr<BloomFilter> bloom_filter,
+                             BloomFilter::Create(expected_entries, fpp));
+        return bloom_filter->GetNumHashFunctions();
+    };
+    ASSERT_EQ(7, get_num_hash_functions(1000, 0.01));
+    ASSERT_EQ(7, get_num_hash_functions(10000, 0.01));
+    ASSERT_EQ(7, get_num_hash_functions(100000, 0.01));
+    ASSERT_EQ(4, get_num_hash_functions(100000, 0.05));
+    ASSERT_EQ(7, get_num_hash_functions(1000000, 0.01));
+    ASSERT_EQ(4, get_num_hash_functions(1000000, 0.05));
+}
+
+TEST(BloomFilterTest, TestInvalidExpectedEntriesAndFpp) {
+    ASSERT_NOK_WITH_MSG(BloomFilter::Create(/*expected_entries=*/0, /*fpp=*/0.1),
+                        "expected entries must be greater than 0");
+    ASSERT_NOK_WITH_MSG(BloomFilter::Create(/*expected_entries=*/-1, /*fpp=*/0.1),
+                        "expected entries must be greater than 0");
+    ASSERT_NOK_WITH_MSG(BloomFilter::Create(/*expected_entries=*/100, /*fpp=*/0.0),
+                        "fpp must be greater than 0 and less than 1");
+    ASSERT_NOK_WITH_MSG(BloomFilter::Create(/*expected_entries=*/100, /*fpp=*/-0.1),
+                        "fpp must be greater than 0 and less than 1");
+    ASSERT_NOK_WITH_MSG(BloomFilter::Create(/*expected_entries=*/100, /*fpp=*/1.0),
+                        "fpp must be greater than 0 and less than 1");
+    ASSERT_NOK_WITH_MSG(BloomFilter::Create(/*expected_entries=*/100, /*fpp=*/1.1),
+                        "fpp must be greater than 0 and less than 1");
 }
 
 TEST(BloomFilterTest, TestBloomNumBits) {
diff --git a/src/paimon/common/utils/math.h b/src/paimon/common/utils/math.h
index 54ad6cf73..6aba6523f 100644
--- a/src/paimon/common/utils/math.h
+++ b/src/paimon/common/utils/math.h
@@ -36,6 +36,7 @@
 
 #include "fmt/format.h"
 #include "paimon/common/utils/options_utils.h"
+#include "paimon/io/byte_order.h"
 #include "paimon/status.h"
 
 namespace paimon {
@@ -136,4 +137,30 @@ inline T EndianSwapValue(T v) {
     }
 }
 
+template <typename T>
+inline T ToBigEndian(T value) {
+    if constexpr (SystemByteOrder() == ByteOrder::PAIMON_LITTLE_ENDIAN) {
+        return EndianSwapValue(value);
+    }
+    return value;
+}
+
+template <typename T>
+inline T ToLittleEndian(T value) {
+    if constexpr (SystemByteOrder() == ByteOrder::PAIMON_BIG_ENDIAN) {
+        return EndianSwapValue(value);
+    }
+    return value;
+}
+
+template <typename T>
+inline T FromBigEndian(T value) {
+    return ToBigEndian(value);
+}
+
+template <typename T>
+inline T FromLittleEndian(T value) {
+    return ToLittleEndian(value);
+}
+
 }  // namespace paimon
diff --git a/src/paimon/common/utils/math_test.cpp b/src/paimon/common/utils/math_test.cpp
index 36df1cbab..49d31d472 100644
--- a/src/paimon/common/utils/math_test.cpp
+++ b/src/paimon/common/utils/math_test.cpp
@@ -19,6 +19,8 @@
 
 #include "paimon/common/utils/math.h"
 
+#include <array>
+#include <cstring>
 #include <limits>
 
 #include "gtest/gtest.h"
@@ -44,6 +46,22 @@ TEST(MathTest, EndianSwapValue) {
     ASSERT_EQ(swapped64, 0xF0DEBC9A78563412);
 }
 
+TEST(MathTest, ToEndian) {
+    constexpr uint32_t kValue = 0x12345678;
+
+    const uint32_t big_endian = ToBigEndian(kValue);
+    std::array<uint8_t, sizeof(big_endian)> big_endian_bytes{};
+    std::memcpy(big_endian_bytes.data(), &big_endian, sizeof(big_endian));
+    ASSERT_EQ((std::array<uint8_t, 4>{0x12, 0x34, 0x56, 0x78}), big_endian_bytes);
+    ASSERT_EQ(kValue, FromBigEndian(big_endian));
+
+    const uint32_t little_endian = ToLittleEndian(kValue);
+    std::array<uint8_t, sizeof(little_endian)> little_endian_bytes{};
+    std::memcpy(little_endian_bytes.data(), &little_endian, sizeof(little_endian));
+    ASSERT_EQ((std::array<uint8_t, 4>{0x78, 0x56, 0x34, 0x12}), little_endian_bytes);
+    ASSERT_EQ(kValue, FromLittleEndian(little_endian));
+}
+
 TEST(MathTest, InRange) {
     // signed -> unsigned: negative values out of range, boundary values in range
     ASSERT_TRUE(InRange<uint32_t>(0));
diff --git a/src/paimon/core/io/data_file_index_writer_test.cpp b/src/paimon/core/io/data_file_index_writer_test.cpp
index e9ab7940b..1e2c9593f 100644
--- a/src/paimon/core/io/data_file_index_writer_test.cpp
+++ b/src/paimon/core/io/data_file_index_writer_test.cpp
@@ -197,11 +197,38 @@ TEST_F(DataFileIndexWriterTest, TestExternalIndexAndAbortCleanup) {
     ASSERT_FALSE(exists);
 }
 
+TEST_F(DataFileIndexWriterTest, TestBsiAndBloomFilterEmbeddedRoundTrip) {
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         CreateWriter({{"file-index.bsi.columns", "f0"},
+                                       {"file-index.bloom-filter.columns", "f1"},
+                                       {"file-index.bloom-filter.f1.items", "100"},
+                                       {"file-index.bloom-filter.f1.fpp", "0.01"},
+                                       {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1MB"}}));
+    ASSERT_OK(writer->AddBatch(CreateBatch(R"([{"f0": 1, "f1": 10},
+                                                {"f0": -2, "f1": 20}])")));
+    ASSERT_OK(writer->AddBatch(CreateBatch(R"([{"f0": null, "f1": 30},
+                                                {"f0": 5, "f1": 40}])")));
+
+    ASSERT_OK_AND_ASSIGN(FileIndexWriteResult result, writer->Finish("unused.orc"));
+    ASSERT_TRUE(result.embedded_index);
+    ASSERT_OK_AND_ASSIGN(auto reader, CreateReader(result.embedded_index));
+
+    ASSERT_OK_AND_ASSIGN(auto bsi_readers, ReadColumn(reader.get(), "f0"));
+    ASSERT_EQ(1, bsi_readers.size());
+    ASSERT_OK_AND_ASSIGN(auto greater_result, bsi_readers[0]->VisitGreaterThan(Literal(1)));
+    ASSERT_EQ("{3}", greater_result->ToString());
+    ASSERT_OK_AND_ASSIGN(auto null_result, bsi_readers[0]->VisitIsNull());
+    ASSERT_EQ("{2}", null_result->ToString());
+
+    ASSERT_OK_AND_ASSIGN(auto bloom_readers, ReadColumn(reader.get(), "f1"));
+    ASSERT_EQ(1, bloom_readers.size());
+    ASSERT_OK_AND_ASSIGN(auto present_result, bloom_readers[0]->VisitEqual(Literal(30)));
+    ASSERT_TRUE(present_result->IsRemain().value());
+}
+
 TEST_F(DataFileIndexWriterTest, TestUnavailableWriterFailsCreation) {
     ASSERT_NOK_WITH_MSG(CreateWriter({{"file-index.unknown.columns", "f0"}}),
                         "File index type 'unknown' is not registered");
-    ASSERT_NOK_WITH_MSG(CreateWriter({{"file-index.bloom-filter.columns", "f0"}}),
-                        "do not support index writer in bloom filter");
 }
 
 TEST_F(DataFileIndexWriterTest, TestRejectSystemFieldIndex) {

From 7d992ca04af66adcef9db12b9367a951f412cc7e Mon Sep 17 00:00:00 2001
From: gripleaf <425797155@qq.com>
Date: Tue, 25 Aug 2026 19:17:06 +0800
Subject: [PATCH 20/93] perf(read): reduce Arrow read-path overhead (#242)

---
 .../common/data/columnar/columnar_row.h       |  6 +++++
 .../data/columnar/columnar_row_test.cpp       | 10 +++++++
 src/paimon/core/manifest/manifest_file.cpp    |  3 ++-
 src/paimon/core/utils/objects_file.h          |  3 ++-
 .../format/avro/avro_direct_decoder.cpp       | 24 +++++++++++++++++
 src/paimon/format/avro/avro_direct_decoder.h  |  6 +++++
 .../avro/avro_direct_encoder_decoder_test.cpp | 27 +++++++++++++++++++
 .../format/avro/avro_file_batch_reader.cpp    |  7 +++++
 src/paimon/format/orc/orc_adapter.cpp         |  2 ++
 9 files changed, 86 insertions(+), 2 deletions(-)

diff --git a/src/paimon/common/data/columnar/columnar_row.h b/src/paimon/common/data/columnar/columnar_row.h
index 2156c814d..f3d20dba0 100644
--- a/src/paimon/common/data/columnar/columnar_row.h
+++ b/src/paimon/common/data/columnar/columnar_row.h
@@ -78,6 +78,12 @@ class ColumnarRow : public InternalRow {
         row_kind_ = kind;
     }
 
+    /// Update the row represented by this view without rebuilding its column pointers.
+    /// @param row_id Zero-based row index in the underlying arrays.
+    void SetRowId(int64_t row_id) {
+        row_id_ = row_id;
+    }
+
     int32_t GetFieldCount() const override {
         return array_vec_.size();
     }
diff --git a/src/paimon/common/data/columnar/columnar_row_test.cpp b/src/paimon/common/data/columnar/columnar_row_test.cpp
index 6301cd293..d1a5c6003 100644
--- a/src/paimon/common/data/columnar/columnar_row_test.cpp
+++ b/src/paimon/common/data/columnar/columnar_row_test.cpp
@@ -70,6 +70,16 @@ TEST(ColumnarRowTest, TestSimple) {
     ASSERT_EQ(row.GetDouble(6), 5.5);
     ASSERT_EQ(row.GetString(7).ToString(), "Hello");
     ASSERT_EQ(std::string(row.GetStringView(7)), "Hello");
+
+    row.SetRowId(3);
+    ASSERT_TRUE(row.GetBoolean(0));
+    ASSERT_EQ(row.GetByte(1), 3);
+    ASSERT_EQ(row.GetShort(2), 7);
+    ASSERT_EQ(row.GetInt(3), 13);
+    ASSERT_EQ(row.GetLong(4), 18);
+    ASSERT_EQ(row.GetFloat(5), 3.3f);
+    ASSERT_EQ(row.GetDouble(6), 8.8);
+    ASSERT_EQ(std::string(row.GetStringView(7)), "WORLD");
 }
 
 TEST(ColumnarRowRefTest, TestSimple) {
diff --git a/src/paimon/core/manifest/manifest_file.cpp b/src/paimon/core/manifest/manifest_file.cpp
index 9f9c8aee2..a556b82ea 100644
--- a/src/paimon/core/manifest/manifest_file.cpp
+++ b/src/paimon/core/manifest/manifest_file.cpp
@@ -93,8 +93,9 @@ Status ManifestFile::ReadBucketEntries(const std::string& file_name, int32_t buc
         file_name,
         [this, bucket, entries](const std::shared_ptr<arrow::StructArray>& batch) -> Status {
             const arrow::ArrayVector& fields = batch->fields();
+            ColumnarRow row(fields, pool_, /*row_id=*/0);
             for (int64_t i = 0; i < batch->length(); i++) {
-                ColumnarRow row(fields, pool_, i);
+                row.SetRowId(i);
                 PAIMON_RETURN_NOT_OK(ManifestEntrySerializer::ValidateVersion(row.GetInt(0)));
                 if (ManifestEntrySerializer::GetBucket(row) != bucket) {
                     continue;
diff --git a/src/paimon/core/utils/objects_file.h b/src/paimon/core/utils/objects_file.h
index b3135b312..43d782019 100644
--- a/src/paimon/core/utils/objects_file.h
+++ b/src/paimon/core/utils/objects_file.h
@@ -135,8 +135,9 @@ Status ObjectsFile<T>::Read(const std::string& file_name,
         [this, &filter, result](const std::shared_ptr<arrow::StructArray>& struct_array) -> Status {
             result->reserve(result->size() + struct_array->length());
             const arrow::ArrayVector& fields = struct_array->fields();
+            ColumnarRow row(fields, pool_, /*row_id=*/0);
             for (int64_t i = 0; i < struct_array->length(); i++) {
-                ColumnarRow row(fields, pool_, i);
+                row.SetRowId(i);
                 PAIMON_ASSIGN_OR_RAISE(T obj, serializer_->FromRow(row));
                 if (filter) {
                     PAIMON_ASSIGN_OR_RAISE(bool filter_res, filter(obj));
diff --git a/src/paimon/format/avro/avro_direct_decoder.cpp b/src/paimon/format/avro/avro_direct_decoder.cpp
index f9c8a9a41..9f9e36426 100644
--- a/src/paimon/format/avro/avro_direct_decoder.cpp
+++ b/src/paimon/format/avro/avro_direct_decoder.cpp
@@ -57,6 +57,20 @@ Status DecodeFieldToBuilder(const ::avro::NodePtr& avro_node,
                             ::avro::Decoder* decoder, arrow::ArrayBuilder* array_builder,
                             AvroDirectDecoder::DecodeContext* ctx);
 
+Status ReserveBuilderCapacityImpl(int64_t capacity, arrow::ArrayBuilder* array_builder) {
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(array_builder->Reserve(capacity));
+    if (array_builder->type()->id() != arrow::Type::STRUCT) {
+        return Status::OK();
+    }
+
+    auto* struct_builder = checked_cast<arrow::StructBuilder*>(array_builder);
+    for (int32_t i = 0; i < struct_builder->num_fields(); ++i) {
+        PAIMON_RETURN_NOT_OK(
+            ReserveBuilderCapacityImpl(capacity, struct_builder->field_builder(i)));
+    }
+    return Status::OK();
+}
+
 /// \brief Skip an Avro value based on its schema without decoding
 Status SkipAvroValue(const ::avro::NodePtr& avro_node, ::avro::Decoder* decoder) {
     switch (avro_node->type()) {
@@ -193,6 +207,7 @@ Status DecodeListToBuilder(const ::avro::NodePtr& avro_node, ::avro::Decoder* de
     // Read array block count
     int64_t block_count = decoder->arrayStart();
     while (block_count != 0) {
+        PAIMON_RETURN_NOT_OK(ReserveBuilderCapacityImpl(block_count, value_builder));
         for (int64_t i = 0; i < block_count; ++i) {
             PAIMON_RETURN_NOT_OK(DecodeFieldToBuilder(element_node, /*projection=*/std::nullopt,
                                                       decoder, value_builder, ctx));
@@ -221,6 +236,8 @@ Status DecodeMapToBuilder(const ::avro::NodePtr& avro_node, ::avro::Decoder* dec
         // Read map block count
         int64_t block_count = decoder->mapStart();
         while (block_count != 0) {
+            PAIMON_RETURN_NOT_OK(ReserveBuilderCapacityImpl(block_count, key_builder));
+            PAIMON_RETURN_NOT_OK(ReserveBuilderCapacityImpl(block_count, item_builder));
             for (int64_t i = 0; i < block_count; ++i) {
                 PAIMON_RETURN_NOT_OK(DecodeFieldToBuilder(key_node, /*projection=*/std::nullopt,
                                                           decoder, key_builder, ctx));
@@ -248,6 +265,8 @@ Status DecodeMapToBuilder(const ::avro::NodePtr& avro_node, ::avro::Decoder* dec
         // Read array block count
         int64_t block_count = decoder->arrayStart();
         while (block_count != 0) {
+            PAIMON_RETURN_NOT_OK(ReserveBuilderCapacityImpl(block_count, key_builder));
+            PAIMON_RETURN_NOT_OK(ReserveBuilderCapacityImpl(block_count, item_builder));
             for (int64_t i = 0; i < block_count; ++i) {
                 PAIMON_RETURN_NOT_OK(DecodeFieldToBuilder(key_node, /*projection=*/std::nullopt,
                                                           decoder, key_builder, ctx));
@@ -447,4 +466,9 @@ Status AvroDirectDecoder::DecodeAvroToBuilder(const ::avro::NodePtr& avro_node,
     return DecodeFieldToBuilder(avro_node, projection, decoder, array_builder, ctx);
 }
 
+Status AvroDirectDecoder::ReserveBuilderCapacity(int64_t capacity,
+                                                 arrow::ArrayBuilder* array_builder) {
+    return ReserveBuilderCapacityImpl(capacity, array_builder);
+}
+
 }  // namespace paimon::avro
diff --git a/src/paimon/format/avro/avro_direct_decoder.h b/src/paimon/format/avro/avro_direct_decoder.h
index 6422f9154..3b316f540 100644
--- a/src/paimon/format/avro/avro_direct_decoder.h
+++ b/src/paimon/format/avro/avro_direct_decoder.h
@@ -81,6 +81,12 @@ class AvroDirectDecoder {
                                       const std::optional<std::set<size_t>>& projection,
                                       ::avro::Decoder* decoder, arrow::ArrayBuilder* array_builder,
                                       DecodeContext* ctx);
+
+    /// Reserve slots for a builder and any struct children with the same cardinality.
+    /// @param capacity Number of additional values to append.
+    /// @param array_builder Builder to reserve.
+    /// @return Status::OK if all reservations succeed.
+    static Status ReserveBuilderCapacity(int64_t capacity, arrow::ArrayBuilder* array_builder);
 };
 
 }  // namespace paimon::avro
diff --git a/src/paimon/format/avro/avro_direct_encoder_decoder_test.cpp b/src/paimon/format/avro/avro_direct_encoder_decoder_test.cpp
index 78f4ca483..8e65520d3 100644
--- a/src/paimon/format/avro/avro_direct_encoder_decoder_test.cpp
+++ b/src/paimon/format/avro/avro_direct_encoder_decoder_test.cpp
@@ -361,6 +361,33 @@ TEST_F(AvroDirectEncoderDecoderTest, TestRecordType) {
     CheckResult(schema_json, input_array, &struct_builder);
 }
 
+TEST_F(AvroDirectEncoderDecoderTest, TestReserveBuilderCapacity) {
+    std::shared_ptr<arrow::DataType> nested_type = arrow::struct_({
+        arrow::field("id", arrow::int32()),
+        arrow::field("nested", arrow::struct_({arrow::field("value", arrow::int64())})),
+        arrow::field("items", arrow::list(arrow::int32())),
+    });
+    arrow::Result<std::unique_ptr<arrow::ArrayBuilder>> builder_result =
+        arrow::MakeBuilder(nested_type);
+    ASSERT_TRUE(builder_result.ok()) << builder_result.status().ToString();
+    std::unique_ptr<arrow::ArrayBuilder> builder = std::move(builder_result).ValueOrDie();
+
+    constexpr int64_t capacity = 1024;
+    ASSERT_OK(AvroDirectDecoder::ReserveBuilderCapacity(capacity, builder.get()));
+
+    auto* root_builder = checked_cast<arrow::StructBuilder*>(builder.get());
+    ASSERT_GE(root_builder->capacity(), capacity);
+    ASSERT_GE(root_builder->field_builder(0)->capacity(), capacity);
+
+    auto* nested_builder = checked_cast<arrow::StructBuilder*>(root_builder->field_builder(1));
+    ASSERT_GE(nested_builder->capacity(), capacity);
+    ASSERT_GE(nested_builder->field_builder(0)->capacity(), capacity);
+
+    auto* list_builder = checked_cast<arrow::ListBuilder*>(root_builder->field_builder(2));
+    ASSERT_GE(list_builder->capacity(), capacity);
+    ASSERT_EQ(list_builder->value_builder()->capacity(), 0);
+}
+
 TEST_F(AvroDirectEncoderDecoderTest, TestDecodeWithProjection) {
     arrow::FieldVector fields = {
         arrow::field("f0", arrow::boolean()),
diff --git a/src/paimon/format/avro/avro_file_batch_reader.cpp b/src/paimon/format/avro/avro_file_batch_reader.cpp
index 1e217f5c4..f8b00c7c6 100644
--- a/src/paimon/format/avro/avro_file_batch_reader.cpp
+++ b/src/paimon/format/avro/avro_file_batch_reader.cpp
@@ -110,6 +110,10 @@ Result<BatchReader::ReadBatch> AvroFileBatchReader::NextBatch() {
             if (!reader_->hasMore()) {
                 break;
             }
+            if (array_builder_->length() == 0) {
+                PAIMON_RETURN_NOT_OK(
+                    AvroDirectDecoder::ReserveBuilderCapacity(batch_size_, array_builder_.get()));
+            }
             reader_->decr();
             PAIMON_RETURN_NOT_OK(AvroDirectDecoder::DecodeAvroToBuilder(
                 reader_->dataSchema().root(), read_fields_projection_, &reader_->decoder(),
@@ -123,7 +127,10 @@ Result<BatchReader::ReadBatch> AvroFileBatchReader::NextBatch() {
         }
         PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
                                           array_builder_->Finish());
+#ifndef NDEBUG
+        // Keep structural validation in debug builds without adding its recursive cost to reads.
         PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate());
+#endif
         std::unique_ptr<ArrowArray> c_array = std::make_unique<ArrowArray>();
         std::unique_ptr<ArrowSchema> c_schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, c_array.get(), c_schema.get()));
diff --git a/src/paimon/format/orc/orc_adapter.cpp b/src/paimon/format/orc/orc_adapter.cpp
index 4a03d7cda..dc99c5e68 100644
--- a/src/paimon/format/orc/orc_adapter.cpp
+++ b/src/paimon/format/orc/orc_adapter.cpp
@@ -940,6 +940,8 @@ Result<std::shared_ptr<arrow::Array>> OrcAdapter::AppendBatch(
                            MakeArrowBuilder(type, batch, pool));
     std::shared_ptr<arrow::Array> array;
     PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&array));
+    // Keep this check in release builds so malformed nested arrays return a Status before they
+    // reach Arrow constructors that enforce their invariants with a process-terminating check.
     PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate());
     return array;
 }

From c3290904d46b5bb876fa404f52192aa9de8dae23 Mon Sep 17 00:00:00 2001
From: Nicholas Jiang <programgeek@163.com>
Date: Wed, 26 Aug 2026 17:22:32 +0800
Subject: [PATCH 21/93] feat(benchmark): add format-level Parquet read/write
 micro-benchmarks (#232)

---
 benchmark/CMakeLists.txt                    |   31 +
 benchmark/parquet_format_benchmark.cpp      | 1444 +++++++++++++++++++
 benchmark/parquet_format_benchmark_test.cpp |  533 +++++++
 docs/source/examples/benchmark.rst          |  112 +-
 4 files changed, 2116 insertions(+), 4 deletions(-)
 create mode 100644 benchmark/parquet_format_benchmark.cpp
 create mode 100644 benchmark/parquet_format_benchmark_test.cpp

diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
index 375e98961..0d4dc28cf 100644
--- a/benchmark/CMakeLists.txt
+++ b/benchmark/CMakeLists.txt
@@ -63,6 +63,20 @@ if(PAIMON_BUILD_BENCHMARKS)
                          ${PAIMON_BENCHMARK_LINK_TOOLCHAIN}
                          EXTRA_INCLUDES
                          ${CMAKE_SOURCE_DIR})
+
+    add_paimon_benchmark(parquet_format_benchmark
+                         SOURCES
+                         parquet_format_benchmark.cpp
+                         STATIC_LINK_LIBS
+                         arrow
+                         parquet
+                         ${PAIMON_BENCHMARK_STATIC_LINK_LIBS}
+                         test_utils_static
+                         Threads::Threads
+                         ${PAIMON_BENCHMARK_PLATFORM_LINK_LIBS}
+                         ${PAIMON_BENCHMARK_LINK_TOOLCHAIN}
+                         EXTRA_INCLUDES
+                         ${CMAKE_SOURCE_DIR})
 endif()
 
 if(PAIMON_BUILD_TESTS)
@@ -74,4 +88,21 @@ if(PAIMON_BUILD_TESTS)
                     STATIC_LINK_LIBS
                     paimon_shared
                     ${GTEST_LINK_TOOLCHAIN})
+
+    # Guards the format-layer assumptions parquet_format_benchmark.cpp is built on. The benchmark
+    # itself is only compiled under PAIMON_BUILD_BENCHMARKS, which CI does not set, so this test is
+    # what keeps those assumptions covered.
+    add_paimon_test(parquet_format_benchmark_test
+                    SOURCES
+                    parquet_format_benchmark_test.cpp
+                    EXTRA_INCLUDES
+                    ${CMAKE_SOURCE_DIR}
+                    STATIC_LINK_LIBS
+                    arrow
+                    parquet
+                    paimon_shared
+                    ${PAIMON_LOCAL_FILE_SYSTEM_SHARED_LINK_LIBS}
+                    ${PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS}
+                    test_utils_static
+                    ${GTEST_LINK_TOOLCHAIN})
 endif()
diff --git a/benchmark/parquet_format_benchmark.cpp b/benchmark/parquet_format_benchmark.cpp
new file mode 100644
index 000000000..e8d3e0026
--- /dev/null
+++ b/benchmark/parquet_format_benchmark.cpp
@@ -0,0 +1,1444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Format-level micro-benchmarks for the Parquet reader and writer. These drive
+// ParquetWriterBuilder / ParquetFileBatchReader directly, so a format-layer change can be
+// attributed without the catalog lookup, split planning, merge/sort and commit that the
+// table-level read_write_benchmark includes. Each case comment says what that case answers.
+//
+// Every axis - type, cardinality, null density, batch size, encoding, selectivity - is swept on
+// its own rather than as a combined matrix, because the point is attributing one change rather
+// than describing a workload.
+//
+// Data is generated outside the timed region and on Arrow's default pool, because the writer
+// cuts a new row group once its own pool crosses parquet.writer.max.memory.use.
+//
+// IO goes through the local FileSystem into a temporary directory - the project has no in-memory
+// FileSystem - so absolute numbers are only meaningful relative to each other.
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <cstdint>
+#include <functional>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/c/abi.h"
+#include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
+#include "arrow/util/bit_util.h"
+#include "benchmark/benchmark.h"
+#include "paimon/common/utils/arrow/arrow_input_stream_adapter.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/defs.h"
+#include "paimon/format/format_writer.h"
+#include "paimon/format/parquet/parquet_field_id_converter.h"
+#include "paimon/format/parquet/parquet_file_batch_reader.h"
+#include "paimon/format/parquet/parquet_format_defs.h"
+#include "paimon/format/parquet/parquet_writer_builder.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/metrics.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/reader/batch_reader.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+#include "paimon/testing/utils/testharness.h"
+#include "paimon/utils/roaring_bitmap32.h"
+
+namespace {
+
+using ::paimon::ArrowInputStreamAdapter;
+using ::paimon::BatchReader;
+using ::paimon::FieldType;
+using ::paimon::FileStatus;
+using ::paimon::FileSystem;
+using ::paimon::FormatWriter;
+using ::paimon::InputStream;
+using ::paimon::Literal;
+using ::paimon::OutputStream;
+using ::paimon::PathUtil;
+using ::paimon::Predicate;
+using ::paimon::PredicateBuilder;
+using ::paimon::Result;
+using ::paimon::RoaringBitmap32;
+using ::paimon::Status;
+using ::paimon::parquet::ParquetFieldIdConverter;
+using ::paimon::parquet::ParquetFileBatchReader;
+using ::paimon::parquet::ParquetWriterBuilder;
+
+constexpr int64_t kRowsPerFile = 100'000;
+constexpr int64_t kRowsPerBatch = 10'000;
+constexpr int32_t kReadBatchSize = 4096;
+constexpr int32_t kWriteBatchSize = 1024;
+// Small enough that a 100K-row file spans many pages, so page-level pruning has something to
+// prune. Arrow's page limit is byte-based; a row-count limit is not available yet.
+constexpr int64_t kPageSizeBytes = 64 * 1024;
+// Four row groups per read fixture, so row-group pruning and page pruning are both in play.
+constexpr int64_t kRowGroupLength = 25'000;
+constexpr int64_t kStringCardinality = 1'000;
+// Few enough distinct values that arrow keeps the column dictionary-encoded for the whole file,
+// which is the shape the wide-schema case wants: per-column work small, per-batch cost visible.
+constexpr int64_t kLowStringCardinality = 10;
+constexpr int32_t kVectorDimension = 16;
+constexpr int32_t kListLength = 4;
+// Same as kListLength, so the MAP and LIST cases differ only in the extra key leaf.
+constexpr int32_t kMapEntries = kListLength;
+constexpr char kDefaultCompression[] = "zstd";
+// Nulls are placed over a 100-row window, so a requested density is exact, not statistical.
+constexpr int64_t kNullWindow = 100;
+
+std::shared_ptr<arrow::Field> MakeField(const std::string& name,
+                                        const std::shared_ptr<arrow::DataType>& type,
+                                        int32_t field_id) {
+    return arrow::field(name, type,
+                        arrow::KeyValueMetadata::Make({ParquetFieldIdConverter::PARQUET_FIELD_ID},
+                                                      {std::to_string(field_id)}));
+}
+
+std::shared_ptr<arrow::DataType> StructColumnType() {
+    return arrow::struct_({arrow::field("a", arrow::int64()), arrow::field("b", arrow::utf8())});
+}
+
+std::shared_ptr<arrow::DataType> ListColumnType() {
+    return arrow::list(arrow::int64());
+}
+
+std::shared_ptr<arrow::DataType> VectorColumnType() {
+    return arrow::fixed_size_list(arrow::field("element", arrow::float32(), /*nullable=*/false),
+                                  kVectorDimension);
+}
+
+std::shared_ptr<arrow::DataType> MapColumnType() {
+    return arrow::map(arrow::utf8(), arrow::int64());
+}
+
+std::shared_ptr<arrow::DataType> DictionaryStringType() {
+    return arrow::dictionary(arrow::int32(), arrow::utf8());
+}
+
+std::shared_ptr<arrow::DataType> DictionaryInt32Type() {
+    return arrow::dictionary(arrow::int32(), arrow::int32());
+}
+
+// The three-column file the flat read cases scan. `id` is ordered so a range predicate maps
+// onto a contiguous row range, which is what makes page-index pruning measurable.
+std::shared_ptr<arrow::Schema> FlatSchema() {
+    return arrow::schema({MakeField("id", arrow::int64(), 0), MakeField("name", arrow::utf8(), 1),
+                          MakeField("amount", arrow::decimal128(18, 4), 2)});
+}
+
+std::shared_ptr<arrow::Schema> DecimalSchema(int32_t precision) {
+    return arrow::schema({MakeField("amount", arrow::decimal128(precision, 4), 0)});
+}
+
+std::shared_ptr<arrow::Schema> DoubleSchema() {
+    return arrow::schema({MakeField("value", arrow::float64(), 0)});
+}
+
+std::shared_ptr<arrow::Schema> NestedSchema() {
+    return arrow::schema(
+        {MakeField("id", arrow::int64(), 0), MakeField("info", StructColumnType(), 1),
+         MakeField("tags", ListColumnType(), 2), MakeField("embedding", VectorColumnType(), 3),
+         MakeField("attrs", MapColumnType(), 4)});
+}
+
+// A VECTOR is stored as a Parquet LIST and the reader hands back the file's own types, so a
+// format-level read asks for the physical type; VectorFileBatchReader restores the view above.
+std::shared_ptr<arrow::Schema> NestedReadSchema() {
+    return arrow::schema({MakeField("id", arrow::int64(), 0),
+                          MakeField("info", StructColumnType(), 1),
+                          MakeField("tags", ListColumnType(), 2),
+                          MakeField("embedding",
+                                    arrow::list(arrow::field("element", arrow::float32(),
+                                                             /*nullable=*/false)),
+                                    3),
+                          MakeField("attrs", MapColumnType(), 4)});
+}
+
+Result<std::shared_ptr<arrow::Array>> MakeInt64Column(int64_t num_rows, int64_t offset) {
+    arrow::Int64Builder builder;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(num_rows));
+    for (int64_t i = 0; i < num_rows; ++i) {
+        builder.UnsafeAppend(offset + i);
+    }
+    std::shared_ptr<arrow::Array> array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&array));
+    return array;
+}
+
+Result<std::shared_ptr<arrow::Array>> MakeDoubleColumn(int64_t num_rows, int64_t offset) {
+    arrow::DoubleBuilder builder;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(num_rows));
+    for (int64_t i = 0; i < num_rows; ++i) {
+        builder.UnsafeAppend(static_cast<double>(offset + i) * 1.5);
+    }
+    std::shared_ptr<arrow::Array> array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&array));
+    return array;
+}
+
+Result<std::shared_ptr<arrow::Array>> MakeBooleanColumn(int64_t num_rows, int64_t offset) {
+    arrow::BooleanBuilder builder;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(num_rows));
+    for (int64_t i = 0; i < num_rows; ++i) {
+        builder.UnsafeAppend(((offset + i) & 1) == 0);
+    }
+    std::shared_ptr<arrow::Array> array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&array));
+    return array;
+}
+
+// Low cardinality is what arrow dictionary-encodes; high cardinality falls back to plain.
+Result<std::shared_ptr<arrow::Array>> MakeStringColumn(int64_t num_rows, int64_t offset,
+                                                       int64_t cardinality) {
+    arrow::StringBuilder builder;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(num_rows));
+    for (int64_t i = 0; i < num_rows; ++i) {
+        const int64_t value = (offset + i) % cardinality;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append("value_" + std::to_string(value)));
+    }
+    std::shared_ptr<arrow::Array> array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&array));
+    return array;
+}
+
+// INT32 cycling through `cardinality` distinct values, the flat control for
+// BM_ParquetWrite_DictionaryInt32: same logical values, same cardinality, same width, so the
+// delta between the two is the dictionary materialization alone.
+Result<std::shared_ptr<arrow::Array>> MakeInt32Column(int64_t num_rows, int64_t offset,
+                                                      int64_t cardinality) {
+    arrow::Int32Builder builder;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(num_rows));
+    for (int64_t i = 0; i < num_rows; ++i) {
+        builder.UnsafeAppend(static_cast<int32_t>(((offset + i) % cardinality) * 7));
+    }
+    std::shared_ptr<arrow::Array> array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&array));
+    return array;
+}
+
+// Dictionary-encoded input: `values->length()` distinct values behind an int32 index array. Arrow
+// hands the indices straight to Parquet when the value type is binary-like
+// (DictionaryDirectWriteSupported) and densifies them otherwise, so the value type alone decides
+// whether the writer materializes anything.
+Result<std::shared_ptr<arrow::Array>> MakeDictionaryColumn(
+    const std::shared_ptr<arrow::Array>& values, int64_t num_rows, int64_t offset) {
+    const int64_t cardinality = values->length();
+    arrow::Int32Builder index_builder;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(index_builder.Reserve(num_rows));
+    for (int64_t i = 0; i < num_rows; ++i) {
+        index_builder.UnsafeAppend(static_cast<int32_t>((offset + i) % cardinality));
+    }
+    std::shared_ptr<arrow::Array> indices;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(index_builder.Finish(&indices));
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
+                                      arrow::DictionaryArray::FromArrays(indices, values));
+    return array;
+}
+
+// STRING values: binary-like, so arrow can write the indices directly.
+Result<std::shared_ptr<arrow::Array>> MakeDictionaryStringColumn(int64_t num_rows, int64_t offset,
+                                                                 int64_t cardinality) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> values,
+                           MakeStringColumn(cardinality, /*offset=*/0, cardinality));
+    return MakeDictionaryColumn(values, num_rows, offset);
+}
+
+// INT32 values: is_base_binary_like excludes them, so arrow densifies before writing. The
+// dictionary holds the same `i * 7` values MakeInt32Column emits inline, so the two are directly
+// comparable.
+Result<std::shared_ptr<arrow::Array>> MakeDictionaryInt32Column(int64_t num_rows, int64_t offset,
+                                                                int64_t cardinality) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> values,
+                           MakeInt32Column(cardinality, /*offset=*/0, cardinality));
+    return MakeDictionaryColumn(values, num_rows, offset);
+}
+
+// Precision drives the Parquet physical type: <= 9 is INT32, <= 18 is INT64, larger is
+// FIXED_LEN_BYTE_ARRAY, and the three take different transfer paths on read.
+Result<std::shared_ptr<arrow::Array>> MakeDecimalColumn(int64_t num_rows, int64_t offset,
+                                                        int32_t precision, int32_t scale) {
+    arrow::Decimal128Builder builder(arrow::decimal128(precision, scale));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(num_rows));
+    for (int64_t i = 0; i < num_rows; ++i) {
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(arrow::Decimal128(offset + i)));
+    }
+    std::shared_ptr<arrow::Array> array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&array));
+    return array;
+}
+
+Result<std::shared_ptr<arrow::Array>> MakeStructArray(
+    const arrow::FieldVector& fields, const std::vector<std::shared_ptr<arrow::Array>>& columns) {
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> array,
+                                      arrow::StructArray::Make(columns, fields));
+    return paimon::checked_pointer_cast<arrow::Array>(array);
+}
+
+Result<std::shared_ptr<arrow::Array>> MakeStructColumn(int64_t num_rows, int64_t offset) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> a, MakeInt64Column(num_rows, offset));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> b,
+                           MakeStringColumn(num_rows, offset, kStringCardinality));
+    return MakeStructArray(StructColumnType()->fields(), {a, b});
+}
+
+// Constant element count per row, so the delta against a flat BIGINT column is the levels.
+Result<std::shared_ptr<arrow::Array>> MakeListColumn(int64_t num_rows, int64_t offset) {
+    auto value_builder = std::make_shared<arrow::Int64Builder>();
+    arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, ListColumnType());
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(num_rows));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Reserve(num_rows * kListLength));
+    for (int64_t i = 0; i < num_rows; ++i) {
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append());
+        for (int32_t j = 0; j < kListLength; ++j) {
+            value_builder->UnsafeAppend(offset + i + j);
+        }
+    }
+    std::shared_ptr<arrow::Array> array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&array));
+    return array;
+}
+
+// The writer converts VECTOR to a Parquet LIST, so this also covers ParquetVectorConverter.
+Result<std::shared_ptr<arrow::Array>> MakeVectorColumn(int64_t num_rows, int64_t offset) {
+    auto value_builder = std::make_shared<arrow::FloatBuilder>();
+    arrow::FixedSizeListBuilder builder(arrow::default_memory_pool(), value_builder,
+                                        VectorColumnType());
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(num_rows));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Reserve(num_rows * kVectorDimension));
+    for (int64_t i = 0; i < num_rows; ++i) {
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append());
+        for (int32_t j = 0; j < kVectorDimension; ++j) {
+            value_builder->UnsafeAppend(static_cast<float>((offset + i) % 1024) +
+                                        static_cast<float>(j));
+        }
+    }
+    std::shared_ptr<arrow::Array> array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&array));
+    return array;
+}
+
+// A Parquet MAP is a LIST of a two-field STRUCT, so against LIST the difference is the key leaf.
+Result<std::shared_ptr<arrow::Array>> MakeMapColumn(int64_t num_rows, int64_t offset,
+                                                    int32_t entries) {
+    auto key_builder = std::make_shared<arrow::StringBuilder>();
+    auto item_builder = std::make_shared<arrow::Int64Builder>();
+    arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, item_builder,
+                              MapColumnType());
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(num_rows));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Reserve(num_rows * entries));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(item_builder->Reserve(num_rows * entries));
+    for (int64_t i = 0; i < num_rows; ++i) {
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append());
+        for (int32_t j = 0; j < entries; ++j) {
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Append("key_" + std::to_string(j)));
+            item_builder->UnsafeAppend(offset + i + j);
+        }
+    }
+    std::shared_ptr<arrow::Array> array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&array));
+    return array;
+}
+
+// Masks `null_pct` percent of the slots null. Doing it after the fact rather than in every
+// generator keeps the values identical across densities, so the delta between two settings is the
+// level machinery alone. Only the top level is masked: a STRUCT or LIST row, not its leaves.
+Result<std::shared_ptr<arrow::Array>> WithNulls(const std::shared_ptr<arrow::Array>& array,
+                                                int64_t offset, int64_t null_pct) {
+    if (null_pct <= 0) {
+        return array;
+    }
+    // Builder output starts at slot 0, so the bitmap below can be indexed by position.
+    if (array->data()->offset != 0) {
+        return Status::Invalid("WithNulls expects an unsliced array");
+    }
+    const int64_t length = array->length();
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::Buffer> validity,
+        arrow::AllocateEmptyBitmap(length, arrow::default_memory_pool()));
+    int64_t null_count = 0;
+    for (int64_t i = 0; i < length; ++i) {
+        if ((offset + i) % kNullWindow < null_pct) {
+            ++null_count;
+        } else {
+            arrow::bit_util::SetBit(validity->mutable_data(), i);
+        }
+    }
+    std::shared_ptr<arrow::ArrayData> data = array->data()->Copy();
+    data->buffers[0] = std::move(validity);
+    data->SetNullCount(null_count);
+    return arrow::MakeArray(data);
+}
+
+using BatchFactory = std::function<Result<std::shared_ptr<arrow::Array>>(
+    const std::shared_ptr<arrow::Schema>& schema, int64_t offset, int64_t rows)>;
+
+using ColumnFactory =
+    std::function<Result<std::shared_ptr<arrow::Array>>(int64_t rows, int64_t offset)>;
+
+ColumnFactory NullableColumnFactory(const ColumnFactory& make_column, int64_t null_pct) {
+    return [make_column, null_pct](int64_t rows,
+                                   int64_t offset) -> Result<std::shared_ptr<arrow::Array>> {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> column, make_column(rows, offset));
+        return WithNulls(column, offset, null_pct);
+    };
+}
+
+// Lifts a one-column generator into a batch factory for a one-field schema.
+BatchFactory SingleColumnBatch(const ColumnFactory& make_column) {
+    return [make_column](const std::shared_ptr<arrow::Schema>& schema, int64_t offset,
+                         int64_t rows) -> Result<std::shared_ptr<arrow::Array>> {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> column, make_column(rows, offset));
+        return MakeStructArray(schema->fields(), {column});
+    };
+}
+
+Result<std::shared_ptr<arrow::Array>> MakeNullableFlatBatch(
+    const std::shared_ptr<arrow::Schema>& schema, int64_t offset, int64_t rows, int64_t null_pct) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> ids, MakeInt64Column(rows, offset));
+    PAIMON_ASSIGN_OR_RAISE(ids, WithNulls(ids, offset, null_pct));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> names,
+                           MakeStringColumn(rows, offset, kStringCardinality));
+    PAIMON_ASSIGN_OR_RAISE(names, WithNulls(names, offset, null_pct));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> amounts,
+                           MakeDecimalColumn(rows, offset, /*precision=*/18, /*scale=*/4));
+    PAIMON_ASSIGN_OR_RAISE(amounts, WithNulls(amounts, offset, null_pct));
+    return MakeStructArray(schema->fields(), {ids, names, amounts});
+}
+
+Result<std::shared_ptr<arrow::Array>> MakeFlatBatch(const std::shared_ptr<arrow::Schema>& schema,
+                                                    int64_t offset, int64_t rows) {
+    return MakeNullableFlatBatch(schema, offset, rows, /*null_pct=*/0);
+}
+
+// One low-cardinality VARCHAR per field, cheap enough that a wide schema leaves the per-column
+// setup holding the measurement. The staggered offset keeps the columns from being identical.
+Result<std::shared_ptr<arrow::Array>> MakeWideBatch(const std::shared_ptr<arrow::Schema>& schema,
+                                                    int64_t offset, int64_t rows) {
+    std::vector<std::shared_ptr<arrow::Array>> columns;
+    columns.reserve(schema->num_fields());
+    for (int32_t i = 0; i < schema->num_fields(); ++i) {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> column,
+                               MakeStringColumn(rows, offset + i, kLowStringCardinality));
+        columns.push_back(std::move(column));
+    }
+    return MakeStructArray(schema->fields(), columns);
+}
+
+Result<std::shared_ptr<arrow::Array>> MakeNestedBatch(const std::shared_ptr<arrow::Schema>& schema,
+                                                      int64_t offset, int64_t rows) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> ids, MakeInt64Column(rows, offset));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> info, MakeStructColumn(rows, offset));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> tags, MakeListColumn(rows, offset));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> embedding, MakeVectorColumn(rows, offset));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> attrs,
+                           MakeMapColumn(rows, offset, kMapEntries));
+    return MakeStructArray(schema->fields(), {ids, info, tags, embedding, attrs});
+}
+
+// Whether every AddBatch call gets its own array or they all share one. It matters for dictionary
+// input: arrow compares each batch's dictionary against the previous one, so N distinct-but-equal
+// dictionaries and one dictionary written N times are not the same write. Only
+// BM_ParquetWrite_MemoryThreshold reuses one batch; every other case generates each batch
+// independently. That is about the arrays, not the values - generators that cycle with a period
+// dividing the batch size, such as the boolean and the low-cardinality string ones, produce
+// batches that are equal in value but separate objects.
+enum class BatchReuse { kFresh, kReused };
+
+Result<std::vector<std::shared_ptr<arrow::Array>>> MakeBatches(
+    const std::shared_ptr<arrow::Schema>& schema, const BatchFactory& make_batch,
+    int64_t rows_per_batch, int64_t total_rows = kRowsPerFile,
+    BatchReuse reuse = BatchReuse::kFresh) {
+    if (reuse == BatchReuse::kReused) {
+        // One array written N times cannot express a short final batch, and silently writing a
+        // full one instead would put more rows in the file than the reported metrics divide by.
+        if (total_rows % rows_per_batch != 0) {
+            return Status::Invalid("BatchReuse::kReused needs total_rows divisible by " +
+                                   std::to_string(rows_per_batch) + ", got " +
+                                   std::to_string(total_rows));
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> batch,
+                               make_batch(schema, /*offset=*/0, rows_per_batch));
+        return std::vector<std::shared_ptr<arrow::Array>>(total_rows / rows_per_batch,
+                                                          std::move(batch));
+    }
+    std::vector<std::shared_ptr<arrow::Array>> batches;
+    for (int64_t offset = 0; offset < total_rows; offset += rows_per_batch) {
+        const int64_t rows = std::min<int64_t>(rows_per_batch, total_rows - offset);
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> batch,
+                               make_batch(schema, offset, rows));
+        batches.push_back(std::move(batch));
+    }
+    return batches;
+}
+
+Result<int64_t> WriteParquetFile(const std::shared_ptr<FileSystem>& fs, const std::string& path,
+                                 const std::shared_ptr<arrow::Schema>& schema,
+                                 const std::vector<std::shared_ptr<arrow::Array>>& batches,
+                                 const std::map<std::string, std::string>& options,
+                                 const std::string& compression) {
+    ParquetWriterBuilder writer_builder(schema, kWriteBatchSize, options);
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OutputStream> out, fs->Create(path, /*overwrite=*/true));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatWriter> writer,
+                           writer_builder.Build(out, compression));
+    for (const auto& batch : batches) {
+        // AddBatch imports - and so consumes - the C array, so each batch needs a fresh export.
+        ArrowArray c_array;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*batch, &c_array));
+        PAIMON_RETURN_NOT_OK(writer->AddBatch(&c_array));
+    }
+    PAIMON_RETURN_NOT_OK(writer->Finish());
+    PAIMON_RETURN_NOT_OK(out->Close());
+    PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, fs->GetFileStatus(path));
+    return file_status.GetLen();
+}
+
+struct ReadStats {
+    int64_t rows = 0;
+    // Reader-side counterpart of file size: makes pruning and skipping visible apart from CPU.
+    uint64_t storage_bytes = 0;
+    // Without these a filtered case shows only that it got faster, not that pruning is why.
+    uint64_t row_groups_total = 0;
+    uint64_t row_groups_after_filter = 0;
+    uint64_t batches = 0;
+};
+
+// Every counter read below is set unconditionally by the reader - the row-group pair in
+// SetReadSchema, the batch count in NextBatch - so an absent one means the metric moved or stopped
+// being recorded. Reporting that as a zero would hide the regression behind a plausible number,
+// so the error propagates and fails the case instead.
+Result<uint64_t> ReadCounter(const std::shared_ptr<paimon::Metrics>& metrics,
+                             const std::string& name) {
+    return metrics->GetCounter(name);
+}
+
+Result<ReadStats> ReadParquetFile(const std::shared_ptr<FileSystem>& fs, const std::string& path,
+                                  const std::shared_ptr<arrow::Schema>& read_schema,
+                                  const std::shared_ptr<Predicate>& predicate,
+                                  const std::optional<RoaringBitmap32>& selection_bitmap,
+                                  const std::map<std::string, std::string>& options,
+                                  int32_t batch_size,
+                                  const std::shared_ptr<arrow::MemoryPool>& pool) {
+    PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, fs->GetFileStatus(path));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> input, fs->Open(path));
+    auto in_stream = std::make_shared<ArrowInputStreamAdapter>(input, file_status.GetLen(), pool);
+    // Held separately so the counter outlives the adapter the reader takes ownership of.
+    std::shared_ptr<std::atomic<uint64_t>> storage_read_bytes = in_stream->StorageReadBytes();
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<ParquetFileBatchReader> reader,
+        ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size,
+                                       /*file_metadata=*/nullptr, storage_read_bytes, pool,
+                                       /*hints=*/std::nullopt));
+
+    // SetReadSchema imports the C schema and takes ownership of it.
+    ArrowSchema c_schema;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, &c_schema));
+    PAIMON_RETURN_NOT_OK(reader->SetReadSchema(&c_schema, predicate, selection_bitmap));
+
+    ReadStats stats;
+    while (true) {
+        PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
+        if (BatchReader::IsEofBatch(batch)) {
+            break;
+        }
+        stats.rows += batch.first->length;
+        ArrowArrayRelease(batch.first.get());
+        ArrowSchemaRelease(batch.second.get());
+    }
+
+    std::shared_ptr<paimon::Metrics> metrics = reader->GetReaderMetrics();
+    PAIMON_ASSIGN_OR_RAISE(
+        stats.row_groups_total,
+        ReadCounter(metrics, paimon::parquet::ParquetMetrics::READ_ROW_GROUPS_TOTAL));
+    PAIMON_ASSIGN_OR_RAISE(
+        stats.row_groups_after_filter,
+        ReadCounter(metrics, paimon::parquet::ParquetMetrics::READ_ROW_GROUPS_AFTER_FILTER));
+    PAIMON_ASSIGN_OR_RAISE(stats.batches,
+                           ReadCounter(metrics, paimon::parquet::ParquetMetrics::READ_BATCH_COUNT));
+    reader->Close();
+    stats.storage_bytes = storage_read_bytes->load();
+    return stats;
+}
+
+// Row groups the written file actually ended up with. The writer decides that itself - by row
+// count, or by its pool crossing parquet.writer.max.memory.use - so the footer is the only
+// reliable source.
+//
+// The write schema is deliberately not used to read back, because for two cases it does not
+// describe the file. A VECTOR is stored as a LIST, and CollectLeafIndices branches on the file
+// type, so a FixedSizeList read type against a file LIST is rejected outright. A dictionary column
+// is stored as its value type; that one survives leaf collection, which compares nothing for
+// atomic fields, and would fail later against read_data_type_ once a batch is read. Create()
+// resolves the file's own schema and sets the row-group counters while doing so, which is all this
+// needs - no read schema, no batch read, and neither trap.
+Result<uint64_t> ReadRowGroupCount(const std::shared_ptr<FileSystem>& fs, const std::string& path,
+                                   const std::shared_ptr<arrow::MemoryPool>& pool) {
+    PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, fs->GetFileStatus(path));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> input, fs->Open(path));
+    auto in_stream = std::make_shared<ArrowInputStreamAdapter>(input, file_status.GetLen(), pool);
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<ParquetFileBatchReader> reader,
+        ParquetFileBatchReader::Create(std::move(in_stream), /*options=*/{}, kReadBatchSize,
+                                       /*file_metadata=*/nullptr, /*storage_read_bytes=*/nullptr,
+                                       pool, /*hints=*/std::nullopt));
+    PAIMON_ASSIGN_OR_RAISE(uint64_t row_groups,
+                           ReadCounter(reader->GetReaderMetrics(),
+                                       paimon::parquet::ParquetMetrics::READ_ROW_GROUPS_TOTAL));
+    reader->Close();
+    return row_groups;
+}
+
+// Owns a temporary directory plus the shared FileSystem / MemoryPool, removed on destruction.
+class BenchmarkEnv {
+ public:
+    static Result<std::unique_ptr<BenchmarkEnv>> Create() {
+        std::unique_ptr<paimon::test::UniqueTestDirectory> dir =
+            paimon::test::UniqueTestDirectory::Create();
+        if (!dir) {
+            return Status::IOError("failed to create a temporary benchmark directory");
+        }
+        return std::unique_ptr<BenchmarkEnv>(new BenchmarkEnv(std::move(dir)));
+    }
+
+    const std::shared_ptr<FileSystem>& fs() const {
+        return fs_;
+    }
+
+    // The reader takes an arrow pool; the writer builder allocates its own from the paimon pool.
+    const std::shared_ptr<arrow::MemoryPool>& arrow_pool() const {
+        return arrow_pool_;
+    }
+
+    std::string PathOf(const std::string& file_name) const {
+        return PathUtil::JoinPath(dir_->Str(), file_name);
+    }
+
+ private:
+    explicit BenchmarkEnv(std::unique_ptr<paimon::test::UniqueTestDirectory> dir)
+        : dir_(std::move(dir)),
+          fs_(dir_->GetFileSystem()),
+          arrow_pool_(paimon::GetArrowPool(paimon::GetDefaultPool())) {}
+
+    std::unique_ptr<paimon::test::UniqueTestDirectory> dir_;
+    std::shared_ptr<FileSystem> fs_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+};
+
+// A file the read cases scan, built once per configuration; rebuilding it would dominate.
+class ReadFixture {
+ public:
+    ReadFixture(const std::string& file_name, const std::shared_ptr<arrow::Schema>& schema,
+                const BatchFactory& make_batch,
+                const std::map<std::string, std::string>& write_options = {}) {
+        status_ = Build(file_name, schema, make_batch, write_options);
+    }
+
+    const Status& status() const {
+        return status_;
+    }
+
+    const std::string& path() const {
+        return path_;
+    }
+
+    const std::shared_ptr<FileSystem>& fs() const {
+        return env_->fs();
+    }
+
+    const std::shared_ptr<arrow::MemoryPool>& arrow_pool() const {
+        return env_->arrow_pool();
+    }
+
+    int64_t file_bytes() const {
+        return file_bytes_;
+    }
+
+ private:
+    Status Build(const std::string& file_name, const std::shared_ptr<arrow::Schema>& schema,
+                 const BatchFactory& make_batch,
+                 const std::map<std::string, std::string>& write_options) {
+        PAIMON_ASSIGN_OR_RAISE(env_, BenchmarkEnv::Create());
+        path_ = env_->PathOf(file_name);
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<arrow::Array>> batches,
+                               MakeBatches(schema, make_batch, kRowsPerBatch));
+        std::map<std::string, std::string> options = write_options;
+        options[paimon::parquet::PARQUET_PAGE_SIZE] = std::to_string(kPageSizeBytes);
+        options[paimon::parquet::PARQUET_WRITE_MAX_ROW_GROUP_LENGTH] =
+            std::to_string(kRowGroupLength);
+        PAIMON_ASSIGN_OR_RAISE(file_bytes_, WriteParquetFile(env_->fs(), path_, schema, batches,
+                                                             options, kDefaultCompression));
+        return Status::OK();
+    }
+
+    std::unique_ptr<BenchmarkEnv> env_;
+    std::string path_;
+    int64_t file_bytes_ = 0;
+    Status status_;
+};
+
+// Built on first use, so a case excluded by --benchmark_filter never pays to write its file.
+// google/benchmark may report from a different thread than it registered on, hence the lock.
+const ReadFixture& GetFixture(const std::string& key,
+                              const std::function<std::unique_ptr<ReadFixture>()>& build) {
+    static std::mutex mutex;
+    static std::map<std::string, std::unique_ptr<ReadFixture>> fixtures;
+    std::lock_guard<std::mutex> guard(mutex);
+    std::unique_ptr<ReadFixture>& fixture = fixtures[key];
+    if (!fixture) {
+        fixture = build();
+    }
+    return *fixture;
+}
+
+const ReadFixture& FlatFixture() {
+    return GetFixture("flat", [] {
+        return std::make_unique<ReadFixture>("flat.parquet", FlatSchema(), &MakeFlatBatch);
+    });
+}
+
+const ReadFixture& NestedFixture() {
+    return GetFixture("nested", [] {
+        return std::make_unique<ReadFixture>("nested.parquet", NestedSchema(), &MakeNestedBatch);
+    });
+}
+
+const ReadFixture& NullableFlatFixture(int64_t null_pct) {
+    const std::string key = "flat_nulls_" + std::to_string(null_pct);
+    return GetFixture(key, [key, null_pct] {
+        return std::make_unique<ReadFixture>(
+            key + ".parquet", FlatSchema(),
+            [null_pct](const std::shared_ptr<arrow::Schema>& schema, int64_t offset, int64_t rows) {
+                return MakeNullableFlatBatch(schema, offset, rows, null_pct);
+            });
+    });
+}
+
+// A one-column file, for read cases that isolate a single decoder.
+const ReadFixture& ColumnFixture(const std::string& key,
+                                 const std::shared_ptr<arrow::Schema>& schema,
+                                 const ColumnFactory& make_column) {
+    return GetFixture(key, [key, schema, make_column] {
+        return std::make_unique<ReadFixture>(key + ".parquet", schema,
+                                             SingleColumnBatch(make_column));
+    });
+}
+
+// The flat fixture only carries precision 18, so without this the FIXED_LEN_BYTE_ARRAY path that
+// precision 38 takes is written but never read.
+const ReadFixture& DecimalFixture(int32_t precision) {
+    return ColumnFixture("decimal_" + std::to_string(precision), DecimalSchema(precision),
+                         [precision](int64_t rows, int64_t offset) {
+                             return MakeDecimalColumn(rows, offset, precision, /*scale=*/4);
+                         });
+}
+
+// The flat schema carries no floating-point column, so a DOUBLE read needs a file of its own.
+const ReadFixture& DoubleFixture() {
+    return ColumnFixture("double", DoubleSchema(), &MakeDoubleColumn);
+}
+
+// The same data with dictionary encoding off, giving the read side a plain baseline.
+const ReadFixture& PlainFlatFixture() {
+    return GetFixture("flat_plain", [] {
+        std::map<std::string, std::string> options;
+        options[paimon::parquet::PARQUET_ENABLE_DICTIONARY] = "false";
+        return std::make_unique<ReadFixture>("flat_plain.parquet", FlatSchema(), &MakeFlatBatch,
+                                             options);
+    });
+}
+
+// google/benchmark's own main exits 0 whatever happened, so a case that called SkipWithError
+// prints as skipped while `ctest -L benchmark` still passes. Nothing here skips on purpose, so
+// the flag every error sets becomes the process exit code. It is recorded here rather than in a
+// custom reporter because passing one to RunSpecifiedBenchmarks would bypass
+// CreateDefaultDisplayReporter and with it --benchmark_format, --benchmark_color and
+// --benchmark_counters_tabular.
+std::atomic<bool> g_failed{false};
+
+bool FailBenchmark(::benchmark::State& state, const Status& status) {
+    if (status.ok()) {
+        return false;
+    }
+    g_failed.store(true);
+    state.SkipWithError(status.ToString().c_str());
+    return true;
+}
+
+class Timer {
+ public:
+    double ElapsedNanos() const {
+        return std::chrono::duration<double, std::nano>(std::chrono::steady_clock::now() - started_)
+            .count();
+    }
+
+ private:
+    std::chrono::steady_clock::time_point started_ = std::chrono::steady_clock::now();
+};
+
+// google/benchmark reports items/s; ns per row is what the issue asks for, so the timed region
+// is also measured directly and divided by the row count.
+void ReportRowRate(::benchmark::State& state, int64_t rows, double elapsed_ns) {
+    state.SetItemsProcessed(static_cast<int64_t>(state.iterations()) * rows);
+    const double total_rows = static_cast<double>(state.iterations()) * static_cast<double>(rows);
+    if (total_rows > 0) {
+        state.counters["ns_per_row"] = ::benchmark::Counter(elapsed_ns / total_rows);
+    }
+}
+
+// Bytes per iteration, plus the per-row form that shows a CPU-for-size trade next to ns_per_row.
+void ReportBytes(::benchmark::State& state, const std::string& name, int64_t bytes, int64_t rows) {
+    state.counters[name] = ::benchmark::Counter(static_cast<double>(bytes));
+    if (rows > 0) {
+        state.counters["bytes_per_row"] =
+            ::benchmark::Counter(static_cast<double>(bytes) / static_cast<double>(rows));
+    }
+}
+
+// Everything the writer needs per file - properties, output stream, schema conversion, footer -
+// is inside the timed region, because that is what a caller pays per data file.
+void RunWriteBenchmark(::benchmark::State& state, const std::shared_ptr<arrow::Schema>& schema,
+                       const BatchFactory& make_batch, int64_t rows_per_batch,
+                       const std::map<std::string, std::string>& options,
+                       const std::string& compression, int64_t total_rows = kRowsPerFile,
+                       BatchReuse reuse = BatchReuse::kFresh) {
+    Result<std::unique_ptr<BenchmarkEnv>> env = BenchmarkEnv::Create();
+    if (FailBenchmark(state, env.status())) {
+        return;
+    }
+    Result<std::vector<std::shared_ptr<arrow::Array>>> batches =
+        MakeBatches(schema, make_batch, rows_per_batch, total_rows, reuse);
+    if (FailBenchmark(state, batches.status())) {
+        return;
+    }
+    const std::string path = env.value()->PathOf("write_case.parquet");
+
+    int64_t file_bytes = 0;
+    Timer timer;
+    for (auto _ : state) {
+        Result<int64_t> written = WriteParquetFile(env.value()->fs(), path, schema, batches.value(),
+                                                   options, compression);
+        if (FailBenchmark(state, written.status())) {
+            return;
+        }
+        file_bytes = written.value();
+    }
+    // Captured before the read-back below, which reopens the file and parses its footer. Reading
+    // the timer after would put that inside ns_per_row but not inside google/benchmark's own
+    // real_time, leaving the two disagreeing by a fixed amount that matters at low iteration
+    // counts.
+    const double elapsed_ns = timer.ElapsedNanos();
+
+    // Read back rather than computed: with a byte-triggered flush the count is not predictable
+    // from the arguments, and where a row-count limit does make it predictable, reporting the real
+    // number is what would catch the prediction being wrong.
+    Result<uint64_t> row_groups =
+        ReadRowGroupCount(env.value()->fs(), path, env.value()->arrow_pool());
+    if (FailBenchmark(state, row_groups.status())) {
+        return;
+    }
+
+    ReportRowRate(state, total_rows, elapsed_ns);
+    ReportBytes(state, "file_bytes", file_bytes, total_rows);
+    state.counters["batches"] = ::benchmark::Counter(
+        static_cast<double>((total_rows + rows_per_batch - 1) / rows_per_batch));
+    state.counters["row_groups"] = ::benchmark::Counter(static_cast<double>(row_groups.value()));
+}
+
+// Single-column variant: one column isolates one encoder.
+void RunColumnWriteBenchmark(::benchmark::State& state, const std::shared_ptr<arrow::Field>& field,
+                             const ColumnFactory& make_column, int64_t rows_per_batch,
+                             const std::map<std::string, std::string>& options,
+                             int64_t total_rows = kRowsPerFile,
+                             BatchReuse reuse = BatchReuse::kFresh) {
+    RunWriteBenchmark(state, arrow::schema({field}), SingleColumnBatch(make_column), rows_per_batch,
+                      options, kDefaultCompression, total_rows, reuse);
+}
+
+void BM_ParquetWrite_Int64(::benchmark::State& state) {
+    RunColumnWriteBenchmark(state, MakeField("id", arrow::int64(), 0), &MakeInt64Column,
+                            kRowsPerBatch, /*options=*/{});
+}
+
+void BM_ParquetWrite_Double(::benchmark::State& state) {
+    RunColumnWriteBenchmark(state, MakeField("value", arrow::float64(), 0), &MakeDoubleColumn,
+                            kRowsPerBatch, /*options=*/{});
+}
+
+void BM_ParquetWrite_Boolean(::benchmark::State& state) {
+    RunColumnWriteBenchmark(state, MakeField("flag", arrow::boolean(), 0), &MakeBooleanColumn,
+                            kRowsPerBatch, /*options=*/{});
+}
+
+void BM_ParquetWrite_String(::benchmark::State& state) {
+    const int64_t cardinality = state.range(0);
+    RunColumnWriteBenchmark(state, MakeField("name", arrow::utf8(), 0),
+                            [cardinality](int64_t rows, int64_t offset) {
+                                return MakeStringColumn(rows, offset, cardinality);
+                            },
+                            kRowsPerBatch, /*options=*/{});
+}
+
+// arg: number of distinct values. The flat control for BM_ParquetWrite_DictionaryInt32.
+void BM_ParquetWrite_FlatInt32(::benchmark::State& state) {
+    const int64_t cardinality = state.range(0);
+    RunColumnWriteBenchmark(state, MakeField("value", arrow::int32(), 0),
+                            [cardinality](int64_t rows, int64_t offset) {
+                                return MakeInt32Column(rows, offset, cardinality);
+                            },
+                            kRowsPerBatch, /*options=*/{});
+}
+
+// arg: dictionary cardinality. Same values as BM_ParquetWrite_String at the same cardinality, but
+// handed to the writer already dictionary-encoded - nothing in ParquetWriterBuilder rejects a
+// dictionary arrow type, so the writer does reach this path - and the pair isolates what it saves
+// when it can pass indices through instead of materializing every value.
+void BM_ParquetWrite_DictionaryString(::benchmark::State& state) {
+    const int64_t cardinality = state.range(0);
+    RunColumnWriteBenchmark(state, MakeField("name", DictionaryStringType(), 0),
+                            [cardinality](int64_t rows, int64_t offset) {
+                                return MakeDictionaryStringColumn(rows, offset, cardinality);
+                            },
+                            kRowsPerBatch, /*options=*/{});
+}
+
+// The same axis on an INTEGER dictionary, which arrow cannot direct-write - is_base_binary_like
+// excludes int32, so it densifies first. Its baseline is BM_ParquetWrite_FlatInt32 at the same
+// cardinality, not the String case: only the flat INT32 control holds value, width and encoding
+// fixed, so only that delta is the materialization cost.
+void BM_ParquetWrite_DictionaryInt32(::benchmark::State& state) {
+    const int64_t cardinality = state.range(0);
+    RunColumnWriteBenchmark(state, MakeField("value", DictionaryInt32Type(), 0),
+                            [cardinality](int64_t rows, int64_t offset) {
+                                return MakeDictionaryInt32Column(rows, offset, cardinality);
+                            },
+                            kRowsPerBatch, /*options=*/{});
+}
+
+// Same data with dictionary encoding turned off, as an encoding baseline.
+void BM_ParquetWrite_StringNoDictionary(::benchmark::State& state) {
+    const int64_t cardinality = state.range(0);
+    std::map<std::string, std::string> options;
+    options[paimon::parquet::PARQUET_ENABLE_DICTIONARY] = "false";
+    RunColumnWriteBenchmark(
+        state, MakeField("name", arrow::utf8(), 0),
+        [cardinality](int64_t rows, int64_t offset) {
+            return MakeStringColumn(rows, offset, cardinality);
+        },
+        kRowsPerBatch, options);
+}
+
+void BM_ParquetWrite_Decimal(::benchmark::State& state) {
+    const auto precision = static_cast<int32_t>(state.range(0));
+    RunColumnWriteBenchmark(state, MakeField("amount", arrow::decimal128(precision, 4), 0),
+                            [precision](int64_t rows, int64_t offset) {
+                                return MakeDecimalColumn(rows, offset, precision, /*scale=*/4);
+                            },
+                            kRowsPerBatch, /*options=*/{});
+}
+
+void BM_ParquetWrite_Struct(::benchmark::State& state) {
+    RunColumnWriteBenchmark(state, MakeField("info", StructColumnType(), 0), &MakeStructColumn,
+                            kRowsPerBatch, /*options=*/{});
+}
+
+void BM_ParquetWrite_List(::benchmark::State& state) {
+    RunColumnWriteBenchmark(state, MakeField("tags", ListColumnType(), 0), &MakeListColumn,
+                            kRowsPerBatch, /*options=*/{});
+}
+
+void BM_ParquetWrite_Vector(::benchmark::State& state) {
+    RunColumnWriteBenchmark(state, MakeField("embedding", VectorColumnType(), 0), &MakeVectorColumn,
+                            kRowsPerBatch, /*options=*/{});
+}
+
+// arg: entries per map row.
+void BM_ParquetWrite_Map(::benchmark::State& state) {
+    const auto entries = static_cast<int32_t>(state.range(0));
+    RunColumnWriteBenchmark(
+        state, MakeField("attrs", MapColumnType(), 0),
+        [entries](int64_t rows, int64_t offset) { return MakeMapColumn(rows, offset, entries); },
+        kRowsPerBatch, /*options=*/{});
+}
+
+// arg: percentage of nulls. Every field is nullable, so the column is `optional` and definition
+// levels are written at every density including zero; the sweep moves what arrow's null-free fast
+// path is worth. BIGINT is the narrowest column, so levels are the largest share of what is left.
+void BM_ParquetWrite_Nulls(::benchmark::State& state) {
+    const int64_t null_pct = state.range(0);
+    RunColumnWriteBenchmark(state, MakeField("id", arrow::int64(), 0),
+                            NullableColumnFactory(&MakeInt64Column, null_pct), kRowsPerBatch,
+                            /*options=*/{});
+}
+
+// arg: rows per AddBatch call, at a fixed total row count. Smaller batches mean the same data
+// carries more per-batch fixed cost, and the file-level setup is amortized over more calls.
+void BM_ParquetWrite_BatchSize(::benchmark::State& state) {
+    RunColumnWriteBenchmark(state, MakeField("id", arrow::int64(), 0), &MakeInt64Column,
+                            state.range(0), /*options=*/{});
+}
+
+// arg: number of columns at a fixed row count. With BM_ParquetWrite_BatchSize this separates the
+// two ways per-AddBatch work grows - more calls, or more columns per call - where that work is
+// AddBatch importing the C array plus arrow walking the schema, neither of which encodes a value.
+void BM_ParquetWrite_ColumnCount(::benchmark::State& state) {
+    const auto columns = static_cast<int32_t>(state.range(0));
+    arrow::FieldVector fields;
+    fields.reserve(columns);
+    for (int32_t i = 0; i < columns; ++i) {
+        fields.push_back(MakeField("c" + std::to_string(i), arrow::utf8(), i));
+    }
+    RunWriteBenchmark(state, arrow::schema(fields), &MakeWideBatch, kRowsPerBatch, /*options=*/{},
+                      kDefaultCompression);
+    state.counters["columns"] = ::benchmark::Counter(static_cast<double>(columns));
+}
+
+// arg: maximum rows per row group, the write-side half of BM_ParquetRead_Filtered - a row group
+// is the unit the reader prunes, so more of them buys finer pruning at the cost of footer
+// metadata, flush work and codec material. The byte-triggered counterpart is
+// BM_ParquetWrite_MemoryThreshold.
+void BM_ParquetWrite_RowGroupSize(::benchmark::State& state) {
+    const int64_t row_group_length = state.range(0);
+    std::map<std::string, std::string> options;
+    options[paimon::parquet::PARQUET_WRITE_MAX_ROW_GROUP_LENGTH] = std::to_string(row_group_length);
+    RunWriteBenchmark(state, FlatSchema(), &MakeFlatBatch, kRowsPerBatch, options,
+                      kDefaultCompression);
+}
+
+// args: the parquet.writer.max.memory.use threshold in KiB, and the number of AddBatch calls.
+// Unlike every other case here the file is not a fixed kRowsPerFile: batches are a fixed 10'000
+// rows and the total grows with the batch count, because the question is whether retained size
+// accumulates across batches until the writer cuts a row group. Holding the total fixed and
+// shrinking the batch would measure AddBatch granularity instead, which is what
+// BM_ParquetWrite_BatchSize already does. The input is low-cardinality dictionary data, where
+// retained and flat size differ most, written as one batch object repeatedly rather than a fresh
+// one each time so arrow sees the same dictionary rather than N equal ones.
+//
+// Nothing here sets a row-group row limit, so the byte threshold is the only thing that can
+// trigger a flush. row_groups reports whether it fired at all; if it reads 1 everywhere the sweep
+// measured nothing, and the unit test only covers the mechanism, not this particular setting.
+void BM_ParquetWrite_MemoryThreshold(::benchmark::State& state) {
+    constexpr int64_t kFlushRowsPerBatch = 10'000;
+    std::map<std::string, std::string> options;
+    options[paimon::parquet::PARQUET_WRITER_MAX_MEMORY_USE] = std::to_string(state.range(0) * 1024);
+    RunColumnWriteBenchmark(
+        state, MakeField("name", DictionaryStringType(), 0),
+        [](int64_t rows, int64_t offset) {
+            return MakeDictionaryStringColumn(rows, offset, kLowStringCardinality);
+        },
+        kFlushRowsPerBatch, options, kFlushRowsPerBatch * state.range(1), BatchReuse::kReused);
+}
+
+// Codec sweep over the mixed flat schema, so the CPU-versus-size trade shows on data that is not
+// uniformly one type. Levels stay at the ParquetWriterBuilder defaults. The names are the ones
+// Parquet accepts, not the ones arrow does: "lz4" resolves to arrow's LZ4_FRAME, which
+// parquet::IsCodecSupported rejects, so the two framings Parquet defines are spelled out.
+void BM_ParquetWrite_Compression(::benchmark::State& state, const char* compression) {
+    RunWriteBenchmark(state, FlatSchema(), &MakeFlatBatch, kRowsPerBatch, /*options=*/{},
+                      compression);
+}
+
+// Rows a read case has to materialize for its number to mean anything. Pruning is not precise, so
+// a filtered case gets a range rather than an exact count. Without the bound, a fixture that
+// silently stopped producing rows would just look fast.
+struct RowExpectation {
+    int64_t min = kRowsPerFile;
+    int64_t max = kRowsPerFile;
+};
+
+void RunReadBenchmark(::benchmark::State& state, const ReadFixture& fixture,
+                      const std::shared_ptr<arrow::Schema>& read_schema,
+                      const std::shared_ptr<Predicate>& predicate,
+                      const std::optional<RoaringBitmap32>& selection_bitmap,
+                      const std::map<std::string, std::string>& options, int32_t batch_size,
+                      const RowExpectation& expected = {}) {
+    if (FailBenchmark(state, fixture.status())) {
+        return;
+    }
+
+    ReadStats stats;
+    Timer timer;
+    for (auto _ : state) {
+        Result<ReadStats> result =
+            ReadParquetFile(fixture.fs(), fixture.path(), read_schema, predicate, selection_bitmap,
+                            options, batch_size, fixture.arrow_pool());
+        if (FailBenchmark(state, result.status())) {
+            return;
+        }
+        stats = result.value();
+    }
+    if (stats.rows < expected.min || stats.rows > expected.max) {
+        FailBenchmark(state, Status::Invalid("read " + std::to_string(stats.rows) +
+                                             " rows, expected " + std::to_string(expected.min) +
+                                             " to " + std::to_string(expected.max)));
+        return;
+    }
+
+    const double elapsed_ns = timer.ElapsedNanos();
+    ReportRowRate(state, stats.rows, elapsed_ns);
+    // Normalized by what the file holds, not what this case materialized: pruning drops the
+    // per-materialized-row numerator and denominator together, so ns_per_row and bytes_per_row can
+    // rise while the run gets faster. Only a denominator every setting shares is comparable across
+    // settings that prune by different amounts.
+    const double total_input_rows =
+        static_cast<double>(state.iterations()) * static_cast<double>(kRowsPerFile);
+    if (total_input_rows > 0) {
+        state.counters["ns_per_input_row"] = ::benchmark::Counter(elapsed_ns / total_input_rows);
+        state.counters["bytes_per_input_row"] = ::benchmark::Counter(
+            static_cast<double>(stats.storage_bytes) / static_cast<double>(kRowsPerFile));
+    }
+    ReportBytes(state, "read_bytes", static_cast<int64_t>(stats.storage_bytes), stats.rows);
+    state.counters["rows_read"] = ::benchmark::Counter(static_cast<double>(stats.rows));
+    state.counters["file_bytes"] = ::benchmark::Counter(static_cast<double>(fixture.file_bytes()));
+    state.counters["row_groups"] =
+        ::benchmark::Counter(static_cast<double>(stats.row_groups_total));
+    state.counters["row_groups_after_filter"] =
+        ::benchmark::Counter(static_cast<double>(stats.row_groups_after_filter));
+    state.counters["batches"] = ::benchmark::Counter(static_cast<double>(stats.batches));
+}
+
+void BM_ParquetRead_FullScan(::benchmark::State& state) {
+    RunReadBenchmark(state, FlatFixture(), FlatSchema(), /*predicate=*/nullptr,
+                     /*selection_bitmap=*/std::nullopt, /*options=*/{}, kReadBatchSize);
+}
+
+// One column at a time, separating per-column transfer from materializing the whole row.
+void BM_ParquetRead_Projection(::benchmark::State& state, const char* column) {
+    std::shared_ptr<arrow::Field> field = FlatSchema()->GetFieldByName(column);
+    if (!field) {
+        FailBenchmark(state, Status::Invalid("unknown projection column"));
+        return;
+    }
+    RunReadBenchmark(state, FlatFixture(), arrow::schema({field}), /*predicate=*/nullptr,
+                     /*selection_bitmap=*/std::nullopt, /*options=*/{}, kReadBatchSize);
+}
+
+// args: percentage of rows the predicate keeps, and whether page-level pruning is on. The `id`
+// column is ordered, so the surviving rows form a prefix; running both page-index settings makes
+// the pruning gain attributable instead of merely visible.
+void BM_ParquetRead_Filtered(::benchmark::State& state) {
+    const int64_t threshold = kRowsPerFile * state.range(0) / 100;
+    // `id` is ordered, so row-group statistics alone must already discard every group past the
+    // threshold - that holds with page-index filtering off too. Bounding at the row-group grain
+    // rather than at kRowsPerFile is what makes a pruning regression fail the case instead of
+    // quietly reading the whole file.
+    const int64_t row_group_bound =
+        ((threshold + kRowGroupLength - 1) / kRowGroupLength) * kRowGroupLength;
+    const bool enable_page_index = state.range(1) != 0;
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::LessThan(
+        /*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT, Literal(threshold));
+    std::map<std::string, std::string> options;
+    options[paimon::parquet::PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] =
+        enable_page_index ? "true" : "false";
+    RunReadBenchmark(state, FlatFixture(), FlatSchema(), predicate,
+                     /*selection_bitmap=*/std::nullopt, options, kReadBatchSize,
+                     RowExpectation{threshold, row_group_bound});
+}
+
+// arg: distance between selected rows. The strides straddle the coalesce hole limit (32 rows by
+// default), which splits this into two regimes: at or below it neighbouring single-row ranges
+// merge into long spans, so rows_read runs far ahead of selected_rows and nothing is skipped;
+// above it every row stays its own range and arrow's Skip decodes and discards each gap instead.
+// Compare read_bytes and ns_per_input_row across the two.
+void BM_ParquetRead_SkipHeavy(::benchmark::State& state) {
+    const int64_t stride = state.range(0);
+    RoaringBitmap32 bitmap;
+    for (int64_t row = 0; row < kRowsPerFile; row += stride) {
+        bitmap.Add(static_cast<int32_t>(row));
+    }
+    const int64_t selected = bitmap.Cardinality();
+    state.counters["selected_rows"] = ::benchmark::Counter(static_cast<double>(selected));
+    RunReadBenchmark(state, FlatFixture(), FlatSchema(), /*predicate=*/nullptr, bitmap,
+                     /*options=*/{}, kReadBatchSize, RowExpectation{selected, kRowsPerFile});
+}
+
+// arg: percentage of nulls, the read side of BM_ParquetWrite_Nulls: definition levels have to be
+// decoded back into a validity bitmap, and past some density arrow may do less work, not more.
+void BM_ParquetRead_Nulls(::benchmark::State& state) {
+    RunReadBenchmark(state, NullableFlatFixture(state.range(0)), FlatSchema(),
+                     /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt, /*options=*/{},
+                     kReadBatchSize);
+}
+
+// The VARCHAR column from a dictionary-encoded file and from a plain-encoded one. Both are
+// decoded by arrow, so this is arrow's dictionary path against its plain path on equal values.
+void BM_ParquetRead_Encoding(::benchmark::State& state, bool enable_dictionary) {
+    std::shared_ptr<arrow::Field> field = FlatSchema()->GetFieldByName("name");
+    RunReadBenchmark(state, enable_dictionary ? FlatFixture() : PlainFlatFixture(),
+                     arrow::schema({field}), /*predicate=*/nullptr,
+                     /*selection_bitmap=*/std::nullopt, /*options=*/{}, kReadBatchSize);
+}
+
+// arg: decimal precision, the read side of BM_ParquetWrite_Decimal. Precision picks the physical
+// type - INT32, INT64 or FIXED_LEN_BYTE_ARRAY, since ParquetWriterBuilder enables
+// store_decimal_as_integer - and the three take different paths back to Decimal128Array.
+void BM_ParquetRead_Decimal(::benchmark::State& state) {
+    const auto precision = static_cast<int32_t>(state.range(0));
+    RunReadBenchmark(state, DecimalFixture(precision), DecimalSchema(precision),
+                     /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt, /*options=*/{},
+                     kReadBatchSize);
+}
+
+void BM_ParquetRead_Double(::benchmark::State& state) {
+    RunReadBenchmark(state, DoubleFixture(), DoubleSchema(), /*predicate=*/nullptr,
+                     /*selection_bitmap=*/std::nullopt, /*options=*/{}, kReadBatchSize);
+}
+
+// arg: rows per NextBatch call. At a fixed row count this turns the per-batch fixed cost of
+// ParquetFileBatchReader::NextBatch - Validate, the two metrics counters, the ArrowSchema
+// export - into a number, separated from the per-row decoding cost.
+void BM_ParquetRead_BatchSize(::benchmark::State& state) {
+    RunReadBenchmark(state, FlatFixture(), FlatSchema(), /*predicate=*/nullptr,
+                     /*selection_bitmap=*/std::nullopt, /*options=*/{},
+                     static_cast<int32_t>(state.range(0)));
+}
+
+// Each nested column costs definition and repetition levels a flat column does not pay.
+void BM_ParquetRead_Nested(::benchmark::State& state, const char* column) {
+    std::shared_ptr<arrow::Schema> full_schema = NestedReadSchema();
+    std::shared_ptr<arrow::Schema> read_schema = full_schema;
+    if (column != nullptr) {
+        std::shared_ptr<arrow::Field> field = full_schema->GetFieldByName(column);
+        if (!field) {
+            FailBenchmark(state, Status::Invalid("unknown nested column"));
+            return;
+        }
+        read_schema = arrow::schema({field});
+    }
+    RunReadBenchmark(state, NestedFixture(), read_schema, /*predicate=*/nullptr,
+                     /*selection_bitmap=*/std::nullopt, /*options=*/{}, kReadBatchSize);
+}
+
+}  // namespace
+
+BENCHMARK(BM_ParquetWrite_Int64)->Unit(benchmark::kMillisecond)->UseRealTime();
+BENCHMARK(BM_ParquetWrite_Double)->Unit(benchmark::kMillisecond)->UseRealTime();
+BENCHMARK(BM_ParquetWrite_Boolean)->Unit(benchmark::kMillisecond)->UseRealTime();
+BENCHMARK(BM_ParquetWrite_String)
+    ->ArgName("cardinality")
+    ->Arg(10)
+    ->Arg(1000)
+    ->Arg(kRowsPerFile)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetWrite_StringNoDictionary)
+    ->ArgName("cardinality")
+    ->Arg(10)
+    ->Arg(1000)
+    ->Arg(kRowsPerFile)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetWrite_FlatInt32)
+    ->ArgName("cardinality")
+    ->Arg(10)
+    ->Arg(100)
+    ->Arg(1000)
+    ->Arg(10000)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetWrite_DictionaryString)
+    ->ArgName("cardinality")
+    ->Arg(10)
+    ->Arg(100)
+    ->Arg(1000)
+    ->Arg(10000)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetWrite_DictionaryInt32)
+    ->ArgName("cardinality")
+    ->Arg(10)
+    ->Arg(100)
+    ->Arg(1000)
+    ->Arg(10000)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetWrite_Decimal)
+    ->ArgName("precision")
+    ->Arg(9)
+    ->Arg(18)
+    ->Arg(38)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetWrite_Struct)->Unit(benchmark::kMillisecond)->UseRealTime();
+BENCHMARK(BM_ParquetWrite_List)->Unit(benchmark::kMillisecond)->UseRealTime();
+BENCHMARK(BM_ParquetWrite_Vector)->Unit(benchmark::kMillisecond)->UseRealTime();
+BENCHMARK(BM_ParquetWrite_Map)
+    ->ArgName("entries")
+    ->Arg(3)
+    ->Arg(5)
+    ->Arg(10)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetWrite_Nulls)
+    ->ArgName("null_pct")
+    ->Arg(0)
+    ->Arg(20)
+    ->Arg(50)
+    ->Arg(70)
+    ->Arg(100)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetWrite_BatchSize)
+    ->ArgName("rows_per_batch")
+    ->Arg(100)
+    ->Arg(1000)
+    ->Arg(10000)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetWrite_ColumnCount)
+    ->ArgName("columns")
+    ->Arg(1)
+    ->Arg(5)
+    ->Arg(10)
+    ->Arg(20)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetWrite_RowGroupSize)
+    ->ArgName("row_group_rows")
+    ->Arg(5000)
+    ->Arg(25000)
+    ->Arg(kRowsPerFile)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetWrite_MemoryThreshold)
+    ->ArgNames({"max_memory_kib", "batches"})
+    ->Args({512, 50})
+    ->Args({512, 200})
+    ->Args({4096, 200})
+    ->Args({65536, 200})
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetWrite_Compression, none, "none")
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetWrite_Compression, snappy, "snappy")
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetWrite_Compression, gzip, "gzip")
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetWrite_Compression, lz4_raw, "lz4_raw")
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetWrite_Compression, lz4_hadoop, "lz4_hadoop")
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetWrite_Compression, brotli, "brotli")
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetWrite_Compression, zstd, "zstd")
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+
+BENCHMARK(BM_ParquetRead_FullScan)->Unit(benchmark::kMillisecond)->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetRead_Projection, id, "id")
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetRead_Projection, name, "name")
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetRead_Projection, amount, "amount")
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetRead_Filtered)
+    ->ArgNames({"keep_pct", "page_index"})
+    ->Args({1, 1})
+    ->Args({1, 0})
+    ->Args({10, 1})
+    ->Args({10, 0})
+    ->Args({50, 1})
+    ->Args({50, 0})
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetRead_SkipHeavy)
+    ->ArgName("stride")
+    ->Arg(8)
+    ->Arg(32)
+    ->Arg(64)
+    ->Arg(512)
+    ->Arg(4096)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetRead_Nulls)
+    ->ArgName("null_pct")
+    ->Arg(0)
+    ->Arg(20)
+    ->Arg(50)
+    ->Arg(70)
+    ->Arg(100)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetRead_Encoding, dictionary, true)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetRead_Encoding, plain, false)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetRead_Decimal)
+    ->ArgName("precision")
+    ->Arg(9)
+    ->Arg(18)
+    ->Arg(38)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK(BM_ParquetRead_Double)->Unit(benchmark::kMillisecond)->UseRealTime();
+BENCHMARK(BM_ParquetRead_BatchSize)
+    ->ArgName("batch_size")
+    ->Arg(512)
+    ->Arg(4096)
+    ->Arg(16384)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetRead_Nested, info_struct, "info")
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetRead_Nested, tags_list, "tags")
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetRead_Nested, embedding_vector, "embedding")
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetRead_Nested, attrs_map, "attrs")
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+BENCHMARK_CAPTURE(BM_ParquetRead_Nested, all_columns, nullptr)
+    ->Unit(benchmark::kMillisecond)
+    ->UseRealTime();
+
+int main(int argc, char** argv) {
+    ::benchmark::Initialize(&argc, argv);
+    if (::benchmark::ReportUnrecognizedArguments(argc, argv)) {
+        return 1;
+    }
+    ::benchmark::RunSpecifiedBenchmarks();
+    ::benchmark::Shutdown();
+    return g_failed.load() ? 1 : 0;
+}
diff --git a/benchmark/parquet_format_benchmark_test.cpp b/benchmark/parquet_format_benchmark_test.cpp
new file mode 100644
index 000000000..e930449d0
--- /dev/null
+++ b/benchmark/parquet_format_benchmark_test.cpp
@@ -0,0 +1,533 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Smoke test for the assumptions parquet_format_benchmark.cpp is built on.
+//
+// The benchmark is only compiled under PAIMON_BUILD_BENCHMARKS, which CI does not set, so nothing
+// there runs in CI. Every assumption it makes about the format layer - that a codec name is
+// accepted, that a dictionary-encoded input array can be written, that a nested or high-precision
+// column survives a round trip, that a predicate and a selection bitmap return the rows the
+// benchmark asserts on, that the reader metrics it reports exist - is checked here instead, at a
+// row count small enough to stay a test. A regression in any of them would otherwise surface as a
+// benchmark that quietly measures the wrong thing.
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/array/concatenate.h"
+#include "arrow/c/abi.h"
+#include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
+#include "gtest/gtest.h"
+#include "paimon/common/utils/arrow/arrow_input_stream_adapter.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/defs.h"
+#include "paimon/format/format_writer.h"
+#include "paimon/format/parquet/parquet_field_id_converter.h"
+#include "paimon/format/parquet/parquet_file_batch_reader.h"
+#include "paimon/format/parquet/parquet_format_defs.h"
+#include "paimon/format/parquet/parquet_writer_builder.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/metrics.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/reader/batch_reader.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+#include "paimon/testing/utils/testharness.h"
+#include "paimon/utils/roaring_bitmap32.h"
+
+namespace paimon::parquet {
+namespace {
+
+// Small enough to stay a test, large enough to span several batches and row groups.
+constexpr int64_t kRows = 2'000;
+constexpr int32_t kBatchSize = 256;
+constexpr int64_t kRowGroupLength = 500;
+
+std::shared_ptr<arrow::Field> MakeField(const std::string& name,
+                                        const std::shared_ptr<arrow::DataType>& type,
+                                        int32_t field_id) {
+    return arrow::field(name, type,
+                        arrow::KeyValueMetadata::Make({ParquetFieldIdConverter::PARQUET_FIELD_ID},
+                                                      {std::to_string(field_id)}));
+}
+
+class ParquetFormatBenchmarkTest : public ::testing::Test {
+ protected:
+    void SetUp() override {
+        dir_ = test::UniqueTestDirectory::Create();
+        ASSERT_TRUE(dir_);
+        fs_ = dir_->GetFileSystem();
+        pool_ = GetArrowPool(GetDefaultPool());
+    }
+
+    std::string PathOf(const std::string& name) const {
+        return PathUtil::JoinPath(dir_->Str(), name);
+    }
+
+    // Writes the same struct array `batch_count` times through the builder the benchmark uses.
+    // More than one call matters for dictionary input: the benchmark always writes several
+    // batches, and every batch after the first hands the writer the same dictionary again.
+    Status Write(const std::string& path, const std::shared_ptr<arrow::Schema>& schema,
+                 const std::shared_ptr<arrow::Array>& batch, const std::string& compression,
+                 const std::map<std::string, std::string>& extra_options = {},
+                 int32_t batch_count = 1) {
+        std::map<std::string, std::string> options = extra_options;
+        // emplace, not assignment: a caller that set its own row-group limit is testing that.
+        options.emplace(PARQUET_WRITE_MAX_ROW_GROUP_LENGTH, std::to_string(kRowGroupLength));
+        ParquetWriterBuilder writer_builder(schema, kBatchSize, options);
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OutputStream> out,
+                               fs_->Create(path, /*overwrite=*/true));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatWriter> writer,
+                               writer_builder.Build(out, compression));
+        for (int32_t i = 0; i < batch_count; ++i) {
+            ArrowArray c_array;
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*batch, &c_array));
+            PAIMON_RETURN_NOT_OK(writer->AddBatch(&c_array));
+        }
+        PAIMON_RETURN_NOT_OK(writer->Finish());
+        return out->Close();
+    }
+
+    // Concatenating `array` with itself `times` times, so a multi-batch write has an expected
+    // value to be compared against.
+    static Result<std::shared_ptr<arrow::Array>> Repeat(const std::shared_ptr<arrow::Array>& array,
+                                                        int32_t times) {
+        std::vector<std::shared_ptr<arrow::Array>> chunks(times, array);
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> repeated,
+                                          arrow::Concatenate(chunks));
+        return repeated;
+    }
+
+    static Result<std::shared_ptr<arrow::Array>> MakeDictionary(
+        const std::shared_ptr<arrow::Array>& indices, const std::shared_ptr<arrow::Array>& values) {
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
+                                          arrow::DictionaryArray::FromArrays(indices, values));
+        return array;
+    }
+
+    struct ReadResult {
+        int64_t rows = 0;
+        uint64_t row_groups_total = 0;
+        uint64_t row_groups_after_filter = 0;
+        uint64_t batches = 0;
+        // Every batch, imported and concatenated. Row counts alone would let a decoding bug
+        // through, so the tests compare this against what was written.
+        std::shared_ptr<arrow::Array> data;
+    };
+
+    // Reads the file back the way the benchmark does, reporting what the benchmark reports on.
+    Result<ReadResult> Read(const std::string& path,
+                            const std::shared_ptr<arrow::Schema>& read_schema,
+                            const std::shared_ptr<Predicate>& predicate = nullptr,
+                            const std::optional<RoaringBitmap32>& selection = std::nullopt) {
+        PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, fs_->GetFileStatus(path));
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> input, fs_->Open(path));
+        auto in_stream =
+            std::make_shared<ArrowInputStreamAdapter>(input, file_status.GetLen(), pool_);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ParquetFileBatchReader> reader,
+                               ParquetFileBatchReader::Create(
+                                   std::move(in_stream), /*options=*/{}, kBatchSize,
+                                   /*file_metadata=*/nullptr, /*storage_read_bytes=*/nullptr, pool_,
+                                   /*hints=*/std::nullopt));
+        ArrowSchema c_schema;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, &c_schema));
+        PAIMON_RETURN_NOT_OK(reader->SetReadSchema(&c_schema, predicate, selection));
+
+        ReadResult result;
+        std::vector<std::shared_ptr<arrow::Array>> chunks;
+        while (true) {
+            PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
+            if (BatchReader::IsEofBatch(batch)) {
+                break;
+            }
+            // ImportArray takes ownership of both C structs, so nothing is released by hand here.
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> chunk,
+                arrow::ImportArray(batch.first.get(), batch.second.get()));
+            result.rows += chunk->length();
+            chunks.push_back(std::move(chunk));
+        }
+        if (!chunks.empty()) {
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(result.data, arrow::Concatenate(chunks));
+        }
+
+        std::shared_ptr<Metrics> metrics = reader->GetReaderMetrics();
+        PAIMON_ASSIGN_OR_RAISE(result.row_groups_total,
+                               metrics->GetCounter(ParquetMetrics::READ_ROW_GROUPS_TOTAL));
+        PAIMON_ASSIGN_OR_RAISE(result.row_groups_after_filter,
+                               metrics->GetCounter(ParquetMetrics::READ_ROW_GROUPS_AFTER_FILTER));
+        PAIMON_ASSIGN_OR_RAISE(result.batches,
+                               metrics->GetCounter(ParquetMetrics::READ_BATCH_COUNT));
+        reader->Close();
+        return result;
+    }
+
+    static Result<std::shared_ptr<arrow::Array>> Wrap(
+        const std::shared_ptr<arrow::Schema>& schema,
+        const std::vector<std::shared_ptr<arrow::Array>>& columns) {
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> array,
+                                          arrow::StructArray::Make(columns, schema->fields()));
+        return checked_pointer_cast<arrow::Array>(array);
+    }
+
+    std::unique_ptr<test::UniqueTestDirectory> dir_;
+    std::shared_ptr<FileSystem> fs_;
+    std::shared_ptr<arrow::MemoryPool> pool_;
+};
+
+// Every codec name the benchmark registers has to be one Parquet accepts. "lz4" is the trap this
+// guards: it resolves to arrow's LZ4_FRAME, which parquet::IsCodecSupported rejects, and the
+// failure only shows up once a column chunk is actually written.
+TEST_F(ParquetFormatBenchmarkTest, RegisteredCodecsWrite) {
+    std::shared_ptr<arrow::Schema> schema = arrow::schema({MakeField("id", arrow::int64(), 0)});
+    arrow::Int64Builder builder;
+    ASSERT_TRUE(builder.Reserve(kRows).ok());
+    for (int64_t i = 0; i < kRows; ++i) {
+        builder.UnsafeAppend(i);
+    }
+    std::shared_ptr<arrow::Array> ids;
+    ASSERT_TRUE(builder.Finish(&ids).ok());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> batch, Wrap(schema, {ids}));
+
+    for (const std::string& codec :
+         {"none", "snappy", "gzip", "brotli", "zstd", "lz4_raw", "lz4_hadoop"}) {
+        const std::string path = PathOf("codec_" + codec + ".parquet");
+        ASSERT_OK(Write(path, schema, batch, codec)) << "codec " << codec;
+        ASSERT_OK_AND_ASSIGN(ReadResult result, Read(path, schema));
+        EXPECT_EQ(kRows, result.rows) << "codec " << codec;
+        ASSERT_TRUE(result.data);
+        EXPECT_TRUE(result.data->Equals(*batch)) << "codec " << codec;
+    }
+
+    // The name the benchmark deliberately does not register still has to be rejected; if arrow
+    // ever starts accepting it, the comment explaining its absence is stale.
+    EXPECT_FALSE(Write(PathOf("codec_lz4.parquet"), schema, batch, "lz4").ok());
+}
+
+// A dictionary-encoded input array must reach the writer intact. VARCHAR takes arrow's direct
+// write path and INT32 gets densified first; both have to produce a readable file whose logical
+// values match the flat equivalent.
+TEST_F(ParquetFormatBenchmarkTest, DictionaryInputRoundTrip) {
+    constexpr int64_t kCardinality = 8;
+    arrow::Int32Builder index_builder;
+    ASSERT_TRUE(index_builder.Reserve(kRows).ok());
+    for (int64_t i = 0; i < kRows; ++i) {
+        index_builder.UnsafeAppend(static_cast<int32_t>(i % kCardinality));
+    }
+    std::shared_ptr<arrow::Array> indices;
+    ASSERT_TRUE(index_builder.Finish(&indices).ok());
+
+    arrow::StringBuilder string_values;
+    arrow::Int32Builder int_values;
+    for (int64_t i = 0; i < kCardinality; ++i) {
+        ASSERT_TRUE(string_values.Append("value_" + std::to_string(i)).ok());
+        ASSERT_TRUE(int_values.Append(static_cast<int32_t>(i * 7)).ok());
+    }
+    std::shared_ptr<arrow::Array> string_dict;
+    std::shared_ptr<arrow::Array> int_dict;
+    ASSERT_TRUE(string_values.Finish(&string_dict).ok());
+    ASSERT_TRUE(int_values.Finish(&int_dict).ok());
+
+    // The flat arrays the dictionary-encoded input has to decode back to.
+    arrow::StringBuilder flat_strings;
+    arrow::Int32Builder flat_ints;
+    for (int64_t i = 0; i < kRows; ++i) {
+        ASSERT_TRUE(flat_strings.Append("value_" + std::to_string(i % kCardinality)).ok());
+        ASSERT_TRUE(flat_ints.Append(static_cast<int32_t>((i % kCardinality) * 7)).ok());
+    }
+    std::shared_ptr<arrow::Array> flat_string_column;
+    std::shared_ptr<arrow::Array> flat_int_column;
+    ASSERT_TRUE(flat_strings.Finish(&flat_string_column).ok());
+    ASSERT_TRUE(flat_ints.Finish(&flat_int_column).ok());
+
+    struct Case {
+        const char* name;
+        std::shared_ptr<arrow::Array> dictionary;
+        std::shared_ptr<arrow::DataType> read_type;
+        std::shared_ptr<arrow::Array> flat;
+    };
+    // The benchmark always writes several batches, so the writer sees the same dictionary more
+    // than once.
+    constexpr int32_t kBatches = 3;
+    for (const Case& c : {Case{"string", string_dict, arrow::utf8(), flat_string_column},
+                          Case{"int32", int_dict, arrow::int32(), flat_int_column}}) {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> dictionary_array,
+                             MakeDictionary(indices, c.dictionary));
+        std::shared_ptr<arrow::Schema> write_schema =
+            arrow::schema({MakeField("v", dictionary_array->type(), 0)});
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> batch,
+                             Wrap(write_schema, {dictionary_array}));
+        const std::string path = PathOf(std::string("dict_") + c.name + ".parquet");
+        ASSERT_OK(Write(path, write_schema, batch, "zstd", /*extra_options=*/{}, kBatches))
+            << c.name;
+
+        // Parquet has no dictionary type: the column comes back as its value type either way, and
+        // has to carry the values the flat equivalent would have.
+        std::shared_ptr<arrow::Schema> read_schema =
+            arrow::schema({MakeField("v", c.read_type, 0)});
+        ASSERT_OK_AND_ASSIGN(ReadResult result, Read(path, read_schema));
+        EXPECT_EQ(kRows * kBatches, result.rows) << c.name;
+        ASSERT_TRUE(result.data);
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> expected, Repeat(c.flat, kBatches));
+        std::shared_ptr<arrow::Array> actual =
+            checked_pointer_cast<arrow::StructArray>(result.data)->field(0);
+        EXPECT_TRUE(actual->Equals(*expected)) << c.name;
+    }
+}
+
+// DECIMAL precision selects the Parquet physical type, and precision 38 is the only one that
+// reaches FIXED_LEN_BYTE_ARRAY. The benchmark sweeps all three on both sides, so all three have
+// to survive a round trip with their type intact.
+TEST_F(ParquetFormatBenchmarkTest, DecimalPrecisionRoundTrip) {
+    for (int32_t precision : {9, 18, 38}) {
+        std::shared_ptr<arrow::DataType> type = arrow::decimal128(precision, 4);
+        std::shared_ptr<arrow::Schema> schema = arrow::schema({MakeField("amount", type, 0)});
+        arrow::Decimal128Builder builder(type);
+        ASSERT_TRUE(builder.Reserve(kRows).ok());
+        for (int64_t i = 0; i < kRows; ++i) {
+            ASSERT_TRUE(builder.Append(arrow::Decimal128(i)).ok());
+        }
+        std::shared_ptr<arrow::Array> amounts;
+        ASSERT_TRUE(builder.Finish(&amounts).ok());
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> batch, Wrap(schema, {amounts}));
+
+        const std::string path = PathOf("decimal_" + std::to_string(precision) + ".parquet");
+        ASSERT_OK(Write(path, schema, batch, "zstd")) << "precision " << precision;
+        ASSERT_OK_AND_ASSIGN(ReadResult result, Read(path, schema));
+        EXPECT_EQ(kRows, result.rows) << "precision " << precision;
+        ASSERT_TRUE(result.data);
+        EXPECT_TRUE(result.data->type()->field(0)->type()->Equals(*type))
+            << "precision " << precision;
+        EXPECT_TRUE(result.data->Equals(*batch)) << "precision " << precision;
+    }
+}
+
+// The nested fixture the benchmark reads is only meaningful if LIST and MAP survive the round
+// trip with the shape the read schema asks for.
+TEST_F(ParquetFormatBenchmarkTest, NestedRoundTrip) {
+    constexpr int32_t kEntries = 4;
+    auto list_values = std::make_shared<arrow::Int64Builder>();
+    arrow::ListBuilder list_builder(arrow::default_memory_pool(), list_values,
+                                    arrow::list(arrow::int64()));
+    auto key_builder = std::make_shared<arrow::StringBuilder>();
+    auto item_builder = std::make_shared<arrow::Int64Builder>();
+    arrow::MapBuilder map_builder(arrow::default_memory_pool(), key_builder, item_builder,
+                                  arrow::map(arrow::utf8(), arrow::int64()));
+    for (int64_t i = 0; i < kRows; ++i) {
+        ASSERT_TRUE(list_builder.Append().ok());
+        ASSERT_TRUE(map_builder.Append().ok());
+        for (int32_t j = 0; j < kEntries; ++j) {
+            ASSERT_TRUE(list_values->Append(i + j).ok());
+            ASSERT_TRUE(key_builder->Append("key_" + std::to_string(j)).ok());
+            ASSERT_TRUE(item_builder->Append(i + j).ok());
+        }
+    }
+    std::shared_ptr<arrow::Array> tags;
+    std::shared_ptr<arrow::Array> attrs;
+    ASSERT_TRUE(list_builder.Finish(&tags).ok());
+    ASSERT_TRUE(map_builder.Finish(&attrs).ok());
+
+    std::shared_ptr<arrow::Schema> schema =
+        arrow::schema({MakeField("tags", arrow::list(arrow::int64()), 0),
+                       MakeField("attrs", arrow::map(arrow::utf8(), arrow::int64()), 1)});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> batch, Wrap(schema, {tags, attrs}));
+    const std::string path = PathOf("nested.parquet");
+    ASSERT_OK(Write(path, schema, batch, "zstd"));
+
+    ASSERT_OK_AND_ASSIGN(ReadResult result, Read(path, schema));
+    EXPECT_EQ(kRows, result.rows);
+    ASSERT_TRUE(result.data);
+    EXPECT_TRUE(result.data->Equals(*batch));
+
+    // Each nested column also has to be readable on its own, which is what the projected nested
+    // cases do.
+    for (const std::string& column : {"tags", "attrs"}) {
+        std::shared_ptr<arrow::Schema> projected = arrow::schema({schema->GetFieldByName(column)});
+        ASSERT_OK_AND_ASSIGN(ReadResult projected_result, Read(path, projected));
+        EXPECT_EQ(kRows, projected_result.rows) << "column " << column;
+    }
+}
+
+// The filtered and skip-heavy cases assert on row counts, so those counts have to mean what the
+// benchmark assumes: a predicate keeps at least the matching rows and no more than the row groups
+// that could hold them, and a selection bitmap keeps at least the rows it selected.
+TEST_F(ParquetFormatBenchmarkTest, FilteredAndBitmapRowCounts) {
+    std::shared_ptr<arrow::Schema> schema = arrow::schema({MakeField("id", arrow::int64(), 0)});
+    arrow::Int64Builder builder;
+    ASSERT_TRUE(builder.Reserve(kRows).ok());
+    for (int64_t i = 0; i < kRows; ++i) {
+        builder.UnsafeAppend(i);
+    }
+    std::shared_ptr<arrow::Array> ids;
+    ASSERT_TRUE(builder.Finish(&ids).ok());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> batch, Wrap(schema, {ids}));
+    const std::string path = PathOf("filtered.parquet");
+    ASSERT_OK(Write(path, schema, batch, "zstd"));
+
+    ASSERT_OK_AND_ASSIGN(ReadResult full, Read(path, schema));
+    EXPECT_EQ(kRows, full.rows);
+    EXPECT_EQ(static_cast<uint64_t>(kRows / kRowGroupLength), full.row_groups_total);
+    EXPECT_EQ(full.row_groups_total, full.row_groups_after_filter);
+    EXPECT_GT(full.batches, 0u);
+
+    // `id` is ordered, so row-group statistics alone must discard everything past the threshold.
+    // This is the bound BM_ParquetRead_Filtered asserts on.
+    constexpr int64_t kThreshold = 300;
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::LessThan(
+        /*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT, Literal(kThreshold));
+    ASSERT_OK_AND_ASSIGN(ReadResult filtered, Read(path, schema, predicate));
+    EXPECT_GE(filtered.rows, kThreshold);
+    EXPECT_LE(filtered.rows, kRowGroupLength);
+    EXPECT_LT(filtered.row_groups_after_filter, filtered.row_groups_total);
+
+    // Selection is not precise, so the bitmap is a lower bound on what comes back.
+    RoaringBitmap32 bitmap;
+    int64_t selected = 0;
+    for (int64_t row = 0; row < kRows; row += 64) {
+        bitmap.Add(static_cast<int32_t>(row));
+        ++selected;
+    }
+    ASSERT_OK_AND_ASSIGN(ReadResult skipped, Read(path, schema, /*predicate=*/nullptr, bitmap));
+    EXPECT_GE(skipped.rows, selected);
+    EXPECT_LE(skipped.rows, kRows);
+}
+
+// The encoding case compares a dictionary-encoded file against a plain one, which is only a
+// comparison if both files read back identically.
+TEST_F(ParquetFormatBenchmarkTest, PlainAndDictionaryFilesAgree) {
+    std::shared_ptr<arrow::Schema> schema = arrow::schema({MakeField("name", arrow::utf8(), 0)});
+    arrow::StringBuilder builder;
+    ASSERT_TRUE(builder.Reserve(kRows).ok());
+    for (int64_t i = 0; i < kRows; ++i) {
+        ASSERT_TRUE(builder.Append("value_" + std::to_string(i % 16)).ok());
+    }
+    std::shared_ptr<arrow::Array> names;
+    ASSERT_TRUE(builder.Finish(&names).ok());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> batch, Wrap(schema, {names}));
+
+    const std::string dict_path = PathOf("encoding_dict.parquet");
+    const std::string plain_path = PathOf("encoding_plain.parquet");
+    ASSERT_OK(Write(dict_path, schema, batch, "zstd"));
+    std::map<std::string, std::string> plain_options;
+    plain_options[PARQUET_ENABLE_DICTIONARY] = "false";
+    ASSERT_OK(Write(plain_path, schema, batch, "zstd", plain_options));
+
+    ASSERT_OK_AND_ASSIGN(ReadResult dict_result, Read(dict_path, schema));
+    ASSERT_OK_AND_ASSIGN(ReadResult plain_result, Read(plain_path, schema));
+    EXPECT_EQ(kRows, dict_result.rows);
+    EXPECT_EQ(dict_result.rows, plain_result.rows);
+    EXPECT_EQ(dict_result.row_groups_total, plain_result.row_groups_total);
+    ASSERT_TRUE(dict_result.data);
+    ASSERT_TRUE(plain_result.data);
+    EXPECT_TRUE(dict_result.data->Equals(*batch));
+    EXPECT_TRUE(plain_result.data->Equals(*batch));
+}
+
+// BM_ParquetWrite_MemoryThreshold rests on one assumption: that a small
+// parquet.writer.max.memory.use actually makes ParquetFormatWriter cut extra row groups. This
+// covers the mechanism on both input shapes that case can present it with, plain and
+// dictionary-encoded, with the row-count limit raised out of the way so the byte threshold is the
+// only thing that can flush.
+//
+// It does not cover the benchmark's own 512 KiB setting: reaching that with cardinality-10
+// dictionary data takes the 500K to 2M rows the benchmark writes, which is not a unit test. What
+// confirms that setting is the benchmark's own row_groups counter reading more than 1.
+TEST_F(ParquetFormatBenchmarkTest, MemoryThresholdFlushesRowGroups) {
+    constexpr int32_t kBatches = 8;
+    constexpr int64_t kDictCardinality = 10;
+
+    arrow::StringBuilder plain_builder;
+    ASSERT_TRUE(plain_builder.Reserve(kRows).ok());
+    for (int64_t i = 0; i < kRows; ++i) {
+        ASSERT_TRUE(plain_builder.Append("value_" + std::to_string(i)).ok());
+    }
+    std::shared_ptr<arrow::Array> plain_column;
+    ASSERT_TRUE(plain_builder.Finish(&plain_column).ok());
+
+    // The shape BM_ParquetWrite_MemoryThreshold writes: low-cardinality dictionary input.
+    arrow::StringBuilder dict_values;
+    for (int64_t i = 0; i < kDictCardinality; ++i) {
+        ASSERT_TRUE(dict_values.Append("value_" + std::to_string(i)).ok());
+    }
+    std::shared_ptr<arrow::Array> dict_value_column;
+    ASSERT_TRUE(dict_values.Finish(&dict_value_column).ok());
+    arrow::Int32Builder dict_indices;
+    ASSERT_TRUE(dict_indices.Reserve(kRows).ok());
+    for (int64_t i = 0; i < kRows; ++i) {
+        dict_indices.UnsafeAppend(static_cast<int32_t>(i % kDictCardinality));
+    }
+    std::shared_ptr<arrow::Array> index_column;
+    ASSERT_TRUE(dict_indices.Finish(&index_column).ok());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> dict_column,
+                         MakeDictionary(index_column, dict_value_column));
+
+    struct Case {
+        const char* name;
+        std::shared_ptr<arrow::Array> column;
+    };
+    for (const Case& c : {Case{"plain", plain_column}, Case{"dictionary", dict_column}}) {
+        std::shared_ptr<arrow::Schema> write_schema =
+            arrow::schema({MakeField("name", c.column->type(), 0)});
+        // Parquet stores a dictionary column as its value type, so both read back as UTF8.
+        std::shared_ptr<arrow::Schema> read_schema =
+            arrow::schema({MakeField("name", arrow::utf8(), 0)});
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> batch, Wrap(write_schema, {c.column}));
+
+        std::map<std::string, std::string> options;
+        options[PARQUET_WRITE_MAX_ROW_GROUP_LENGTH] = std::to_string(kRows * kBatches * 10);
+
+        options[PARQUET_WRITER_MAX_MEMORY_USE] = std::to_string(uint64_t{8} * 1024);
+        const std::string small_path =
+            PathOf(std::string("threshold_small_") + c.name + ".parquet");
+        ASSERT_OK(Write(small_path, write_schema, batch, "zstd", options, kBatches)) << c.name;
+        ASSERT_OK_AND_ASSIGN(ReadResult small, Read(small_path, read_schema));
+        EXPECT_EQ(kRows * kBatches, small.rows) << c.name;
+        EXPECT_GT(small.row_groups_total, 1u)
+            << c.name << ": byte threshold never triggered a flush";
+
+        options[PARQUET_WRITER_MAX_MEMORY_USE] =
+            std::to_string(DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE);
+        const std::string large_path =
+            PathOf(std::string("threshold_large_") + c.name + ".parquet");
+        ASSERT_OK(Write(large_path, write_schema, batch, "zstd", options, kBatches)) << c.name;
+        ASSERT_OK_AND_ASSIGN(ReadResult large, Read(large_path, read_schema));
+        EXPECT_EQ(kRows * kBatches, large.rows) << c.name;
+        EXPECT_EQ(1u, large.row_groups_total) << c.name;
+
+        // Row-group boundaries must not change the data.
+        ASSERT_TRUE(small.data) << c.name;
+        ASSERT_TRUE(large.data) << c.name;
+        EXPECT_TRUE(small.data->Equals(*large.data)) << c.name;
+    }
+}
+
+}  // namespace
+}  // namespace paimon::parquet
diff --git a/docs/source/examples/benchmark.rst b/docs/source/examples/benchmark.rst
index d2ff1cdef..b82114eea 100644
--- a/docs/source/examples/benchmark.rst
+++ b/docs/source/examples/benchmark.rst
@@ -19,8 +19,17 @@
 Benchmark Usage
 ================
 
-Paimon C++ provides Google Benchmark based cases for append-table write/read and
-primary-key table write/MOR read paths. Benchmarks are disabled by default.
+Paimon C++ provides Google Benchmark based cases at two levels:
+
+``paimon-read-write-benchmark``
+   Table-level cases for append-table write/read and primary-key table write/MOR
+   read paths.
+
+``paimon-parquet-format-benchmark``
+   Format-level cases that drive the Parquet writer and reader directly, without
+   catalog lookup, split planning, merge/sort or commit.
+
+Benchmarks are disabled by default.
 
 Build
 =====
@@ -29,13 +38,14 @@ Enable benchmarks when configuring CMake::
 
    cmake -S . -B build -DPAIMON_BUILD_BENCHMARKS=ON
    cmake --build build --target paimon-read-write-benchmark
+   cmake --build build --target paimon-parquet-format-benchmark
 
 Run all benchmark cases through CTest::
 
    cmake --build build --target benchmark
 
-Custom Options
-==============
+Table-level Custom Options
+==========================
 
 ``paimon-read-write-benchmark`` accepts Google Benchmark options plus the Paimon
 specific options below:
@@ -88,3 +98,97 @@ MOR read from an existing table::
        --paimon_source_table_path /path/table \
        --paimon_pk_columns=id \
        --benchmark_filter=BM_MOR_Read/4
+
+Parquet Format Benchmark
+========================
+
+``paimon-parquet-format-benchmark`` takes only Google Benchmark options. It
+generates its own data and writes it to a temporary directory, so it needs no
+source file or table.
+
+Two things shape how the results should be read:
+
+- Every axis is swept on its own rather than as a combined matrix, so each
+  case answers one question and a change can be attributed to it.
+- Writes go through the local FileSystem into a temporary directory, so
+  absolute numbers carry the cost of that path. Comparisons are meaningful
+  only under the same environment and methodology - the same machine, build
+  configuration and options - which is what makes a before/after comparison
+  useful.
+
+Writer cases (``BM_ParquetWrite_*``) cover flat primitives, ``VARCHAR`` at low /
+medium / high cardinality with and without file-level dictionary encoding,
+already dictionary-encoded ``VARCHAR`` / ``INTEGER`` input arrays against their
+flat equivalents, ``DECIMAL`` at precision 9 / 18 / 38, nested ``STRUCT`` /
+``LIST`` / ``VECTOR`` / ``MAP``, null density from 0 to 100 percent, rows per
+``AddBatch`` call, column count at a fixed row count, row group size, the
+writer memory threshold that triggers a byte-based row-group flush, and the
+codecs Parquet accepts - ``none``, ``snappy``, ``gzip``, ``brotli``, ``zstd``,
+``lz4_raw`` and ``lz4_hadoop``. Note that ``lz4`` is deliberately not among
+them: it resolves to Arrow's ``LZ4_FRAME``, which
+``parquet::IsCodecSupported`` rejects.
+
+The two dictionary axes are different questions. ``BM_ParquetWrite_String`` and
+``BM_ParquetWrite_StringNoDictionary`` vary whether the *file* is dictionary
+encoded; ``BM_ParquetWrite_Dictionary*`` vary whether the *input array* already
+is, which is what decides whether Arrow can pass indices through to Parquet or
+has to materialize them first.
+
+Reader cases (``BM_ParquetRead_*``) cover full scan, single-column projection,
+predicate-filtered reads at varying selectivity with page-index filtering on and
+off, skip-heavy reads driven by a strided selection bitmap, null density,
+``DECIMAL`` at precision 9 / 18 / 38, ``DOUBLE``, dictionary-encoded against
+plain-encoded files, rows per ``NextBatch`` call, and nested column reads.
+
+Every case reports ``ns_per_row`` next to ``bytes_per_row`` - ``file_bytes`` for
+writes, ``read_bytes`` for reads - so a change that trades CPU for size is
+visible in both directions. Read cases additionally report ``rows_read``,
+``batches``, and ``row_groups`` / ``row_groups_after_filter`` from the reader's
+own metrics.
+
+Compare filtered cases on ``ns_per_input_row`` and ``bytes_per_input_row``, not
+``ns_per_row`` and ``bytes_per_row``. The latter pair divides by the rows a case
+actually materialized, so pruning shrinks numerator and denominator together and
+they can rise even as the run gets faster; the ``_input_row`` pair divides by the
+rows the file holds, which every setting shares.
+
+``row_groups_after_filter`` counts row groups only. It does not show page-level
+pruning: on the ordered ``id`` column both page-index settings usually keep the
+same row groups, and the page-index gain shows up in ``rows_read``,
+``read_bytes`` and ``ns_per_input_row`` instead.
+
+A case that cannot run - an unsupported codec, a schema the reader rejects -
+calls ``SkipWithError`` and makes the process exit non-zero, so ``ctest -L
+benchmark`` fails instead of reporting a silent skip. Read cases also assert on
+the number of rows they materialized, so a fixture that stopped producing rows
+fails rather than looking fast.
+
+Because the benchmark is only compiled under ``PAIMON_BUILD_BENCHMARKS``, the
+format-layer assumptions it relies on are covered separately by
+``paimon-parquet-format-benchmark-test``, which builds with the normal test
+suite.
+
+Each read case scans a file that is generated once on first use and reused for
+the rest of the run, so a filtered run only pays to build the fixtures its own
+cases need.
+
+All Parquet writer cases::
+
+   paimon-parquet-format-benchmark --benchmark_filter=BM_ParquetWrite
+
+Page-index filtering at 1% selectivity, on and off - compare ``rows_read``,
+``read_bytes`` and ``ns_per_input_row`` between the two::
+
+   paimon-parquet-format-benchmark \
+       --benchmark_filter='BM_ParquetRead_Filtered/keep_pct:1/'
+
+Read batch size sweep, repeated for a stable comparison::
+
+   paimon-parquet-format-benchmark \
+       --benchmark_filter=BM_ParquetRead_BatchSize \
+       --benchmark_repetitions=5 \
+       --benchmark_report_aggregates_only=true
+
+Null density on both sides, to see what definition levels cost::
+
+   paimon-parquet-format-benchmark --benchmark_filter='Parquet(Write|Read)_Nulls'

From d76c2a6f9ae93ee26cd5ebbecc9adaaf451a4f94 Mon Sep 17 00:00:00 2001
From: lszskye <57179283+lszskye@users.noreply.github.com>
Date: Wed, 26 Aug 2026 06:08:42 -0700
Subject: [PATCH 22/93] fix(compaction): persist keys in lookup SST files while
 first-row merge function (#250)

---
 ...ookup_merge_tree_compact_rewriter_test.cpp | 39 +++++++++++++++++++
 src/paimon/core/mergetree/lookup_levels.cpp   | 14 +------
 2 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
index bbc1608c4..eeb7e0ce2 100644
--- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
+++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
@@ -484,6 +484,45 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowRewrite) {
     CheckResult(compact_file_name, table_schema, "orc", expected_array);
 }
 
+TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowLooksUpExistingKeys) {
+    std::map<std::string, std::string> options = {{Options::MERGE_ENGINE, "first-row"},
+                                                  {Options::FILE_FORMAT, "orc"}};
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options));
+    ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options));
+    auto schema_manager = std::make_shared<SchemaManager>(fs_, table_path);
+    ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager->ReadSchema(0));
+
+    ASSERT_OK_AND_ASSIGN(auto level0_file,
+                         NewFiles(/*level=*/0, /*last_sequence_number=*/0, table_path, core_options,
+                                  "[[1, 111], [2, 22]]"));
+    ASSERT_OK_AND_ASSIGN(auto high_level_file, NewFiles(/*level=*/2, /*last_sequence_number=*/-1,
+                                                        table_path, core_options, "[[1, 11]]"));
+    auto processor_factory = std::make_shared<PersistEmptyProcessor::Factory>();
+    ASSERT_OK_AND_ASSIGN(auto lookup_levels,
+                         CreateLookupLevels<bool>(table_path, table_schema, processor_factory,
+                                                  std::vector<std::shared_ptr<DataFileMeta>>{
+                                                      level0_file, high_level_file}));
+    ASSERT_OK_AND_ASSIGN(auto rewriter,
+                         CreateCompactRewriterForFirstRow(table_path, table_schema, core_options,
+                                                          std::move(lookup_levels)));
+    ASSERT_OK_AND_ASSIGN(
+        auto runs, GenerateSortedRuns(std::vector<std::shared_ptr<DataFileMeta>>{level0_file}));
+    ASSERT_OK_AND_ASSIGN(auto compact_result, rewriter->Rewrite(
+                                                  /*output_level=*/1, /*drop_delete=*/true, runs));
+
+    ASSERT_EQ(1, compact_result.After().size());
+    ASSERT_EQ(1, compact_result.After()[0]->row_count);
+
+    auto type_with_special_fields =
+        arrow::struct_(SpecialFields::CompleteSequenceAndValueKindField(arrow_schema_)->fields());
+    std::shared_ptr<arrow::ChunkedArray> expected;
+    ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(type_with_special_fields,
+                                                                 {"[[2, 0, 2, 22]]"}, &expected)
+                    .ok());
+    CheckResult(table_path + "/bucket-0/" + compact_result.After()[0]->file_name, table_schema,
+                "orc", expected);
+}
+
 TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowUpgrade) {
     std::map<std::string, std::string> options = {{Options::MERGE_ENGINE, "first-row"},
                                                   {Options::FILE_FORMAT, "orc"}};
diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp
index 8b5f69e77..a931fea10 100644
--- a/src/paimon/core/mergetree/lookup_levels.cpp
+++ b/src/paimon/core/mergetree/lookup_levels.cpp
@@ -162,8 +162,8 @@ LookupLevels<T>::LookupLevels(
       lookup_store_factory_(lookup_store_factory),
       lookup_file_cache_(lookup_file_cache),
       remote_lookup_file_manager_(remote_lookup_file_manager) {
-    if constexpr (std::is_same_v<T, FilePosition>) {
-        // if T is FilePosition, only read key fields to create sst file is enough
+    if constexpr (std::is_same_v<T, FilePosition> || std::is_same_v<T, bool>) {
+        // FilePosition and first-row lookup do not persist values, so reading key fields is enough.
         value_schema_ = key_schema_;
     } else {
         value_schema_ = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
@@ -332,16 +332,6 @@ std::optional<std::string> LookupLevels<T>::TryToDownloadRemoteSst(
 template <typename T>
 Status LookupLevels<T>::CreateSstFileFromDataFile(const std::shared_ptr<DataFileMeta>& file,
                                                   const std::string& kv_file_path) {
-    if constexpr (std::is_same_v<T, bool>) {
-        // Short-circuit logic: if T is bool, just write empty lookup file.
-        PAIMON_ASSIGN_OR_RAISE(
-            std::shared_ptr<BloomFilter> bloom_filter,
-            LookupStoreFactory::BfGenerator(file->row_count, options_, pool_.get()));
-        PAIMON_ASSIGN_OR_RAISE(
-            std::unique_ptr<LookupStoreWriter> kv_writer,
-            lookup_store_factory_->CreateWriter(fs_, kv_file_path, bloom_filter, pool_));
-        return kv_writer->Close();
-    }
     // Prepare reader to iterate KeyValue
     PAIMON_ASSIGN_OR_RAISE(
         std::vector<std::unique_ptr<FileBatchReader>> raw_readers,

From 50d4207e439723ed1888a3b5d5846a1cf9a50967 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 19 Aug 2026 22:34:33 +0800
Subject: [PATCH 23/93] feat(realtime): add primary-key in-memory writes

Add typed primary-key store creation, an in-memory PK store, and a no-spill writer that materializes sealed mutations through MergeTreeWriter.

Keep writer-local compaction disabled, preserve sequence progress across sequential writer handoff, and reject unsupported V1 table options.
---
 .../realtime/arrow_realtime_store_factory.h   |   7 +-
 include/paimon/realtime/realtime_store.h      |  47 +-
 src/paimon/CMakeLists.txt                     |   5 +
 .../core/operation/file_store_write.cpp       |  25 +-
 .../operation/key_value_file_store_write.cpp  |  77 ++-
 .../operation/key_value_file_store_write.h    |   8 +
 .../key_value_file_store_write_test.cpp       |  48 ++
 .../realtime/arrow_realtime_store_factory.cpp |  55 +-
 .../realtime/primary_key_realtime_options.cpp |  58 ++
 .../realtime/primary_key_realtime_options.h   |  31 +
 .../primary_key_realtime_options_test.cpp     |  56 ++
 .../realtime/primary_key_realtime_store.cpp   | 563 ++++++++++++++++++
 .../realtime/primary_key_realtime_store.h     |  84 +++
 .../primary_key_realtime_store_test.cpp       | 244 ++++++++
 .../realtime/realtime_append_only_writer.cpp  |  11 +-
 .../core/realtime/realtime_context_impl.cpp   |  67 ++-
 .../core/realtime/realtime_context_impl.h     |  18 +-
 .../core/realtime/realtime_context_test.cpp   | 126 +---
 .../realtime/realtime_primary_key_writer.cpp  | 249 ++++++++
 .../realtime/realtime_primary_key_writer.h    |  89 +++
 20 files changed, 1690 insertions(+), 178 deletions(-)
 create mode 100644 src/paimon/core/realtime/primary_key_realtime_options.cpp
 create mode 100644 src/paimon/core/realtime/primary_key_realtime_options.h
 create mode 100644 src/paimon/core/realtime/primary_key_realtime_options_test.cpp
 create mode 100644 src/paimon/core/realtime/primary_key_realtime_store.cpp
 create mode 100644 src/paimon/core/realtime/primary_key_realtime_store.h
 create mode 100644 src/paimon/core/realtime/primary_key_realtime_store_test.cpp
 create mode 100644 src/paimon/core/realtime/realtime_primary_key_writer.cpp
 create mode 100644 src/paimon/core/realtime/realtime_primary_key_writer.h

diff --git a/include/paimon/realtime/arrow_realtime_store_factory.h b/include/paimon/realtime/arrow_realtime_store_factory.h
index 4d65743ab..da1b8de36 100644
--- a/include/paimon/realtime/arrow_realtime_store_factory.h
+++ b/include/paimon/realtime/arrow_realtime_store_factory.h
@@ -26,11 +26,8 @@ namespace paimon {
 /// Factory for Paimon's default Arrow-backed `RealtimeStore`.
 class PAIMON_EXPORT ArrowRealtimeStoreFactory : public RealtimeStoreFactory {
  public:
-    /// Creates an Arrow-backed store for one partition and bucket.
-    Result<std::shared_ptr<RealtimeStore>> Create(
-        std::unique_ptr<::ArrowSchema> write_schema, StatisticsMode statistics_mode,
-        const std::map<std::string, std::string>& options,
-        const std::shared_ptr<MemoryPool>& memory_pool) override;
+    /// Creates the built-in append or in-memory primary-key store.
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override;
 };
 
 }  // namespace paimon
diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index d02952acd..1e53c173e 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -25,8 +25,10 @@
 #include <optional>
 #include <string>
 #include <utility>
+#include <variant>
 #include <vector>
 
+#include "arrow/c/abi.h"
 #include "paimon/reader/batch_reader.h"
 #include "paimon/realtime/offset_range.h"
 #include "paimon/record_batch.h"
@@ -41,6 +43,31 @@ namespace paimon {
 class MemoryPool;
 class Predicate;
 
+struct PAIMON_EXPORT AppendRealtimeStoreCreateConfig {
+    StatisticsMode statistics_mode;
+};
+
+struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {
+    std::vector<std::string> primary_keys;
+    /// Largest sequence restored from the committed snapshot. A PK store assigns one contiguous
+    /// sequence to every mutation in `Write` order, starting at the next value, and rejects
+    /// `Write` before the assigned sequence would exceed `INT64_MAX - 1`.
+    int64_t restore_max_sequence_number;
+};
+
+using RealtimeStoreCreateConfig =
+    std::variant<AppendRealtimeStoreCreateConfig, PrimaryKeyRealtimeStoreCreateConfig>;
+
+struct PAIMON_EXPORT RealtimeStoreCreateRequest {
+    /// Complete table write schema whose ownership is transferred to the factory.
+    std::unique_ptr<::ArrowSchema> write_schema;
+    std::map<std::string, std::string> options;
+    std::shared_ptr<MemoryPool> memory_pool;
+    std::map<std::string, std::string> partition;
+    int32_t bucket = -1;
+    RealtimeStoreCreateConfig mode_config;
+};
+
 /// A table record batch and its framework-assigned contiguous offset range.
 ///
 /// The batch contains only table write fields. Row `i` is associated with
@@ -133,8 +160,11 @@ class PAIMON_EXPORT RealtimeStore {
     ///
     /// Each output batch contains `_VALUE_KIND` first, followed by the fields requested by
     /// `context.read_schema` except a duplicate `_VALUE_KIND`. Concatenating all returned readers
-    /// must produce every matching row once. Paimon retains `view` for the lifetime of the
-    /// resulting framework reader.
+    /// must produce every matching row once. Primary-key readers additionally provide a non-null
+    /// `_SEQUENCE_NUMBER` when requested, are individually sorted by primary key, and contain at
+    /// most one mutation per key. Assigned sequences remain stable across views and queries;
+    /// readers need not be globally sorted with one another. Paimon retains `view` for the lifetime
+    /// of the resulting framework reader.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) = 0;
@@ -157,16 +187,9 @@ class PAIMON_EXPORT RealtimeStoreFactory {
  public:
     virtual ~RealtimeStoreFactory() = default;
 
-    /// Creates a store configured with the supplied schema, statistics, options, and memory pool.
-    /// @param write_schema Complete table write schema whose ownership is transferred to the
-    /// factory. The factory may consume it or retain it in the created store.
-    /// @param statistics_mode Framework-parsed statistics collection mode.
-    /// @param options Effective table options available to the store.
-    /// @param memory_pool Memory pool provided by the write context.
-    virtual Result<std::shared_ptr<RealtimeStore>> Create(
-        std::unique_ptr<::ArrowSchema> write_schema, StatisticsMode statistics_mode,
-        const std::map<std::string, std::string>& options,
-        const std::shared_ptr<MemoryPool>& memory_pool) = 0;
+    /// Creates a store for the requested table mode and partition-bucket.
+    /// The factory consumes `request`, including ownership of `request.write_schema`.
+    virtual Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) = 0;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 61fd7c96d..ea6159821 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -382,9 +382,12 @@ set(PAIMON_CORE_SRCS
     core/operation/write_restore.cpp
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
+    core/realtime/primary_key_realtime_store.cpp
+    core/realtime/primary_key_realtime_options.cpp
     core/realtime/realtime_append_only_writer.cpp
     core/realtime/realtime_context.cpp
     core/realtime/realtime_context_impl.cpp
+    core/realtime/realtime_primary_key_writer.cpp
     core/postpone/postpone_bucket_writer.cpp
     core/schema/arrow_schema_validator.cpp
     core/schema/schema_manager.cpp
@@ -787,6 +790,8 @@ if(PAIMON_BUILD_TESTS)
                     core/manifest/index_manifest_file_handler_test.cpp
                     core/memory/writer_memory_manager_test.cpp
                     core/realtime/arrow_realtime_store_test.cpp
+                    core/realtime/primary_key_realtime_store_test.cpp
+                    core/realtime/primary_key_realtime_options_test.cpp
                     core/realtime/realtime_context_test.cpp
                     core/realtime/realtime_reader_test.cpp
                     core/mergetree/levels_test.cpp
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 6807ae35e..fb83c254c 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -36,6 +36,7 @@
 #include "paimon/core/operation/key_value_file_store_write.h"
 #include "paimon/core/options/merge_engine.h"
 #include "paimon/core/postpone/postpone_bucket_file_store_write.h"
+#include "paimon/core/realtime/primary_key_realtime_options.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
@@ -197,7 +198,26 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     } else {
         // pk table
         if (ctx->GetRealtimeContext()) {
-            return Status::Invalid("real-time write currently supports append tables only");
+            PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(options));
+            if (ignore_previous_files) {
+                return Status::NotImplemented(
+                    "PK realtime v1 requires restore from the latest snapshot");
+            }
+            if (!ctx->GetWriteSchema().empty()) {
+                return Status::NotImplemented(
+                    "PK realtime v1 does not support a custom write schema");
+            }
+            PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
+                                   snapshot_manager->LatestSnapshot());
+            if (latest_snapshot) {
+                PAIMON_ASSIGN_OR_RAISE(RealtimeOffsetMap realtime_committed_offsets,
+                                       RealtimeCommitProperties::ReadOffsets(
+                                           latest_snapshot, options.GetFileSystem()));
+                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                                       RealtimeContextImpl::Cast(ctx->GetRealtimeContext()));
+                PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress(
+                    latest_snapshot->Id(), realtime_committed_offsets));
+            }
         }
         if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) {
             return PostponeBucketFileStoreWrite::Create(
@@ -253,7 +273,8 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
             ctx->GetRootPath(), schema, arrow_schema, partition_schema, dv_maintainer_factory,
             io_manager, key_comparator, sequence_fields_comparator, merge_function_wrapper, options,
             ignore_previous_files, ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(),
-            ctx->EnableMultiThreadSpill(), ctx->GetExecutor(), ctx->GetMemoryPool());
+            ctx->EnableMultiThreadSpill(), ctx->GetRealtimeContext(), ctx->GetExecutor(),
+            ctx->GetMemoryPool());
     }
 }
 
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 08c5ea0c3..4456ee1c2 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -18,21 +18,29 @@
 
 #include "paimon/core/operation/key_value_file_store_write.h"
 
+#include <limits>
+#include <optional>
 #include <vector>
 
+#include "arrow/c/bridge.h"
 #include "paimon/common/data/binary_row.h"
+#include "paimon/core/compact/noop_compact_manager.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/manifest/manifest_file.h"
 #include "paimon/core/manifest/manifest_list.h"
 #include "paimon/core/mergetree/levels.h"
 #include "paimon/core/mergetree/merge_tree_writer.h"
+#include "paimon/core/operation/commit/realtime_commit_properties.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
+#include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_primary_key_writer.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/primary_key_table_utils.h"
 #include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/realtime/realtime_context.h"
 
 namespace arrow {
 class Schema;
@@ -60,6 +68,7 @@ KeyValueFileStoreWrite::KeyValueFileStoreWrite(
     const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper,
     const CoreOptions& options, bool ignore_previous_files, bool is_streaming_mode,
     bool ignore_num_bucket_check, bool enable_multi_thread_spill,
+    const std::shared_ptr<RealtimeContext>& realtime_context,
     const std::shared_ptr<Executor>& executor, const std::shared_ptr<MemoryPool>& pool)
     : AbstractFileStoreWrite(file_store_path_factory, snapshot_manager, schema_manager, commit_user,
                              root_path, table_schema, schema, /*write_schema=*/schema,
@@ -67,6 +76,7 @@ KeyValueFileStoreWrite::KeyValueFileStoreWrite(
                              ignore_previous_files, is_streaming_mode, ignore_num_bucket_check,
                              executor, pool),
       enable_multi_thread_spill_(enable_multi_thread_spill),
+      realtime_context_(realtime_context),
       key_comparator_(key_comparator),
       user_defined_seq_comparator_(user_defined_seq_comparator),
       merge_function_wrapper_(merge_function_wrapper),
@@ -74,7 +84,25 @@ KeyValueFileStoreWrite::KeyValueFileStoreWrite(
           options_, key_comparator_, user_defined_seq_comparator_, compaction_metrics_,
           table_schema_, schema_, schema_manager_, io_manager_, cache_manager_,
           file_store_path_factory_, root_path_, pool_)),
-      logger_(Logger::GetLogger("KeyValueFileStoreWrite")) {}
+      logger_(Logger::GetLogger("KeyValueFileStoreWrite")) {
+    if (realtime_context_) {
+        writer_memory_manager_ = std::make_unique<NoopWriterMemoryManager>();
+    }
+}
+
+Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
+    if (!realtime_context_) {
+        return Status::Invalid("refresh committed snapshot requires a real-time writer");
+    }
+    PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, snapshot_manager_->LoadSnapshot(snapshot_id));
+    PAIMON_ASSIGN_OR_RAISE(
+        RealtimeOffsetMap committed_offsets,
+        RealtimeCommitProperties::ReadOffsets(std::optional<Snapshot>(std::move(snapshot)),
+                                              options_.GetFileSystem()));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                           RealtimeContextImpl::Cast(realtime_context_));
+    return realtime_context_impl->AdvanceCommittedProgress(snapshot_id, committed_offsets);
+}
 
 Result<std::unique_ptr<FileStoreScan>> KeyValueFileStoreWrite::CreateFileStoreScan(
     const std::shared_ptr<ScanFilter>& scan_filter) const {
@@ -109,19 +137,48 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr<Levels> levels,
         Levels::Create(key_comparator_, restore_data_files, options_.GetNumLevels()));
-    auto compact_strategy = compact_manager_factory_->CreateCompactStrategy();
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<CompactManager> compact_manager,
-        compact_manager_factory_->CreateCompactManager(partition, bucket, compact_strategy,
-                                                       compact_executor_, levels, dv_maintainer));
+    std::map<std::string, std::string> partition_map;
+    int64_t materialized_max_sequence_number = restore_max_seq_number;
+    if (realtime_context_) {
+        std::vector<std::pair<std::string, std::string>> partition_values;
+        PAIMON_ASSIGN_OR_RAISE(partition_values,
+                               file_store_path_factory_->GeneratePartitionVector(partition));
+        partition_map =
+            std::map<std::string, std::string>(partition_values.begin(), partition_values.end());
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                               RealtimeContextImpl::Cast(realtime_context_));
+        const RealtimePartitionBucket partition_bucket(partition_map, bucket);
+        materialized_max_sequence_number = realtime_context_impl->GetMaterializedMaxSequenceNumber(
+            partition_bucket, restore_max_seq_number);
+        if (materialized_max_sequence_number == std::numeric_limits<int64_t>::max()) {
+            return Status::Invalid("PK sequence number has reached INT64_MAX");
+        }
+    }
+    std::shared_ptr<CompactManager> compact_manager;
+    if (realtime_context_) {
+        compact_manager = std::make_shared<NoopCompactManager>();
+    } else {
+        auto compact_strategy = compact_manager_factory_->CreateCompactStrategy();
+        PAIMON_ASSIGN_OR_RAISE(compact_manager, compact_manager_factory_->CreateCompactManager(
+                                                    partition, bucket, compact_strategy,
+                                                    compact_executor_, levels, dv_maintainer));
+    }
 
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr<MergeTreeWriter> writer,
         MergeTreeWriter::Create(
-            restore_max_seq_number, trimmed_primary_keys, data_file_path_factory, key_comparator_,
-            user_defined_seq_comparator_, merge_function_wrapper_, table_schema_->Id(), schema_,
-            options_, compact_manager, io_manager_, enable_multi_thread_spill_, pool_));
-    return writer;
+            materialized_max_sequence_number, trimmed_primary_keys, data_file_path_factory,
+            key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_,
+            table_schema_->Id(), schema_, options_, compact_manager,
+            realtime_context_ ? nullptr : io_manager_, enable_multi_thread_spill_, pool_));
+    if (!realtime_context_) {
+        return std::shared_ptr<BatchWriter>(std::move(writer));
+    }
+    auto c_write_schema = std::make_unique<ArrowSchema>();
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, c_write_schema.get()));
+    return RealtimePrimaryKeyWriter::Create(
+        partition_map, bucket, std::move(c_write_schema), trimmed_primary_keys, realtime_context_,
+        writer, options_.ToMap(), pool_, materialized_max_sequence_number);
 }
 
 Status KeyValueFileStoreWrite::Close() {
diff --git a/src/paimon/core/operation/key_value_file_store_write.h b/src/paimon/core/operation/key_value_file_store_write.h
index 14457590f..66c362f2e 100644
--- a/src/paimon/core/operation/key_value_file_store_write.h
+++ b/src/paimon/core/operation/key_value_file_store_write.h
@@ -45,6 +45,7 @@ class SnapshotManager;
 class SchemaManager;
 class TableSchema;
 class IOManager;
+class RealtimeContext;
 struct KeyValue;
 template <typename T>
 class MergeFunctionWrapper;
@@ -65,8 +66,10 @@ class KeyValueFileStoreWrite : public AbstractFileStoreWrite {
         const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper,
         const CoreOptions& options, bool ignore_previous_files, bool is_streaming_mode,
         bool ignore_num_bucket_check, bool enable_multi_thread_spill,
+        const std::shared_ptr<RealtimeContext>& realtime_context,
         const std::shared_ptr<Executor>& executor, const std::shared_ptr<MemoryPool>& pool);
 
+    Status RefreshCommittedSnapshot(int64_t snapshot_id) override;
     Status Close() override;
 
  private:
@@ -79,8 +82,13 @@ class KeyValueFileStoreWrite : public AbstractFileStoreWrite {
     Result<std::unique_ptr<FileStoreScan>> CreateFileStoreScan(
         const std::shared_ptr<ScanFilter>& filter) const override;
 
+    bool IsRealtimeWrite() const override {
+        return realtime_context_ != nullptr;
+    }
+
  private:
     bool enable_multi_thread_spill_;
+    std::shared_ptr<RealtimeContext> realtime_context_;
     std::shared_ptr<FieldsComparator> key_comparator_;
     std::shared_ptr<FieldsComparator> user_defined_seq_comparator_;
     std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper_;
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index 35d938af7..45462ea6e 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -53,6 +53,7 @@
 #include "paimon/format/reader_builder.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/reader/file_batch_reader.h"
+#include "paimon/realtime/realtime_context.h"
 #include "paimon/record_batch.h"
 #include "paimon/status.h"
 #include "paimon/testing/utils/test_helper.h"
@@ -303,6 +304,53 @@ TEST_F(KeyValueFileStoreWriteTest, TestPrepareCommitShouldSucceedWhenLookupEnabl
     ASSERT_EQ(commit_messages.size(), 1);
 }
 
+TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
+    const std::map<std::string, std::string> options = {
+        {Options::BUCKET, "1"},
+        {Options::WRITE_BUFFER_SIZE, "1"},
+    };
+    const std::shared_ptr<arrow::Schema> schema = arrow::schema({
+        arrow::field("id", arrow::int64(), /*nullable=*/false),
+        arrow::field("value", arrow::utf8()),
+    });
+    std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), schema, options);
+    const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    WriteContextBuilder builder(table_path, "test");
+    builder.SetOptions(options)
+        .WithStreamingMode(true)
+        .WithRealtimeContext(realtime_context)
+        .WithTempDirectory(dir->Str());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         FileStoreWrite::Create(std::move(write_context)));
+
+    ASSERT_OK(writer->Write(MakeBatch(schema, R"([
+        [1, "old"],
+        [2, "two"],
+        [1, "new"]
+    ])")));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progresses,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, progresses.size());
+    ASSERT_EQ(OffsetRange(0, 3), progresses[0].offset_range);
+    std::shared_ptr<CommitMessageImpl> commit_message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(progresses[0].commit_message);
+    ASSERT_NE(nullptr, commit_message);
+    int64_t row_count = 0;
+    for (const std::shared_ptr<DataFileMeta>& file :
+         commit_message->GetNewFilesIncrement().NewFiles()) {
+        row_count += file->row_count;
+    }
+    ASSERT_EQ(2, row_count);
+    ASSERT_EQ(0, TestHelper::CountChannelFiles(dir->GetFileSystem(), dir->Str()));
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(KeyValueFileStoreWriteTest,
        TestPrepareCommitShouldSucceedWhenDefaultCompactRewriterPathEnabled) {
     ASSERT_OK_AND_ASSIGN(
diff --git a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
index 1d7219c41..e6e22edfd 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
@@ -21,29 +21,66 @@
 
 #include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
+#include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
+#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/realtime/arrow_realtime_store.h"
+#include "paimon/core/realtime/primary_key_realtime_store.h"
 #include "paimon/macros.h"
 
 namespace paimon {
 
 Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
-    std::unique_ptr<ArrowSchema> write_schema, StatisticsMode statistics_mode,
-    const std::map<std::string, std::string>&, const std::shared_ptr<MemoryPool>& memory_pool) {
-    if (!write_schema || !write_schema->release) {
+    RealtimeStoreCreateRequest&& request) {
+    if (!request.write_schema || !request.write_schema->release) {
         return Status::Invalid("real-time store write schema is null");
     }
-    ScopeGuard schema_guard([schema = write_schema.get()]() { ArrowSchemaRelease(schema); });
-    if (!memory_pool) {
+    ScopeGuard schema_guard(
+        [schema = request.write_schema.get()]() { ArrowSchemaRelease(schema); });
+    if (!request.memory_pool) {
         return Status::Invalid("real-time store memory pool is null");
     }
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> imported_schema,
-                                      arrow::ImportSchema(write_schema.get()));
-    std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
-    return std::make_shared<ArrowRealtimeStore>(imported_schema, statistics_mode, memory_pool,
-                                                arrow_pool);
+                                      arrow::ImportSchema(request.write_schema.get()));
+    if (std::holds_alternative<AppendRealtimeStoreCreateConfig>(request.mode_config)) {
+        const AppendRealtimeStoreCreateConfig& append_config =
+            std::get<AppendRealtimeStoreCreateConfig>(request.mode_config);
+        std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(request.memory_pool);
+        return std::make_shared<ArrowRealtimeStore>(imported_schema, append_config.statistics_mode,
+                                                    request.memory_pool, arrow_pool);
+    }
+
+    const PrimaryKeyRealtimeStoreCreateConfig& primary_key_config =
+        std::get<PrimaryKeyRealtimeStoreCreateConfig>(request.mode_config);
+    std::vector<DataField> key_fields;
+    key_fields.reserve(primary_key_config.primary_keys.size());
+    for (const std::string& primary_key : primary_key_config.primary_keys) {
+        const int32_t field_index = imported_schema->GetFieldIndex(primary_key);
+        if (field_index < 0) {
+            return Status::Invalid("primary key ", primary_key, " is missing from write schema");
+        }
+        key_fields.emplace_back(field_index, imported_schema->field(field_index));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
+                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
+    auto merge_function_wrapper_factory = []() {
+        auto merge_function = std::make_unique<DeduplicateMergeFunction>(
+            /*ignore_delete=*/false);
+        return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
+    };
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(request.options));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<PrimaryKeyRealtimeStore> store,
+        PrimaryKeyRealtimeStore::Create(imported_schema, primary_key_config.primary_keys,
+                                        key_comparator, merge_function_wrapper_factory,
+                                        primary_key_config.restore_max_sequence_number,
+                                        core_options.GetReadBatchSize(), request.memory_pool));
+    return std::shared_ptr<RealtimeStore>(std::move(store));
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_options.cpp b/src/paimon/core/realtime/primary_key_realtime_options.cpp
new file mode 100644
index 000000000..e9779a59e
--- /dev/null
+++ b/src/paimon/core/realtime/primary_key_realtime_options.cpp
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/primary_key_realtime_options.h"
+
+#include "paimon/core/core_options.h"
+
+namespace paimon {
+
+Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options) {
+    if (options.GetBucket() <= 0) {
+        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
+    }
+    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
+        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
+    }
+    if (options.DataEvolutionEnabled()) {
+        return Status::NotImplemented("PK realtime v1 does not support data evolution");
+    }
+    if (!options.GetFieldsSequenceGroups().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
+    }
+    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
+        options.AggregationRemoveRecordOnDelete() ||
+        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
+        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
+    }
+    if (!options.GetSequenceField().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
+    }
+    if (!options.SequenceFieldSortOrderIsAscending()) {
+        return Status::NotImplemented(
+            "PK realtime v1 supports only ascending sequence.field.sort-order");
+    }
+    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
+        options.GetChangelogProducer() != ChangelogProducer::NONE) {
+        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
+    }
+    return Status::OK();
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_options.h b/src/paimon/core/realtime/primary_key_realtime_options.h
new file mode 100644
index 000000000..a16d35778
--- /dev/null
+++ b/src/paimon/core/realtime/primary_key_realtime_options.h
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include "paimon/status.h"
+
+namespace paimon {
+
+class CoreOptions;
+
+/// Validates the table options supported by the in-memory PK realtime V1 path.
+Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options);
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_options_test.cpp b/src/paimon/core/realtime/primary_key_realtime_options_test.cpp
new file mode 100644
index 000000000..5d3ea7f67
--- /dev/null
+++ b/src/paimon/core/realtime/primary_key_realtime_options_test.cpp
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/primary_key_realtime_options.h"
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "paimon/core/core_options.h"
+#include "paimon/defs.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+TEST(PrimaryKeyRealtimeOptionsTest, TestSupportedOptions) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_OK(ValidatePrimaryKeyRealtimeOptions(options));
+}
+
+TEST(PrimaryKeyRealtimeOptionsTest, TestUnsupportedOptions) {
+    const std::string sequence_group =
+        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
+    const std::vector<std::map<std::string, std::string>> unsupported_options = {
+        {{Options::BUCKET, "0"}},
+        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
+        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
+        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
+    };
+    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
+        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+        ASSERT_NOK(ValidatePrimaryKeyRealtimeOptions(options));
+    }
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
new file mode 100644
index 000000000..84afb97a4
--- /dev/null
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -0,0 +1,563 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/primary_key_realtime_store.h"
+
+#include <algorithm>
+#include <limits>
+#include <mutex>
+#include <utility>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "paimon/common/data/binary_row_writer.h"
+#include "paimon/common/data/columnar/columnar_row_ref.h"
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/types/row_kind.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/fields_comparator.h"
+#include "paimon/core/io/key_value_in_memory_record_reader.h"
+#include "paimon/core/io/key_value_projection_consumer.h"
+#include "paimon/core/io/key_value_projection_reader.h"
+#include "paimon/core/io/merged_key_value_record_reader.h"
+#include "paimon/core/key_value.h"
+#include "paimon/core/mergetree/compact/sort_merge_reader_with_loser_tree.h"
+#include "paimon/macros.h"
+
+namespace paimon {
+namespace {
+
+uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
+    uint64_t result = 0;
+    for (const std::shared_ptr<arrow::Buffer>& buffer : data->buffers) {
+        if (buffer) {
+            result += static_cast<uint64_t>(buffer->size());
+        }
+    }
+    for (const std::shared_ptr<arrow::ArrayData>& child : data->child_data) {
+        result += GetArrayMemoryUsage(child);
+    }
+    if (data->dictionary) {
+        result += GetArrayMemoryUsage(data->dictionary);
+    }
+    return result;
+}
+
+struct StoredBatch {
+    std::shared_ptr<arrow::StructArray> data;
+    std::vector<RecordBatch::RowKind> row_kinds;
+    OffsetRange offset_range;
+    int64_t first_sequence_number;
+    uint64_t memory_usage;
+};
+using BatchGroup = std::vector<std::shared_ptr<const StoredBatch>>;
+
+class Segment final : public RealtimeSegmentHandle {
+ public:
+    Segment(const OffsetRange& offset_range,
+            std::vector<std::shared_ptr<const StoredBatch>>&& batches)
+        : offset_range_(offset_range), batches_(std::move(batches)) {}
+
+    OffsetRange GetOffsetRange() const override {
+        return offset_range_;
+    }
+
+    const std::vector<std::shared_ptr<const StoredBatch>>& Batches() const {
+        return batches_;
+    }
+
+    uint64_t GetMemoryUsage() const {
+        uint64_t result = 0;
+        for (const std::shared_ptr<const StoredBatch>& batch : batches_) {
+            result += batch->memory_usage;
+        }
+        return result;
+    }
+
+ private:
+    OffsetRange offset_range_;
+    std::vector<std::shared_ptr<const StoredBatch>> batches_;
+};
+
+class PrimaryKeyRealtimeReadView final : public RealtimeReadView {
+ public:
+    explicit PrimaryKeyRealtimeReadView(std::vector<BatchGroup>&& groups)
+        : groups_(std::move(groups)) {
+        if (!groups_.empty()) {
+            offset_range_ = OffsetRange(groups_.front().front()->offset_range.begin,
+                                        groups_.back().back()->offset_range.end);
+        }
+    }
+
+    std::optional<OffsetRange> GetOffsetRange() const override {
+        return offset_range_;
+    }
+
+    const std::vector<BatchGroup>& Groups() const {
+        return groups_;
+    }
+
+ private:
+    std::vector<BatchGroup> groups_;
+    std::optional<OffsetRange> offset_range_;
+};
+
+class CommitBatchReader final : public BatchReader {
+ public:
+    CommitBatchReader(const std::shared_ptr<Segment>& segment,
+                      const std::shared_ptr<arrow::MemoryPool>& arrow_pool)
+        : segment_(segment), arrow_pool_(arrow_pool), metrics_(std::make_shared<MetricsImpl>()) {}
+
+    Result<ReadBatch> NextBatch() override {
+        if (!segment_ || next_batch_ >= static_cast<int32_t>(segment_->Batches().size())) {
+            return MakeEofBatch();
+        }
+        const std::shared_ptr<const StoredBatch>& stored = segment_->Batches()[next_batch_++];
+        const int64_t row_count = stored->data->length();
+        arrow::Int8Builder row_kind_builder(arrow_pool_.get());
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(row_kind_builder.Reserve(row_count));
+        if (stored->row_kinds.empty()) {
+            for (int64_t i = 0; i < row_count; ++i) {
+                row_kind_builder.UnsafeAppend(static_cast<int8_t>(RecordBatch::RowKind::INSERT));
+            }
+        } else {
+            for (RecordBatch::RowKind row_kind : stored->row_kinds) {
+                row_kind_builder.UnsafeAppend(static_cast<int8_t>(row_kind));
+            }
+        }
+        std::shared_ptr<arrow::Array> row_kind_array;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(row_kind_builder.Finish(&row_kind_array));
+        arrow::ArrayVector arrays = {std::move(row_kind_array)};
+        arrays.insert(arrays.end(), stored->data->fields().begin(), stored->data->fields().end());
+        arrow::FieldVector fields = {
+            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())};
+        const arrow::FieldVector& value_fields = stored->data->struct_type()->fields();
+        fields.insert(fields.end(), value_fields.begin(), value_fields.end());
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> output,
+                                          arrow::StructArray::Make(arrays, fields));
+        auto c_array = std::make_unique<ArrowArray>();
+        auto c_schema = std::make_unique<ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*output, c_array.get(), c_schema.get()));
+        return ReadBatch(std::move(c_array), std::move(c_schema));
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return metrics_;
+    }
+
+    void Close() override {
+        segment_.reset();
+    }
+
+ private:
+    std::shared_ptr<Segment> segment_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    std::shared_ptr<Metrics> metrics_;
+    int32_t next_batch_ = 0;
+};
+
+class KeyRangeBatchReader final : public BatchReader, public PrimaryKeyRangeProvider {
+ public:
+    KeyRangeBatchReader(std::unique_ptr<BatchReader>&& reader,
+                        const std::shared_ptr<InternalRow>& min_key,
+                        const std::shared_ptr<InternalRow>& max_key)
+        : reader_(std::move(reader)), min_key_(min_key), max_key_(max_key) {}
+
+    Result<ReadBatch> NextBatch() override {
+        return reader_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return reader_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        reader_->Close();
+    }
+
+    std::shared_ptr<InternalRow> GetMinKey() const override {
+        return min_key_;
+    }
+
+    std::shared_ptr<InternalRow> GetMaxKey() const override {
+        return max_key_;
+    }
+
+ private:
+    std::unique_ptr<BatchReader> reader_;
+    std::shared_ptr<InternalRow> min_key_;
+    std::shared_ptr<InternalRow> max_key_;
+};
+
+}  // namespace
+
+class PrimaryKeyRealtimeStore::Impl {
+ public:
+    Impl(const std::shared_ptr<arrow::Schema>& write_schema, std::vector<std::string> primary_keys,
+         const std::shared_ptr<FieldsComparator>& key_comparator,
+         const std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>&
+             merge_function_wrapper_factory,
+         int64_t next_sequence_number, int32_t read_batch_size,
+         const std::shared_ptr<MemoryPool>& memory_pool)
+        : write_schema_(write_schema),
+          primary_keys_(std::move(primary_keys)),
+          key_comparator_(key_comparator),
+          merge_function_wrapper_factory_(merge_function_wrapper_factory),
+          next_sequence_number_(next_sequence_number),
+          read_batch_size_(read_batch_size),
+          memory_pool_(memory_pool),
+          arrow_pool_(GetArrowPool(memory_pool)) {}
+
+    Result<std::shared_ptr<InternalRow>> CopyKey(const InternalRow& key) const {
+        auto result = std::make_shared<BinaryRow>(static_cast<int32_t>(primary_keys_.size()));
+        BinaryRowWriter writer(result.get(), /*initial_size=*/128, memory_pool_.get());
+        writer.Reset();
+        for (int32_t index = 0; index < static_cast<int32_t>(primary_keys_.size()); ++index) {
+            std::shared_ptr<arrow::Field> field =
+                write_schema_->GetFieldByName(primary_keys_[index]);
+            PAIMON_ASSIGN_OR_RAISE(InternalRow::FieldGetterFunc getter,
+                                   InternalRow::CreateFieldGetter(index, field->type(),
+                                                                  /*use_view=*/true));
+            PAIMON_ASSIGN_OR_RAISE(BinaryRowWriter::FieldSetterFunc setter,
+                                   BinaryRowWriter::CreateFieldSetter(index, field->type()));
+            setter(getter(key), &writer);
+        }
+        writer.Complete();
+        return std::static_pointer_cast<InternalRow>(result);
+    }
+
+    Result<std::pair<std::shared_ptr<InternalRow>, std::shared_ptr<InternalRow>>> GetKeyRange(
+        const std::shared_ptr<arrow::StructArray>& values) const {
+        arrow::ArrayVector key_arrays;
+        key_arrays.reserve(primary_keys_.size());
+        for (const std::string& primary_key : primary_keys_) {
+            std::shared_ptr<arrow::Array> key_array = values->GetFieldByName(primary_key);
+            if (!key_array) {
+                return Status::Invalid("primary key is missing from PK query batch: ", primary_key);
+            }
+            key_arrays.push_back(std::move(key_array));
+        }
+        auto context = std::make_shared<ColumnarBatchContext>(key_arrays, memory_pool_);
+        int64_t min_row = 0;
+        int64_t max_row = 0;
+        for (int64_t row = 1; row < values->length(); ++row) {
+            ColumnarRowRef current(context, row);
+            ColumnarRowRef min_key(context, min_row);
+            ColumnarRowRef max_key(context, max_row);
+            if (key_comparator_->CompareTo(current, min_key) < 0) {
+                min_row = row;
+            }
+            if (key_comparator_->CompareTo(current, max_key) > 0) {
+                max_row = row;
+            }
+        }
+        ColumnarRowRef min_key(context, min_row);
+        ColumnarRowRef max_key(context, max_row);
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalRow> copied_min, CopyKey(min_key));
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalRow> copied_max, CopyKey(max_key));
+        return std::make_pair(std::move(copied_min), std::move(copied_max));
+    }
+
+    Status Write(RealtimeWriteBatch&& write_batch) {
+        if (!write_batch.batch || !write_batch.batch->GetData()) {
+            return Status::Invalid("PK real-time write batch is null");
+        }
+        const int64_t row_count = write_batch.batch->GetData()->length;
+        if (row_count <= 0 || write_batch.offset_range.begin < 0 ||
+            write_batch.offset_range.Count() != row_count) {
+            return Status::Invalid("PK real-time offset range does not match batch row count");
+        }
+        const std::vector<RecordBatch::RowKind>& row_kinds = write_batch.batch->GetRowKind();
+        if (!row_kinds.empty() && static_cast<int64_t>(row_kinds.size()) != row_count) {
+            return Status::Invalid("PK real-time row-kind count does not match batch row count");
+        }
+        for (RecordBatch::RowKind row_kind : row_kinds) {
+            PAIMON_ASSIGN_OR_RAISE(const RowKind* validated,
+                                   RowKind::FromByteValue(static_cast<int8_t>(row_kind)));
+            static_cast<void>(validated);
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> imported,
+            arrow::ImportArray(write_batch.batch->GetData(),
+                               arrow::struct_(write_schema_->fields())));
+        if (!imported || imported->type_id() != arrow::Type::STRUCT) {
+            return Status::Invalid("PK real-time write data is not a StructArray");
+        }
+        std::shared_ptr<arrow::StructArray> values =
+            checked_pointer_cast<arrow::StructArray>(imported);
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(values->ValidateFull());
+
+        std::lock_guard<std::mutex> lock(mutex_);
+        if (last_offset_ && write_batch.offset_range.begin != last_offset_.value()) {
+            return Status::Invalid("PK real-time offset ranges must be contiguous");
+        }
+        if (row_count > std::numeric_limits<int64_t>::max() - next_sequence_number_) {
+            return Status::Invalid("PK sequence range exceeds INT64_MAX");
+        }
+        auto stored = std::make_shared<const StoredBatch>(
+            StoredBatch{std::move(values), row_kinds, write_batch.offset_range,
+                        next_sequence_number_, GetArrayMemoryUsage(imported->data())});
+        building_batches_.push_back(std::move(stored));
+        building_memory_usage_ += building_batches_.back()->memory_usage;
+        last_offset_ = write_batch.offset_range.end;
+        next_sequence_number_ += row_count;
+        return Status::OK();
+    }
+
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() {
+        std::lock_guard<std::mutex> lock(mutex_);
+        if (building_batches_.empty()) {
+            return std::optional<std::shared_ptr<RealtimeSegmentHandle>>();
+        }
+        const OffsetRange range(building_batches_.front()->offset_range.begin,
+                                building_batches_.back()->offset_range.end);
+        auto segment = std::make_shared<Segment>(range, std::move(building_batches_));
+        sealed_segments_.push_back(segment);
+        building_batches_.clear();
+        building_memory_usage_ = 0;
+        return std::optional<std::shared_ptr<RealtimeSegmentHandle>>(std::move(segment));
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) {
+        std::shared_ptr<Segment> typed = std::dynamic_pointer_cast<Segment>(segment);
+        if (!typed) {
+            return Status::Invalid("segment was not created by the PK real-time store");
+        }
+        std::vector<std::unique_ptr<BatchReader>> result;
+        result.push_back(std::make_unique<CommitBatchReader>(typed, arrow_pool_));
+        return result;
+    }
+
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() {
+        std::lock_guard<std::mutex> lock(mutex_);
+        std::vector<BatchGroup> groups;
+        groups.reserve(sealed_segments_.size() + (building_batches_.empty() ? 0 : 1));
+        for (const std::shared_ptr<Segment>& segment : sealed_segments_) {
+            groups.push_back(segment->Batches());
+        }
+        if (!building_batches_.empty()) {
+            groups.push_back(building_batches_);
+        }
+        return std::shared_ptr<RealtimeReadView>(new PrimaryKeyRealtimeReadView(std::move(groups)));
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t lower,
+        const RealtimeQueryContext& context) {
+        std::shared_ptr<PrimaryKeyRealtimeReadView> typed =
+            std::dynamic_pointer_cast<PrimaryKeyRealtimeReadView>(view);
+        if (!typed) {
+            return Status::Invalid("read view was not created by the PK real-time store");
+        }
+        if (!context.read_schema || !context.read_schema->release) {
+            return Status::Invalid("PK real-time query read schema is null");
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> requested,
+                                          arrow::ImportSchema(context.read_schema));
+        arrow::FieldVector output_fields = {
+            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())};
+        std::vector<int32_t> projection = {KeyValueProjectionConsumer::kValueKindProjection};
+        for (const std::shared_ptr<arrow::Field>& field : requested->fields()) {
+            if (field->name() == SpecialFields::ValueKind().Name()) {
+                continue;
+            }
+            output_fields.push_back(field);
+            if (field->name() == SpecialFields::SequenceNumber().Name()) {
+                projection.push_back(KeyValueProjectionConsumer::kSequenceNumberProjection);
+                continue;
+            }
+            const int32_t index = write_schema_->GetFieldIndex(field->name());
+            if (index < 0) {
+                return Status::Invalid("PK real-time query field is missing from write schema: ",
+                                       field->name());
+            }
+            projection.push_back(index);
+        }
+
+        std::vector<std::unique_ptr<BatchReader>> result;
+        for (const BatchGroup& group : typed->Groups()) {
+            std::vector<std::unique_ptr<KeyValueRecordReader>> batch_readers;
+            std::shared_ptr<InternalRow> min_key;
+            std::shared_ptr<InternalRow> max_key;
+            for (const std::shared_ptr<const StoredBatch>& batch : group) {
+                if (batch->offset_range.end <= lower) {
+                    continue;
+                }
+                const int64_t offset = std::max<int64_t>(0, lower - batch->offset_range.begin);
+                const int64_t length = batch->data->length() - offset;
+                std::shared_ptr<arrow::Array> sliced = batch->data->Slice(offset, length);
+                std::shared_ptr<arrow::StructArray> selected =
+                    checked_pointer_cast<arrow::StructArray>(sliced);
+                using KeyRange =
+                    std::pair<std::shared_ptr<InternalRow>, std::shared_ptr<InternalRow>>;
+                PAIMON_ASSIGN_OR_RAISE(KeyRange key_range, GetKeyRange(selected));
+                if (!min_key || key_comparator_->CompareTo(*key_range.first, *min_key) < 0) {
+                    min_key = key_range.first;
+                }
+                if (!max_key || key_comparator_->CompareTo(*key_range.second, *max_key) > 0) {
+                    max_key = key_range.second;
+                }
+                std::vector<RecordBatch::RowKind> selected_kinds;
+                if (!batch->row_kinds.empty()) {
+                    selected_kinds.assign(batch->row_kinds.begin() + offset,
+                                          batch->row_kinds.end());
+                }
+                std::unique_ptr<KeyValueRecordReader> reader =
+                    std::make_unique<KeyValueInMemoryRecordReader>(
+                        batch->first_sequence_number + offset, selected, selected_kinds,
+                        primary_keys_, /*user_defined_sequence_fields=*/std::vector<std::string>(),
+                        /*sequence_fields_ascending=*/true, key_comparator_, memory_pool_);
+                std::shared_ptr<MergeFunctionWrapper<KeyValue>> batch_merge =
+                    merge_function_wrapper_factory_();
+                if (!batch_merge) {
+                    return Status::Invalid("merge function wrapper factory returned null");
+                }
+                batch_readers.push_back(std::make_unique<MergedKeyValueRecordReader>(
+                    std::move(reader), key_comparator_, batch_merge));
+            }
+            if (batch_readers.empty()) {
+                continue;
+            }
+            std::shared_ptr<MergeFunctionWrapper<KeyValue>> group_merge =
+                merge_function_wrapper_factory_();
+            if (!group_merge) {
+                return Status::Invalid("merge function wrapper factory returned null");
+            }
+            auto merged = std::make_unique<SortMergeReaderWithLoserTree>(
+                std::move(batch_readers), key_comparator_,
+                /*user_defined_seq_comparator=*/nullptr, group_merge);
+            PAIMON_ASSIGN_OR_RAISE(
+                std::unique_ptr<KeyValueProjectionReader> projected,
+                KeyValueProjectionReader::Create(std::move(merged), arrow::schema(output_fields),
+                                                 projection, read_batch_size_, memory_pool_));
+            result.push_back(
+                std::make_unique<KeyRangeBatchReader>(std::move(projected), min_key, max_key));
+        }
+        return result;
+    }
+
+    Status AdvanceCommittedOffset(int64_t committed_end_offset) {
+        std::lock_guard<std::mutex> lock(mutex_);
+        sealed_segments_.erase(
+            std::remove_if(sealed_segments_.begin(), sealed_segments_.end(),
+                           [committed_end_offset](const std::shared_ptr<Segment>& segment) {
+                               return segment->GetOffsetRange().end <= committed_end_offset;
+                           }),
+            sealed_segments_.end());
+        return Status::OK();
+    }
+
+    uint64_t GetMemoryUsage() const {
+        std::lock_guard<std::mutex> lock(mutex_);
+        uint64_t result = building_memory_usage_;
+        for (const std::shared_ptr<Segment>& segment : sealed_segments_) {
+            result += segment->GetMemoryUsage();
+        }
+        return result;
+    }
+
+ private:
+    std::shared_ptr<arrow::Schema> write_schema_;
+    std::vector<std::string> primary_keys_;
+    std::shared_ptr<FieldsComparator> key_comparator_;
+    std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>
+        merge_function_wrapper_factory_;
+    int64_t next_sequence_number_;
+    int32_t read_batch_size_;
+    std::shared_ptr<MemoryPool> memory_pool_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    mutable std::mutex mutex_;
+    std::vector<std::shared_ptr<const StoredBatch>> building_batches_;
+    std::vector<std::shared_ptr<Segment>> sealed_segments_;
+    uint64_t building_memory_usage_ = 0;
+    std::optional<int64_t> last_offset_;
+};
+
+Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
+    const std::shared_ptr<arrow::Schema>& write_schema,
+    const std::vector<std::string>& primary_keys,
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>&
+        merge_function_wrapper_factory,
+    int64_t restore_max_sequence_number, int32_t read_batch_size,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    if (!write_schema || primary_keys.empty() || !key_comparator ||
+        !merge_function_wrapper_factory || !memory_pool || read_batch_size <= 0) {
+        return Status::Invalid("PK real-time store requires schema, keys, merge helpers, and pool");
+    }
+    if (restore_max_sequence_number < -1) {
+        return Status::Invalid("PK restore max sequence number must be at least -1");
+    }
+    if (restore_max_sequence_number == std::numeric_limits<int64_t>::max()) {
+        return Status::Invalid("PK sequence number has reached INT64_MAX");
+    }
+    for (const std::string& key : primary_keys) {
+        if (write_schema->GetFieldIndex(key) < 0) {
+            return Status::Invalid("primary key ", key, " is missing from write schema");
+        }
+    }
+    auto impl = std::make_unique<Impl>(
+        write_schema, primary_keys, key_comparator, merge_function_wrapper_factory,
+        restore_max_sequence_number + 1, read_batch_size, memory_pool);
+    return std::shared_ptr<PrimaryKeyRealtimeStore>(new PrimaryKeyRealtimeStore(std::move(impl)));
+}
+
+PrimaryKeyRealtimeStore::PrimaryKeyRealtimeStore(std::unique_ptr<Impl>&& impl)
+    : impl_(std::move(impl)) {}
+
+PrimaryKeyRealtimeStore::~PrimaryKeyRealtimeStore() = default;
+
+Status PrimaryKeyRealtimeStore::Write(RealtimeWriteBatch&& batch) {
+    return impl_->Write(std::move(batch));
+}
+
+Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>>
+PrimaryKeyRealtimeStore::SealForCommit() {
+    return impl_->SealForCommit();
+}
+
+Result<std::vector<std::unique_ptr<BatchReader>>> PrimaryKeyRealtimeStore::CreateCommitReaders(
+    const std::shared_ptr<RealtimeSegmentHandle>& segment) {
+    return impl_->CreateCommitReaders(segment);
+}
+
+Result<std::shared_ptr<RealtimeReadView>> PrimaryKeyRealtimeStore::AcquireReadView() {
+    return impl_->AcquireReadView();
+}
+
+Result<std::vector<std::unique_ptr<BatchReader>>> PrimaryKeyRealtimeStore::CreateQueryReaders(
+    const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+    const RealtimeQueryContext& context) {
+    return impl_->CreateQueryReaders(view, offset_begin, context);
+}
+
+Status PrimaryKeyRealtimeStore::AdvanceCommittedOffset(int64_t committed_offset) {
+    return impl_->AdvanceCommittedOffset(committed_offset);
+}
+
+uint64_t PrimaryKeyRealtimeStore::GetMemoryUsage() const {
+    return impl_->GetMemoryUsage();
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
new file mode 100644
index 000000000..05225ed19
--- /dev/null
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "paimon/realtime/realtime_store.h"
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
+namespace paimon {
+
+class FieldsComparator;
+struct KeyValue;
+class MemoryPool;
+class InternalRow;
+template <typename T>
+class MergeFunctionWrapper;
+
+/// Optional metadata exposed by PK query readers with a known inclusive key range.
+class PrimaryKeyRangeProvider {
+ public:
+    virtual ~PrimaryKeyRangeProvider() = default;
+
+    virtual std::shared_ptr<InternalRow> GetMinKey() const = 0;
+    virtual std::shared_ptr<InternalRow> GetMaxKey() const = 0;
+};
+
+/// In-memory store for primary-key real-time writes.
+class PrimaryKeyRealtimeStore final : public RealtimeStore {
+ public:
+    static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
+        const std::shared_ptr<arrow::Schema>& write_schema,
+        const std::vector<std::string>& primary_keys,
+        const std::shared_ptr<FieldsComparator>& key_comparator,
+        const std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>&
+            merge_function_wrapper_factory,
+        int64_t restore_max_sequence_number, int32_t read_batch_size,
+        const std::shared_ptr<MemoryPool>& memory_pool);
+
+    ~PrimaryKeyRealtimeStore() override;
+
+    Status Write(RealtimeWriteBatch&& batch) override;
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override;
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override;
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override;
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override;
+    Status AdvanceCommittedOffset(int64_t committed_offset) override;
+    uint64_t GetMemoryUsage() const override;
+
+ private:
+    class Impl;
+    explicit PrimaryKeyRealtimeStore(std::unique_ptr<Impl>&& impl);
+
+    std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
new file mode 100644
index 000000000..9da272e0f
--- /dev/null
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/primary_key_realtime_store.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "arrow/ipc/json_simple.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/fields_comparator.h"
+#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
+#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+class PrimaryKeyRealtimeStoreTest : public testing::Test {
+ public:
+    void SetUp() override {
+        pool_ = std::shared_ptr<MemoryPool>(GetMemoryPool());
+        schema_ = arrow::schema(
+            {arrow::field("id", arrow::int64()), arrow::field("value", arrow::utf8())});
+        ASSERT_OK_AND_ASSIGN(key_comparator_,
+                             FieldsComparator::Create({DataField(0, schema_->field(0))},
+                                                      /*is_ascending_order=*/true));
+        auto merge_factory = []() {
+            auto merge_function =
+                std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
+            return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
+        };
+        ASSERT_OK_AND_ASSIGN(
+            store_, PrimaryKeyRealtimeStore::Create(schema_, {"id"}, key_comparator_, merge_factory,
+                                                    /*restore_max_sequence_number=*/4,
+                                                    /*read_batch_size=*/1024, pool_));
+    }
+
+    std::unique_ptr<RecordBatch> MakeBatch(
+        const std::string& json, const std::vector<RecordBatch::RowKind>& row_kinds = {}) const {
+        std::shared_ptr<arrow::Array> array =
+            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema_->fields()), json)
+                .ValueOrDie();
+        ArrowArray c_array;
+        EXPECT_TRUE(arrow::ExportArray(*array, &c_array).ok());
+        RecordBatchBuilder builder(&c_array);
+        builder.SetRowKinds(row_kinds);
+        return builder.Finish().value();
+    }
+
+    std::unique_ptr<ArrowSchema> MakeReadSchema(bool include_sequence) const {
+        arrow::FieldVector fields;
+        if (include_sequence) {
+            fields.push_back(
+                DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()));
+        }
+        fields.insert(fields.end(), schema_->fields().begin(), schema_->fields().end());
+        auto c_schema = std::make_unique<ArrowSchema>();
+        EXPECT_TRUE(arrow::ExportSchema(*arrow::schema(fields), c_schema.get()).ok());
+        return c_schema;
+    }
+
+    void AssertReaderOutput(BatchReader* reader, const std::shared_ptr<arrow::DataType>& type,
+                            const std::string& json) const {
+        ASSERT_NE(nullptr, reader);
+        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
+        ASSERT_FALSE(BatchReader::IsEofBatch(batch));
+        arrow::Result<std::shared_ptr<arrow::Array>> imported_result =
+            arrow::ImportArray(batch.first.get(), batch.second.get());
+        ASSERT_TRUE(imported_result.ok()) << imported_result.status().ToString();
+        std::shared_ptr<arrow::Array> actual = std::move(imported_result).ValueOrDie();
+        std::shared_ptr<arrow::Array> expected =
+            arrow::ipc::internal::json::ArrayFromJSON(type, json).ValueOrDie();
+        ASSERT_TRUE(actual->Equals(*expected))
+            << "expected: " << expected->ToString() << ", actual: " << actual->ToString();
+
+        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof, reader->NextBatch());
+        ASSERT_TRUE(BatchReader::IsEofBatch(eof));
+        reader->Close();
+    }
+
+    std::shared_ptr<arrow::DataType> CommitType() const {
+        return arrow::struct_({
+            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
+            schema_->field(0),
+            schema_->field(1),
+        });
+    }
+
+    std::shared_ptr<arrow::DataType> QueryType() const {
+        return arrow::struct_({
+            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
+            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()),
+            schema_->field(0),
+            schema_->field(1),
+        });
+    }
+
+ protected:
+    std::shared_ptr<MemoryPool> pool_;
+    std::shared_ptr<arrow::Schema> schema_;
+    std::shared_ptr<FieldsComparator> key_comparator_;
+    std::shared_ptr<PrimaryKeyRealtimeStore> store_;
+};
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestWriteAndSeal) {
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store_->SealForCommit());
+    ASSERT_FALSE(segment.has_value());
+    ASSERT_NOK_WITH_MSG(store_->Write(RealtimeWriteBatch{nullptr, OffsetRange(0, 0)}),
+                        "write batch is null");
+    ASSERT_NOK_WITH_MSG(
+        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[1, "a"], [2, "b"]])"), OffsetRange(0, 0)}),
+        "offset range does not match batch row count");
+
+    ASSERT_OK(
+        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[1, "a"], [2, "b"]])"), OffsetRange(0, 2)}));
+    ASSERT_NOK_WITH_MSG(
+        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[4, "d"]])"), OffsetRange(3, 4)}),
+        "offset ranges must be contiguous");
+    ASSERT_OK(store_->Write(RealtimeWriteBatch{MakeBatch(R"([[3, "c"]])"), OffsetRange(2, 3)}));
+
+    ASSERT_OK_AND_ASSIGN(segment, store_->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_EQ(OffsetRange(0, 3), segment.value()->GetOffsetRange());
+    ASSERT_GT(store_->GetMemoryUsage(), 0);
+
+    auto merge_factory = []() {
+        auto merge_function = std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
+        return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
+    };
+    ASSERT_NOK_WITH_MSG(PrimaryKeyRealtimeStore::Create(
+                            schema_, {"id"}, key_comparator_, merge_factory,
+                            /*restore_max_sequence_number=*/-2, /*read_batch_size=*/1024, pool_),
+                        "restore max sequence number must be at least -1");
+}
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestCommitReaderPreservesMutations) {
+    ASSERT_OK(store_->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[2, "old"], [1, "one"], [2, "new"]])",
+                                     {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT,
+                                      RecordBatch::RowKind::UPDATE_AFTER}),
+                           OffsetRange(0, 3)}));
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store_->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store_->CreateCommitReaders(segment.value()));
+    ASSERT_EQ(1, readers.size());
+    AssertReaderOutput(readers[0].get(), CommitType(),
+                       R"([[0, 2, "old"], [0, 1, "one"], [2, 2, "new"]])");
+}
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestMutationMerge) {
+    ASSERT_OK(store_->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[1, "old"], [2, "two"]])"), OffsetRange(0, 2)}));
+    ASSERT_OK(store_->Write(RealtimeWriteBatch{
+        MakeBatch(R"([[1, "new"], [2, "gone"]])",
+                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE}),
+        OffsetRange(2, 4)}));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(/*include_sequence=*/true);
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(1, readers.size());
+    AssertReaderOutput(readers[0].get(), QueryType(), R"([[2, 7, 1, "new"], [3, 8, 2, "gone"]])");
+}
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestReadViewLifecycle) {
+    ASSERT_OK(store_->Write(RealtimeWriteBatch{MakeBatch(R"([[10, "a"], [11, "b"], [12, "c"]])"),
+                                               OffsetRange(10, 13)}));
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store_->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
+    ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(10, 13)), view->GetOffsetRange());
+
+    ASSERT_OK(store_->AdvanceCommittedOffset(13));
+    ASSERT_EQ(0, store_->GetMemoryUsage());
+    ASSERT_OK(
+        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[13, "later"]])"), OffsetRange(13, 14)}));
+
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(/*include_sequence=*/true);
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store_->CreateQueryReaders(view, /*offset_begin=*/11, context));
+    ASSERT_EQ(1, readers.size());
+    AssertReaderOutput(readers[0].get(), QueryType(), R"([[0, 6, 11, "b"], [0, 7, 12, "c"]])");
+
+    std::unique_ptr<ArrowSchema> empty_schema = MakeReadSchema(/*include_sequence=*/true);
+    context.read_schema = empty_schema.get();
+    ASSERT_OK_AND_ASSIGN(readers, store_->CreateQueryReaders(view, /*offset_begin=*/13, context));
+    ASSERT_TRUE(readers.empty());
+}
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryKeyRange) {
+    ASSERT_OK(store_->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[5, "five"], [1, "one"]])"), OffsetRange(0, 2)}));
+    ASSERT_OK(store_->SealForCommit());
+    ASSERT_OK(store_->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[9, "nine"], [7, "seven"]])"), OffsetRange(2, 4)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(/*include_sequence=*/true);
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(2, readers.size());
+    auto* first_range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[0].get());
+    auto* second_range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[1].get());
+    ASSERT_NE(nullptr, first_range);
+    ASSERT_NE(nullptr, second_range);
+    ASSERT_EQ(1, first_range->GetMinKey()->GetLong(0));
+    ASSERT_EQ(5, first_range->GetMaxKey()->GetLong(0));
+    ASSERT_EQ(7, second_range->GetMinKey()->GetLong(0));
+    ASSERT_EQ(9, second_range->GetMaxKey()->GetLong(0));
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_append_only_writer.cpp b/src/paimon/core/realtime/realtime_append_only_writer.cpp
index 9d519d791..21d6cfb74 100644
--- a/src/paimon/core/realtime/realtime_append_only_writer.cpp
+++ b/src/paimon/core/realtime/realtime_append_only_writer.cpp
@@ -47,7 +47,7 @@ Result<std::shared_ptr<RealtimeAppendOnlyWriter>> RealtimeAppendOnlyWriter::Crea
     std::unique_ptr<::ArrowSchema> write_schema,
     const std::shared_ptr<RealtimeContext>& realtime_context,
     const std::shared_ptr<AppendOnlyWriter>& file_writer,
-    const std::shared_ptr<arrow::Schema>& input_schema, StatisticsMode statistics_mode,
+    const std::shared_ptr<arrow::Schema>& input_schema,
     const std::map<std::string, std::string>& options,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     if (!realtime_context) {
@@ -55,10 +55,11 @@ Result<std::shared_ptr<RealtimeAppendOnlyWriter>> RealtimeAppendOnlyWriter::Crea
     }
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
                            RealtimeContextImpl::Cast(realtime_context));
-    PAIMON_ASSIGN_OR_RAISE(
-        RealtimeStoreState store_state,
-        realtime_context_impl->GetOrCreateRealtimeStore(partition, bucket, std::move(write_schema),
-                                                        statistics_mode, options, memory_pool));
+    RealtimeStoreCreateRequest request{
+        std::move(write_schema),          options, memory_pool, partition, bucket,
+        AppendRealtimeStoreCreateConfig{statistics_mode}};
+    PAIMON_ASSIGN_OR_RAISE(RealtimeStoreState store_state,
+                           realtime_context_impl->GetOrCreateRealtimeStore(std::move(request)));
     return std::shared_ptr<RealtimeAppendOnlyWriter>(new RealtimeAppendOnlyWriter(
         store_state.store, file_writer, input_schema, store_state.initial_offset, memory_pool));
 }
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index f6bad5cf1..0a367b2cd 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -78,19 +78,16 @@ Status RealtimeContextImpl::Start() {
 }
 
 Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
-    const std::map<std::string, std::string>& partition, int32_t bucket,
-    std::unique_ptr<ArrowSchema> write_schema, StatisticsMode statistics_mode,
-    const std::map<std::string, std::string>& options,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
+    RealtimeStoreCreateRequest&& request) {
     std::lock_guard<std::mutex> progress_lock(progress_mutex_);
     std::lock_guard<std::mutex> registry_lock(mutex_);
-    const RealtimePartitionBucket key(partition, bucket);
+    const RealtimePartitionBucket key(request.partition, request.bucket);
     int64_t initial_offset = 0;
     auto offset_iter = committed_offsets_.find(key);
     if (offset_iter != committed_offsets_.end()) {
         if (offset_iter->second == std::numeric_limits<int64_t>::max()) {
-            if (write_schema) {
-                ArrowSchemaRelease(write_schema.get());
+            if (request.write_schema) {
+                ArrowSchemaRelease(request.write_schema.get());
             }
             return Status::Invalid("real-time offset has reached INT64_MAX");
         }
@@ -98,8 +95,8 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     }
     auto iter = stores_.find(key);
     if (iter != stores_.end()) {
-        if (write_schema) {
-            ArrowSchemaRelease(write_schema.get());
+        if (request.write_schema) {
+            ArrowSchemaRelease(request.write_schema.get());
         }
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
                                iter->second->AcquireReadView());
@@ -119,9 +116,8 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
         }
         return RealtimeStoreState{iter->second, initial_offset};
     }
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<RealtimeStore> store,
-        factory_->Create(std::move(write_schema), statistics_mode, options, memory_pool));
+    Result<std::shared_ptr<RealtimeStore>> store_result = factory_->Create(std::move(request));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
     stores_.emplace(key, store);
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
@@ -129,6 +125,27 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     return RealtimeStoreState{std::move(store), initial_offset};
 }
 
+int64_t RealtimeContextImpl::GetMaterializedMaxSequenceNumber(
+    const RealtimePartitionBucket& partition_bucket, int64_t restored_max_sequence_number) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    auto [iter, inserted] =
+        materialized_max_sequence_numbers_.emplace(partition_bucket, restored_max_sequence_number);
+    if (!inserted && restored_max_sequence_number > iter->second) {
+        iter->second = restored_max_sequence_number;
+    }
+    return iter->second;
+}
+
+void RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
+    const RealtimePartitionBucket& partition_bucket, int64_t max_sequence_number) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    auto [iter, inserted] =
+        materialized_max_sequence_numbers_.emplace(partition_bucket, max_sequence_number);
+    if (!inserted && max_sequence_number > iter->second) {
+        iter->second = max_sequence_number;
+    }
+}
+
 Result<std::vector<RealtimePartitionBucketView>> RealtimeContextImpl::AcquireReadViews() {
     std::lock_guard<std::mutex> lock(mutex_);
     std::vector<RealtimePartitionBucketView> result;
@@ -230,28 +247,12 @@ Status RealtimeContextImpl::AdvanceCommittedProgress(int64_t snapshot_id,
             if (partition_bucket.bucket < 0 || committed_end_offset < 0) {
                 return Status::Invalid("invalid partition-bucket committed offset");
             }
-        }
-        // Only stores created by this context can contain state which cannot be restored in
-        // place. Offsets for other partition-buckets are reference state for lazy store creation
-        // and may be removed or rolled back without rebuilding the context.
-        std::lock_guard<std::mutex> registry_lock(mutex_);
-        for (const auto& store_entry : stores_) {
-            const RealtimePartitionBucket& partition_bucket = store_entry.first;
             auto previous_iter = committed_offsets_.find(partition_bucket);
-            if (previous_iter == committed_offsets_.end()) {
-                continue;
-            }
-
-            auto current_iter = committed_offsets.find(partition_bucket);
-            if (current_iter == committed_offsets.end()) {
-                return Status::Invalid(
-                    "real-time committed progress removed an active partition-bucket; recreate "
-                    "RealtimeContext");
-            }
-            if (current_iter->second < previous_iter->second) {
-                return Status::Invalid(
-                    "real-time committed offset moved backwards for an active partition-bucket; "
-                    "recreate RealtimeContext");
+            if (previous_iter != committed_offsets_.end()) {
+                if (committed_end_offset < previous_iter->second) {
+                    return Status::Invalid(
+                        "real-time partition-bucket committed offset cannot move backwards");
+                }
             }
         }
         committed_offsets_ = committed_offsets;
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index 66c324cab..45d07deeb 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -32,8 +32,8 @@
 #include <vector>
 
 #include "paimon/realtime/realtime_context.h"
+#include "paimon/realtime/realtime_store.h"
 #include "paimon/result.h"
-#include "paimon/statistics_mode.h"
 #include "paimon/visibility.h"
 
 struct ArrowSchema;
@@ -65,11 +65,13 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     static Result<std::shared_ptr<RealtimeContextImpl>> Cast(
         const std::shared_ptr<RealtimeContext>& context);
 
-    Result<RealtimeStoreState> GetOrCreateRealtimeStore(
-        const std::map<std::string, std::string>& partition, int32_t bucket,
-        std::unique_ptr<::ArrowSchema> write_schema, StatisticsMode statistics_mode,
-        const std::map<std::string, std::string>& options,
-        const std::shared_ptr<MemoryPool>& memory_pool);
+    Result<RealtimeStoreState> GetOrCreateRealtimeStore(RealtimeStoreCreateRequest&& request);
+
+    int64_t GetMaterializedMaxSequenceNumber(const RealtimePartitionBucket& partition_bucket,
+                                             int64_t restored_max_sequence_number);
+
+    void AdvanceMaterializedMaxSequenceNumber(const RealtimePartitionBucket& partition_bucket,
+                                              int64_t max_sequence_number);
 
     Result<std::vector<RealtimePartitionBucketView>> AcquireReadViews();
 
@@ -79,9 +81,6 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
 
     Status ReleaseReadView(const std::string& opaque_ticket);
 
-    // Returns an error requiring a new context if a newer snapshot removes or moves committed
-    // progress backwards for a store created by this context. Progress for inactive stores is
-    // only reference state and can be replaced in place.
     Status AdvanceCommittedProgress(int64_t snapshot_id,
                                     const RealtimeOffsetMap& committed_offsets);
 
@@ -103,6 +102,7 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     std::mutex mutex_;
     std::mutex progress_mutex_;
     std::map<RealtimePartitionBucket, std::shared_ptr<RealtimeStore>> stores_;
+    std::map<RealtimePartitionBucket, int64_t> materialized_max_sequence_numbers_;
     // Full-table progress used as the initial offset when a store is created lazily.
     RealtimeOffsetMap committed_offsets_;
     // Progress already reflected in stores owned by this context.
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 017820fd4..33701afac 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -91,14 +91,11 @@ class TestingRealtimeStore : public RealtimeStore {
 
 class TestingRealtimeStoreFactory : public RealtimeStoreFactory {
  public:
-    Result<std::shared_ptr<RealtimeStore>> Create(std::unique_ptr<ArrowSchema> write_schema,
-                                                  StatisticsMode,
-                                                  const std::map<std::string, std::string>&,
-                                                  const std::shared_ptr<MemoryPool>&) override {
-        if (!write_schema || !write_schema->release) {
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        if (!request.write_schema || !request.write_schema->release) {
             return Status::Invalid("testing write schema is null");
         }
-        ArrowSchemaRelease(write_schema.get());
+        ArrowSchemaRelease(request.write_schema.get());
         auto store = std::make_shared<TestingRealtimeStore>();
         stores.push_back(store);
         return store;
@@ -122,20 +119,28 @@ Result<std::shared_ptr<RealtimeContextImpl>> CreateContext(
     return RealtimeContextImpl::Cast(context);
 }
 
-TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
+Result<RealtimeStoreState> GetOrCreateAppendStore(
+    const std::shared_ptr<RealtimeContextImpl>& context,
+    const std::map<std::string, std::string>& partition, int32_t bucket,
+    std::unique_ptr<ArrowSchema> write_schema, const std::map<std::string, std::string>& options,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    return context->GetOrCreateRealtimeStore(
+        RealtimeStoreCreateRequest{std::move(write_schema), options, memory_pool, partition, bucket,
+                                   AppendRealtimeStoreCreateConfig{StatisticsMode::NONE}});
+}
+
+TEST(RealtimeContextTest, TestReusesIndexerAndCapturesRegisteredViews) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
     std::shared_ptr<MemoryPool> pool = GetDefaultPool();
 
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState first_state,
-        context->GetOrCreateRealtimeStore({{"dt", "2026-08-02"}}, 0, MakeWriteSchema(),
-                                          StatisticsMode::NONE, {{"k", "v"}}, pool));
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState first_state,
+                         GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0,
+                                                MakeWriteSchema(), {{"k", "v"}}, pool));
     ASSERT_EQ(0, first_state.initial_offset);
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState first_again_state,
-        context->GetOrCreateRealtimeStore({{"dt", "2026-08-02"}}, 0, MakeWriteSchema(),
-                                          StatisticsMode::NONE, {}, pool));
+        GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0, MakeWriteSchema(), {}, pool));
     ASSERT_EQ(first_state.store, first_again_state.store);
     ASSERT_EQ(0, first_again_state.initial_offset);
     ASSERT_EQ(1, factory->stores.size());
@@ -143,12 +148,10 @@ TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
 
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState second_state,
-        context->GetOrCreateRealtimeStore({{"dt", "2026-08-02"}}, 1, MakeWriteSchema(),
-                                          StatisticsMode::NONE, {}, pool));
+        GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 1, MakeWriteSchema(), {}, pool));
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState third_state,
-        context->GetOrCreateRealtimeStore({{"dt", "2026-08-03"}}, 0, MakeWriteSchema(),
-                                          StatisticsMode::NONE, {}, pool));
+        GetOrCreateAppendStore(context, {{"dt", "2026-08-03"}}, 0, MakeWriteSchema(), {}, pool));
     ASSERT_NE(first_state.store, second_state.store);
     ASSERT_NE(first_state.store, third_state.store);
     ASSERT_EQ(3, factory->stores.size());
@@ -171,10 +174,8 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     std::shared_ptr<MemoryPool> pool = GetDefaultPool();
     const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
 
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 0, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, pool));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 1, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, pool));
+    ASSERT_OK(GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, pool));
+    ASSERT_OK(GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, pool));
     ASSERT_EQ(2, factory->stores.size());
 
     ASSERT_NOK_WITH_MSG(context->AdvanceCommittedProgress(-1, {}),
@@ -193,8 +194,7 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
 
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState restored_state,
-        context->GetOrCreateRealtimeStore({{"dt", "unknown"}}, 0, MakeWriteSchema(),
-                                          StatisticsMode::NONE, {}, pool));
+        GetOrCreateAppendStore(context, {{"dt", "unknown"}}, 0, MakeWriteSchema(), {}, pool));
     ASSERT_EQ(9, restored_state.initial_offset);
 
     ASSERT_OK(context->AdvanceCommittedProgress(
@@ -211,41 +211,15 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     ASSERT_EQ(std::vector<int64_t>({8}), factory->stores[1]->committed_offsets);
 }
 
-TEST(RealtimeContextTest, TestRemovedInactivePartitionDoesNotRequireReopen) {
-    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
-    const std::map<std::string, std::string> active_partition = {{"dt", "2026-08-02"}};
-    const std::map<std::string, std::string> inactive_partition = {{"dt", "2026-08-03"}};
-    const RealtimePartitionBucket active_partition_bucket(active_partition, /*bucket=*/0);
-    const RealtimePartitionBucket inactive_partition_bucket(inactive_partition, /*bucket=*/0);
-
-    ASSERT_OK(context->AdvanceCommittedProgress(
-        5, {{active_partition_bucket, /*offset=*/7}, {inactive_partition_bucket, /*offset=*/9}}));
-    ASSERT_OK_AND_ASSIGN(RealtimeStoreState active_state,
-                         context->GetOrCreateRealtimeStore(active_partition, 0, MakeWriteSchema(),
-                                                           StatisticsMode::NONE, {}, pool));
-    ASSERT_EQ(7, active_state.initial_offset);
-
-    ASSERT_OK(context->AdvanceCommittedProgress(6, {{active_partition_bucket, /*offset=*/7}}));
-    ASSERT_OK_AND_ASSIGN(RealtimeStoreState inactive_state,
-                         context->GetOrCreateRealtimeStore(inactive_partition, 0, MakeWriteSchema(),
-                                                           StatisticsMode::NONE, {}, pool));
-    ASSERT_EQ(0, inactive_state.initial_offset);
-}
-
 TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
     std::shared_ptr<MemoryPool> pool = GetDefaultPool();
     const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
 
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 0, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, pool));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 1, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, pool));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(partition, 2, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, pool));
+    ASSERT_OK(GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, pool));
+    ASSERT_OK(GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, pool));
+    ASSERT_OK(GetOrCreateAppendStore(context, partition, 2, MakeWriteSchema(), {}, pool));
     ASSERT_EQ(3, factory->stores.size());
     factory->stores[1]->fail_next_advance = true;
 
@@ -259,9 +233,9 @@ TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     ASSERT_TRUE(factory->stores[1]->committed_offsets.empty());
     ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[2]->committed_offsets);
 
-    ASSERT_OK_AND_ASSIGN(RealtimeStoreState failed_store_state,
-                         context->GetOrCreateRealtimeStore(partition, 1, MakeWriteSchema(),
-                                                           StatisticsMode::NONE, {}, pool));
+    ASSERT_OK_AND_ASSIGN(
+        RealtimeStoreState failed_store_state,
+        GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, pool));
     ASSERT_EQ(8, failed_store_state.initial_offset);
 
     ASSERT_OK(context->AdvanceCommittedProgress(5, committed_offsets));
@@ -271,45 +245,11 @@ TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     ASSERT_EQ(std::vector<int64_t>({8}), factory->stores[1]->committed_offsets);
 }
 
-TEST(RealtimeContextTest, TestRequiresReopenWhenCommittedProgressMovesBackwards) {
-    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
-    const std::map<std::string, std::string> first_partition = {{"dt", "2026-08-02"}};
-    const std::map<std::string, std::string> second_partition = {{"dt", "2026-08-03"}};
-    const RealtimePartitionBucket first_partition_bucket(first_partition, /*bucket=*/0);
-    const RealtimePartitionBucket second_partition_bucket(second_partition, /*bucket=*/0);
-
-    ASSERT_OK(context->GetOrCreateRealtimeStore(first_partition, 0, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, pool));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(second_partition, 0, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, pool));
-    ASSERT_OK(context->AdvanceCommittedProgress(
-        5, {{first_partition_bucket, /*offset=*/7}, {second_partition_bucket, /*offset=*/9}}));
-    ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
-    ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[1]->committed_offsets);
-
-    ASSERT_NOK_WITH_MSG(
-        context->AdvanceCommittedProgress(
-            6, {{first_partition_bucket, /*offset=*/6}, {second_partition_bucket, /*offset=*/10}}),
-        "recreate RealtimeContext");
-    ASSERT_NOK_WITH_MSG(
-        context->AdvanceCommittedProgress(6, {{first_partition_bucket, /*offset=*/10}}),
-        "recreate RealtimeContext");
-    ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
-    ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[1]->committed_offsets);
-
-    ASSERT_OK(context->AdvanceCommittedProgress(
-        6, {{first_partition_bucket, /*offset=*/10}, {second_partition_bucket, /*offset=*/11}}));
-    ASSERT_EQ(std::vector<int64_t>({7, 10}), factory->stores[0]->committed_offsets);
-    ASSERT_EQ(std::vector<int64_t>({9, 11}), factory->stores[1]->committed_offsets);
-}
-
 TEST(RealtimeContextTest, TestPinsResolvesAndReleasesReadViewTicket) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(/*partition=*/{}, /*bucket=*/0, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, GetDefaultPool()));
+    ASSERT_OK(GetOrCreateAppendStore(context, /*partition=*/{}, /*bucket=*/0, MakeWriteSchema(), {},
+                                     GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimePartitionBucketView> views,
                          context->AcquireReadViews());
     ASSERT_EQ(1, views.size());
@@ -332,8 +272,8 @@ TEST(RealtimeContextTest, TestPinsResolvesAndReleasesReadViewTicket) {
 TEST(RealtimeContextTest, TestExpiresAbandonedReadViewTicket) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    ASSERT_OK(context->GetOrCreateRealtimeStore(/*partition=*/{}, /*bucket=*/0, MakeWriteSchema(),
-                                                StatisticsMode::NONE, {}, GetDefaultPool()));
+    ASSERT_OK(GetOrCreateAppendStore(context, /*partition=*/{}, /*bucket=*/0, MakeWriteSchema(), {},
+                                     GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimePartitionBucketView> views,
                          context->AcquireReadViews());
     ASSERT_EQ(1, views.size());
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
new file mode 100644
index 000000000..2ebcede82
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/realtime_primary_key_writer.h"
+
+#include <limits>
+#include <optional>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "paimon/common/reader/concat_batch_reader.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/row_kind.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/mergetree/merge_tree_writer.h"
+#include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/utils/commit_increment.h"
+#include "paimon/macros.h"
+#include "paimon/realtime/realtime_context.h"
+
+namespace paimon {
+
+Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Create(
+    const std::map<std::string, std::string>& partition, int32_t bucket,
+    std::unique_ptr<::ArrowSchema> write_schema,
+    const std::vector<std::string>& trimmed_primary_keys,
+    const std::shared_ptr<RealtimeContext>& realtime_context,
+    const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+    const std::map<std::string, std::string>& options,
+    const std::shared_ptr<MemoryPool>& memory_pool, int64_t restore_max_sequence_number) {
+    ScopeGuard schema_guard([schema = write_schema.get()]() {
+        if (schema && schema->release) {
+            ArrowSchemaRelease(schema);
+        }
+    });
+    if (!realtime_context) {
+        return Status::Invalid("PK real-time context is null");
+    }
+    if (!merge_tree_writer) {
+        return Status::Invalid("PK real-time merge-tree writer is null");
+    }
+    if (!write_schema || !write_schema->release) {
+        return Status::Invalid("PK real-time write schema is null");
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> imported_schema,
+                                      arrow::ImportSchema(write_schema.get()));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*imported_schema, write_schema.get()));
+    RealtimeStoreCreateRequest request{
+        std::move(write_schema),
+        options,
+        memory_pool,
+        partition,
+        bucket,
+        PrimaryKeyRealtimeStoreCreateConfig{trimmed_primary_keys, restore_max_sequence_number}};
+    schema_guard.Release();
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                           RealtimeContextImpl::Cast(realtime_context));
+    PAIMON_ASSIGN_OR_RAISE(RealtimeStoreState store_state,
+                           realtime_context_impl->GetOrCreateRealtimeStore(std::move(request)));
+    return std::shared_ptr<RealtimePrimaryKeyWriter>(
+        new RealtimePrimaryKeyWriter(store_state.store, merge_tree_writer, realtime_context_impl,
+                                     RealtimePartitionBucket(partition, bucket), imported_schema,
+                                     store_state.initial_offset, memory_pool));
+}
+
+RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
+    const std::shared_ptr<RealtimeStore>& realtime_store,
+    const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+    const std::shared_ptr<RealtimeContextImpl>& realtime_context,
+    const RealtimePartitionBucket& partition_bucket,
+    const std::shared_ptr<arrow::Schema>& write_schema, int64_t next_offset,
+    const std::shared_ptr<MemoryPool>& memory_pool)
+    : memory_pool_(memory_pool),
+      realtime_store_(realtime_store),
+      merge_tree_writer_(merge_tree_writer),
+      realtime_context_(realtime_context),
+      partition_bucket_(partition_bucket),
+      write_schema_(write_schema),
+      next_offset_(next_offset) {}
+
+Status RealtimePrimaryKeyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
+    if (!batch || !batch->GetData()) {
+        return Status::Invalid("PK real-time write batch is null");
+    }
+    const int64_t row_count = batch->GetData()->length;
+    if (row_count == 0) {
+        return Status::OK();
+    }
+    std::lock_guard<std::mutex> lock(realtime_store_mutex_);
+    if (row_count > std::numeric_limits<int64_t>::max() - next_offset_) {
+        return Status::Invalid("real-time offset range exceeds INT64_MAX");
+    }
+    const OffsetRange range(next_offset_, next_offset_ + row_count);
+    PAIMON_RETURN_NOT_OK(realtime_store_->Write(RealtimeWriteBatch{std::move(batch), range}));
+    next_offset_ += row_count;
+    return Status::OK();
+}
+
+Result<CommitIncrement> RealtimePrimaryKeyWriter::PrepareCommit(bool wait_compaction) {
+    std::lock_guard<std::mutex> lock(prepare_mutex_);
+    std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment;
+    {
+        std::lock_guard<std::mutex> realtime_store_lock(realtime_store_mutex_);
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<RealtimeSegmentHandle>> sealed_segment,
+                               realtime_store_->SealForCommit());
+        segment = std::move(sealed_segment);
+    }
+    if (segment) {
+        PAIMON_RETURN_NOT_OK(FlushSegment(segment.value()));
+    }
+    PAIMON_ASSIGN_OR_RAISE(CommitIncrement increment,
+                           merge_tree_writer_->PrepareCommit(wait_compaction));
+    if (segment) {
+        const std::vector<std::shared_ptr<DataFileMeta>>& new_files =
+            increment.GetNewFilesIncrement().NewFiles();
+        if (!new_files.empty()) {
+            realtime_context_->AdvanceMaterializedMaxSequenceNumber(
+                partition_bucket_, DataFileMeta::GetMaxSequenceNumber(new_files));
+        }
+        increment.SetRealtimeOffsetRange(segment.value()->GetOffsetRange());
+    }
+    return increment;
+}
+
+Status RealtimePrimaryKeyWriter::FlushSegment(
+    const std::shared_ptr<RealtimeSegmentHandle>& segment) {
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                           realtime_store_->CreateCommitReaders(segment));
+    for (const std::unique_ptr<BatchReader>& reader : readers) {
+        if (!reader) {
+            return Status::Invalid("PK real-time store returned a null commit reader");
+        }
+    }
+    ConcatBatchReader reader(std::move(readers), memory_pool_);
+    ScopeGuard reader_guard([&reader]() { reader.Close(); });
+    const OffsetRange offset_range = segment->GetOffsetRange();
+    int64_t emitted_rows = 0;
+    while (true) {
+        PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader.NextBatch());
+        if (BatchReader::IsEofBatch(batch)) {
+            break;
+        }
+        auto& [c_array, c_schema] = batch;
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> imported,
+                                          arrow::ImportArray(c_array.get(), c_schema.get()));
+        if (!imported || imported->type_id() != arrow::Type::STRUCT) {
+            return Status::Invalid("PK real-time store commit reader returned a non-StructArray");
+        }
+        std::shared_ptr<arrow::StructArray> struct_array =
+            checked_pointer_cast<arrow::StructArray>(imported);
+        std::shared_ptr<arrow::Array> value_kind =
+            struct_array->GetFieldByName(SpecialFields::ValueKind().Name());
+        if (!value_kind || value_kind->type_id() != arrow::Type::INT8) {
+            return Status::Invalid(
+                "PK real-time store commit reader must return an INT8 _VALUE_KIND field");
+        }
+        std::shared_ptr<arrow::Int8Array> encoded_row_kinds =
+            checked_pointer_cast<arrow::Int8Array>(value_kind);
+        std::vector<RecordBatch::RowKind> row_kinds;
+        row_kinds.reserve(static_cast<size_t>(encoded_row_kinds->length()));
+        for (int64_t i = 0; i < encoded_row_kinds->length(); ++i) {
+            if (encoded_row_kinds->IsNull(i)) {
+                return Status::Invalid("PK real-time store commit reader returned a null row kind");
+            }
+            PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind,
+                                   RowKind::FromByteValue(encoded_row_kinds->Value(i)));
+            row_kinds.push_back(static_cast<RecordBatch::RowKind>(row_kind->ToByteValue()));
+        }
+        PAIMON_ASSIGN_OR_RAISE(struct_array, ArrowUtils::RemoveFieldFromStructArray(
+                                                 struct_array, SpecialFields::ValueKind().Name()));
+        if (!struct_array->type()->Equals(arrow::struct_(write_schema_->fields()))) {
+            return Status::Invalid(
+                "PK real-time store commit reader schema does not match table write schema");
+        }
+        const int64_t row_count = struct_array->length();
+        if (row_count > offset_range.Count() - emitted_rows) {
+            return Status::Invalid(
+                "PK real-time store commit readers returned more rows than the sealed offset "
+                "range");
+        }
+        emitted_rows += row_count;
+        if (row_count == 0) {
+            continue;
+        }
+        auto output = std::make_unique<ArrowArray>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, output.get()));
+        RecordBatchBuilder builder(output.get());
+        builder.SetRowKinds(row_kinds);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> record_batch, builder.Finish());
+        PAIMON_RETURN_NOT_OK(merge_tree_writer_->Write(std::move(record_batch)));
+    }
+    if (emitted_rows != offset_range.Count()) {
+        return Status::Invalid(
+            "PK real-time store commit readers returned fewer rows than the sealed offset range");
+    }
+    return Status::OK();
+}
+
+Status RealtimePrimaryKeyWriter::Compact(bool) {
+    return Status::Invalid("PK real-time write does not support explicit compaction");
+}
+
+uint64_t RealtimePrimaryKeyWriter::GetMemoryUsage() const {
+    return realtime_store_->GetMemoryUsage();
+}
+
+Status RealtimePrimaryKeyWriter::FlushMemory() {
+    return Status::OK();
+}
+
+Result<bool> RealtimePrimaryKeyWriter::CompactNotCompleted() {
+    return merge_tree_writer_->CompactNotCompleted();
+}
+
+Status RealtimePrimaryKeyWriter::Sync() {
+    return merge_tree_writer_->Sync();
+}
+
+Status RealtimePrimaryKeyWriter::Close() {
+    return merge_tree_writer_->Close();
+}
+
+std::shared_ptr<Metrics> RealtimePrimaryKeyWriter::GetMetrics() const {
+    return merge_tree_writer_->GetMetrics();
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
new file mode 100644
index 000000000..fa057e079
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#include "paimon/core/utils/batch_writer.h"
+#include "paimon/realtime/realtime_context.h"
+#include "paimon/realtime/realtime_store.h"
+
+struct ArrowSchema;
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
+namespace paimon {
+
+class MemoryPool;
+class MergeTreeWriter;
+class RealtimeContext;
+class RealtimeContextImpl;
+
+/// Primary-key real-time writer backed by an in-memory mutation indexer.
+class RealtimePrimaryKeyWriter final : public BatchWriter {
+ public:
+    static Result<std::shared_ptr<RealtimePrimaryKeyWriter>> Create(
+        const std::map<std::string, std::string>& partition, int32_t bucket,
+        std::unique_ptr<::ArrowSchema> write_schema,
+        const std::vector<std::string>& trimmed_primary_keys,
+        const std::shared_ptr<RealtimeContext>& realtime_context,
+        const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+        const std::map<std::string, std::string>& options,
+        const std::shared_ptr<MemoryPool>& memory_pool, int64_t restore_max_sequence_number);
+
+    Status Write(std::unique_ptr<RecordBatch>&& batch) override;
+    Result<CommitIncrement> PrepareCommit(bool wait_compaction) override;
+    Status Compact(bool full_compaction) override;
+    uint64_t GetMemoryUsage() const override;
+    Status FlushMemory() override;
+    Result<bool> CompactNotCompleted() override;
+    Status Sync() override;
+    Status Close() override;
+    std::shared_ptr<Metrics> GetMetrics() const override;
+
+ private:
+    RealtimePrimaryKeyWriter(const std::shared_ptr<RealtimeStore>& realtime_store,
+                             const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+                             const std::shared_ptr<RealtimeContextImpl>& realtime_context,
+                             const RealtimePartitionBucket& partition_bucket,
+                             const std::shared_ptr<arrow::Schema>& write_schema,
+                             int64_t next_offset, const std::shared_ptr<MemoryPool>& memory_pool);
+
+    Status FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment);
+
+    std::shared_ptr<MemoryPool> memory_pool_;
+    std::shared_ptr<RealtimeStore> realtime_store_;
+    std::shared_ptr<MergeTreeWriter> merge_tree_writer_;
+    std::shared_ptr<RealtimeContextImpl> realtime_context_;
+    RealtimePartitionBucket partition_bucket_;
+    std::shared_ptr<arrow::Schema> write_schema_;
+    int64_t next_offset_;
+    std::mutex realtime_store_mutex_;
+    std::mutex prepare_mutex_;
+};
+
+}  // namespace paimon

From 82949b30a63f96645d36ded2d1987f2cedc91c4f Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 19 Aug 2026 22:34:33 +0800
Subject: [PATCH 24/93] feat(read): merge primary-key realtime memory with
 snapshots

Capture partition-bucket read views in realtime splits and merge PK memory readers with snapshot data by key range.

Retain read views for reader lifetime, defer ticket consumption until vector reader construction succeeds, and apply predicates after PK deduplication.
---
 .../core/operation/merge_file_split_read.cpp  | 274 ++++++++++++++++++
 .../core/operation/merge_file_split_read.h    |  18 ++
 .../table/source/key_value_table_read.cpp     | 264 +++++++++++++++++
 .../core/table/source/key_value_table_read.h  |   7 +
 .../core/table/source/realtime_table_scan.cpp |   2 +-
 .../core/table/source/realtime_table_scan.h   |   2 +-
 src/paimon/core/table/source/table_scan.cpp   |   7 +-
 7 files changed, 569 insertions(+), 5 deletions(-)

diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index b753ea431..8d8367e39 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -30,6 +30,7 @@
 #include "arrow/c/bridge.h"
 #include "arrow/type.h"
 #include "fmt/format.h"
+#include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/reader/complete_row_kind_batch_reader.h"
 #include "paimon/common/reader/concat_batch_reader.h"
 #include "paimon/common/table/special_fields.h"
@@ -78,6 +79,273 @@ struct KeyValue;
 template <typename T>
 class MergeFunctionWrapper;
 
+namespace {
+
+/// Concatenates merge readers whose key ranges are ordered and non-overlapping, preserving one
+/// projection pipeline without merging independent disk-only components.
+class ConcatNonOverlappingMergeReaders final : public SortMergeReader {
+ public:
+    explicit ConcatNonOverlappingMergeReaders(
+        std::vector<std::unique_ptr<SortMergeReader>>&& readers)
+        : readers_(std::move(readers)) {}
+
+    Result<std::unique_ptr<SortMergeReader::Iterator>> NextBatch() override {
+        while (current_ < readers_.size()) {
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader::Iterator> iterator,
+                                   readers_[current_]->NextBatch());
+            if (iterator) {
+                return iterator;
+            }
+            readers_[current_]->Close();
+            ++current_;
+        }
+        return std::unique_ptr<SortMergeReader::Iterator>();
+    }
+
+    void Close() override {
+        while (current_ < readers_.size()) {
+            readers_[current_++]->Close();
+        }
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return MetricsImpl::CollectReadMetrics(readers_);
+    }
+
+ private:
+    std::vector<std::unique_ptr<SortMergeReader>> readers_;
+    size_t current_ = 0;
+};
+
+}  // namespace
+
+class MergeFileSplitRead::RealtimeReaderBuilder {
+ public:
+    static Result<std::unique_ptr<BatchReader>> Create(
+        MergeFileSplitRead* owner, const std::vector<std::shared_ptr<Split>>& disk_splits,
+        std::vector<AdditionalKeyValueReader>&& additional_readers) {
+        RealtimeReaderBuilder builder(owner);
+        if (disk_splits.empty()) {
+            std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
+            readers.reserve(additional_readers.size());
+            for (AdditionalKeyValueReader& additional : additional_readers) {
+                readers.push_back(std::move(additional.reader));
+            }
+            return builder.CreateMergedReader(std::move(readers));
+        }
+
+        PAIMON_RETURN_NOT_OK(builder.CollectDiskInputs(disk_splits));
+        builder.AddRangeInputs(std::move(additional_readers));
+        return builder.CreateReader();
+    }
+
+ private:
+    struct RangeInput {
+        std::shared_ptr<InternalRow> min_key;
+        std::shared_ptr<InternalRow> max_key;
+        std::vector<SortedRun> disk_runs;
+        std::unique_ptr<KeyValueRecordReader> additional_reader;
+    };
+
+    explicit RealtimeReaderBuilder(MergeFileSplitRead* owner) : owner_(owner) {}
+
+    Status CollectDiskInputs(const std::vector<std::shared_ptr<Split>>& disk_splits) {
+        first_split_ = std::dynamic_pointer_cast<DataSplitImpl>(disk_splits.front());
+        if (!first_split_) {
+            return Status::Invalid("merge input disk split is not a data split");
+        }
+        const BinaryRow& partition = first_split_->Partition();
+        const int32_t bucket = first_split_->Bucket();
+        PAIMON_ASSIGN_OR_RAISE(data_file_path_factory_,
+                               owner_->path_factory_->CreateDataFilePathFactory(partition, bucket));
+
+        std::vector<std::shared_ptr<DataFileMeta>> data_files;
+        std::vector<std::optional<DeletionFile>> deletion_files;
+        for (const std::shared_ptr<Split>& disk_split : disk_splits) {
+            std::shared_ptr<DataSplitImpl> data_split =
+                std::dynamic_pointer_cast<DataSplitImpl>(disk_split);
+            if (!data_split || !(data_split->Partition() == partition) ||
+                data_split->Bucket() != bucket) {
+                return Status::Invalid("merge input disk splits do not share a partition-bucket");
+            }
+            if (!data_split->BeforeFiles().empty() || data_split->IsStreaming() ||
+                data_split->Bucket() == BucketModeDefine::POSTPONE_BUCKET) {
+                return Status::Invalid("additional merge input requires fixed-bucket batch splits");
+            }
+            const std::vector<std::shared_ptr<DataFileMeta>>& split_files = data_split->DataFiles();
+            const std::vector<std::optional<DeletionFile>>& split_deletion_files =
+                data_split->DeletionFiles();
+            if (!split_deletion_files.empty() &&
+                split_deletion_files.size() != split_files.size()) {
+                return Status::Invalid(
+                    "merge input disk split deletion files must be empty or match data files");
+            }
+            data_files.insert(data_files.end(), split_files.begin(), split_files.end());
+            if (split_deletion_files.empty()) {
+                deletion_files.insert(deletion_files.end(), split_files.size(), std::nullopt);
+            } else {
+                deletion_files.insert(deletion_files.end(), split_deletion_files.begin(),
+                                      split_deletion_files.end());
+            }
+        }
+
+        dv_factory_ = DeletionVector::CreateFactory(
+            owner_->options_.GetFileSystem(),
+            DeletionVector::CreateDeletionFileMap(data_files, deletion_files), owner_->pool_);
+        std::vector<std::vector<SortedRun>> disk_sections =
+            IntervalPartition(data_files, owner_->key_comparator_).Partition();
+        inputs_.reserve(disk_sections.size());
+        for (std::vector<SortedRun>& section : disk_sections) {
+            std::shared_ptr<DataFileMeta> min_file = section.front().Files().front();
+            std::shared_ptr<DataFileMeta> max_file = min_file;
+            for (const SortedRun& run : section) {
+                for (const std::shared_ptr<DataFileMeta>& file : run.Files()) {
+                    if (owner_->key_comparator_->CompareTo(file->min_key, min_file->min_key) < 0) {
+                        min_file = file;
+                    }
+                    if (owner_->key_comparator_->CompareTo(file->max_key, max_file->max_key) > 0) {
+                        max_file = file;
+                    }
+                }
+            }
+            inputs_.push_back(RangeInput{std::shared_ptr<InternalRow>(min_file, &min_file->min_key),
+                                         std::shared_ptr<InternalRow>(max_file, &max_file->max_key),
+                                         std::move(section), nullptr});
+        }
+        return Status::OK();
+    }
+
+    void AddRangeInputs(std::vector<AdditionalKeyValueReader>&& additional_readers) {
+        inputs_.reserve(inputs_.size() + additional_readers.size());
+        for (AdditionalKeyValueReader& additional : additional_readers) {
+            has_unknown_range_ |= !additional.min_key || !additional.max_key;
+            inputs_.push_back(RangeInput{additional.min_key, additional.max_key,
+                                         /*disk_runs=*/{}, std::move(additional.reader)});
+        }
+    }
+
+    Result<std::unique_ptr<KeyValueRecordReader>> CreateDiskReader(const SortedRun& run) {
+        return owner_->CreateReaderForRun(first_split_->Partition(), run, dv_factory_,
+                                          owner_->predicate_for_keys_, data_file_path_factory_);
+    }
+
+    Result<std::unique_ptr<BatchReader>> CreateMergedReader(
+        std::vector<std::unique_ptr<KeyValueRecordReader>>&& record_readers) {
+        if (record_readers.empty()) {
+            return std::make_unique<ConcatBatchReader>(std::vector<std::unique_ptr<BatchReader>>{},
+                                                       owner_->pool_);
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> sort_merge_reader,
+                               owner_->CreateSortMergeReader(std::move(record_readers)));
+        return CreateProjectedReader(std::move(sort_merge_reader));
+    }
+
+    Result<std::unique_ptr<BatchReader>> CreateProjectedReader(
+        std::unique_ptr<SortMergeReader>&& sort_merge_reader) {
+        if (!owner_->force_keep_delete_) {
+            sort_merge_reader = std::make_unique<DropDeleteReader>(std::move(sort_merge_reader));
+        }
+
+        std::unique_ptr<BatchReader> projection_reader;
+        if (!owner_->context_->EnableMultiThreadRowToBatch()) {
+            PAIMON_ASSIGN_OR_RAISE(
+                projection_reader,
+                KeyValueProjectionReader::Create(
+                    std::move(sort_merge_reader), owner_->raw_read_schema_, owner_->projection_,
+                    owner_->options_.GetReadBatchSize(), owner_->pool_));
+        } else {
+            const int32_t thread_number = owner_->context_->GetRowToBatchThreadNumber();
+            assert(thread_number > 0);
+            projection_reader = std::make_unique<AsyncKeyValueProjectionReader>(
+                std::move(sort_merge_reader), owner_->raw_read_schema_, owner_->projection_,
+                owner_->options_.GetReadBatchSize(), thread_number, owner_->pool_);
+        }
+        PAIMON_ASSIGN_OR_RAISE(projection_reader,
+                               owner_->ApplyPredicateFilterIfNeeded(
+                                   std::move(projection_reader), owner_->context_->GetPredicate()));
+        return std::make_unique<CompleteRowKindBatchReader>(std::move(projection_reader),
+                                                            owner_->pool_);
+    }
+
+    Result<std::unique_ptr<BatchReader>> CreateUnknownRangeReader() {
+        std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
+        for (RangeInput& input : inputs_) {
+            for (const SortedRun& run : input.disk_runs) {
+                PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> disk_reader,
+                                       CreateDiskReader(run));
+                readers.push_back(std::move(disk_reader));
+            }
+            if (input.additional_reader) {
+                readers.push_back(std::move(input.additional_reader));
+            }
+        }
+        return CreateMergedReader(std::move(readers));
+    }
+
+    Result<std::unique_ptr<BatchReader>> CreateKnownRangeReader() {
+        std::sort(inputs_.begin(), inputs_.end(),
+                  [this](const RangeInput& lhs, const RangeInput& rhs) {
+                      return owner_->key_comparator_->CompareTo(*lhs.min_key, *rhs.min_key) < 0;
+                  });
+        std::vector<std::vector<RangeInput>> components;
+        std::shared_ptr<InternalRow> component_max_key;
+        for (RangeInput& input : inputs_) {
+            if (components.empty() ||
+                owner_->key_comparator_->CompareTo(*input.min_key, *component_max_key) > 0) {
+                components.emplace_back();
+                component_max_key = input.max_key;
+            } else if (owner_->key_comparator_->CompareTo(*input.max_key, *component_max_key) > 0) {
+                component_max_key = input.max_key;
+            }
+            components.back().push_back(std::move(input));
+        }
+
+        std::vector<std::unique_ptr<SortMergeReader>> component_readers;
+        component_readers.reserve(components.size());
+        for (std::vector<RangeInput>& component : components) {
+            if (component.size() == 1 && !component.front().additional_reader) {
+                PAIMON_ASSIGN_OR_RAISE(
+                    std::unique_ptr<SortMergeReader> disk_component,
+                    owner_->CreateSortMergeReaderForSection(
+                        component.front().disk_runs, first_split_->Partition(), dv_factory_,
+                        component.front().disk_runs.size() == 1 ? owner_->context_->GetPredicate()
+                                                                : owner_->predicate_for_keys_,
+                        data_file_path_factory_, /*drop_delete=*/false));
+                component_readers.push_back(std::move(disk_component));
+                continue;
+            }
+
+            std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
+            for (RangeInput& input : component) {
+                for (const SortedRun& run : input.disk_runs) {
+                    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> disk_reader,
+                                           CreateDiskReader(run));
+                    readers.push_back(std::move(disk_reader));
+                }
+                if (input.additional_reader) {
+                    readers.push_back(std::move(input.additional_reader));
+                }
+            }
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> component_reader,
+                                   owner_->CreateSortMergeReader(std::move(readers)));
+            component_readers.push_back(std::move(component_reader));
+        }
+        return CreateProjectedReader(
+            std::make_unique<ConcatNonOverlappingMergeReaders>(std::move(component_readers)));
+    }
+
+    Result<std::unique_ptr<BatchReader>> CreateReader() {
+        return has_unknown_range_ ? CreateUnknownRangeReader() : CreateKnownRangeReader();
+    }
+
+    MergeFileSplitRead* owner_;
+    std::shared_ptr<DataSplitImpl> first_split_;
+    std::shared_ptr<DataFilePathFactory> data_file_path_factory_;
+    DeletionVector::Factory dv_factory_;
+    std::vector<RangeInput> inputs_;
+    bool has_unknown_range_ = false;
+};
+
 Result<std::unique_ptr<MergeFileSplitRead>> MergeFileSplitRead::Create(
     const std::shared_ptr<FileStorePathFactory>& path_factory,
     const std::shared_ptr<InternalReadContext>& context,
@@ -158,6 +426,12 @@ Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateReader(
     return std::make_unique<CompleteRowKindBatchReader>(std::move(batch_reader), pool_);
 }
 
+Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateRealtimeReader(
+    const std::vector<std::shared_ptr<Split>>& disk_splits,
+    std::vector<AdditionalKeyValueReader>&& additional_readers) {
+    return RealtimeReaderBuilder::Create(this, disk_splits, std::move(additional_readers));
+}
+
 void MergeFileSplitRead::SetMergeFunctionWrapper(
     const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper) {
     merge_function_wrapper_ = merge_function_wrapper;
diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h
index 5003cb55a..0824254b7 100644
--- a/src/paimon/core/operation/merge_file_split_read.h
+++ b/src/paimon/core/operation/merge_file_split_read.h
@@ -55,6 +55,7 @@ class FieldsComparator;
 class FileBatchReader;
 class FileStorePathFactory;
 class InternalReadContext;
+class InternalRow;
 class MemoryPool;
 class SchemaManager;
 class SortedRun;
@@ -65,6 +66,12 @@ struct KeyValue;
 template <typename T>
 class MergeFunctionWrapper;
 
+struct AdditionalKeyValueReader {
+    std::unique_ptr<KeyValueRecordReader> reader;
+    std::shared_ptr<InternalRow> min_key;
+    std::shared_ptr<InternalRow> max_key;
+};
+
 /// If the class name below is enclosed in parentheses, it might be present in the read path;
 /// otherwise, it must be present in the read path.
 ///
@@ -116,10 +123,21 @@ class MergeFileSplitRead : public AbstractSplitRead {
         return value_schema_;
     }
 
+    std::shared_ptr<arrow::Schema> GetKeySchema() const {
+        return key_schema_;
+    }
+
+    /// Merges ordinary disk splits with generic additional sorted KeyValue readers.
+    Result<std::unique_ptr<BatchReader>> CreateRealtimeReader(
+        const std::vector<std::shared_ptr<Split>>& disk_splits,
+        std::vector<AdditionalKeyValueReader>&& additional_readers);
+
     void SetMergeFunctionWrapper(
         const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper);
 
  private:
+    class RealtimeReaderBuilder;
+
     Result<std::unique_ptr<BatchReader>> CreateMergeReader(
         const std::shared_ptr<DataSplitImpl>& data_split,
         const std::shared_ptr<DataFilePathFactory>& data_file_path_factory);
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 208807493..770caf1ca 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -20,12 +20,28 @@
 #include "paimon/core/table/source/key_value_table_read.h"
 
 #include <utility>
+#include <vector>
 
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "paimon/common/data/columnar/columnar_row_ref.h"
+#include "paimon/common/reader/concat_batch_reader.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/types/row_kind.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/global_index/indexed_split_impl.h"
+#include "paimon/core/key_value.h"
 #include "paimon/core/operation/merge_file_split_read.h"
 #include "paimon/core/operation/raw_file_split_read.h"
+#include "paimon/core/realtime/primary_key_realtime_store.h"
+#include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_reader.h"
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/table/source/pk_count_reader.h"
+#include "paimon/core/table/source/realtime_split.h"
 #include "paimon/status.h"
 
 namespace paimon {
@@ -34,6 +50,163 @@ class Executor;
 class FileStorePathFactory;
 class InternalReadContext;
 class MemoryPool;
+struct ColumnarBatchContext;
+
+namespace {
+
+class QueryBatchKeyValueReader final : public KeyValueRecordReader {
+ public:
+    QueryBatchKeyValueReader(std::unique_ptr<BatchReader>&& reader,
+                             const std::shared_ptr<arrow::Schema>& key_schema,
+                             const std::shared_ptr<arrow::Schema>& value_schema,
+                             const std::shared_ptr<MemoryPool>& pool)
+        : reader_(std::move(reader)),
+          key_schema_(key_schema),
+          value_schema_(value_schema),
+          pool_(pool) {}
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override;
+    std::shared_ptr<Metrics> GetReaderMetrics() const override;
+    void Close() override;
+
+ private:
+    class Iterator;
+
+    std::unique_ptr<BatchReader> reader_;
+    std::shared_ptr<arrow::Schema> key_schema_;
+    std::shared_ptr<arrow::Schema> value_schema_;
+    std::shared_ptr<MemoryPool> pool_;
+    std::shared_ptr<arrow::StructArray> values_;
+    std::shared_ptr<arrow::Int64Array> sequences_;
+    std::shared_ptr<arrow::Int8Array> row_kinds_;
+    std::shared_ptr<ColumnarBatchContext> key_context_;
+    std::shared_ptr<ColumnarBatchContext> value_context_;
+};
+
+class QueryBatchKeyValueReader::Iterator final : public KeyValueRecordReader::Iterator {
+ public:
+    explicit Iterator(QueryBatchKeyValueReader* reader) : reader_(reader) {}
+
+    Result<bool> HasNext() const override {
+        return cursor_ < reader_->values_->length();
+    }
+
+    Result<KeyValue> Next() override {
+        if (reader_->sequences_->IsNull(cursor_) || reader_->row_kinds_->IsNull(cursor_)) {
+            return Status::Invalid("PK merge metadata must not be null");
+        }
+        PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind,
+                               RowKind::FromByteValue(reader_->row_kinds_->Value(cursor_)));
+        const int64_t sequence = reader_->sequences_->Value(cursor_);
+        std::shared_ptr<InternalRow> key =
+            std::make_shared<ColumnarRowRef>(reader_->key_context_, cursor_);
+        auto value = std::make_unique<ColumnarRowRef>(reader_->value_context_, cursor_++);
+        return KeyValue(row_kind, sequence, KeyValue::UNKNOWN_LEVEL, std::move(key),
+                        std::move(value));
+    }
+
+ private:
+    QueryBatchKeyValueReader* reader_;
+    int64_t cursor_ = 0;
+};
+
+Result<std::unique_ptr<KeyValueRecordReader::Iterator>> QueryBatchKeyValueReader::NextBatch() {
+    PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
+    if (BatchReader::IsEofBatch(batch)) {
+        return std::unique_ptr<KeyValueRecordReader::Iterator>();
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> imported,
+                                      arrow::ImportArray(batch.first.get(), batch.second.get()));
+    std::shared_ptr<arrow::StructArray> input =
+        std::dynamic_pointer_cast<arrow::StructArray>(imported);
+    if (!input) {
+        return Status::Invalid("PK merge input is not a StructArray");
+    }
+    sequences_ = std::dynamic_pointer_cast<arrow::Int64Array>(
+        input->GetFieldByName(SpecialFields::SequenceNumber().Name()));
+    row_kinds_ = std::dynamic_pointer_cast<arrow::Int8Array>(
+        input->GetFieldByName(SpecialFields::ValueKind().Name()));
+    if (!sequences_ || !row_kinds_) {
+        return Status::Invalid("PK merge input is missing sequence or value-kind metadata");
+    }
+    PAIMON_ASSIGN_OR_RAISE(input, ArrowUtils::RemoveFieldFromStructArray(
+                                      input, SpecialFields::SequenceNumber().Name()));
+    PAIMON_ASSIGN_OR_RAISE(
+        values_, ArrowUtils::RemoveFieldFromStructArray(input, SpecialFields::ValueKind().Name()));
+    if (!ArrowUtils::EqualsIgnoreNullable(values_->type(),
+                                          arrow::struct_(value_schema_->fields()))) {
+        return Status::Invalid("PK merge input value schema does not match the table read schema");
+    }
+    arrow::ArrayVector key_fields;
+    key_fields.reserve(key_schema_->num_fields());
+    for (const std::shared_ptr<arrow::Field>& field : key_schema_->fields()) {
+        std::shared_ptr<arrow::Array> key = values_->GetFieldByName(field->name());
+        if (!key) {
+            return Status::Invalid("PK merge input is missing key field ", field->name());
+        }
+        key_fields.push_back(std::move(key));
+    }
+    key_context_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
+    value_context_ = std::make_shared<ColumnarBatchContext>(values_->fields(), pool_);
+    return std::make_unique<Iterator>(this);
+}
+
+std::shared_ptr<Metrics> QueryBatchKeyValueReader::GetReaderMetrics() const {
+    return reader_->GetReaderMetrics();
+}
+
+void QueryBatchKeyValueReader::Close() {
+    values_.reset();
+    sequences_.reset();
+    row_kinds_.reset();
+    key_context_.reset();
+    value_context_.reset();
+    reader_->Close();
+}
+
+Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
+    const std::shared_ptr<RealtimeSplit>& split, const RealtimePartitionBucketView& memory,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<InternalReadContext>& context,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    arrow::FieldVector requested_fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())};
+    requested_fields.insert(requested_fields.end(), value_schema->fields().begin(),
+                            value_schema->fields().end());
+    auto c_schema = std::make_unique<ArrowSchema>();
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(
+        arrow::ExportSchema(*arrow::schema(requested_fields), c_schema.get()));
+    ScopeGuard schema_guard([schema = c_schema.get()]() { ArrowSchemaRelease(schema); });
+    RealtimeQueryContext query_context{c_schema.get(), /*predicate=*/nullptr,
+                                       /*enable_predicate_pushdown=*/false};
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> batch_readers,
+                           memory.store->CreateQueryReaders(
+                               memory.read_view, split->CommittedEndOffset(), query_context));
+    if (batch_readers.empty()) {
+        return Status::Invalid("PK real-time store returned no query readers for active memory");
+    }
+    std::vector<AdditionalKeyValueReader> result;
+    result.reserve(batch_readers.size());
+    for (std::unique_ptr<BatchReader>& reader : batch_readers) {
+        if (!reader) {
+            return Status::Invalid("PK real-time store returned a null query reader");
+        }
+        std::shared_ptr<InternalRow> min_key;
+        std::shared_ptr<InternalRow> max_key;
+        if (auto* provider = dynamic_cast<PrimaryKeyRangeProvider*>(reader.get())) {
+            min_key = provider->GetMinKey();
+            max_key = provider->GetMaxKey();
+        }
+        result.push_back(
+            AdditionalKeyValueReader{std::make_unique<QueryBatchKeyValueReader>(
+                                         std::move(reader), key_schema, value_schema, memory_pool),
+                                     std::move(min_key), std::move(max_key)});
+    }
+    return result;
+}
+
+}  // namespace
 
 KeyValueTableRead::KeyValueTableRead(std::vector<std::unique_ptr<SplitRead>>&& split_reads,
                                      const std::shared_ptr<FileStorePathFactory>& path_factory,
@@ -75,6 +248,11 @@ void KeyValueTableRead::ForceKeepDelete(bool force_keep_delete) {
 
 Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
     const std::shared_ptr<Split>& split) {
+    std::shared_ptr<RealtimeSplit> realtime_split = std::dynamic_pointer_cast<RealtimeSplit>(split);
+    if (realtime_split) {
+        return CreateRealtimeReader(realtime_split, /*release_ticket=*/true);
+    }
+
     std::shared_ptr<Split> dispatch_split = split;
     if (auto indexed_split = std::dynamic_pointer_cast<IndexedSplitImpl>(split)) {
         PAIMON_RETURN_NOT_OK(indexed_split->Validate());
@@ -126,8 +304,94 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
     return Status::Invalid("create reader failed, not read match with data split.");
 }
 
+Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
+    const std::vector<std::shared_ptr<Split>>& splits) {
+    std::vector<std::unique_ptr<BatchReader>> readers;
+    readers.reserve(splits.size());
+    std::vector<std::shared_ptr<RealtimeSplit>> realtime_splits;
+    for (const std::shared_ptr<Split>& split : splits) {
+        std::shared_ptr<RealtimeSplit> realtime_split =
+            std::dynamic_pointer_cast<RealtimeSplit>(split);
+        if (realtime_split) {
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
+                                   CreateRealtimeReader(realtime_split, /*release_ticket=*/false));
+            readers.push_back(std::move(reader));
+            realtime_splits.push_back(std::move(realtime_split));
+        } else {
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader, CreateReader(split));
+            readers.push_back(std::move(reader));
+        }
+    }
+
+    std::unique_ptr<BatchReader> result =
+        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
+    if (!realtime_splits.empty()) {
+        const std::shared_ptr<RealtimeContext> realtime_context = context_->GetRealtimeContext();
+        if (!realtime_context) {
+            return Status::Invalid("reading a real-time split requires a real-time context");
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                               RealtimeContextImpl::Cast(realtime_context));
+        for (const std::shared_ptr<RealtimeSplit>& realtime_split : realtime_splits) {
+            PAIMON_RETURN_NOT_OK(
+                realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
+        }
+    }
+    return result;
+}
+
+Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
+    const std::shared_ptr<RealtimeSplit>& realtime_split, bool release_ticket) {
+    if (realtime_split->Version() != RealtimeSplit::kCurrentVersion) {
+        return Status::Invalid("unsupported real-time split version");
+    }
+    const std::shared_ptr<RealtimeContext> realtime_context = context_->GetRealtimeContext();
+    if (!realtime_context) {
+        return Status::Invalid("reading a real-time split requires a real-time context");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                           RealtimeContextImpl::Cast(realtime_context));
+    PAIMON_ASSIGN_OR_RAISE(RealtimePartitionBucketView memory,
+                           realtime_context_impl->ResolveReadView(realtime_split->OpaqueTicket()));
+    const RealtimePartitionBucket expected_partition_bucket(realtime_split->Partition(),
+                                                            realtime_split->Bucket());
+    if (memory.partition_bucket != expected_partition_bucket) {
+        return Status::Invalid("real-time read-view ticket belongs to another partition-bucket");
+    }
+    const std::optional<OffsetRange> memory_range = memory.read_view->GetOffsetRange();
+    if (!memory_range || memory_range->end != realtime_split->MemoryEndOffset()) {
+        return Status::Invalid("real-time read-view ticket does not match the split offset range");
+    }
+    for (const std::unique_ptr<SplitRead>& read : split_reads_) {
+        auto* merge_read = dynamic_cast<MergeFileSplitRead*>(read.get());
+        if (merge_read) {
+            PAIMON_ASSIGN_OR_RAISE(
+                std::vector<AdditionalKeyValueReader> memory_readers,
+                CreateMemoryReaders(realtime_split, memory, merge_read->GetKeySchema(),
+                                    merge_read->GetValueSchema(), context_, GetMemoryPool()));
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
+                                   merge_read->CreateRealtimeReader(realtime_split->DiskSplits(),
+                                                                    std::move(memory_readers)));
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RealtimeReader> realtime_reader,
+                                   RealtimeReader::Create(memory.read_view, std::move(reader)));
+            if (release_ticket) {
+                PAIMON_RETURN_NOT_OK(
+                    realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
+            }
+            return std::unique_ptr<BatchReader>(std::move(realtime_reader));
+        }
+    }
+    return Status::Invalid("create reader failed, merge file split read not found");
+}
+
 Result<std::unique_ptr<CountReader>> KeyValueTableRead::CreateCountReader(
     const std::vector<std::shared_ptr<Split>>& splits) {
+    for (const std::shared_ptr<Split>& split : splits) {
+        if (std::dynamic_pointer_cast<RealtimeSplit>(split)) {
+            return Status::NotImplemented(
+                "CreateCountReader does not support process-local real-time splits");
+        }
+    }
     if (context_->GetPredicate() != nullptr) {
         return Status::NotImplemented(
             "CreateCountReader with predicate pushdown is not supported yet");
diff --git a/src/paimon/core/table/source/key_value_table_read.h b/src/paimon/core/table/source/key_value_table_read.h
index d6a1c83d3..6824ae59e 100644
--- a/src/paimon/core/table/source/key_value_table_read.h
+++ b/src/paimon/core/table/source/key_value_table_read.h
@@ -35,6 +35,7 @@ class Executor;
 class FileStorePathFactory;
 class InternalReadContext;
 class MemoryPool;
+class RealtimeSplit;
 
 class KeyValueTableRead : public TableRead {
  public:
@@ -45,6 +46,9 @@ class KeyValueTableRead : public TableRead {
 
     Result<std::unique_ptr<BatchReader>> CreateReader(const std::shared_ptr<Split>& split) override;
 
+    Result<std::unique_ptr<BatchReader>> CreateReader(
+        const std::vector<std::shared_ptr<Split>>& splits) override;
+
     Result<std::unique_ptr<CountReader>> CreateCountReader(
         const std::vector<std::shared_ptr<Split>>& splits) override;
 
@@ -57,6 +61,9 @@ class KeyValueTableRead : public TableRead {
                       const std::shared_ptr<MemoryPool>& memory_pool,
                       const std::shared_ptr<Executor>& executor);
 
+    Result<std::unique_ptr<BatchReader>> CreateRealtimeReader(
+        const std::shared_ptr<RealtimeSplit>& realtime_split, bool release_ticket);
+
     std::vector<std::unique_ptr<SplitRead>> split_reads_;
     std::shared_ptr<FileStorePathFactory> path_factory_;
     std::shared_ptr<InternalReadContext> context_;
diff --git a/src/paimon/core/table/source/realtime_table_scan.cpp b/src/paimon/core/table/source/realtime_table_scan.cpp
index c275208c5..1b496d8a1 100644
--- a/src/paimon/core/table/source/realtime_table_scan.cpp
+++ b/src/paimon/core/table/source/realtime_table_scan.cpp
@@ -108,7 +108,7 @@ Result<std::vector<std::shared_ptr<Split>>> RealtimeTableScan::CreateRealtimeSpl
     for (const std::shared_ptr<Split>& split : disk_splits) {
         std::shared_ptr<DataSplitImpl> data_split = std::dynamic_pointer_cast<DataSplitImpl>(split);
         if (!data_split) {
-            return Status::Invalid("real-time append scan requires process-local data splits");
+            return Status::Invalid("real-time scan requires process-local data splits");
         }
         std::vector<std::pair<std::string, std::string>> partition_values;
         PAIMON_ASSIGN_OR_RAISE(partition_values,
diff --git a/src/paimon/core/table/source/realtime_table_scan.h b/src/paimon/core/table/source/realtime_table_scan.h
index 7d036d420..959203ca4 100644
--- a/src/paimon/core/table/source/realtime_table_scan.h
+++ b/src/paimon/core/table/source/realtime_table_scan.h
@@ -35,7 +35,7 @@ class FileSystem;
 class ScanFilter;
 class SnapshotManager;
 
-/// Adds process-local memory splits to a normal append-table batch scan.
+/// Adds process-local memory splits to a normal data-table batch scan.
 class RealtimeTableScan : public TableScan {
  public:
     RealtimeTableScan(std::unique_ptr<TableScan>&& disk_scan,
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 2dda955ac..b12e59a84 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -41,6 +41,7 @@
 #include "paimon/core/operation/data_evolution_file_store_scan.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
+#include "paimon/core/realtime/primary_key_realtime_options.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/schema_validation.h"
@@ -225,15 +226,15 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
     if (!core_options.RealtimeEnabled()) {
         return Status::Invalid("real-time scan requires realtime.enabled=true");
     }
-    if (!table_schema.PrimaryKeys().empty()) {
-        return Status::Invalid("real-time union read currently supports append tables only");
-    }
     if (core_options.GetBucket() <= 0) {
         return Status::Invalid("real-time union read requires fixed bucket mode");
     }
     if (core_options.DataEvolutionEnabled()) {
         return Status::Invalid("real-time union read does not support data evolution");
     }
+    if (!table_schema.PrimaryKeys().empty()) {
+        PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(core_options));
+    }
     if (context.IsStreamingMode()) {
         return Status::Invalid("real-time union read currently supports batch scans only");
     }

From f8078a5588ca4ab3dcbef00d37607a2191dcecfc Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 19 Aug 2026 22:34:33 +0800
Subject: [PATCH 25/93] test(realtime): cover primary-key realtime lifecycle

Cover PK write and read, recovery, external compaction, supported concurrency, writer handoff, ticket lifecycle, plugin contracts, rolling files, and multi-partition and bucket restore.
---
 .../operation/key_value_file_store_write.cpp  |   35 +-
 test/inte/realtime_write_inte_test.cpp        | 1005 ++++++++++++++++-
 2 files changed, 977 insertions(+), 63 deletions(-)

diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 4456ee1c2..e94c45a15 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -90,20 +90,6 @@ KeyValueFileStoreWrite::KeyValueFileStoreWrite(
     }
 }
 
-Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
-    if (!realtime_context_) {
-        return Status::Invalid("refresh committed snapshot requires a real-time writer");
-    }
-    PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, snapshot_manager_->LoadSnapshot(snapshot_id));
-    PAIMON_ASSIGN_OR_RAISE(
-        RealtimeOffsetMap committed_offsets,
-        RealtimeCommitProperties::ReadOffsets(std::optional<Snapshot>(std::move(snapshot)),
-                                              options_.GetFileSystem()));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
-                           RealtimeContextImpl::Cast(realtime_context_));
-    return realtime_context_impl->AdvanceCommittedProgress(snapshot_id, committed_offsets);
-}
-
 Result<std::unique_ptr<FileStoreScan>> KeyValueFileStoreWrite::CreateFileStoreScan(
     const std::shared_ptr<ScanFilter>& scan_filter) const {
     PAIMON_ASSIGN_OR_RAISE(
@@ -139,6 +125,7 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         Levels::Create(key_comparator_, restore_data_files, options_.GetNumLevels()));
     std::map<std::string, std::string> partition_map;
     int64_t materialized_max_sequence_number = restore_max_seq_number;
+    std::shared_ptr<CompactManager> compact_manager;
     if (realtime_context_) {
         std::vector<std::pair<std::string, std::string>> partition_values;
         PAIMON_ASSIGN_OR_RAISE(partition_values,
@@ -147,15 +134,11 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
             std::map<std::string, std::string>(partition_values.begin(), partition_values.end());
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
                                RealtimeContextImpl::Cast(realtime_context_));
-        const RealtimePartitionBucket partition_bucket(partition_map, bucket);
         materialized_max_sequence_number = realtime_context_impl->GetMaterializedMaxSequenceNumber(
-            partition_bucket, restore_max_seq_number);
+            RealtimePartitionBucket(partition_map, bucket), restore_max_seq_number);
         if (materialized_max_sequence_number == std::numeric_limits<int64_t>::max()) {
             return Status::Invalid("PK sequence number has reached INT64_MAX");
         }
-    }
-    std::shared_ptr<CompactManager> compact_manager;
-    if (realtime_context_) {
         compact_manager = std::make_shared<NoopCompactManager>();
     } else {
         auto compact_strategy = compact_manager_factory_->CreateCompactStrategy();
@@ -181,6 +164,20 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         writer, options_.ToMap(), pool_, materialized_max_sequence_number);
 }
 
+Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
+    if (!realtime_context_) {
+        return Status::Invalid("refresh committed snapshot requires a real-time writer");
+    }
+    PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, snapshot_manager_->LoadSnapshot(snapshot_id));
+    PAIMON_ASSIGN_OR_RAISE(
+        RealtimeOffsetMap committed_offsets,
+        RealtimeCommitProperties::ReadOffsets(std::optional<Snapshot>(std::move(snapshot)),
+                                              options_.GetFileSystem()));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                           RealtimeContextImpl::Cast(realtime_context_));
+    return realtime_context_impl->AdvanceCommittedProgress(snapshot_id, committed_offsets);
+}
+
 Status KeyValueFileStoreWrite::Close() {
     PAIMON_RETURN_NOT_OK(AbstractFileStoreWrite::Close());
     compact_manager_factory_->Close();
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 6298137ea..f18c3f1e4 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -23,6 +23,7 @@
 #include <condition_variable>
 #include <cstdint>
 #include <deque>
+#include <future>
 #include <iterator>
 #include <limits>
 #include <map>
@@ -42,7 +43,11 @@
 #include "paimon/catalog/catalog.h"
 #include "paimon/catalog/identifier.h"
 #include "paimon/commit_context.h"
+#include "paimon/common/factories/io_hook.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/operation/commit/realtime_commit_properties.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
@@ -59,6 +64,7 @@
 #include "paimon/predicate/predicate_builder.h"
 #include "paimon/read_context.h"
 #include "paimon/reader/count_reader.h"
+#include "paimon/realtime/arrow_realtime_store_factory.h"
 #include "paimon/realtime/realtime_context.h"
 #include "paimon/realtime/realtime_store.h"
 #include "paimon/record_batch.h"
@@ -71,6 +77,308 @@
 #include "paimon/write_context.h"
 
 namespace paimon::test {
+namespace {
+
+class BlockingState {
+ public:
+    void Block() {
+        std::unique_lock<std::mutex> lock(mutex_);
+        entered_ = true;
+        entered_cv_.notify_all();
+        release_cv_.wait(lock, [this]() { return released_; });
+    }
+
+    bool WaitUntilBlocked() {
+        std::unique_lock<std::mutex> lock(mutex_);
+        return entered_cv_.wait_for(lock, std::chrono::seconds(30), [this]() { return entered_; });
+    }
+
+    void Release() {
+        std::lock_guard<std::mutex> lock(mutex_);
+        released_ = true;
+        release_cv_.notify_all();
+    }
+
+ private:
+    std::mutex mutex_;
+    std::condition_variable entered_cv_;
+    std::condition_variable release_cv_;
+    bool entered_ = false;
+    bool released_ = false;
+};
+
+class BlockingBatchReader final : public BatchReader {
+ public:
+    BlockingBatchReader(std::unique_ptr<BatchReader>&& reader,
+                        const std::shared_ptr<BlockingState>& state)
+        : reader_(std::move(reader)), state_(state) {}
+
+    Result<ReadBatch> NextBatch() override {
+        if (!blocked_) {
+            blocked_ = true;
+            state_->Block();
+        }
+        return reader_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return reader_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        reader_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> reader_;
+    std::shared_ptr<BlockingState> state_;
+    bool blocked_ = false;
+};
+
+class BlockingRealtimeStore final : public RealtimeStore {
+ public:
+    BlockingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
+                          const std::shared_ptr<BlockingState>& state)
+        : delegate_(delegate), state_(state) {}
+
+    Status Write(RealtimeWriteBatch&& batch) override {
+        return delegate_->Write(std::move(batch));
+    }
+
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
+        return delegate_->SealForCommit();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               delegate_->CreateCommitReaders(segment));
+        if (!readers.empty()) {
+            readers[0] = std::make_unique<BlockingBatchReader>(std::move(readers[0]), state_);
+        }
+        return readers;
+    }
+
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
+        return delegate_->AcquireReadView();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        return delegate_->CreateQueryReaders(view, offset_begin, context);
+    }
+
+    Status AdvanceCommittedOffset(int64_t committed_offset) override {
+        return delegate_->AdvanceCommittedOffset(committed_offset);
+    }
+
+    uint64_t GetMemoryUsage() const override {
+        return delegate_->GetMemoryUsage();
+    }
+
+ private:
+    std::shared_ptr<RealtimeStore> delegate_;
+    std::shared_ptr<BlockingState> state_;
+};
+
+class BlockingRealtimeStoreFactory final : public RealtimeStoreFactory {
+ public:
+    explicit BlockingRealtimeStoreFactory(const std::shared_ptr<BlockingState>& state)
+        : state_(state) {}
+
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
+                               delegate_.Create(std::move(request)));
+        return std::shared_ptr<RealtimeStore>(
+            std::make_shared<BlockingRealtimeStore>(delegate, state_));
+    }
+
+ private:
+    ArrowRealtimeStoreFactory delegate_;
+    std::shared_ptr<BlockingState> state_;
+};
+
+class TrackingRealtimeReadView final : public RealtimeReadView {
+ public:
+    explicit TrackingRealtimeReadView(std::shared_ptr<RealtimeReadView> delegate)
+        : delegate_(std::move(delegate)) {}
+
+    std::optional<OffsetRange> GetOffsetRange() const override {
+        return delegate_->GetOffsetRange();
+    }
+
+    const std::shared_ptr<RealtimeReadView>& Delegate() const {
+        return delegate_;
+    }
+
+ private:
+    std::shared_ptr<RealtimeReadView> delegate_;
+};
+
+class ReadViewCheckingBatchReader final : public BatchReader {
+ public:
+    ReadViewCheckingBatchReader(std::unique_ptr<BatchReader> delegate,
+                                std::weak_ptr<RealtimeReadView> read_view)
+        : delegate_(std::move(delegate)), read_view_(std::move(read_view)) {}
+
+    Result<ReadBatch> NextBatch() override {
+        if (read_view_.expired()) {
+            return Status::Invalid("real-time read view was released before reader completion");
+        }
+        return delegate_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    std::weak_ptr<RealtimeReadView> read_view_;
+};
+
+class QueryTrackingRealtimeStore final : public RealtimeStore {
+ public:
+    QueryTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
+                               const std::shared_ptr<std::atomic<bool>>& saw_query_predicate,
+                               const std::shared_ptr<std::weak_ptr<RealtimeReadView>>& query_view)
+        : delegate_(delegate), saw_query_predicate_(saw_query_predicate), query_view_(query_view) {}
+
+    Status Write(RealtimeWriteBatch&& batch) override {
+        return delegate_->Write(std::move(batch));
+    }
+
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
+        return delegate_->SealForCommit();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
+        return delegate_->CreateCommitReaders(segment);
+    }
+
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> delegate_view,
+                               delegate_->AcquireReadView());
+        return std::shared_ptr<RealtimeReadView>(
+            std::make_shared<TrackingRealtimeReadView>(delegate_view));
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        if (context.predicate) {
+            saw_query_predicate_->store(true, std::memory_order_release);
+        }
+        *query_view_ = view;
+        std::shared_ptr<TrackingRealtimeReadView> tracking_view =
+            std::dynamic_pointer_cast<TrackingRealtimeReadView>(view);
+        if (!tracking_view) {
+            return Status::Invalid("query tracking store received an unexpected read view");
+        }
+        PAIMON_ASSIGN_OR_RAISE(
+            std::vector<std::unique_ptr<BatchReader>> readers,
+            delegate_->CreateQueryReaders(tracking_view->Delegate(), offset_begin, context));
+        for (std::unique_ptr<BatchReader>& reader : readers) {
+            reader = std::make_unique<ReadViewCheckingBatchReader>(std::move(reader), view);
+        }
+        return readers;
+    }
+
+    Status AdvanceCommittedOffset(int64_t committed_offset) override {
+        return delegate_->AdvanceCommittedOffset(committed_offset);
+    }
+
+    uint64_t GetMemoryUsage() const override {
+        return delegate_->GetMemoryUsage();
+    }
+
+ private:
+    std::shared_ptr<RealtimeStore> delegate_;
+    std::shared_ptr<std::atomic<bool>> saw_query_predicate_;
+    std::shared_ptr<std::weak_ptr<RealtimeReadView>> query_view_;
+};
+
+class QueryTrackingRealtimeStoreFactory final : public RealtimeStoreFactory {
+ public:
+    explicit QueryTrackingRealtimeStoreFactory(
+        const std::shared_ptr<std::atomic<bool>>& saw_query_predicate,
+        const std::shared_ptr<std::weak_ptr<RealtimeReadView>>& query_view)
+        : saw_query_predicate_(saw_query_predicate), query_view_(query_view) {}
+
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
+                               delegate_.Create(std::move(request)));
+        return std::shared_ptr<RealtimeStore>(std::make_shared<QueryTrackingRealtimeStore>(
+            delegate, saw_query_predicate_, query_view_));
+    }
+
+ private:
+    ArrowRealtimeStoreFactory delegate_;
+    std::shared_ptr<std::atomic<bool>> saw_query_predicate_;
+    std::shared_ptr<std::weak_ptr<RealtimeReadView>> query_view_;
+};
+
+class InvalidReaderRealtimeStore final : public RealtimeStore {
+ public:
+    explicit InvalidReaderRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
+        : delegate_(delegate) {}
+
+    Status Write(RealtimeWriteBatch&& batch) override {
+        return delegate_->Write(std::move(batch));
+    }
+
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
+        return delegate_->SealForCommit();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>&) override {
+        std::vector<std::unique_ptr<BatchReader>> readers;
+        readers.push_back(nullptr);
+        return readers;
+    }
+
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
+        return delegate_->AcquireReadView();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>&, int64_t, const RealtimeQueryContext&) override {
+        return std::vector<std::unique_ptr<BatchReader>>();
+    }
+
+    Status AdvanceCommittedOffset(int64_t committed_offset) override {
+        return delegate_->AdvanceCommittedOffset(committed_offset);
+    }
+
+    uint64_t GetMemoryUsage() const override {
+        return delegate_->GetMemoryUsage();
+    }
+
+ private:
+    std::shared_ptr<RealtimeStore> delegate_;
+};
+
+class InvalidReaderRealtimeStoreFactory final : public RealtimeStoreFactory {
+ public:
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
+                               delegate_.Create(std::move(request)));
+        return std::shared_ptr<RealtimeStore>(
+            std::make_shared<InvalidReaderRealtimeStore>(delegate));
+    }
+
+ private:
+    ArrowRealtimeStoreFactory delegate_;
+};
+
+}  // namespace
 
 namespace {
 
@@ -219,6 +527,18 @@ class RealtimeWriteInteTest : public ::testing::Test {
                                        /*ignore_if_exists=*/false));
     }
 
+    void CreatePkTable(const std::vector<std::string>& partition_keys = {}) const {
+        auto c_schema = std::make_unique<ArrowSchema>();
+        ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok());
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<Catalog> catalog,
+                             Catalog::Create(dir_->Str(), options_));
+        ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false));
+        std::vector<std::string> primary_keys = partition_keys;
+        primary_keys.push_back("id");
+        ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), c_schema.get(), partition_keys,
+                                       primary_keys, options_, /*ignore_if_exists=*/false));
+    }
+
     Result<std::unique_ptr<FileStoreWrite>> CreateRealtimeWriter(
         const std::shared_ptr<RealtimeContext>& realtime_context) const {
         WriteContextBuilder builder(table_path_, commit_user_);
@@ -240,6 +560,12 @@ class RealtimeWriteInteTest : public ::testing::Test {
 
     Result<std::unique_ptr<RecordBatch>> MakeBatch(const std::vector<Row>& rows, bool partitioned,
                                                    int32_t bucket) const {
+        return MakeBatch(rows, partitioned, bucket, /*row_kinds=*/{});
+    }
+
+    Result<std::unique_ptr<RecordBatch>> MakeBatch(
+        const std::vector<Row>& rows, bool partitioned, int32_t bucket,
+        const std::vector<RecordBatch::RowKind>& row_kinds) const {
         if (rows.empty()) {
             return Status::Invalid("cannot create an empty test batch");
         }
@@ -263,6 +589,7 @@ class RealtimeWriteInteTest : public ::testing::Test {
         ArrowArray c_array;
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
         RecordBatchBuilder builder(&c_array);
+        builder.SetRowKinds(row_kinds);
         if (partitioned) {
             builder.SetPartition({{"pt", partition}});
         }
@@ -426,6 +753,16 @@ class RealtimeWriteInteTest : public ::testing::Test {
         return commit->Expire();
     }
 
+    Status CommitMessages(const std::vector<std::shared_ptr<CommitMessage>>& messages,
+                          int64_t commit_identifier) const {
+        CommitContextBuilder builder(table_path_, commit_user_);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
+                               builder.SetOptions(options_).Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
+                               FileStoreCommit::Create(std::move(context)));
+        return commit->Commit(messages, commit_identifier);
+    }
+
     Result<std::shared_ptr<Plan>> CreatePlan(
         const std::shared_ptr<RealtimeContext>& realtime_context,
         const std::shared_ptr<Predicate>& predicate) const {
@@ -573,6 +910,75 @@ class RealtimeWriteInteTest : public ::testing::Test {
         return memory_usage;
     }
 
+    Result<std::vector<int64_t>> ReadPkSequences(
+        const std::shared_ptr<RealtimeContext>& realtime_context) const {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                               RealtimeContextImpl::Cast(realtime_context));
+        PAIMON_ASSIGN_OR_RAISE(std::vector<RealtimePartitionBucketView> views,
+                               realtime_context_impl->AcquireReadViews());
+        if (views.size() != 1) {
+            return Status::Invalid("expected one PK real-time read view");
+        }
+        auto read_schema = std::make_unique<ArrowSchema>();
+        arrow::FieldVector requested_fields = {
+            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())};
+        requested_fields.insert(requested_fields.end(), schema_->fields().begin(),
+                                schema_->fields().end());
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(
+            arrow::ExportSchema(*arrow::schema(requested_fields), read_schema.get()));
+        ScopeGuard schema_guard([schema = read_schema.get()]() { ArrowSchemaRelease(schema); });
+        RealtimeQueryContext query_context{read_schema.get(), /*predicate=*/nullptr,
+                                           /*enable_predicate_pushdown=*/false};
+        PAIMON_ASSIGN_OR_RAISE(
+            std::vector<std::unique_ptr<BatchReader>> readers,
+            views[0].store->CreateQueryReaders(views[0].read_view,
+                                               /*offset_begin=*/0, query_context));
+        std::vector<int64_t> sequences;
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            while (true) {
+                PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
+                if (BatchReader::IsEofBatch(batch)) {
+                    break;
+                }
+                PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                    std::shared_ptr<arrow::Array> imported,
+                    arrow::ImportArray(batch.first.get(), batch.second.get()));
+                std::shared_ptr<arrow::StructArray> values =
+                    std::dynamic_pointer_cast<arrow::StructArray>(imported);
+                if (!values) {
+                    return Status::Invalid("PK query reader did not return a StructArray");
+                }
+                std::shared_ptr<arrow::Int64Array> sequence_array =
+                    std::dynamic_pointer_cast<arrow::Int64Array>(
+                        values->GetFieldByName(SpecialFields::SequenceNumber().Name()));
+                if (!sequence_array) {
+                    return Status::Invalid("PK query reader did not return sequence numbers");
+                }
+                for (int64_t row = 0; row < sequence_array->length(); ++row) {
+                    sequences.push_back(sequence_array->Value(row));
+                }
+            }
+            reader->Close();
+        }
+        return sequences;
+    }
+
+    static std::vector<std::shared_ptr<DataFileMeta>> NewFiles(
+        const std::vector<RealtimeCommitProgress>& progresses) {
+        std::vector<std::shared_ptr<DataFileMeta>> files;
+        for (const RealtimeCommitProgress& progress : progresses) {
+            std::shared_ptr<CommitMessageImpl> message =
+                std::dynamic_pointer_cast<CommitMessageImpl>(progress.commit_message);
+            if (!message) {
+                continue;
+            }
+            const std::vector<std::shared_ptr<DataFileMeta>>& new_files =
+                message->GetNewFilesIncrement().NewFiles();
+            files.insert(files.end(), new_files.begin(), new_files.end());
+        }
+        return files;
+    }
+
     static Status ValidateReadPrefix(const std::vector<Row>& rows, int64_t total_rows) {
         std::vector<bool> seen(static_cast<size_t>(total_rows), false);
         int64_t max_id = -1;
@@ -627,7 +1033,6 @@ class RealtimeWriteInteTest : public ::testing::Test {
         options_[Options::PARTITION_GENERATE_LEGACY_NAME] =
             legacy_partition_name_enabled ? "true" : "false";
         CreateTable(/*partition_keys=*/{"pt"});
-
         ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                              RealtimeContext::Create());
         ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -666,6 +1071,57 @@ class RealtimeWriteInteTest : public ::testing::Test {
         ASSERT_OK(writer->Close());
     }
 
+    void CheckVectorReaderRetry(bool primary_key) {
+        if (primary_key) {
+            CreatePkTable(/*partition_keys=*/{"pt"});
+        } else {
+            CreateTable(/*partition_keys=*/{"pt"});
+        }
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                             RealtimeContext::Create());
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                             CreateRealtimeWriter(realtime_context));
+        std::vector<Row> p0_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0_batch,
+                             MakeBatch(p0_rows, /*partitioned=*/true));
+        ASSERT_OK(writer->Write(std::move(p0_batch)));
+        std::vector<Row> p1_rows = MakeRows(/*first_id=*/10, /*count=*/3, /*partition=*/"p1");
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p1_batch,
+                             MakeBatch(p1_rows, /*partitioned=*/true));
+        ASSERT_OK(writer->Write(std::move(p1_batch)));
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                             CreatePlan(realtime_context, /*predicate=*/nullptr));
+        ASSERT_EQ(2, plan->Splits().size());
+
+        std::vector<std::shared_ptr<Split>> invalid_splits = plan->Splits();
+        std::shared_ptr<RealtimeSplit> second_split =
+            std::dynamic_pointer_cast<RealtimeSplit>(invalid_splits[1]);
+        ASSERT_NE(nullptr, second_split);
+        std::vector<std::shared_ptr<Split>> second_disk_splits = second_split->DiskSplits();
+        invalid_splits[1] = std::make_shared<RealtimeSplit>(
+            RealtimeSplit::kCurrentVersion + 1, second_split->SnapshotId(),
+            second_split->Partition(), second_split->Bucket(), std::move(second_disk_splits),
+            second_split->CommittedEndOffset(), second_split->MemoryEndOffset(),
+            second_split->OpaqueTicket());
+
+        ReadContextBuilder read_builder(table_path_);
+        read_builder.SetOptions(options_)
+            .SetReadFieldNames({"id", "payload", "pt"})
+            .WithRealtimeContext(realtime_context)
+            .WithMemoryPool(pool_);
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+                             TableRead::Create(std::move(read_context)));
+        ASSERT_NOK_WITH_MSG(table_read->CreateReader(invalid_splits),
+                            "unsupported real-time split version");
+
+        std::vector<Row> expected_rows = p0_rows;
+        expected_rows.insert(expected_rows.end(), p1_rows.begin(), p1_rows.end());
+        ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
+        ASSERT_EQ(expected_rows, actual_rows);
+        ASSERT_OK(writer->Close());
+    }
+
     std::unique_ptr<UniqueTestDirectory> dir_;
     std::string table_path_;
     std::string commit_user_ = "realtime_commit_user";
@@ -723,6 +1179,459 @@ TEST_F(RealtimeWriteInteTest, TestAppendCommitAndRead) {
     FinalizeCommitAndCheck(writer.get(), /*realtime_commits=*/{}, /*prepare_identifier=*/0, rows);
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkRead) {
+    CreatePkTable();
+    auto saw_query_predicate = std::make_shared<std::atomic<bool>>(false);
+    auto query_view = std::make_shared<std::weak_ptr<RealtimeReadView>>();
+    auto factory =
+        std::make_shared<QueryTrackingRealtimeStoreFactory>(saw_query_predicate, query_view);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    std::vector<Row> first_rows = {{1, "old", "p0"}, {2, "two", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+                         MakeBatch(first_rows, /*partitioned=*/false, /*bucket=*/0,
+                                   {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(writer->Write(std::move(first_batch)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> update_batch,
+                         MakeBatch({Row{1, "new", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
+                                   {RecordBatch::RowKind::UPDATE_AFTER}));
+    ASSERT_OK(writer->Write(std::move(update_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> memory_rows, ReadRows(realtime_context));
+    ASSERT_EQ((std::vector<Row>{{1, "new", "p0"}, {2, "two", "p0"}}), memory_rows);
+
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, progress.size());
+    ASSERT_OK(Commit(progress, /*commit_identifier=*/0));
+
+    std::vector<Row> second_rows = {{1, "latest", "p0"}, {2, "gone", "p0"}, {3, "three", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+                         MakeBatch(second_rows, /*partitioned=*/false, /*bucket=*/0,
+                                   {RecordBatch::RowKind::UPDATE_AFTER,
+                                    RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(writer->Write(std::move(second_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> union_rows, ReadRows(realtime_context));
+    ASSERT_EQ((std::vector<Row>{{1, "latest", "p0"}, {3, "three", "p0"}}), union_rows);
+
+    const std::string expected_payload = "new";
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::Equal(
+        /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
+        Literal(FieldType::STRING, expected_payload.data(), expected_payload.size()));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> filtered_plan,
+                         CreatePlan(realtime_context, predicate));
+    ASSERT_OK_AND_ASSIGN(
+        CollectedReadResult filtered_result,
+        ReadPlan(filtered_plan, realtime_context, {"id", "payload", "pt"}, predicate,
+                 /*enable_predicate_filter=*/true));
+    ASSERT_EQ(nullptr, filtered_result.data);
+    ASSERT_FALSE(saw_query_predicate->load(std::memory_order_acquire));
+    filtered_result.reader->Close();
+    filtered_result.reader.reset();
+    ASSERT_OK(writer->Close());
+    writer.reset();
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> lifetime_plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ReadContextBuilder read_builder(table_path_);
+    read_builder.SetOptions(options_)
+        .SetReadFieldNames({"id", "payload", "pt"})
+        .WithRealtimeContext(realtime_context)
+        .WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+                         TableRead::Create(std::move(read_context)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader,
+                         table_read->CreateReader(lifetime_plan->Splits()));
+    ASSERT_FALSE(query_view->expired());
+
+    std::weak_ptr<RealtimeContext> weak_context = realtime_context;
+    table_read.reset();
+    lifetime_plan.reset();
+    realtime_context.reset();
+    ASSERT_TRUE(weak_context.expired());
+    ASSERT_FALSE(query_view->expired());
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch());
+    ASSERT_FALSE(BatchReader::IsEofBatch(read_batch));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> read_array,
+                         ReadResultCollector::GetArray(std::move(read_batch)));
+    ASSERT_NE(nullptr, read_array);
+    read_array.reset();
+    reader->Close();
+    reader.reset();
+    ASSERT_TRUE(query_view->expired());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkRecovery) {
+    CreatePkTable();
+
+    WriteContextBuilder seed_builder(table_path_, commit_user_);
+    seed_builder.SetOptions(options_).WithStreamingMode(true);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> seed_context, seed_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> seed_writer,
+                         FileStoreWrite::Create(std::move(seed_context)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> seed_batch,
+                         MakeBatch({Row{99, "seed", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(seed_writer->Write(std::move(seed_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> seed_messages,
+                         seed_writer->PrepareCommit(/*wait_compaction=*/false,
+                                                    /*commit_identifier=*/0));
+    CommitContextBuilder seed_commit_builder(table_path_, commit_user_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> seed_commit_context,
+                         seed_commit_builder.SetOptions(options_).Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> seed_commit,
+                         FileStoreCommit::Create(std::move(seed_commit_context)));
+    ASSERT_OK(seed_commit->Commit(seed_messages));
+    ASSERT_OK(seed_writer->Close());
+    const std::vector<Row> mutations = {
+        {1, "one", "p0"}, {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
+    const std::vector<RecordBatch::RowKind> mutation_kinds = {
+        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
+        RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT};
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
+                         CreateRealtimeWriter(failed_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
+                         MakeBatch(mutations, /*partitioned=*/false, /*bucket=*/0, mutation_kinds));
+    ASSERT_OK(failed_writer->Write(std::move(failed_batch)));
+    IOHook* io_hook = IOHook::GetInstance();
+    ScopeGuard hook_guard([io_hook]() { io_hook->Clear(); });
+    io_hook->Reset(/*pos=*/0, IOHook::Mode::RETURN_ERROR);
+    Result<std::vector<RealtimeCommitProgress>> failed_prepare =
+        failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/1);
+    io_hook->Clear();
+    ASSERT_TRUE(failed_prepare.status().IsIOError()) << failed_prepare.status().ToString();
+    ASSERT_OK(failed_writer->Close());
+    failed_writer.reset();
+    failed_context.reset();
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failed_prepare, ReadRows());
+    ASSERT_EQ((std::vector<Row>{{99, "seed", "p0"}}), rows_after_failed_prepare);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> first_context, RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
+                         CreateRealtimeWriter(first_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch(mutations, /*partitioned=*/false, /*bucket=*/0, mutation_kinds));
+    ASSERT_OK(first_writer->Write(std::move(batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<int64_t> memory_sequences, ReadPkSequences(first_context));
+    ASSERT_EQ((std::vector<int64_t>{2, 3, 4}), memory_sequences);
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, progress.size());
+    ASSERT_EQ(OffsetRange(0, 4), progress[0].offset_range);
+    ASSERT_EQ(1, NewFiles(progress).size());
+    ASSERT_EQ(memory_sequences.front(), NewFiles(progress)[0]->min_sequence_number);
+    ASSERT_EQ(memory_sequences.back(), NewFiles(progress)[0]->max_sequence_number);
+    ASSERT_OK(Commit(progress, /*commit_identifier=*/1));
+    ASSERT_OK(first_writer->Close());
+    first_context.reset();
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_replay, ReadRows());
+    ASSERT_EQ((std::vector<Row>{{1, "one-new", "p0"}, {3, "three", "p0"}, {99, "seed", "p0"}}),
+              rows_after_replay);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> second_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
+                         CreateRealtimeWriter(second_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> restart_batch,
+                         MakeBatch({Row{4, "four", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(second_writer->Write(std::move(restart_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<int64_t> restart_sequences, ReadPkSequences(second_context));
+    ASSERT_EQ((std::vector<int64_t>{5}), restart_sequences);
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> restart_progress,
+                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_EQ(1, restart_progress.size());
+    ASSERT_EQ(OffsetRange(4, 5), restart_progress[0].offset_range);
+    ASSERT_EQ(5, NewFiles(restart_progress)[0]->min_sequence_number);
+    ASSERT_EQ(5, NewFiles(restart_progress)[0]->max_sequence_number);
+    ASSERT_OK(second_writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
+    options_[Options::NUM_SORTED_RUNS_COMPACTION_TRIGGER] = "1";
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    int64_t latest_snapshot_id = -1;
+    constexpr int64_t kCommitRoundsBeforeCompaction = 4;
+    std::set<std::string> committed_file_names;
+    for (int64_t round = 0; round < kCommitRoundsBeforeCompaction; ++round) {
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch({Row{round, "value-" + std::to_string(round), "p0"}},
+                                       /*partitioned=*/false));
+        ASSERT_OK(writer->Write(std::move(batch)));
+        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                             writer->PrepareCommitWithProgress(round));
+        ASSERT_EQ(1, progress.size());
+        std::shared_ptr<CommitMessageImpl> message =
+            std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
+        ASSERT_NE(nullptr, message);
+        ASSERT_TRUE(message->GetCompactIncrement().IsEmpty());
+        ASSERT_EQ(1, NewFiles(progress).size());
+        committed_file_names.insert(NewFiles(progress)[0]->file_name);
+        ASSERT_OK_AND_ASSIGN(latest_snapshot_id, Commit(progress, round));
+        ASSERT_OK(writer->RefreshCommittedSnapshot(latest_snapshot_id));
+        ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
+        ASSERT_EQ(0, memory_usage);
+    }
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> next_batch,
+                         MakeBatch({Row{4, "value-4", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(next_batch)));
+
+    WriteContextBuilder compact_builder(table_path_, commit_user_);
+    compact_builder.SetOptions(options_).WithStreamingMode(true);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> compact_context, compact_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> compact_writer,
+                         FileStoreWrite::Create(std::move(compact_context)));
+    ASSERT_OK(compact_writer->Compact(/*partition=*/{}, /*bucket=*/0,
+                                      /*full_compaction=*/true));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<std::shared_ptr<CommitMessage>> compact_messages,
+        compact_writer->PrepareCommit(/*wait_compaction=*/true, /*commit_identifier=*/4));
+    ASSERT_EQ(1, compact_messages.size());
+    std::shared_ptr<CommitMessageImpl> compact_message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(compact_messages[0]);
+    ASSERT_NE(nullptr, compact_message);
+    ASSERT_TRUE(compact_message->GetNewFilesIncrement().IsEmpty());
+    ASSERT_EQ(kCommitRoundsBeforeCompaction,
+              compact_message->GetCompactIncrement().CompactBefore().size());
+    std::set<std::string> compacted_file_names;
+    for (const std::shared_ptr<DataFileMeta>& file :
+         compact_message->GetCompactIncrement().CompactBefore()) {
+        compacted_file_names.insert(file->file_name);
+    }
+    ASSERT_EQ(committed_file_names, compacted_file_names);
+    ASSERT_FALSE(compact_message->GetCompactIncrement().CompactAfter().empty());
+    ASSERT_OK(CommitMessages(compact_messages, /*commit_identifier=*/4));
+    ASSERT_OK(compact_writer->Close());
+
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(options_));
+    SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_);
+    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> compact_snapshot,
+                         snapshot_manager.LatestSnapshot());
+    ASSERT_TRUE(compact_snapshot);
+    ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot->GetCommitKind());
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets, ReadCommittedOffsets());
+    ASSERT_EQ(4, offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(compact_snapshot->Id()));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> compacted_rows, ReadRows(realtime_context));
+    ASSERT_EQ((std::vector<Row>{{0, "value-0", "p0"},
+                                {1, "value-1", "p0"},
+                                {2, "value-2", "p0"},
+                                {3, "value-3", "p0"},
+                                {4, "value-4", "p0"}}),
+              compacted_rows);
+
+    constexpr int64_t kCommitRoundsAfterCompaction = 2;
+    for (int64_t round = 0; round < kCommitRoundsAfterCompaction; ++round) {
+        if (round > 0) {
+            ASSERT_OK_AND_ASSIGN(
+                std::unique_ptr<RecordBatch> batch,
+                MakeBatch({Row{4 + round, "value-" + std::to_string(4 + round), "p0"}},
+                          /*partitioned=*/false));
+            ASSERT_OK(writer->Write(std::move(batch)));
+        }
+        const int64_t commit_identifier = 5 + round;
+        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                             writer->PrepareCommitWithProgress(commit_identifier));
+        ASSERT_EQ(1, progress.size());
+        ASSERT_EQ(OffsetRange(4 + round, 5 + round), progress[0].offset_range);
+        ASSERT_OK_AND_ASSIGN(latest_snapshot_id, Commit(progress, commit_identifier));
+        ASSERT_OK(writer->RefreshCommittedSnapshot(latest_snapshot_id));
+        ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
+        ASSERT_EQ(0, memory_usage);
+    }
+    ASSERT_OK_AND_ASSIGN(offsets, ReadCommittedOffsets());
+    ASSERT_EQ(6, offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows(realtime_context));
+    ASSERT_EQ((std::vector<Row>{{0, "value-0", "p0"},
+                                {1, "value-1", "p0"},
+                                {2, "value-2", "p0"},
+                                {3, "value-3", "p0"},
+                                {4, "value-4", "p0"},
+                                {5, "value-5", "p0"}}),
+              final_rows);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkConcurrency) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    constexpr int64_t kRowCount = 20;
+    constexpr int32_t kReaderCount = 2;
+    std::atomic<bool> writer_done{false};
+    std::atomic<bool> control_done{false};
+    std::atomic<int32_t> commit_count{0};
+    ConcurrentTestState state;
+    std::vector<int32_t> read_counts(kReaderCount, 0);
+
+    std::thread write_thread([&]() {
+        state.WaitForStart();
+        for (int64_t id = 0; id < kRowCount && !state.ShouldStop(); ++id) {
+            Result<std::unique_ptr<RecordBatch>> batch =
+                MakeBatch(MakeRows(id, /*count=*/1, /*partition=*/"p0"),
+                          /*partitioned=*/false);
+            if (state.RecordErrorIfNotOk(batch) ||
+                state.RecordErrorIfNotOk(writer->Write(std::move(batch).value()))) {
+                break;
+            }
+            std::this_thread::sleep_for(std::chrono::milliseconds(2));
+        }
+        writer_done.store(true, std::memory_order_release);
+    });
+
+    std::thread control_thread([&]() {
+        state.WaitForStart();
+        int64_t commit_identifier = 0;
+        do {
+            Result<std::vector<RealtimeCommitProgress>> progress =
+                writer->PrepareCommitWithProgress(commit_identifier);
+            if (state.RecordErrorIfNotOk(progress)) {
+                break;
+            }
+            if (!progress.value().empty()) {
+                Result<int64_t> snapshot = Commit(progress.value(), commit_identifier++);
+                if (state.RecordErrorIfNotOk(snapshot) ||
+                    state.RecordErrorIfNotOk(writer->RefreshCommittedSnapshot(snapshot.value()))) {
+                    break;
+                }
+                commit_count.fetch_add(1, std::memory_order_relaxed);
+            }
+            std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        } while (!writer_done.load(std::memory_order_acquire) && !state.ShouldStop());
+        if (!state.ShouldStop()) {
+            Result<std::vector<RealtimeCommitProgress>> progress =
+                writer->PrepareCommitWithProgress(commit_identifier);
+            if (!state.RecordErrorIfNotOk(progress) && !progress.value().empty()) {
+                Result<int64_t> snapshot = Commit(progress.value(), commit_identifier);
+                if (!state.RecordErrorIfNotOk(snapshot) &&
+                    !state.RecordErrorIfNotOk(writer->RefreshCommittedSnapshot(snapshot.value()))) {
+                    commit_count.fetch_add(1, std::memory_order_relaxed);
+                }
+            }
+        }
+        control_done.store(true, std::memory_order_release);
+    });
+
+    std::vector<std::thread> read_threads;
+    read_threads.reserve(kReaderCount);
+    for (int32_t reader_index = 0; reader_index < kReaderCount; ++reader_index) {
+        read_threads.emplace_back([&, reader_index]() {
+            state.WaitForStart();
+            while (!control_done.load(std::memory_order_acquire) && !state.ShouldStop()) {
+                Result<std::vector<Row>> rows = ReadRows(realtime_context);
+                ++read_counts[reader_index];
+                if (state.RecordErrorIfNotOk(rows) ||
+                    state.RecordErrorIfNotOk(ValidateReadPrefix(rows.value(), kRowCount))) {
+                    break;
+                }
+                std::this_thread::sleep_for(std::chrono::milliseconds(1));
+            }
+        });
+    }
+
+    state.StartWhenReady(/*worker_count=*/2 + kReaderCount);
+    write_thread.join();
+    control_thread.join();
+    for (std::thread& read_thread : read_threads) {
+        read_thread.join();
+    }
+
+    ASSERT_TRUE(state.Errors().empty()) << (state.Errors().empty() ? "" : state.Errors().front());
+    ASSERT_GT(commit_count.load(), 0);
+    for (int32_t read_count : read_counts) {
+        ASSERT_GT(read_count, 0);
+    }
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows(realtime_context));
+    ASSERT_EQ(kRowCount, static_cast<int64_t>(final_rows.size()));
+    ASSERT_OK(ValidateReadPrefix(final_rows, kRowCount));
+    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
+    ASSERT_EQ(0, memory_usage);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkWriteDuringPrepare) {
+    CreatePkTable();
+    auto state = std::make_shared<BlockingState>();
+    auto factory = std::make_shared<BlockingRealtimeStoreFactory>(state);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(first_batch)));
+
+    Result<std::vector<RealtimeCommitProgress>> prepare_result =
+        Status::Invalid("prepare did not run");
+    std::thread prepare_thread(
+        [&]() { prepare_result = writer->PrepareCommitWithProgress(/*commit_identifier=*/0); });
+    const bool prepare_blocked = state->WaitUntilBlocked();
+    if (!prepare_blocked) {
+        state->Release();
+        prepare_thread.join();
+        ASSERT_TRUE(prepare_blocked);
+    }
+
+    std::promise<Status> write_promise;
+    std::future<Status> write_future = write_promise.get_future();
+    std::thread write_thread([&]() {
+        Result<std::unique_ptr<RecordBatch>> batch =
+            MakeBatch({Row{2, "two", "p0"}}, /*partitioned=*/false);
+        if (!batch.ok()) {
+            write_promise.set_value(batch.status());
+            return;
+        }
+        write_promise.set_value(writer->Write(std::move(batch).value()));
+    });
+    const bool write_completed =
+        write_future.wait_for(std::chrono::seconds(5)) == std::future_status::ready;
+    state->Release();
+    prepare_thread.join();
+    write_thread.join();
+
+    ASSERT_TRUE(write_completed);
+    ASSERT_OK(write_future.get());
+    ASSERT_OK(prepare_result);
+    ASSERT_EQ(1, prepare_result.value().size());
+    ASSERT_EQ(OffsetRange(0, 1), prepare_result.value()[0].offset_range);
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, second_progress.size());
+    ASSERT_EQ(OffsetRange(1, 2), second_progress[0].offset_range);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
+    CreatePkTable();
+    auto factory = std::make_shared<InvalidReaderRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+
+    ASSERT_NOK_WITH_MSG(ReadRows(realtime_context),
+                        "PK real-time store returned no query readers for active memory");
+    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
+                        "PK real-time store returned a null commit reader");
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestRollingFilesPreserveProgress) {
     options_[Options::TARGET_FILE_ROW_NUM] = "10";
     CreateTable(/*partition_keys=*/{});
@@ -1235,50 +2144,12 @@ TEST_F(RealtimeWriteInteTest, TestFailedReaderCreationPreservesRealtimeSplitTick
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestVectorReaderFailurePreservesEarlierSplitTicket) {
-    CreateTable(/*partition_keys=*/{"pt"});
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    std::vector<Row> p0_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0_batch,
-                         MakeBatch(p0_rows, /*partitioned=*/true));
-    ASSERT_OK(writer->Write(std::move(p0_batch)));
-    std::vector<Row> p1_rows = MakeRows(/*first_id=*/10, /*count=*/3, /*partition=*/"p1");
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p1_batch,
-                         MakeBatch(p1_rows, /*partitioned=*/true));
-    ASSERT_OK(writer->Write(std::move(p1_batch)));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
-                         CreatePlan(realtime_context, /*predicate=*/nullptr));
-    ASSERT_EQ(2, plan->Splits().size());
-
-    std::vector<std::shared_ptr<Split>> invalid_splits = plan->Splits();
-    std::shared_ptr<RealtimeSplit> second_split =
-        std::dynamic_pointer_cast<RealtimeSplit>(invalid_splits[1]);
-    ASSERT_NE(nullptr, second_split);
-    std::vector<std::shared_ptr<Split>> second_disk_splits = second_split->DiskSplits();
-    invalid_splits[1] = std::make_shared<RealtimeSplit>(
-        RealtimeSplit::kCurrentVersion + 1, second_split->SnapshotId(), second_split->Partition(),
-        second_split->Bucket(), std::move(second_disk_splits), second_split->CommittedEndOffset(),
-        second_split->MemoryEndOffset(), second_split->OpaqueTicket());
-
-    ReadContextBuilder read_builder(table_path_);
-    read_builder.SetOptions(options_)
-        .SetReadFieldNames({"id", "payload", "pt"})
-        .WithRealtimeContext(realtime_context)
-        .WithMemoryPool(pool_);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
-                         TableRead::Create(std::move(read_context)));
-    ASSERT_NOK_WITH_MSG(table_read->CreateReader(invalid_splits),
-                        "unsupported real-time split version");
+TEST_F(RealtimeWriteInteTest, TestAppendVectorRetry) {
+    CheckVectorReaderRetry(/*primary_key=*/false);
+}
 
-    std::vector<Row> expected_rows = p0_rows;
-    expected_rows.insert(expected_rows.end(), p1_rows.begin(), p1_rows.end());
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
-    ASSERT_EQ(expected_rows, actual_rows);
-    ASSERT_OK(writer->Close());
+TEST_F(RealtimeWriteInteTest, TestPkVectorRetry) {
+    CheckVectorReaderRetry(/*primary_key=*/true);
 }
 
 TEST_F(RealtimeWriteInteTest, TestCloseWriterKeepsContextReadable) {
@@ -1351,6 +2222,52 @@ TEST_F(RealtimeWriteInteTest, TestCloseWriterAllowsContextReuseByLaterWriter) {
     ASSERT_OK(second_writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkWriterHandoff) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
+                         CreateRealtimeWriter(realtime_context));
+    std::vector<Row> first_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+                         MakeBatch(first_rows, /*partitioned=*/false));
+    ASSERT_OK(first_writer->Write(std::move(first_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
+                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, commits.size());
+    ASSERT_EQ(OffsetRange(0, 3), commits[0].offset_range);
+    ASSERT_EQ(1, NewFiles(commits).size());
+    ASSERT_EQ(0, NewFiles(commits)[0]->min_sequence_number);
+    ASSERT_EQ(2, NewFiles(commits)[0]->max_sequence_number);
+    ASSERT_OK(first_writer->Close());
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
+                         CreateRealtimeWriter(realtime_context));
+    std::vector<Row> second_rows = {
+        Row{0, "updated-0", "p0"},
+        Row{3, "value-3", "p0"},
+    };
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+                         MakeBatch(second_rows, /*partitioned=*/false));
+    ASSERT_OK(second_writer->Write(std::move(second_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
+                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, second_commits.size());
+    ASSERT_EQ(OffsetRange(3, 5), second_commits[0].offset_range);
+    ASSERT_EQ(1, NewFiles(second_commits).size());
+    ASSERT_EQ(3, NewFiles(second_commits)[0]->min_sequence_number);
+    ASSERT_EQ(4, NewFiles(second_commits)[0]->max_sequence_number);
+
+    commits.push_back(std::move(second_commits[0]));
+    ASSERT_OK(Commit(commits, /*commit_identifier=*/1));
+    std::vector<Row> expected_rows = first_rows;
+    expected_rows[0] = second_rows[0];
+    expected_rows.push_back(second_rows[1]);
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
+    ASSERT_EQ(expected_rows, actual_rows);
+    ASSERT_OK(second_writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestReadCommittedDiskAndBuildingMemory) {
     CreateTable(/*partition_keys=*/{});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,

From 949356aa24dc83150f612067180dbdad749bd037 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 20 Aug 2026 14:01:40 +0800
Subject: [PATCH 26/93] refactor(realtime): consolidate PK state and validation

---
 src/paimon/CMakeLists.txt                     |  2 -
 .../core/operation/file_store_write.cpp       |  3 +-
 .../operation/key_value_file_store_write.cpp  | 30 ++++++----
 .../realtime/primary_key_realtime_options.cpp | 58 -------------------
 .../realtime/primary_key_realtime_options.h   | 31 ----------
 .../primary_key_realtime_options_test.cpp     | 56 ------------------
 .../core/realtime/realtime_context_impl.cpp   | 27 ++++-----
 .../core/realtime/realtime_context_impl.h     |  4 +-
 .../core/realtime/realtime_context_test.cpp   | 38 ++++++++++++
 .../realtime/realtime_primary_key_writer.cpp  | 41 ++-----------
 .../realtime/realtime_primary_key_writer.h    | 13 ++---
 src/paimon/core/table/source/table_scan.cpp   |  4 +-
 .../core/utils/primary_key_table_utils.cpp    | 32 ++++++++++
 .../core/utils/primary_key_table_utils.h      |  3 +
 .../utils/primary_key_table_utils_test.cpp    | 26 +++++++++
 15 files changed, 144 insertions(+), 224 deletions(-)
 delete mode 100644 src/paimon/core/realtime/primary_key_realtime_options.cpp
 delete mode 100644 src/paimon/core/realtime/primary_key_realtime_options.h
 delete mode 100644 src/paimon/core/realtime/primary_key_realtime_options_test.cpp

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index ea6159821..d2c0a2b4f 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -383,7 +383,6 @@ set(PAIMON_CORE_SRCS
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
     core/realtime/primary_key_realtime_store.cpp
-    core/realtime/primary_key_realtime_options.cpp
     core/realtime/realtime_append_only_writer.cpp
     core/realtime/realtime_context.cpp
     core/realtime/realtime_context_impl.cpp
@@ -791,7 +790,6 @@ if(PAIMON_BUILD_TESTS)
                     core/memory/writer_memory_manager_test.cpp
                     core/realtime/arrow_realtime_store_test.cpp
                     core/realtime/primary_key_realtime_store_test.cpp
-                    core/realtime/primary_key_realtime_options_test.cpp
                     core/realtime/realtime_context_test.cpp
                     core/realtime/realtime_reader_test.cpp
                     core/mergetree/levels_test.cpp
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index fb83c254c..f216476bd 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -36,7 +36,6 @@
 #include "paimon/core/operation/key_value_file_store_write.h"
 #include "paimon/core/options/merge_engine.h"
 #include "paimon/core/postpone/postpone_bucket_file_store_write.h"
-#include "paimon/core/realtime/primary_key_realtime_options.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
@@ -198,7 +197,7 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     } else {
         // pk table
         if (ctx->GetRealtimeContext()) {
-            PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(options));
+            PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options));
             if (ignore_previous_files) {
                 return Status::NotImplemented(
                     "PK realtime v1 requires restore from the latest snapshot");
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index e94c45a15..492161cf8 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -124,19 +124,28 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         std::shared_ptr<Levels> levels,
         Levels::Create(key_comparator_, restore_data_files, options_.GetNumLevels()));
     std::map<std::string, std::string> partition_map;
-    int64_t materialized_max_sequence_number = restore_max_seq_number;
+    int64_t initial_max_sequence_number = restore_max_seq_number;
     std::shared_ptr<CompactManager> compact_manager;
+    std::shared_ptr<RealtimeContextImpl> realtime_context_impl;
+    std::optional<RealtimeStoreState> realtime_store_state;
     if (realtime_context_) {
         std::vector<std::pair<std::string, std::string>> partition_values;
         PAIMON_ASSIGN_OR_RAISE(partition_values,
                                file_store_path_factory_->GeneratePartitionVector(partition));
         partition_map =
             std::map<std::string, std::string>(partition_values.begin(), partition_values.end());
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
-                               RealtimeContextImpl::Cast(realtime_context_));
-        materialized_max_sequence_number = realtime_context_impl->GetMaterializedMaxSequenceNumber(
-            RealtimePartitionBucket(partition_map, bucket), restore_max_seq_number);
-        if (materialized_max_sequence_number == std::numeric_limits<int64_t>::max()) {
+        PAIMON_ASSIGN_OR_RAISE(realtime_context_impl, RealtimeContextImpl::Cast(realtime_context_));
+        auto c_write_schema = std::make_unique<ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, c_write_schema.get()));
+        PAIMON_ASSIGN_OR_RAISE(
+            RealtimeStoreState store_state,
+            realtime_context_impl->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
+                std::move(c_write_schema), options_.ToMap(), pool_, partition_map, bucket,
+                PrimaryKeyRealtimeStoreCreateConfig{trimmed_primary_keys,
+                                                    restore_max_seq_number}}));
+        realtime_store_state = std::move(store_state);
+        initial_max_sequence_number = realtime_store_state->initial_max_sequence_number.value();
+        if (initial_max_sequence_number == std::numeric_limits<int64_t>::max()) {
             return Status::Invalid("PK sequence number has reached INT64_MAX");
         }
         compact_manager = std::make_shared<NoopCompactManager>();
@@ -150,18 +159,15 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr<MergeTreeWriter> writer,
         MergeTreeWriter::Create(
-            materialized_max_sequence_number, trimmed_primary_keys, data_file_path_factory,
+            initial_max_sequence_number, trimmed_primary_keys, data_file_path_factory,
             key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_,
             table_schema_->Id(), schema_, options_, compact_manager,
             realtime_context_ ? nullptr : io_manager_, enable_multi_thread_spill_, pool_));
     if (!realtime_context_) {
         return std::shared_ptr<BatchWriter>(std::move(writer));
     }
-    auto c_write_schema = std::make_unique<ArrowSchema>();
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, c_write_schema.get()));
-    return RealtimePrimaryKeyWriter::Create(
-        partition_map, bucket, std::move(c_write_schema), trimmed_primary_keys, realtime_context_,
-        writer, options_.ToMap(), pool_, materialized_max_sequence_number);
+    return RealtimePrimaryKeyWriter::Create(partition_map, bucket, schema_, realtime_context_impl,
+                                            writer, pool_, realtime_store_state.value());
 }
 
 Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
diff --git a/src/paimon/core/realtime/primary_key_realtime_options.cpp b/src/paimon/core/realtime/primary_key_realtime_options.cpp
deleted file mode 100644
index e9779a59e..000000000
--- a/src/paimon/core/realtime/primary_key_realtime_options.cpp
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "paimon/core/realtime/primary_key_realtime_options.h"
-
-#include "paimon/core/core_options.h"
-
-namespace paimon {
-
-Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options) {
-    if (options.GetBucket() <= 0) {
-        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
-    }
-    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
-        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
-    }
-    if (options.DataEvolutionEnabled()) {
-        return Status::NotImplemented("PK realtime v1 does not support data evolution");
-    }
-    if (!options.GetFieldsSequenceGroups().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
-    }
-    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
-        options.AggregationRemoveRecordOnDelete() ||
-        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
-        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
-    }
-    if (!options.GetSequenceField().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
-    }
-    if (!options.SequenceFieldSortOrderIsAscending()) {
-        return Status::NotImplemented(
-            "PK realtime v1 supports only ascending sequence.field.sort-order");
-    }
-    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
-        options.GetChangelogProducer() != ChangelogProducer::NONE) {
-        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
-    }
-    return Status::OK();
-}
-
-}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_options.h b/src/paimon/core/realtime/primary_key_realtime_options.h
deleted file mode 100644
index a16d35778..000000000
--- a/src/paimon/core/realtime/primary_key_realtime_options.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include "paimon/status.h"
-
-namespace paimon {
-
-class CoreOptions;
-
-/// Validates the table options supported by the in-memory PK realtime V1 path.
-Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options);
-
-}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_options_test.cpp b/src/paimon/core/realtime/primary_key_realtime_options_test.cpp
deleted file mode 100644
index 5d3ea7f67..000000000
--- a/src/paimon/core/realtime/primary_key_realtime_options_test.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "paimon/core/realtime/primary_key_realtime_options.h"
-
-#include <map>
-#include <string>
-#include <vector>
-
-#include "paimon/core/core_options.h"
-#include "paimon/defs.h"
-#include "paimon/testing/utils/testharness.h"
-
-namespace paimon::test {
-
-TEST(PrimaryKeyRealtimeOptionsTest, TestSupportedOptions) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_OK(ValidatePrimaryKeyRealtimeOptions(options));
-}
-
-TEST(PrimaryKeyRealtimeOptionsTest, TestUnsupportedOptions) {
-    const std::string sequence_group =
-        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
-    const std::vector<std::map<std::string, std::string>> unsupported_options = {
-        {{Options::BUCKET, "0"}},
-        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
-        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
-        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
-    };
-    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
-        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-        ASSERT_NOK(ValidatePrimaryKeyRealtimeOptions(options));
-    }
-}
-
-}  // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 0a367b2cd..6624059a6 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -82,6 +82,18 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     std::lock_guard<std::mutex> progress_lock(progress_mutex_);
     std::lock_guard<std::mutex> registry_lock(mutex_);
     const RealtimePartitionBucket key(request.partition, request.bucket);
+    std::optional<int64_t> initial_max_sequence_number;
+    PrimaryKeyRealtimeStoreCreateConfig* primary_key_config =
+        std::get_if<PrimaryKeyRealtimeStoreCreateConfig>(&request.mode_config);
+    if (primary_key_config) {
+        auto [sequence_iter, inserted] = materialized_max_sequence_numbers_.emplace(
+            key, primary_key_config->restore_max_sequence_number);
+        if (!inserted && primary_key_config->restore_max_sequence_number > sequence_iter->second) {
+            sequence_iter->second = primary_key_config->restore_max_sequence_number;
+        }
+        initial_max_sequence_number = sequence_iter->second;
+        primary_key_config->restore_max_sequence_number = sequence_iter->second;
+    }
     int64_t initial_offset = 0;
     auto offset_iter = committed_offsets_.find(key);
     if (offset_iter != committed_offsets_.end()) {
@@ -114,7 +126,7 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
                 initial_offset = memory_range->end;
             }
         }
-        return RealtimeStoreState{iter->second, initial_offset};
+        return RealtimeStoreState{iter->second, initial_offset, initial_max_sequence_number};
     }
     Result<std::shared_ptr<RealtimeStore>> store_result = factory_->Create(std::move(request));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
@@ -122,18 +134,7 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
     }
-    return RealtimeStoreState{std::move(store), initial_offset};
-}
-
-int64_t RealtimeContextImpl::GetMaterializedMaxSequenceNumber(
-    const RealtimePartitionBucket& partition_bucket, int64_t restored_max_sequence_number) {
-    std::lock_guard<std::mutex> lock(mutex_);
-    auto [iter, inserted] =
-        materialized_max_sequence_numbers_.emplace(partition_bucket, restored_max_sequence_number);
-    if (!inserted && restored_max_sequence_number > iter->second) {
-        iter->second = restored_max_sequence_number;
-    }
-    return iter->second;
+    return RealtimeStoreState{std::move(store), initial_offset, initial_max_sequence_number};
 }
 
 void RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index 45d07deeb..f4cd3866e 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -47,6 +47,7 @@ class MemoryPool;
 struct RealtimeStoreState {
     std::shared_ptr<RealtimeStore> store;
     int64_t initial_offset;
+    std::optional<int64_t> initial_max_sequence_number;
 };
 
 struct RealtimePartitionBucketView {
@@ -67,9 +68,6 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
 
     Result<RealtimeStoreState> GetOrCreateRealtimeStore(RealtimeStoreCreateRequest&& request);
 
-    int64_t GetMaterializedMaxSequenceNumber(const RealtimePartitionBucket& partition_bucket,
-                                             int64_t restored_max_sequence_number);
-
     void AdvanceMaterializedMaxSequenceNumber(const RealtimePartitionBucket& partition_bucket,
                                               int64_t max_sequence_number);
 
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 33701afac..b4d2c6718 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -129,6 +129,15 @@ Result<RealtimeStoreState> GetOrCreateAppendStore(
                                    AppendRealtimeStoreCreateConfig{StatisticsMode::NONE}});
 }
 
+Result<RealtimeStoreState> GetOrCreatePrimaryKeyStore(
+    const std::shared_ptr<RealtimeContextImpl>& context,
+    const std::map<std::string, std::string>& partition, int32_t bucket,
+    int64_t restore_max_sequence_number, const std::shared_ptr<MemoryPool>& memory_pool) {
+    return context->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
+        MakeWriteSchema(), /*options=*/{}, memory_pool, partition, bucket,
+        PrimaryKeyRealtimeStoreCreateConfig{{"id"}, restore_max_sequence_number}});
+}
+
 TEST(RealtimeContextTest, TestReusesIndexerAndCapturesRegisteredViews) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
@@ -138,6 +147,7 @@ TEST(RealtimeContextTest, TestReusesIndexerAndCapturesRegisteredViews) {
                          GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0,
                                                 MakeWriteSchema(), {{"k", "v"}}, pool));
     ASSERT_EQ(0, first_state.initial_offset);
+    ASSERT_FALSE(first_state.initial_max_sequence_number);
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState first_again_state,
         GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0, MakeWriteSchema(), {}, pool));
@@ -168,6 +178,34 @@ TEST(RealtimeContextTest, TestReusesIndexerAndCapturesRegisteredViews) {
     ASSERT_EQ(1, factory->stores[2]->acquire_count);
 }
 
+TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
+
+    ASSERT_OK_AND_ASSIGN(
+        RealtimeStoreState first_state,
+        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/0,
+                                   /*restore_max_sequence_number=*/4, GetDefaultPool()));
+    ASSERT_EQ(4, first_state.initial_max_sequence_number);
+
+    const RealtimePartitionBucket partition_bucket(partition, /*bucket=*/0);
+    context->AdvanceMaterializedMaxSequenceNumber(partition_bucket, /*max_sequence_number=*/8);
+    ASSERT_OK_AND_ASSIGN(
+        RealtimeStoreState retained_state,
+        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/0,
+                                   /*restore_max_sequence_number=*/6, GetDefaultPool()));
+    ASSERT_EQ(first_state.store, retained_state.store);
+    ASSERT_EQ(8, retained_state.initial_max_sequence_number);
+
+    ASSERT_OK_AND_ASSIGN(
+        RealtimeStoreState restored_state,
+        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/0,
+                                   /*restore_max_sequence_number=*/10, GetDefaultPool()));
+    ASSERT_EQ(first_state.store, restored_state.store);
+    ASSERT_EQ(10, restored_state.initial_max_sequence_number);
+}
+
 TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 2ebcede82..e33f48bba 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -44,44 +44,13 @@ namespace paimon {
 
 Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Create(
     const std::map<std::string, std::string>& partition, int32_t bucket,
-    std::unique_ptr<::ArrowSchema> write_schema,
-    const std::vector<std::string>& trimmed_primary_keys,
-    const std::shared_ptr<RealtimeContext>& realtime_context,
+    const std::shared_ptr<arrow::Schema>& write_schema,
+    const std::shared_ptr<RealtimeContextImpl>& realtime_context,
     const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
-    const std::map<std::string, std::string>& options,
-    const std::shared_ptr<MemoryPool>& memory_pool, int64_t restore_max_sequence_number) {
-    ScopeGuard schema_guard([schema = write_schema.get()]() {
-        if (schema && schema->release) {
-            ArrowSchemaRelease(schema);
-        }
-    });
-    if (!realtime_context) {
-        return Status::Invalid("PK real-time context is null");
-    }
-    if (!merge_tree_writer) {
-        return Status::Invalid("PK real-time merge-tree writer is null");
-    }
-    if (!write_schema || !write_schema->release) {
-        return Status::Invalid("PK real-time write schema is null");
-    }
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> imported_schema,
-                                      arrow::ImportSchema(write_schema.get()));
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*imported_schema, write_schema.get()));
-    RealtimeStoreCreateRequest request{
-        std::move(write_schema),
-        options,
-        memory_pool,
-        partition,
-        bucket,
-        PrimaryKeyRealtimeStoreCreateConfig{trimmed_primary_keys, restore_max_sequence_number}};
-    schema_guard.Release();
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
-                           RealtimeContextImpl::Cast(realtime_context));
-    PAIMON_ASSIGN_OR_RAISE(RealtimeStoreState store_state,
-                           realtime_context_impl->GetOrCreateRealtimeStore(std::move(request)));
+    const std::shared_ptr<MemoryPool>& memory_pool, const RealtimeStoreState& store_state) {
     return std::shared_ptr<RealtimePrimaryKeyWriter>(
-        new RealtimePrimaryKeyWriter(store_state.store, merge_tree_writer, realtime_context_impl,
-                                     RealtimePartitionBucket(partition, bucket), imported_schema,
+        new RealtimePrimaryKeyWriter(store_state.store, merge_tree_writer, realtime_context,
+                                     RealtimePartitionBucket(partition, bucket), write_schema,
                                      store_state.initial_offset, memory_pool));
 }
 
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index fa057e079..c1e893c85 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -24,14 +24,11 @@
 #include <memory>
 #include <mutex>
 #include <string>
-#include <vector>
 
 #include "paimon/core/utils/batch_writer.h"
 #include "paimon/realtime/realtime_context.h"
 #include "paimon/realtime/realtime_store.h"
 
-struct ArrowSchema;
-
 namespace arrow {
 class Schema;
 }  // namespace arrow
@@ -40,20 +37,18 @@ namespace paimon {
 
 class MemoryPool;
 class MergeTreeWriter;
-class RealtimeContext;
 class RealtimeContextImpl;
+struct RealtimeStoreState;
 
 /// Primary-key real-time writer backed by an in-memory mutation indexer.
 class RealtimePrimaryKeyWriter final : public BatchWriter {
  public:
     static Result<std::shared_ptr<RealtimePrimaryKeyWriter>> Create(
         const std::map<std::string, std::string>& partition, int32_t bucket,
-        std::unique_ptr<::ArrowSchema> write_schema,
-        const std::vector<std::string>& trimmed_primary_keys,
-        const std::shared_ptr<RealtimeContext>& realtime_context,
+        const std::shared_ptr<arrow::Schema>& write_schema,
+        const std::shared_ptr<RealtimeContextImpl>& realtime_context,
         const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
-        const std::map<std::string, std::string>& options,
-        const std::shared_ptr<MemoryPool>& memory_pool, int64_t restore_max_sequence_number);
+        const std::shared_ptr<MemoryPool>& memory_pool, const RealtimeStoreState& store_state);
 
     Status Write(std::unique_ptr<RecordBatch>&& batch) override;
     Result<CommitIncrement> PrepareCommit(bool wait_compaction) override;
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index b12e59a84..92155de3b 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -41,7 +41,6 @@
 #include "paimon/core/operation/data_evolution_file_store_scan.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
-#include "paimon/core/realtime/primary_key_realtime_options.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/schema_validation.h"
@@ -64,6 +63,7 @@
 #include "paimon/core/utils/field_mapping.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/index_file_path_factories.h"
+#include "paimon/core/utils/primary_key_table_utils.h"
 #include "paimon/core/utils/snapshot_manager.h"
 #include "paimon/format/file_format.h"
 #include "paimon/realtime/realtime_context.h"
@@ -233,7 +233,7 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
         return Status::Invalid("real-time union read does not support data evolution");
     }
     if (!table_schema.PrimaryKeys().empty()) {
-        PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(core_options));
+        PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options));
     }
     if (context.IsStreamingMode()) {
         return Status::Invalid("real-time union read currently supports batch scans only");
diff --git a/src/paimon/core/utils/primary_key_table_utils.cpp b/src/paimon/core/utils/primary_key_table_utils.cpp
index cf72da4ae..823d48c41 100644
--- a/src/paimon/core/utils/primary_key_table_utils.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils.cpp
@@ -96,4 +96,36 @@ Result<std::unique_ptr<FieldsComparator>> PrimaryKeyTableUtils::CreateSequenceFi
                                     options.SequenceFieldSortOrderIsAscending());
 }
 
+Status PrimaryKeyTableUtils::ValidateRealtimeOptions(const CoreOptions& options) {
+    if (options.GetBucket() <= 0) {
+        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
+    }
+    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
+        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
+    }
+    if (options.DataEvolutionEnabled()) {
+        return Status::NotImplemented("PK realtime v1 does not support data evolution");
+    }
+    if (!options.GetFieldsSequenceGroups().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
+    }
+    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
+        options.AggregationRemoveRecordOnDelete() ||
+        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
+        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
+    }
+    if (!options.GetSequenceField().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
+    }
+    if (!options.SequenceFieldSortOrderIsAscending()) {
+        return Status::NotImplemented(
+            "PK realtime v1 supports only ascending sequence.field.sort-order");
+    }
+    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
+        options.GetChangelogProducer() != ChangelogProducer::NONE) {
+        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
+    }
+    return Status::OK();
+}
+
 }  // namespace paimon
diff --git a/src/paimon/core/utils/primary_key_table_utils.h b/src/paimon/core/utils/primary_key_table_utils.h
index 82a108ab7..7877ee4ab 100644
--- a/src/paimon/core/utils/primary_key_table_utils.h
+++ b/src/paimon/core/utils/primary_key_table_utils.h
@@ -24,6 +24,7 @@
 
 #include "arrow/type.h"
 #include "paimon/result.h"
+#include "paimon/status.h"
 
 namespace arrow {
 class Schema;
@@ -57,6 +58,8 @@ class PrimaryKeyTableUtils {
 
     static Result<std::unique_ptr<FieldsComparator>> CreateSequenceFieldsComparator(
         const std::vector<DataField>& value_fields, const CoreOptions& options);
+
+    static Status ValidateRealtimeOptions(const CoreOptions& options);
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/utils/primary_key_table_utils_test.cpp b/src/paimon/core/utils/primary_key_table_utils_test.cpp
index 12713ca5b..072965cff 100644
--- a/src/paimon/core/utils/primary_key_table_utils_test.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils_test.cpp
@@ -22,7 +22,9 @@
 #include <map>
 #include <memory>
 #include <optional>
+#include <string>
 #include <utility>
+#include <vector>
 
 #include "arrow/type.h"
 #include "gtest/gtest.h"
@@ -112,4 +114,28 @@ TEST(PrimaryKeyTableUtilsTest, TestCreateFirstRowMergeFunctionWithIgnoreDelete)
                         "First row merge engine can not accept DELETE/UPDATE_BEFORE records");
 }
 
+TEST(PrimaryKeyTableUtilsTest, TestSupportedRealtimeOptions) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options));
+}
+
+TEST(PrimaryKeyTableUtilsTest, TestUnsupportedRealtimeOptions) {
+    const std::string sequence_group =
+        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
+    const std::vector<std::map<std::string, std::string>> unsupported_options = {
+        {{Options::BUCKET, "0"}},
+        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
+        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
+        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
+    };
+    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
+        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+        ASSERT_NOK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options));
+    }
+}
+
 }  // namespace paimon::test

From 7f4b60b6e9ad8ea165845c72f89074195e51fe23 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 20 Aug 2026 14:55:20 +0800
Subject: [PATCH 27/93] fix(read): close PK realtime query readers

---
 .../table/source/key_value_table_read.cpp     |   9 +-
 test/inte/realtime_write_inte_test.cpp        | 138 ++++++++++++++++++
 2 files changed, 146 insertions(+), 1 deletion(-)

diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 770caf1ca..59041b78e 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -65,6 +65,10 @@ class QueryBatchKeyValueReader final : public KeyValueRecordReader {
           value_schema_(value_schema),
           pool_(pool) {}
 
+    ~QueryBatchKeyValueReader() override {
+        Close();
+    }
+
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override;
     std::shared_ptr<Metrics> GetReaderMetrics() const override;
     void Close() override;
@@ -161,7 +165,10 @@ void QueryBatchKeyValueReader::Close() {
     row_kinds_.reset();
     key_context_.reset();
     value_context_.reset();
-    reader_->Close();
+    if (reader_) {
+        reader_->Close();
+        reader_.reset();
+    }
 }
 
 Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index f18c3f1e4..cee96301f 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -324,6 +324,101 @@ class QueryTrackingRealtimeStoreFactory final : public RealtimeStoreFactory {
     std::shared_ptr<std::weak_ptr<RealtimeReadView>> query_view_;
 };
 
+class CloseTrackingBatchReader final : public BatchReader {
+ public:
+    CloseTrackingBatchReader(std::unique_ptr<BatchReader> delegate,
+                             const std::shared_ptr<std::atomic<int32_t>>& close_count)
+        : delegate_(std::move(delegate)), close_count_(close_count) {}
+
+    Result<ReadBatch> NextBatch() override {
+        return delegate_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        close_count_->fetch_add(1, std::memory_order_release);
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    std::shared_ptr<std::atomic<int32_t>> close_count_;
+};
+
+class CloseTrackingRealtimeStore final : public RealtimeStore {
+ public:
+    CloseTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
+                               const std::shared_ptr<std::atomic<int32_t>>& close_count,
+                               const std::shared_ptr<std::atomic<bool>>& append_null_reader)
+        : delegate_(delegate), close_count_(close_count), append_null_reader_(append_null_reader) {}
+
+    Status Write(RealtimeWriteBatch&& batch) override {
+        return delegate_->Write(std::move(batch));
+    }
+
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
+        return delegate_->SealForCommit();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
+        return delegate_->CreateCommitReaders(segment);
+    }
+
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
+        return delegate_->AcquireReadView();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               delegate_->CreateQueryReaders(view, offset_begin, context));
+        for (std::unique_ptr<BatchReader>& reader : readers) {
+            reader = std::make_unique<CloseTrackingBatchReader>(std::move(reader), close_count_);
+        }
+        if (append_null_reader_->load(std::memory_order_acquire)) {
+            readers.push_back(nullptr);
+        }
+        return readers;
+    }
+
+    Status AdvanceCommittedOffset(int64_t committed_offset) override {
+        return delegate_->AdvanceCommittedOffset(committed_offset);
+    }
+
+    uint64_t GetMemoryUsage() const override {
+        return delegate_->GetMemoryUsage();
+    }
+
+ private:
+    std::shared_ptr<RealtimeStore> delegate_;
+    std::shared_ptr<std::atomic<int32_t>> close_count_;
+    std::shared_ptr<std::atomic<bool>> append_null_reader_;
+};
+
+class CloseTrackingRealtimeStoreFactory final : public RealtimeStoreFactory {
+ public:
+    CloseTrackingRealtimeStoreFactory(const std::shared_ptr<std::atomic<int32_t>>& close_count,
+                                      const std::shared_ptr<std::atomic<bool>>& append_null_reader)
+        : close_count_(close_count), append_null_reader_(append_null_reader) {}
+
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
+                               delegate_.Create(std::move(request)));
+        return std::shared_ptr<RealtimeStore>(std::make_shared<CloseTrackingRealtimeStore>(
+            delegate, close_count_, append_null_reader_));
+    }
+
+ private:
+    ArrowRealtimeStoreFactory delegate_;
+    std::shared_ptr<std::atomic<int32_t>> close_count_;
+    std::shared_ptr<std::atomic<bool>> append_null_reader_;
+};
+
 class InvalidReaderRealtimeStore final : public RealtimeStore {
  public:
     explicit InvalidReaderRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
@@ -1632,6 +1727,49 @@ TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkPluginQueryReaderCloseLifecycle) {
+    CreatePkTable();
+    auto close_count = std::make_shared<std::atomic<int32_t>>(0);
+    auto append_null_reader = std::make_shared<std::atomic<bool>>(false);
+    auto factory =
+        std::make_shared<CloseTrackingRealtimeStoreFactory>(close_count, append_null_reader);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+
+    auto create_reader = [&]() -> Result<std::unique_ptr<BatchReader>> {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
+                               CreatePlan(realtime_context, /*predicate=*/nullptr));
+        ReadContextBuilder read_builder(table_path_);
+        read_builder.SetOptions(options_)
+            .SetReadFieldNames({"id", "payload", "pt"})
+            .WithRealtimeContext(realtime_context)
+            .WithMemoryPool(pool_);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
+                               TableRead::Create(std::move(read_context)));
+        return table_read->CreateReader(plan->Splits());
+    };
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> explicitly_closed_reader, create_reader());
+    explicitly_closed_reader->Close();
+    explicitly_closed_reader.reset();
+    ASSERT_EQ(1, close_count->load(std::memory_order_acquire));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> destroyed_reader, create_reader());
+    destroyed_reader.reset();
+    ASSERT_EQ(2, close_count->load(std::memory_order_acquire));
+
+    append_null_reader->store(true, std::memory_order_release);
+    ASSERT_NOK_WITH_MSG(create_reader(), "PK real-time store returned a null query reader");
+    ASSERT_EQ(3, close_count->load(std::memory_order_acquire));
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestRollingFilesPreserveProgress) {
     options_[Options::TARGET_FILE_ROW_NUM] = "10";
     CreateTable(/*partition_keys=*/{});

From 3dee339038f793f079dfb0af03a0e7117f35f927 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 20 Aug 2026 15:41:07 +0800
Subject: [PATCH 28/93] fix(realtime): close rejected plugin readers

---
 .../realtime/realtime_primary_key_writer.cpp  |   7 +
 .../table/source/key_value_table_read.cpp     |   7 +
 test/inte/realtime_write_inte_test.cpp        | 132 ++++++++++++++----
 3 files changed, 119 insertions(+), 27 deletions(-)

diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index e33f48bba..65bcebcad 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -117,6 +117,13 @@ Status RealtimePrimaryKeyWriter::FlushSegment(
     const std::shared_ptr<RealtimeSegmentHandle>& segment) {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                            realtime_store_->CreateCommitReaders(segment));
+    ScopeGuard readers_guard([&readers]() {
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            if (reader) {
+                reader->Close();
+            }
+        }
+    });
     for (const std::unique_ptr<BatchReader>& reader : readers) {
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null commit reader");
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 59041b78e..9b5f6ee83 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -190,6 +190,13 @@ Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> batch_readers,
                            memory.store->CreateQueryReaders(
                                memory.read_view, split->CommittedEndOffset(), query_context));
+    ScopeGuard reader_guard([&batch_readers]() {
+        for (const std::unique_ptr<BatchReader>& reader : batch_readers) {
+            if (reader) {
+                reader->Close();
+            }
+        }
+    });
     if (batch_readers.empty()) {
         return Status::Invalid("PK real-time store returned no query readers for active memory");
     }
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index cee96301f..e6000561f 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -348,12 +348,20 @@ class CloseTrackingBatchReader final : public BatchReader {
     std::shared_ptr<std::atomic<int32_t>> close_count_;
 };
 
+struct CloseTrackingReaderState {
+    std::shared_ptr<std::atomic<int32_t>> query_close_count =
+        std::make_shared<std::atomic<int32_t>>(0);
+    std::shared_ptr<std::atomic<int32_t>> commit_close_count =
+        std::make_shared<std::atomic<int32_t>>(0);
+    int32_t query_null_index = -1;
+    int32_t commit_null_index = -1;
+};
+
 class CloseTrackingRealtimeStore final : public RealtimeStore {
  public:
     CloseTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                               const std::shared_ptr<std::atomic<int32_t>>& close_count,
-                               const std::shared_ptr<std::atomic<bool>>& append_null_reader)
-        : delegate_(delegate), close_count_(close_count), append_null_reader_(append_null_reader) {}
+                               const std::shared_ptr<CloseTrackingReaderState>& state)
+        : delegate_(delegate), state_(state) {}
 
     Status Write(RealtimeWriteBatch&& batch) override {
         return delegate_->Write(std::move(batch));
@@ -365,7 +373,14 @@ class CloseTrackingRealtimeStore final : public RealtimeStore {
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
-        return delegate_->CreateCommitReaders(segment);
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               delegate_->CreateCommitReaders(segment));
+        for (std::unique_ptr<BatchReader>& reader : readers) {
+            reader = std::make_unique<CloseTrackingBatchReader>(std::move(reader),
+                                                                state_->commit_close_count);
+        }
+        PAIMON_RETURN_NOT_OK(InsertNullReader(state_->commit_null_index, &readers));
+        return readers;
     }
 
     Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
@@ -378,11 +393,10 @@ class CloseTrackingRealtimeStore final : public RealtimeStore {
         PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                                delegate_->CreateQueryReaders(view, offset_begin, context));
         for (std::unique_ptr<BatchReader>& reader : readers) {
-            reader = std::make_unique<CloseTrackingBatchReader>(std::move(reader), close_count_);
-        }
-        if (append_null_reader_->load(std::memory_order_acquire)) {
-            readers.push_back(nullptr);
+            reader = std::make_unique<CloseTrackingBatchReader>(std::move(reader),
+                                                                state_->query_close_count);
         }
+        PAIMON_RETURN_NOT_OK(InsertNullReader(state_->query_null_index, &readers));
         return readers;
     }
 
@@ -395,28 +409,38 @@ class CloseTrackingRealtimeStore final : public RealtimeStore {
     }
 
  private:
+    static Status InsertNullReader(int32_t index,
+                                   std::vector<std::unique_ptr<BatchReader>>* readers) {
+        if (index < 0) {
+            return Status::OK();
+        }
+        if (index > static_cast<int32_t>(readers->size())) {
+            return Status::Invalid("null reader index exceeds reader count");
+        }
+        readers->insert(readers->begin() + index, nullptr);
+        return Status::OK();
+    }
+
     std::shared_ptr<RealtimeStore> delegate_;
-    std::shared_ptr<std::atomic<int32_t>> close_count_;
-    std::shared_ptr<std::atomic<bool>> append_null_reader_;
+    std::shared_ptr<CloseTrackingReaderState> state_;
 };
 
 class CloseTrackingRealtimeStoreFactory final : public RealtimeStoreFactory {
  public:
-    CloseTrackingRealtimeStoreFactory(const std::shared_ptr<std::atomic<int32_t>>& close_count,
-                                      const std::shared_ptr<std::atomic<bool>>& append_null_reader)
-        : close_count_(close_count), append_null_reader_(append_null_reader) {}
+    explicit CloseTrackingRealtimeStoreFactory(
+        const std::shared_ptr<CloseTrackingReaderState>& state)
+        : state_(state) {}
 
     Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
                                delegate_.Create(std::move(request)));
-        return std::shared_ptr<RealtimeStore>(std::make_shared<CloseTrackingRealtimeStore>(
-            delegate, close_count_, append_null_reader_));
+        return std::shared_ptr<RealtimeStore>(
+            std::make_shared<CloseTrackingRealtimeStore>(delegate, state_));
     }
 
  private:
     ArrowRealtimeStoreFactory delegate_;
-    std::shared_ptr<std::atomic<int32_t>> close_count_;
-    std::shared_ptr<std::atomic<bool>> append_null_reader_;
+    std::shared_ptr<CloseTrackingReaderState> state_;
 };
 
 class InvalidReaderRealtimeStore final : public RealtimeStore {
@@ -1727,12 +1751,10 @@ TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkPluginQueryReaderCloseLifecycle) {
+TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
     CreatePkTable();
-    auto close_count = std::make_shared<std::atomic<int32_t>>(0);
-    auto append_null_reader = std::make_shared<std::atomic<bool>>(false);
-    auto factory =
-        std::make_shared<CloseTrackingRealtimeStoreFactory>(close_count, append_null_reader);
+    auto state = std::make_shared<CloseTrackingReaderState>();
+    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -1758,15 +1780,71 @@ TEST_F(RealtimeWriteInteTest, TestPkPluginQueryReaderCloseLifecycle) {
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> explicitly_closed_reader, create_reader());
     explicitly_closed_reader->Close();
     explicitly_closed_reader.reset();
-    ASSERT_EQ(1, close_count->load(std::memory_order_acquire));
+    ASSERT_EQ(1, state->query_close_count->load(std::memory_order_acquire));
 
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> destroyed_reader, create_reader());
     destroyed_reader.reset();
-    ASSERT_EQ(2, close_count->load(std::memory_order_acquire));
+    ASSERT_EQ(2, state->query_close_count->load(std::memory_order_acquire));
+
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkQueryReaderCloseFailure) {
+    CreatePkTable();
+    auto state = std::make_shared<CloseTrackingReaderState>();
+    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(first_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, progress.size());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+                         MakeBatch({Row{2, "two", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(second_batch)));
 
-    append_null_reader->store(true, std::memory_order_release);
-    ASSERT_NOK_WITH_MSG(create_reader(), "PK real-time store returned a null query reader");
-    ASSERT_EQ(3, close_count->load(std::memory_order_acquire));
+    auto create_reader = [&]() -> Result<std::unique_ptr<BatchReader>> {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
+                               CreatePlan(realtime_context, /*predicate=*/nullptr));
+        ReadContextBuilder read_builder(table_path_);
+        read_builder.SetOptions(options_)
+            .SetReadFieldNames({"id", "payload", "pt"})
+            .WithRealtimeContext(realtime_context)
+            .WithMemoryPool(pool_);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
+                               TableRead::Create(std::move(read_context)));
+        return table_read->CreateReader(plan->Splits());
+    };
+
+    for (int32_t null_index = 0; null_index <= 2; ++null_index) {
+        state->query_null_index = null_index;
+        ASSERT_NOK_WITH_MSG(create_reader(), "PK real-time store returned a null query reader");
+        ASSERT_EQ(2 * (null_index + 1), state->query_close_count->load(std::memory_order_acquire));
+    }
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkCommitReaderCloseFailure) {
+    CreatePkTable();
+    auto state = std::make_shared<CloseTrackingReaderState>();
+    state->commit_null_index = 1;
+    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+
+    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
+                        "PK real-time store returned a null commit reader");
+    ASSERT_EQ(1, state->commit_close_count->load(std::memory_order_acquire));
     ASSERT_OK(writer->Close());
 }
 

From b36b24f0873ebdb77fbcc6c7792d0129fcf052f6 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 20 Aug 2026 18:05:54 +0800
Subject: [PATCH 29/93] fix(read): preserve PK reader metrics after close

---
 src/paimon/core/table/source/key_value_table_read.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 9b5f6ee83..76160ac93 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -85,6 +85,7 @@ class QueryBatchKeyValueReader final : public KeyValueRecordReader {
     std::shared_ptr<arrow::Int8Array> row_kinds_;
     std::shared_ptr<ColumnarBatchContext> key_context_;
     std::shared_ptr<ColumnarBatchContext> value_context_;
+    bool closed_ = false;
 };
 
 class QueryBatchKeyValueReader::Iterator final : public KeyValueRecordReader::Iterator {
@@ -160,6 +161,10 @@ std::shared_ptr<Metrics> QueryBatchKeyValueReader::GetReaderMetrics() const {
 }
 
 void QueryBatchKeyValueReader::Close() {
+    if (closed_) {
+        return;
+    }
+    closed_ = true;
     values_.reset();
     sequences_.reset();
     row_kinds_.reset();
@@ -167,7 +172,6 @@ void QueryBatchKeyValueReader::Close() {
     value_context_.reset();
     if (reader_) {
         reader_->Close();
-        reader_.reset();
     }
 }
 

From 20b834ea7bbc24be751546f7349a71ea9b5d97c8 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 20 Aug 2026 18:06:13 +0800
Subject: [PATCH 30/93] refactor(realtime): colocate PK realtime option
 validation

---
 .../core/operation/file_store_write.cpp       |  3 +-
 .../realtime/primary_key_realtime_store.cpp   | 34 +++++++++++++++++++
 .../realtime/primary_key_realtime_store.h     |  3 ++
 .../primary_key_realtime_store_test.cpp       | 26 ++++++++++++++
 src/paimon/core/table/source/table_scan.cpp   |  4 +--
 .../core/utils/primary_key_table_utils.cpp    | 32 -----------------
 .../core/utils/primary_key_table_utils.h      |  2 --
 .../utils/primary_key_table_utils_test.cpp    | 25 --------------
 8 files changed, 67 insertions(+), 62 deletions(-)

diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index f216476bd..4d4f45156 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -36,6 +36,7 @@
 #include "paimon/core/operation/key_value_file_store_write.h"
 #include "paimon/core/options/merge_engine.h"
 #include "paimon/core/postpone/postpone_bucket_file_store_write.h"
+#include "paimon/core/realtime/primary_key_realtime_store.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
@@ -197,7 +198,7 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     } else {
         // pk table
         if (ctx->GetRealtimeContext()) {
-            PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options));
+            PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(options));
             if (ignore_previous_files) {
                 return Status::NotImplemented(
                     "PK realtime v1 requires restore from the latest snapshot");
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 84afb97a4..afdc0c73c 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -36,6 +36,7 @@
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/fields_comparator.h"
+#include "paimon/core/core_options.h"
 #include "paimon/core/io/key_value_in_memory_record_reader.h"
 #include "paimon/core/io/key_value_projection_consumer.h"
 #include "paimon/core/io/key_value_projection_reader.h"
@@ -45,6 +46,39 @@
 #include "paimon/macros.h"
 
 namespace paimon {
+
+Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options) {
+    if (options.GetBucket() <= 0) {
+        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
+    }
+    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
+        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
+    }
+    if (options.DataEvolutionEnabled()) {
+        return Status::NotImplemented("PK realtime v1 does not support data evolution");
+    }
+    if (!options.GetFieldsSequenceGroups().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
+    }
+    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
+        options.AggregationRemoveRecordOnDelete() ||
+        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
+        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
+    }
+    if (!options.GetSequenceField().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
+    }
+    if (!options.SequenceFieldSortOrderIsAscending()) {
+        return Status::NotImplemented(
+            "PK realtime v1 supports only ascending sequence.field.sort-order");
+    }
+    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
+        options.GetChangelogProducer() != ChangelogProducer::NONE) {
+        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
+    }
+    return Status::OK();
+}
+
 namespace {
 
 uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index 05225ed19..017864c04 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -33,6 +33,7 @@ class Schema;
 
 namespace paimon {
 
+class CoreOptions;
 class FieldsComparator;
 struct KeyValue;
 class MemoryPool;
@@ -40,6 +41,8 @@ class InternalRow;
 template <typename T>
 class MergeFunctionWrapper;
 
+Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options);
+
 /// Optional metadata exposed by PK query readers with a known inclusive key range.
 class PrimaryKeyRangeProvider {
  public:
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 9da272e0f..cbbf9c82a 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -19,6 +19,7 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
+#include <map>
 #include <memory>
 #include <string>
 #include <vector>
@@ -30,6 +31,7 @@
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/fields_comparator.h"
+#include "paimon/core/core_options.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/memory/memory_pool.h"
@@ -37,6 +39,30 @@
 
 namespace paimon::test {
 
+TEST(PrimaryKeyRealtimeStoreOptionsTest, TestSupportedOptions) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_OK(ValidatePrimaryKeyRealtimeOptions(options));
+}
+
+TEST(PrimaryKeyRealtimeStoreOptionsTest, TestUnsupportedOptions) {
+    const std::string sequence_group =
+        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
+    const std::vector<std::map<std::string, std::string>> unsupported_options = {
+        {{Options::BUCKET, "0"}},
+        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
+        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
+        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
+    };
+    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
+        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+        ASSERT_NOK(ValidatePrimaryKeyRealtimeOptions(options));
+    }
+}
+
 class PrimaryKeyRealtimeStoreTest : public testing::Test {
  public:
     void SetUp() override {
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 92155de3b..dcf10e90c 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -41,6 +41,7 @@
 #include "paimon/core/operation/data_evolution_file_store_scan.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
+#include "paimon/core/realtime/primary_key_realtime_store.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/schema_validation.h"
@@ -63,7 +64,6 @@
 #include "paimon/core/utils/field_mapping.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/index_file_path_factories.h"
-#include "paimon/core/utils/primary_key_table_utils.h"
 #include "paimon/core/utils/snapshot_manager.h"
 #include "paimon/format/file_format.h"
 #include "paimon/realtime/realtime_context.h"
@@ -233,7 +233,7 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
         return Status::Invalid("real-time union read does not support data evolution");
     }
     if (!table_schema.PrimaryKeys().empty()) {
-        PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options));
+        PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(core_options));
     }
     if (context.IsStreamingMode()) {
         return Status::Invalid("real-time union read currently supports batch scans only");
diff --git a/src/paimon/core/utils/primary_key_table_utils.cpp b/src/paimon/core/utils/primary_key_table_utils.cpp
index 823d48c41..cf72da4ae 100644
--- a/src/paimon/core/utils/primary_key_table_utils.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils.cpp
@@ -96,36 +96,4 @@ Result<std::unique_ptr<FieldsComparator>> PrimaryKeyTableUtils::CreateSequenceFi
                                     options.SequenceFieldSortOrderIsAscending());
 }
 
-Status PrimaryKeyTableUtils::ValidateRealtimeOptions(const CoreOptions& options) {
-    if (options.GetBucket() <= 0) {
-        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
-    }
-    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
-        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
-    }
-    if (options.DataEvolutionEnabled()) {
-        return Status::NotImplemented("PK realtime v1 does not support data evolution");
-    }
-    if (!options.GetFieldsSequenceGroups().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
-    }
-    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
-        options.AggregationRemoveRecordOnDelete() ||
-        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
-        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
-    }
-    if (!options.GetSequenceField().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
-    }
-    if (!options.SequenceFieldSortOrderIsAscending()) {
-        return Status::NotImplemented(
-            "PK realtime v1 supports only ascending sequence.field.sort-order");
-    }
-    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
-        options.GetChangelogProducer() != ChangelogProducer::NONE) {
-        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
-    }
-    return Status::OK();
-}
-
 }  // namespace paimon
diff --git a/src/paimon/core/utils/primary_key_table_utils.h b/src/paimon/core/utils/primary_key_table_utils.h
index 7877ee4ab..c40e92cda 100644
--- a/src/paimon/core/utils/primary_key_table_utils.h
+++ b/src/paimon/core/utils/primary_key_table_utils.h
@@ -58,8 +58,6 @@ class PrimaryKeyTableUtils {
 
     static Result<std::unique_ptr<FieldsComparator>> CreateSequenceFieldsComparator(
         const std::vector<DataField>& value_fields, const CoreOptions& options);
-
-    static Status ValidateRealtimeOptions(const CoreOptions& options);
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/utils/primary_key_table_utils_test.cpp b/src/paimon/core/utils/primary_key_table_utils_test.cpp
index 072965cff..1a7345fdf 100644
--- a/src/paimon/core/utils/primary_key_table_utils_test.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils_test.cpp
@@ -19,7 +19,6 @@
 #include "paimon/core/utils/primary_key_table_utils.h"
 
 #include <cstdint>
-#include <map>
 #include <memory>
 #include <optional>
 #include <string>
@@ -114,28 +113,4 @@ TEST(PrimaryKeyTableUtilsTest, TestCreateFirstRowMergeFunctionWithIgnoreDelete)
                         "First row merge engine can not accept DELETE/UPDATE_BEFORE records");
 }
 
-TEST(PrimaryKeyTableUtilsTest, TestSupportedRealtimeOptions) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options));
-}
-
-TEST(PrimaryKeyTableUtilsTest, TestUnsupportedRealtimeOptions) {
-    const std::string sequence_group =
-        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
-    const std::vector<std::map<std::string, std::string>> unsupported_options = {
-        {{Options::BUCKET, "0"}},
-        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
-        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
-        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
-    };
-    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
-        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-        ASSERT_NOK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options));
-    }
-}
-
 }  // namespace paimon::test

From b89a9fc44380283ce5d7c5e415042f3c301370da Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 20 Aug 2026 20:45:39 +0800
Subject: [PATCH 31/93] test(realtime): improve primary key coverage

---
 .../primary_key_realtime_store_test.cpp       | 281 ++++++++++++++----
 test/inte/realtime_write_inte_test.cpp        | 248 +++++++++++++++-
 2 files changed, 463 insertions(+), 66 deletions(-)

diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index cbbf9c82a..5c04d4310 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -19,9 +19,13 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
+#include <cstdint>
+#include <limits>
 #include <map>
 #include <memory>
+#include <optional>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "arrow/api.h"
@@ -29,7 +33,6 @@
 #include "arrow/ipc/json_simple.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
-#include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
@@ -69,24 +72,37 @@ class PrimaryKeyRealtimeStoreTest : public testing::Test {
         pool_ = std::shared_ptr<MemoryPool>(GetMemoryPool());
         schema_ = arrow::schema(
             {arrow::field("id", arrow::int64()), arrow::field("value", arrow::utf8())});
-        ASSERT_OK_AND_ASSIGN(key_comparator_,
-                             FieldsComparator::Create({DataField(0, schema_->field(0))},
-                                                      /*is_ascending_order=*/true));
+        ASSERT_OK_AND_ASSIGN(store_, CreateStore(schema_, {"id"}, /*restore_max_sequence=*/4));
+    }
+
+    Result<std::shared_ptr<PrimaryKeyRealtimeStore>> CreateStore(
+        const std::shared_ptr<arrow::Schema>& schema, const std::vector<std::string>& primary_keys,
+        int64_t restore_max_sequence) const {
+        std::vector<DataField> key_fields;
+        key_fields.reserve(primary_keys.size());
+        for (const std::string& primary_key : primary_keys) {
+            const int32_t index = schema->GetFieldIndex(primary_key);
+            key_fields.emplace_back(index, schema->field(index));
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
+                               FieldsComparator::Create(key_fields,
+                                                        /*is_ascending_order=*/true));
         auto merge_factory = []() {
             auto merge_function =
                 std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
             return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
         };
-        ASSERT_OK_AND_ASSIGN(
-            store_, PrimaryKeyRealtimeStore::Create(schema_, {"id"}, key_comparator_, merge_factory,
-                                                    /*restore_max_sequence_number=*/4,
-                                                    /*read_batch_size=*/1024, pool_));
+        return PrimaryKeyRealtimeStore::Create(schema, primary_keys, key_comparator, merge_factory,
+                                               restore_max_sequence,
+                                               /*read_batch_size=*/2, pool_);
     }
 
     std::unique_ptr<RecordBatch> MakeBatch(
-        const std::string& json, const std::vector<RecordBatch::RowKind>& row_kinds = {}) const {
+        const std::string& json, const std::vector<RecordBatch::RowKind>& row_kinds = {},
+        const std::shared_ptr<arrow::Schema>& schema = nullptr) const {
+        const std::shared_ptr<arrow::Schema>& batch_schema = schema ? schema : schema_;
         std::shared_ptr<arrow::Array> array =
-            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema_->fields()), json)
+            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(batch_schema->fields()), json)
                 .ValueOrDie();
         ArrowArray c_array;
         EXPECT_TRUE(arrow::ExportArray(*array, &c_array).ok());
@@ -95,35 +111,39 @@ class PrimaryKeyRealtimeStoreTest : public testing::Test {
         return builder.Finish().value();
     }
 
-    std::unique_ptr<ArrowSchema> MakeReadSchema(bool include_sequence) const {
-        arrow::FieldVector fields;
-        if (include_sequence) {
-            fields.push_back(
-                DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()));
-        }
-        fields.insert(fields.end(), schema_->fields().begin(), schema_->fields().end());
+    std::unique_ptr<ArrowSchema> MakeReadSchema(const arrow::FieldVector& fields) const {
         auto c_schema = std::make_unique<ArrowSchema>();
         EXPECT_TRUE(arrow::ExportSchema(*arrow::schema(fields), c_schema.get()).ok());
         return c_schema;
     }
 
-    void AssertReaderOutput(BatchReader* reader, const std::shared_ptr<arrow::DataType>& type,
+    void AssertReaderOutput(const std::vector<std::unique_ptr<BatchReader>>& readers,
+                            const std::shared_ptr<arrow::DataType>& type,
                             const std::string& json) const {
-        ASSERT_NE(nullptr, reader);
-        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
-        ASSERT_FALSE(BatchReader::IsEofBatch(batch));
-        arrow::Result<std::shared_ptr<arrow::Array>> imported_result =
-            arrow::ImportArray(batch.first.get(), batch.second.get());
-        ASSERT_TRUE(imported_result.ok()) << imported_result.status().ToString();
-        std::shared_ptr<arrow::Array> actual = std::move(imported_result).ValueOrDie();
+        std::vector<std::shared_ptr<arrow::Array>> batches;
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            while (true) {
+                ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
+                if (BatchReader::IsEofBatch(batch)) {
+                    break;
+                }
+                arrow::Result<std::shared_ptr<arrow::Array>> imported =
+                    arrow::ImportArray(batch.first.get(), batch.second.get());
+                ASSERT_TRUE(imported.ok()) << imported.status().ToString();
+                batches.push_back(std::move(imported).ValueOrDie());
+            }
+        }
+        ASSERT_FALSE(batches.empty());
+        arrow::Result<std::shared_ptr<arrow::Array>> concatenated = arrow::Concatenate(batches);
+        ASSERT_TRUE(concatenated.ok()) << concatenated.status().ToString();
+        std::shared_ptr<arrow::Array> actual = std::move(concatenated).ValueOrDie();
         std::shared_ptr<arrow::Array> expected =
             arrow::ipc::internal::json::ArrayFromJSON(type, json).ValueOrDie();
         ASSERT_TRUE(actual->Equals(*expected))
             << "expected: " << expected->ToString() << ", actual: " << actual->ToString();
-
-        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof, reader->NextBatch());
-        ASSERT_TRUE(BatchReader::IsEofBatch(eof));
-        reader->Close();
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            reader->Close();
+        }
     }
 
     std::shared_ptr<arrow::DataType> CommitType() const {
@@ -143,10 +163,18 @@ class PrimaryKeyRealtimeStoreTest : public testing::Test {
         });
     }
 
+    arrow::FieldVector FullQueryFields(
+        const std::shared_ptr<arrow::Schema>& schema = nullptr) const {
+        const std::shared_ptr<arrow::Schema>& query_schema = schema ? schema : schema_;
+        arrow::FieldVector fields = {
+            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())};
+        fields.insert(fields.end(), query_schema->fields().begin(), query_schema->fields().end());
+        return fields;
+    }
+
  protected:
     std::shared_ptr<MemoryPool> pool_;
     std::shared_ptr<arrow::Schema> schema_;
-    std::shared_ptr<FieldsComparator> key_comparator_;
     std::shared_ptr<PrimaryKeyRealtimeStore> store_;
 };
 
@@ -172,30 +200,48 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestWriteAndSeal) {
     ASSERT_EQ(OffsetRange(0, 3), segment.value()->GetOffsetRange());
     ASSERT_GT(store_->GetMemoryUsage(), 0);
 
-    auto merge_factory = []() {
-        auto merge_function = std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
-        return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
+    struct ValidationCase {
+        int64_t restore_max_sequence;
+        std::string error;
+    };
+    const std::vector<ValidationCase> cases = {
+        {-2, "restore max sequence number must be at least -1"},
+        {std::numeric_limits<int64_t>::max(), "sequence number has reached INT64_MAX"},
     };
-    ASSERT_NOK_WITH_MSG(PrimaryKeyRealtimeStore::Create(
-                            schema_, {"id"}, key_comparator_, merge_factory,
-                            /*restore_max_sequence_number=*/-2, /*read_batch_size=*/1024, pool_),
-                        "restore max sequence number must be at least -1");
+    for (const ValidationCase& test_case : cases) {
+        ASSERT_NOK_WITH_MSG(CreateStore(schema_, {"id"}, test_case.restore_max_sequence),
+                            test_case.error);
+    }
 }
 
-TEST_F(PrimaryKeyRealtimeStoreTest, TestCommitReaderPreservesMutations) {
+TEST_F(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
+    ASSERT_OK(store_->Write(RealtimeWriteBatch{
+        MakeBatch(R"([[3, "three"], [1, "before"]])",
+                  {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_BEFORE}),
+        OffsetRange(0, 2)}));
+    ASSERT_OK(store_->Write(RealtimeWriteBatch{
+        MakeBatch(R"([[2, "after"]])", {RecordBatch::RowKind::UPDATE_AFTER}), OffsetRange(2, 3)}));
     ASSERT_OK(store_->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[2, "old"], [1, "one"], [2, "new"]])",
-                                     {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT,
-                                      RecordBatch::RowKind::UPDATE_AFTER}),
-                           OffsetRange(0, 3)}));
+        RealtimeWriteBatch{MakeBatch(R"([[4, "deleted"], [0, "zero"]])",
+                                     {RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT}),
+                           OffsetRange(3, 5)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store_->SealForCommit());
     ASSERT_TRUE(segment.has_value());
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store_->CreateCommitReaders(segment.value()));
-    ASSERT_EQ(1, readers.size());
-    AssertReaderOutput(readers[0].get(), CommitType(),
-                       R"([[0, 2, "old"], [0, 1, "one"], [2, 2, "new"]])");
+    AssertReaderOutput(readers, CommitType(),
+                       R"([[0, 3, "three"], [1, 1, "before"], [2, 2, "after"],
+                            [3, 4, "deleted"], [0, 0, "zero"]])");
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(readers, store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    AssertReaderOutput(readers, QueryType(),
+                       R"([[0, 9, 0, "zero"], [1, 6, 1, "before"], [2, 7, 2, "after"],
+                            [0, 5, 3, "three"], [3, 8, 4, "deleted"]])");
 }
 
 TEST_F(PrimaryKeyRealtimeStoreTest, TestMutationMerge) {
@@ -207,13 +253,12 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestMutationMerge) {
         OffsetRange(2, 4)}));
 
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(/*include_sequence=*/true);
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
     RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    ASSERT_EQ(1, readers.size());
-    AssertReaderOutput(readers[0].get(), QueryType(), R"([[2, 7, 1, "new"], [3, 8, 2, "gone"]])");
+    AssertReaderOutput(readers, QueryType(), R"([[2, 7, 1, "new"], [3, 8, 2, "gone"]])");
 }
 
 TEST_F(PrimaryKeyRealtimeStoreTest, TestReadViewLifecycle) {
@@ -230,15 +275,14 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestReadViewLifecycle) {
     ASSERT_OK(
         store_->Write(RealtimeWriteBatch{MakeBatch(R"([[13, "later"]])"), OffsetRange(13, 14)}));
 
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(/*include_sequence=*/true);
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
     RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store_->CreateQueryReaders(view, /*offset_begin=*/11, context));
-    ASSERT_EQ(1, readers.size());
-    AssertReaderOutput(readers[0].get(), QueryType(), R"([[0, 6, 11, "b"], [0, 7, 12, "c"]])");
+    AssertReaderOutput(readers, QueryType(), R"([[0, 6, 11, "b"], [0, 7, 12, "c"]])");
 
-    std::unique_ptr<ArrowSchema> empty_schema = MakeReadSchema(/*include_sequence=*/true);
+    std::unique_ptr<ArrowSchema> empty_schema = MakeReadSchema(FullQueryFields());
     context.read_schema = empty_schema.get();
     ASSERT_OK_AND_ASSIGN(readers, store_->CreateQueryReaders(view, /*offset_begin=*/13, context));
     ASSERT_TRUE(readers.empty());
@@ -251,20 +295,135 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryKeyRange) {
     ASSERT_OK(store_->Write(
         RealtimeWriteBatch{MakeBatch(R"([[9, "nine"], [7, "seven"]])"), OffsetRange(2, 4)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(/*include_sequence=*/true);
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
     RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
     ASSERT_EQ(2, readers.size());
-    auto* first_range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[0].get());
-    auto* second_range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[1].get());
-    ASSERT_NE(nullptr, first_range);
-    ASSERT_NE(nullptr, second_range);
-    ASSERT_EQ(1, first_range->GetMinKey()->GetLong(0));
-    ASSERT_EQ(5, first_range->GetMaxKey()->GetLong(0));
-    ASSERT_EQ(7, second_range->GetMinKey()->GetLong(0));
-    ASSERT_EQ(9, second_range->GetMaxKey()->GetLong(0));
+    const std::vector<std::pair<int64_t, int64_t>> key_ranges = {{1, 5}, {7, 9}};
+    for (size_t i = 0; i < readers.size(); ++i) {
+        auto* range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[i].get());
+        ASSERT_NE(nullptr, range);
+        ASSERT_EQ(key_ranges[i].first, range->GetMinKey()->GetLong(0));
+        ASSERT_EQ(key_ranges[i].second, range->GetMaxKey()->GetLong(0));
+    }
+    AssertReaderOutput(readers, QueryType(),
+                       R"([[0, 6, 1, "one"], [0, 5, 5, "five"], [0, 8, 7, "seven"],
+                            [0, 7, 9, "nine"]])");
+
+    ASSERT_OK(store_->AdvanceCommittedOffset(2));
+    ASSERT_OK_AND_ASSIGN(view, store_->AcquireReadView());
+    read_schema = MakeReadSchema(FullQueryFields());
+    context.read_schema = read_schema.get();
+    ASSERT_OK_AND_ASSIGN(readers, store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(1, readers.size());
+    auto* range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[0].get());
+    ASSERT_NE(nullptr, range);
+    ASSERT_EQ(7, range->GetMinKey()->GetLong(0));
+    ASSERT_EQ(9, range->GetMaxKey()->GetLong(0));
+    AssertReaderOutput(readers, QueryType(), R"([[0, 8, 7, "seven"], [0, 7, 9, "nine"]])");
+}
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestSequenceExhaustion) {
+    const int64_t max_sequence = std::numeric_limits<int64_t>::max();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         CreateStore(schema_, {"id"}, max_sequence - 3));
+    ASSERT_OK(store->Write(RealtimeWriteBatch{MakeBatch(R"([[1, "kept"]])"), OffsetRange(10, 11)}));
+    ASSERT_NOK_WITH_MSG(
+        store->Write(RealtimeWriteBatch{
+            MakeBatch(R"([[7, "rejected-a"], [8, "rejected-b"], [9, "rejected-c"]])"),
+            OffsetRange(11, 14)}),
+        "sequence range exceeds INT64_MAX");
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[2, "also-kept"]])"), OffsetRange(11, 12)}));
+
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_EQ(OffsetRange(10, 12), segment.value()->GetOffsetRange());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+    ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(10, 12)), view->GetOffsetRange());
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/10, context));
+    AssertReaderOutput(readers, QueryType(),
+                       R"([[0, 9223372036854775805, 1, "kept"],
+                            [0, 9223372036854775806, 2, "also-kept"]])");
+}
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryProjection) {
+    ASSERT_OK(
+        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[2, "b"], [1, "a"]])"), OffsetRange(0, 2)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
+    const std::shared_ptr<arrow::Field> value_kind =
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind());
+    const std::shared_ptr<arrow::Field> sequence =
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber());
+    struct ProjectionCase {
+        arrow::FieldVector requested;
+        std::shared_ptr<arrow::DataType> expected_type;
+        std::string expected_json;
+    };
+    const std::vector<ProjectionCase> cases = {
+        {{schema_->field(1), value_kind, sequence, schema_->field(0)},
+         arrow::struct_({value_kind, schema_->field(1), sequence, schema_->field(0)}),
+         R"([[0, "a", 6, 1], [0, "b", 5, 2]])"},
+        {{schema_->field(0), value_kind},
+         arrow::struct_({value_kind, schema_->field(0)}),
+         R"([[0, 1], [0, 2]])"},
+    };
+    for (const ProjectionCase& test_case : cases) {
+        std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(test_case.requested);
+        RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                     /*enable_predicate_pushdown=*/false};
+        ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                             store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
+        AssertReaderOutput(readers, test_case.expected_type, test_case.expected_json);
+    }
+
+    std::unique_ptr<ArrowSchema> read_schema =
+        MakeReadSchema({arrow::field("unknown", arrow::int64())});
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_NOK_WITH_MSG(store_->CreateQueryReaders(view, /*offset_begin=*/0, context),
+                        "query field is missing from write schema: unknown");
+}
+
+TEST_F(PrimaryKeyRealtimeStoreTest, TestCompositeKeyClipping) {
+    std::shared_ptr<arrow::Schema> composite_schema =
+        arrow::schema({arrow::field("id", arrow::int64()), arrow::field("region", arrow::utf8()),
+                       arrow::field("value", arrow::utf8())});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         CreateStore(composite_schema, {"id", "region"},
+                                     /*restore_max_sequence=*/4));
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[9, "z", "clipped"], [2, "b", "two-b"], [1, "c", "one-c"],
+                       [2, "a", "two-a"]])",
+                                     {}, composite_schema),
+                           OffsetRange(20, 24)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields(composite_schema));
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/21, context));
+    ASSERT_EQ(1, readers.size());
+    auto* range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[0].get());
+    ASSERT_NE(nullptr, range);
+    ASSERT_EQ(1, range->GetMinKey()->GetLong(0));
+    ASSERT_EQ("c", range->GetMinKey()->GetString(1).ToString());
+    ASSERT_EQ(2, range->GetMaxKey()->GetLong(0));
+    ASSERT_EQ("b", range->GetMaxKey()->GetString(1).ToString());
+    std::shared_ptr<arrow::DataType> query_type = arrow::struct_(
+        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
+         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()),
+         composite_schema->field(0), composite_schema->field(1), composite_schema->field(2)});
+    AssertReaderOutput(readers, query_type,
+                       R"([[0, 7, 1, "c", "one-c"], [0, 8, 2, "a", "two-a"],
+                            [0, 6, 2, "b", "two-b"]])");
 }
 
 }  // namespace paimon::test
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index e6000561f..aad9dc2ac 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -646,16 +646,18 @@ class RealtimeWriteInteTest : public ::testing::Test {
                                        /*ignore_if_exists=*/false));
     }
 
-    void CreatePkTable(const std::vector<std::string>& partition_keys = {}) const {
+    void CreatePkTable(const std::vector<std::string>& partition_keys = {},
+                       const std::vector<std::string>& primary_keys = {"id"}) const {
         auto c_schema = std::make_unique<ArrowSchema>();
         ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok());
         ASSERT_OK_AND_ASSIGN(std::unique_ptr<Catalog> catalog,
                              Catalog::Create(dir_->Str(), options_));
         ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false));
-        std::vector<std::string> primary_keys = partition_keys;
-        primary_keys.push_back("id");
+        std::vector<std::string> table_primary_keys = partition_keys;
+        table_primary_keys.insert(table_primary_keys.end(), primary_keys.begin(),
+                                  primary_keys.end());
         ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), c_schema.get(), partition_keys,
-                                       primary_keys, options_, /*ignore_if_exists=*/false));
+                                       table_primary_keys, options_, /*ignore_if_exists=*/false));
     }
 
     Result<std::unique_ptr<FileStoreWrite>> CreateRealtimeWriter(
@@ -692,7 +694,7 @@ class RealtimeWriteInteTest : public ::testing::Test {
         std::string json = "[";
         for (size_t i = 0; i < rows.size(); ++i) {
             const auto& [id, payload, pt] = rows[i];
-            if (pt != partition) {
+            if (partitioned && pt != partition) {
                 return Status::Invalid("one test batch must contain only one partition");
             }
             if (i > 0) {
@@ -1383,6 +1385,242 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) {
     ASSERT_TRUE(query_view->expired());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkMergeDiskSealedAndActive) {
+    options_[Options::READ_BATCH_SIZE] = "2";
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    const std::vector<std::vector<Row>> disk_batches = {
+        {{1, "disk-1", "p0"}, {2, "disk-2", "p0"}, {3, "disk-3", "p0"}},
+        {{10, "disk-10", "p0"}, {11, "disk-11", "p0"}},
+    };
+    int64_t commit_identifier = 0;
+    for (const std::vector<Row>& disk_rows : disk_batches) {
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch(disk_rows, /*partitioned=*/false));
+        ASSERT_OK(writer->Write(std::move(batch)));
+        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                             writer->PrepareCommitWithProgress(commit_identifier));
+        ASSERT_EQ(1, progress.size());
+        ASSERT_EQ(1, NewFiles(progress).size());
+        ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, commit_identifier));
+        ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+        ++commit_identifier;
+    }
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> sealed_batch,
+        MakeBatch({Row{1, "sealed-1", "p0"}, Row{2, "deleted-2", "p0"}, Row{4, "sealed-4", "p0"}},
+                  /*partitioned=*/false, /*bucket=*/0,
+                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE,
+                   RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(writer->Write(std::move(sealed_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> sealed_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_EQ(1, sealed_progress.size());
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> active_batch,
+        MakeBatch({Row{1, "active-1", "p0"}, Row{4, "deleted-4", "p0"}, Row{5, "active-5", "p0"}},
+                  /*partitioned=*/false, /*bucket=*/0,
+                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE,
+                   RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(writer->Write(std::move(active_batch)));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult result,
+                         ReadPlan(plan, realtime_context, {"payload", "id"}, /*predicate=*/nullptr,
+                                  /*enable_predicate_filter=*/false));
+    ASSERT_NE(nullptr, result.data);
+    ASSERT_GT(result.data->num_chunks(), 1);
+    for (const std::shared_ptr<arrow::Array>& chunk : result.data->chunks()) {
+        ASSERT_LE(chunk->length(), 2);
+    }
+    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
+        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("payload", arrow::utf8()),
+         arrow::field("id", arrow::int64())});
+    std::shared_ptr<arrow::Array> expected =
+        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
+            [0, "active-1", 1],
+            [0, "disk-3", 3],
+            [0, "active-5", 5],
+            [0, "disk-10", 10],
+            [0, "disk-11", 11]
+        ])")
+            .ValueOrDie();
+    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*result.data))
+        << result.data->ToString();
+    result.reader->Close();
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkCompositeMerge) {
+    CreatePkTable(/*partition_keys=*/{}, /*primary_keys=*/{"id", "payload"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
+                         MakeBatch({Row{1, "a", "disk-1a"}, Row{1, "b", "disk-1b"},
+                                    Row{2, "a", "disk-2a"}, Row{3, "c", "disk-3c"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(disk_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, disk_progress.size());
+    ASSERT_EQ(OffsetRange(0, 4), disk_progress[0].offset_range);
+    ASSERT_EQ(1, NewFiles(disk_progress).size());
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(disk_progress, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> sealed_batch,
+        MakeBatch({Row{1, "a", "sealed-1a"}, Row{1, "b", "deleted-1b"}, Row{2, "b", "sealed-2b"}},
+                  /*partitioned=*/false, /*bucket=*/0,
+                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE,
+                   RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(writer->Write(std::move(sealed_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> sealed_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, sealed_progress.size());
+    ASSERT_EQ(OffsetRange(4, 7), sealed_progress[0].offset_range);
+    ASSERT_EQ(1, NewFiles(sealed_progress).size());
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> active_batch,
+        MakeBatch({Row{1, "a", "active-1a"}, Row{1, "c", "active-1c"}, Row{2, "a", "active-2a"}},
+                  /*partitioned=*/false, /*bucket=*/0,
+                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::INSERT,
+                   RecordBatch::RowKind::UPDATE_AFTER}));
+    ASSERT_OK(writer->Write(std::move(active_batch)));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_EQ(1, plan->Splits().size());
+    std::shared_ptr<RealtimeSplit> split =
+        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
+    ASSERT_NE(nullptr, split);
+    ASSERT_FALSE(split->DiskSplits().empty());
+    ASSERT_EQ(4, split->CommittedEndOffset());
+    ASSERT_EQ(10, split->MemoryEndOffset());
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
+    ASSERT_EQ((std::vector<Row>{{1, "a", "active-1a"},
+                                {1, "c", "active-1c"},
+                                {2, "a", "active-2a"},
+                                {2, "b", "sealed-2b"},
+                                {3, "c", "disk-3c"}}),
+              actual_rows);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkPartitionBucketRecovery) {
+    options_[Options::BUCKET] = "2";
+    CreatePkTable(/*partition_keys=*/{"pt"});
+    const RealtimePartitionBucket p0b0({{"pt", "p0"}}, /*bucket=*/0);
+    const RealtimePartitionBucket p1b1({{"pt", "p1"}}, /*bucket=*/1);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> first_context, RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
+                         CreateRealtimeWriter(first_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0_first_batch,
+                         MakeBatch({Row{0, "p0-zero", "p0"}, Row{1, "p0-one", "p0"}},
+                                   /*partitioned=*/true, /*bucket=*/0));
+    ASSERT_OK(first_writer->Write(std::move(p0_first_batch)));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> p1_first_batch,
+        MakeBatch({Row{10, "p1-ten", "p1"}, Row{11, "p1-eleven", "p1"}, Row{12, "p1-twelve", "p1"}},
+                  /*partitioned=*/true, /*bucket=*/1));
+    ASSERT_OK(first_writer->Write(std::move(p1_first_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_progress,
+                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(2, first_progress.size());
+    std::map<RealtimePartitionBucket, OffsetRange> first_ranges;
+    std::map<RealtimePartitionBucket, std::pair<int64_t, int64_t>> first_sequences;
+    for (const RealtimeCommitProgress& progress : first_progress) {
+        first_ranges.emplace(progress.partition_bucket, progress.offset_range);
+        std::shared_ptr<CommitMessageImpl> message =
+            std::dynamic_pointer_cast<CommitMessageImpl>(progress.commit_message);
+        ASSERT_NE(nullptr, message);
+        const std::vector<std::shared_ptr<DataFileMeta>>& files =
+            message->GetNewFilesIncrement().NewFiles();
+        ASSERT_EQ(1, files.size());
+        first_sequences.emplace(
+            progress.partition_bucket,
+            std::make_pair(files[0]->min_sequence_number, files[0]->max_sequence_number));
+    }
+    ASSERT_EQ(OffsetRange(0, 2), first_ranges.at(p0b0));
+    ASSERT_EQ(OffsetRange(0, 3), first_ranges.at(p1b1));
+    ASSERT_EQ((std::make_pair<int64_t, int64_t>(0, 1)), first_sequences.at(p0b0));
+    ASSERT_EQ((std::make_pair<int64_t, int64_t>(0, 2)), first_sequences.at(p1b1));
+    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id,
+                         Commit(first_progress, /*commit_identifier=*/0));
+    ASSERT_OK(first_writer->RefreshCommittedSnapshot(first_snapshot_id));
+    ASSERT_OK(first_writer->Close());
+    first_writer.reset();
+    first_context.reset();
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> second_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
+                         CreateRealtimeWriter(second_context));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> p0_second_batch,
+        MakeBatch({Row{0, "p0-zero-new", "p0"}, Row{2, "p0-two", "p0"}},
+                  /*partitioned=*/true, /*bucket=*/0,
+                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(second_writer->Write(std::move(p0_second_batch)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p1_second_batch,
+                         MakeBatch({Row{10, "p1-ten-deleted", "p1"}, Row{13, "p1-thirteen", "p1"}},
+                                   /*partitioned=*/true, /*bucket=*/1,
+                                   {RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(second_writer->Write(std::move(p1_second_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
+                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(2, second_progress.size());
+    std::map<RealtimePartitionBucket, OffsetRange> second_ranges;
+    std::map<RealtimePartitionBucket, std::pair<int64_t, int64_t>> second_sequences;
+    for (const RealtimeCommitProgress& progress : second_progress) {
+        second_ranges.emplace(progress.partition_bucket, progress.offset_range);
+        std::shared_ptr<CommitMessageImpl> message =
+            std::dynamic_pointer_cast<CommitMessageImpl>(progress.commit_message);
+        ASSERT_NE(nullptr, message);
+        const std::vector<std::shared_ptr<DataFileMeta>>& files =
+            message->GetNewFilesIncrement().NewFiles();
+        ASSERT_EQ(1, files.size());
+        second_sequences.emplace(
+            progress.partition_bucket,
+            std::make_pair(files[0]->min_sequence_number, files[0]->max_sequence_number));
+    }
+    ASSERT_EQ(OffsetRange(2, 4), second_ranges.at(p0b0));
+    ASSERT_EQ(OffsetRange(3, 5), second_ranges.at(p1b1));
+    ASSERT_EQ((std::make_pair<int64_t, int64_t>(2, 3)), second_sequences.at(p0b0));
+    ASSERT_EQ((std::make_pair<int64_t, int64_t>(3, 4)), second_sequences.at(p1b1));
+    ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
+                         Commit(second_progress, /*commit_identifier=*/1));
+    ASSERT_OK(second_writer->RefreshCommittedSnapshot(second_snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(second_context));
+    std::sort(actual_rows.begin(), actual_rows.end());
+    ASSERT_EQ((std::vector<Row>{{0, "p0-zero-new", "p0"},
+                                {1, "p0-one", "p0"},
+                                {2, "p0-two", "p0"},
+                                {11, "p1-eleven", "p1"},
+                                {12, "p1-twelve", "p1"},
+                                {13, "p1-thirteen", "p1"}}),
+              actual_rows);
+    ASSERT_OK(second_writer->Close());
+
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets, ReadCommittedOffsets());
+    ASSERT_EQ(2, offsets.size());
+    ASSERT_EQ(4, offsets.at(p0b0));
+    ASSERT_EQ(5, offsets.at(p1b1));
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkRecovery) {
     CreatePkTable();
 

From b3862d119eb26004a769808ce3f2cb8e37e97c87 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 21 Aug 2026 12:38:57 +0800
Subject: [PATCH 32/93] fix(realtime): prevent sequence reuse and align nested
 projections

---
 .../realtime/primary_key_realtime_store.cpp   |  9 +++
 .../primary_key_realtime_store_test.cpp       | 28 +++++++
 .../core/realtime/realtime_context_impl.cpp   | 10 ++-
 .../core/realtime/realtime_context_test.cpp   | 16 +++-
 test/inte/realtime_write_inte_test.cpp        | 81 +++++++++++++++++++
 5 files changed, 139 insertions(+), 5 deletions(-)

diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index afdc0c73c..7999de75d 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -43,6 +43,7 @@
 #include "paimon/core/io/merged_key_value_record_reader.h"
 #include "paimon/core/key_value.h"
 #include "paimon/core/mergetree/compact/sort_merge_reader_with_loser_tree.h"
+#include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
 
 namespace paimon {
@@ -411,6 +412,7 @@ class PrimaryKeyRealtimeStore::Impl {
                                           arrow::ImportSchema(context.read_schema));
         arrow::FieldVector output_fields = {
             DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())};
+        arrow::FieldVector aligned_value_fields = write_schema_->fields();
         std::vector<int32_t> projection = {KeyValueProjectionConsumer::kValueKindProjection};
         for (const std::shared_ptr<arrow::Field>& field : requested->fields()) {
             if (field->name() == SpecialFields::ValueKind().Name()) {
@@ -426,8 +428,11 @@ class PrimaryKeyRealtimeStore::Impl {
                 return Status::Invalid("PK real-time query field is missing from write schema: ",
                                        field->name());
             }
+            aligned_value_fields[index] = field;
             projection.push_back(index);
         }
+        const std::shared_ptr<arrow::DataType> aligned_value_type =
+            arrow::struct_(aligned_value_fields);
 
         std::vector<std::unique_ptr<BatchReader>> result;
         for (const BatchGroup& group : typed->Groups()) {
@@ -452,6 +457,10 @@ class PrimaryKeyRealtimeStore::Impl {
                 if (!max_key || key_comparator_->CompareTo(*key_range.second, *max_key) > 0) {
                     max_key = key_range.second;
                 }
+                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> aligned,
+                                       NestedProjectionUtils::AlignArrayToReadType(
+                                           selected, aligned_value_type, arrow_pool_.get()));
+                selected = checked_pointer_cast<arrow::StructArray>(aligned);
                 std::vector<RecordBatch::RowKind> selected_kinds;
                 if (!batch->row_kinds.empty()) {
                     selected_kinds.assign(batch->row_kinds.begin() + offset,
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 5c04d4310..ef293e54b 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -392,6 +392,34 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryProjection) {
                         "query field is missing from write schema: unknown");
 }
 
+TEST_F(PrimaryKeyRealtimeStoreTest, TestNestedProjection) {
+    const std::shared_ptr<arrow::Field> id =
+        DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64())));
+    const std::shared_ptr<arrow::Field> a =
+        DataField::ConvertDataFieldToArrowField(DataField(10, arrow::field("a", arrow::int64())));
+    const std::shared_ptr<arrow::Field> b =
+        DataField::ConvertDataFieldToArrowField(DataField(11, arrow::field("b", arrow::int64())));
+    const std::shared_ptr<arrow::Field> payload = DataField::ConvertDataFieldToArrowField(
+        DataField(1, arrow::field("payload", arrow::struct_({a, b}))));
+    const std::shared_ptr<arrow::Schema> nested_schema = arrow::schema({id, payload});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         CreateStore(nested_schema, {"id"}, /*restore_max_sequence=*/4));
+    ASSERT_OK(store->Write(RealtimeWriteBatch{
+        MakeBatch(R"([[2, [200, 2000]], [1, [100, null]], [3, [300, 3000]]])", {}, nested_schema),
+        OffsetRange(0, 3)}));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+    const std::shared_ptr<arrow::Field> projected_payload = payload->WithType(arrow::struct_({b}));
+    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema({projected_payload});
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    const std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
+        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()), projected_payload});
+    AssertReaderOutput(readers, result_type, R"([[0, [null]], [0, [2000]], [0, [3000]]])");
+}
+
 TEST_F(PrimaryKeyRealtimeStoreTest, TestCompositeKeyClipping) {
     std::shared_ptr<arrow::Schema> composite_schema =
         arrow::schema({arrow::field("id", arrow::int64()), arrow::field("region", arrow::utf8()),
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 6624059a6..066e54e8a 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -82,6 +82,7 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     std::lock_guard<std::mutex> progress_lock(progress_mutex_);
     std::lock_guard<std::mutex> registry_lock(mutex_);
     const RealtimePartitionBucket key(request.partition, request.bucket);
+    auto iter = stores_.find(key);
     std::optional<int64_t> initial_max_sequence_number;
     PrimaryKeyRealtimeStoreCreateConfig* primary_key_config =
         std::get_if<PrimaryKeyRealtimeStoreCreateConfig>(&request.mode_config);
@@ -89,6 +90,14 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
         auto [sequence_iter, inserted] = materialized_max_sequence_numbers_.emplace(
             key, primary_key_config->restore_max_sequence_number);
         if (!inserted && primary_key_config->restore_max_sequence_number > sequence_iter->second) {
+            if (iter != stores_.end()) {
+                if (request.write_schema) {
+                    ArrowSchemaRelease(request.write_schema.get());
+                }
+                return Status::Invalid(
+                    "restore max sequence number exceeds the materialized watermark of an "
+                    "existing PK real-time store");
+            }
             sequence_iter->second = primary_key_config->restore_max_sequence_number;
         }
         initial_max_sequence_number = sequence_iter->second;
@@ -105,7 +114,6 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
         }
         initial_offset = offset_iter->second;
     }
-    auto iter = stores_.find(key);
     if (iter != stores_.end()) {
         if (request.write_schema) {
             ArrowSchemaRelease(request.write_schema.get());
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index b4d2c6718..ab0abe4a7 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -198,12 +198,20 @@ TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
     ASSERT_EQ(first_state.store, retained_state.store);
     ASSERT_EQ(8, retained_state.initial_max_sequence_number);
 
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState restored_state,
+    ASSERT_NOK_WITH_MSG(
         GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/0,
+                                   /*restore_max_sequence_number=*/10, GetDefaultPool()),
+        "restore max sequence number exceeds the materialized watermark of an "
+        "existing PK real-time store");
+
+    const RealtimePartitionBucket new_partition_bucket(partition, /*bucket=*/1);
+    context->AdvanceMaterializedMaxSequenceNumber(new_partition_bucket,
+                                                  /*max_sequence_number=*/8);
+    ASSERT_OK_AND_ASSIGN(
+        RealtimeStoreState new_state,
+        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/1,
                                    /*restore_max_sequence_number=*/10, GetDefaultPool()));
-    ASSERT_EQ(first_state.store, restored_state.store);
-    ASSERT_EQ(10, restored_state.initial_max_sequence_number);
+    ASSERT_EQ(10, new_state.initial_max_sequence_number);
 }
 
 TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index aad9dc2ac..9f302eb37 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -1458,6 +1458,87 @@ TEST_F(RealtimeWriteInteTest, TestPkMergeDiskSealedAndActive) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) {
+    const std::shared_ptr<arrow::Field> projected_b = arrow::field("b", arrow::int64());
+    fields_ = {
+        arrow::field("id", arrow::int64()),
+        arrow::field("payload", arrow::struct_({arrow::field("a", arrow::int64()), projected_b})),
+        arrow::field("pt", arrow::utf8()),
+    };
+    schema_ = arrow::schema(fields_);
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    auto make_batch = [&](const std::string& json) -> Result<std::unique_ptr<RecordBatch>> {
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> array,
+            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), json));
+        ArrowArray c_array;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
+        RecordBatchBuilder builder(&c_array);
+        return builder.SetBucket(0).Finish();
+    };
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
+                         make_batch(R"([[1, [101, 1001], "p0"], [2, [102, 1002], "p0"]])"));
+    ASSERT_OK(writer->Write(std::move(disk_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(disk_progress, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> sealed_batch,
+                         make_batch(R"([[1, [201, 2001], "p0"], [3, [203, 2003], "p0"]])"));
+    ASSERT_OK(writer->Write(std::move(sealed_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> sealed_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, sealed_progress.size());
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> active_batch,
+                         make_batch(R"([[1, [301, 3001], "p0"], [4, [304, null], "p0"]])"));
+    ASSERT_OK(writer->Write(std::move(active_batch)));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+
+    auto projected_schema = arrow::schema({
+        arrow::field("payload", arrow::struct_({projected_b})),
+        arrow::field("id", arrow::int64()),
+    });
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok());
+    ReadContextBuilder read_builder(table_path_);
+    read_builder.SetOptions(options_)
+        .SetReadSchema(std::move(c_schema))
+        .WithRealtimeContext(realtime_context)
+        .WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+                         TableRead::Create(std::move(read_context)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader,
+                         table_read->CreateReader(plan->Splits()));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> actual,
+                         ReadResultCollector::CollectResult(reader.get()));
+    const std::shared_ptr<arrow::DataType> result_type = arrow::struct_({
+        arrow::field("_VALUE_KIND", arrow::int8()),
+        arrow::field("payload", arrow::struct_({projected_b})),
+        arrow::field("id", arrow::int64()),
+    });
+    const std::shared_ptr<arrow::Array> expected =
+        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
+            [0, [3001], 1],
+            [0, [1002], 2],
+            [0, [2003], 3],
+            [0, [null], 4]
+        ])")
+            .ValueOrDie();
+    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*actual))
+        << actual->ToString();
+    reader->Close();
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkCompositeMerge) {
     CreatePkTable(/*partition_keys=*/{}, /*primary_keys=*/{"id", "payload"});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,

From 3acae038e0446cfb4d92f8572e51b13d3e00b63e Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 21 Aug 2026 13:20:03 +0800
Subject: [PATCH 33/93] fix(realtime): align PK reads across schema changes

---
 .../realtime/primary_key_realtime_store.cpp   |   28 +-
 test/inte/CMakeLists.txt                      |    7 +
 ...chema_evolution_write_verify_inte_test.cpp | 1110 +++++++++++++++++
 3 files changed, 1136 insertions(+), 9 deletions(-)
 create mode 100644 test/inte/schema_evolution_write_verify_inte_test.cpp

diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 7999de75d..6565ed8d7 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -423,12 +423,18 @@ class PrimaryKeyRealtimeStore::Impl {
                 projection.push_back(KeyValueProjectionConsumer::kSequenceNumberProjection);
                 continue;
             }
-            const int32_t index = write_schema_->GetFieldIndex(field->name());
+            int32_t index = write_schema_->GetFieldIndex(field->name());
             if (index < 0) {
-                return Status::Invalid("PK real-time query field is missing from write schema: ",
-                                       field->name());
+                Result<int32_t> field_id = NestedProjectionUtils::GetPaimonFieldId(field);
+                if (!field_id.ok()) {
+                    return Status::Invalid(
+                        "PK real-time query field is missing from write schema: ", field->name());
+                }
+                index = static_cast<int32_t>(aligned_value_fields.size());
+                aligned_value_fields.push_back(field);
+            } else {
+                aligned_value_fields[index] = field;
             }
-            aligned_value_fields[index] = field;
             projection.push_back(index);
         }
         const std::shared_ptr<arrow::DataType> aligned_value_type =
@@ -446,8 +452,16 @@ class PrimaryKeyRealtimeStore::Impl {
                 const int64_t offset = std::max<int64_t>(0, lower - batch->offset_range.begin);
                 const int64_t length = batch->data->length() - offset;
                 std::shared_ptr<arrow::Array> sliced = batch->data->Slice(offset, length);
+                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> aligned,
+                                       NestedProjectionUtils::AlignArrayToReadType(
+                                           sliced, aligned_value_type, arrow_pool_.get()));
+                if (!aligned || aligned->type_id() != arrow::Type::STRUCT) {
+                    return Status::Invalid(
+                        "PK real-time query projection did not produce a "
+                        "StructArray");
+                }
                 std::shared_ptr<arrow::StructArray> selected =
-                    checked_pointer_cast<arrow::StructArray>(sliced);
+                    checked_pointer_cast<arrow::StructArray>(aligned);
                 using KeyRange =
                     std::pair<std::shared_ptr<InternalRow>, std::shared_ptr<InternalRow>>;
                 PAIMON_ASSIGN_OR_RAISE(KeyRange key_range, GetKeyRange(selected));
@@ -457,10 +471,6 @@ class PrimaryKeyRealtimeStore::Impl {
                 if (!max_key || key_comparator_->CompareTo(*key_range.second, *max_key) > 0) {
                     max_key = key_range.second;
                 }
-                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> aligned,
-                                       NestedProjectionUtils::AlignArrayToReadType(
-                                           selected, aligned_value_type, arrow_pool_.get()));
-                selected = checked_pointer_cast<arrow::StructArray>(aligned);
                 std::vector<RecordBatch::RowKind> selected_kinds;
                 if (!batch->row_kinds.empty()) {
                     selected_kinds.assign(batch->row_kinds.begin() + offset,
diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt
index 75147ce60..f1b3f8ce6 100644
--- a/test/inte/CMakeLists.txt
+++ b/test/inte/CMakeLists.txt
@@ -43,6 +43,13 @@ if(PAIMON_BUILD_TESTS)
                     test_utils_static
                     ${GTEST_LINK_TOOLCHAIN})
 
+    add_paimon_test(schema_evolution_write_verify_inte_test
+                    STATIC_LINK_LIBS
+                    paimon_shared
+                    ${TEST_STATIC_LINK_LIBS}
+                    test_utils_static
+                    ${GTEST_LINK_TOOLCHAIN})
+
     add_paimon_test(global_index_test
                     STATIC_LINK_LIBS
                     paimon_shared
diff --git a/test/inte/schema_evolution_write_verify_inte_test.cpp b/test/inte/schema_evolution_write_verify_inte_test.cpp
new file mode 100644
index 000000000..dcadbd9e1
--- /dev/null
+++ b/test/inte/schema_evolution_write_verify_inte_test.cpp
@@ -0,0 +1,1110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstddef>
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "arrow/ipc/json_simple.h"
+#include "gtest/gtest.h"
+#include "paimon/catalog/catalog.h"
+#include "paimon/catalog/identifier.h"
+#include "paimon/commit_context.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/global_index/indexed_split_impl.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/table/sink/commit_message_impl.h"
+#include "paimon/core/table/source/data_split_impl.h"
+#include "paimon/defs.h"
+#include "paimon/file_index/file_index_format.h"
+#include "paimon/file_index/file_index_reader.h"
+#include "paimon/file_index/file_index_result.h"
+#include "paimon/file_store_commit.h"
+#include "paimon/file_store_write.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/io/byte_array_input_stream.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/read_context.h"
+#include "paimon/reader/batch_reader.h"
+#include "paimon/realtime/realtime_context.h"
+#include "paimon/record_batch.h"
+#include "paimon/scan_context.h"
+#include "paimon/table/source/plan.h"
+#include "paimon/table/source/startup_mode.h"
+#include "paimon/table/source/table_read.h"
+#include "paimon/table/source/table_scan.h"
+#include "paimon/testing/utils/read_result_collector.h"
+#include "paimon/testing/utils/test_helper.h"
+#include "paimon/testing/utils/testharness.h"
+#include "paimon/write_context.h"
+
+namespace paimon::test {
+namespace {
+
+std::map<std::string, std::string> BaseOptions() {
+    return {{Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"},
+            {Options::FILE_SYSTEM, "local"},   {Options::BUCKET, "1"},
+            {Options::BUCKET_KEY, "id"},       {Options::TARGET_FILE_SIZE, "1MB"}};
+}
+
+std::map<std::string, std::string> DataEvolutionOptions() {
+    return {{Options::MANIFEST_FORMAT, "orc"},       {Options::FILE_FORMAT, "orc"},
+            {Options::FILE_SYSTEM, "local"},         {Options::TARGET_FILE_SIZE, "1MB"},
+            {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}};
+}
+
+arrow::FieldVector BaseFields() {
+    return {arrow::field("id", arrow::int64()), arrow::field("payload", arrow::utf8())};
+}
+
+arrow::FieldVector EvolvedFields() {
+    return {arrow::field("id", arrow::int64()), arrow::field("payload", arrow::utf8()),
+            arrow::field("extra", arrow::int32())};
+}
+
+arrow::FieldVector DataEvolutionFields() {
+    return {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()),
+            arrow::field("f2", arrow::utf8())};
+}
+
+Result<std::unique_ptr<RecordBatch>> MakeBatch(
+    const arrow::FieldVector& fields, const std::string& json,
+    const std::map<std::string, std::string>& partition, int32_t bucket,
+    const std::vector<RecordBatch::RowKind>& row_kinds = {}) {
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::Array> array,
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), json));
+    ArrowArray c_array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
+    RecordBatchBuilder builder(&c_array);
+    return builder.SetPartition(partition).SetBucket(bucket).SetRowKinds(row_kinds).Finish();
+}
+
+Result<std::unique_ptr<RecordBatch>> MakeUnbucketedBatch(
+    const arrow::FieldVector& fields, const std::string& json,
+    const std::map<std::string, std::string>& partition,
+    const std::vector<RecordBatch::RowKind>& row_kinds = {}) {
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::Array> array,
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), json));
+    ArrowArray c_array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
+    RecordBatchBuilder builder(&c_array);
+    return builder.SetPartition(partition).SetRowKinds(row_kinds).Finish();
+}
+
+Result<std::unique_ptr<FileStoreWrite>> CreateWriter(
+    const std::string& table_path, const std::map<std::string, std::string>& options,
+    const std::shared_ptr<RealtimeContext>& realtime_context = nullptr,
+    const std::vector<std::string>& write_schema = {}) {
+    WriteContextBuilder builder(table_path, "schema_evolution_verify");
+    builder.SetOptions(options).WithStreamingMode(true);
+    if (realtime_context) {
+        builder.WithRealtimeContext(realtime_context);
+    }
+    if (!write_schema.empty()) {
+        builder.WithWriteSchema(write_schema);
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<WriteContext> context, builder.Finish());
+    return FileStoreWrite::Create(std::move(context));
+}
+
+Result<std::vector<std::shared_ptr<CommitMessage>>> WriteWithNewWriter(
+    const std::string& table_path, const std::map<std::string, std::string>& options,
+    std::unique_ptr<RecordBatch> batch, int64_t commit_identifier,
+    const std::vector<std::string>& write_schema = {}) {
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreWrite> writer,
+                           CreateWriter(table_path, options, nullptr, write_schema));
+    PAIMON_RETURN_NOT_OK(writer->Write(std::move(batch)));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<CommitMessage>> messages,
+                           writer->PrepareCommit(/*wait_compaction=*/false, commit_identifier));
+    PAIMON_RETURN_NOT_OK(writer->Close());
+    return messages;
+}
+
+Result<std::unique_ptr<FileStoreCommit>> CreateCommit(
+    const std::string& table_path, const std::map<std::string, std::string>& options) {
+    CommitContextBuilder builder(table_path, "schema_evolution_verify");
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
+                           builder.SetOptions(options).IgnoreEmptyCommit(false).Finish());
+    return FileStoreCommit::Create(std::move(context));
+}
+
+Status CommitMessages(const std::string& table_path,
+                      const std::map<std::string, std::string>& options,
+                      const std::vector<std::shared_ptr<CommitMessage>>& messages,
+                      int64_t commit_identifier) {
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
+                           CreateCommit(table_path, options));
+    return commit->Commit(messages, commit_identifier);
+}
+
+Result<int64_t> CommitRealtimeMessages(const std::string& table_path,
+                                       const std::map<std::string, std::string>& options,
+                                       const std::vector<RealtimeCommitProgress>& messages,
+                                       int64_t commit_identifier) {
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
+                           CreateCommit(table_path, options));
+    return commit->CommitWithProgress(messages, commit_identifier, /*watermark=*/std::nullopt);
+}
+
+Result<std::optional<Snapshot>> LatestSnapshot(const std::string& table_path,
+                                               const std::map<std::string, std::string>& options,
+                                               const std::shared_ptr<FileSystem>& file_system) {
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system));
+    SnapshotManager snapshot_manager(core_options.GetFileSystem(), table_path);
+    return snapshot_manager.LatestSnapshot();
+}
+
+Result<std::shared_ptr<Plan>> ScanTable(
+    const std::string& table_path, const std::map<std::string, std::string>& options,
+    const std::shared_ptr<MemoryPool>& pool,
+    const std::shared_ptr<RealtimeContext>& realtime_context = nullptr,
+    const std::shared_ptr<Predicate>& predicate = nullptr) {
+    ScanContextBuilder scan_builder(table_path);
+    scan_builder.SetOptions(options)
+        .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString())
+        .SetPredicate(predicate)
+        .WithMemoryPool(pool);
+    if (realtime_context) {
+        scan_builder.WithRealtimeContext(realtime_context);
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ScanContext> scan_context, scan_builder.Finish());
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableScan> table_scan,
+                           TableScan::Create(std::move(scan_context)));
+    return table_scan->CreatePlan();
+}
+
+std::vector<std::shared_ptr<DataFileMeta>> DataFilesFromPlan(const std::shared_ptr<Plan>& plan) {
+    std::vector<std::shared_ptr<DataFileMeta>> files;
+    for (const std::shared_ptr<Split>& split : plan->Splits()) {
+        std::shared_ptr<Split> data_split = split;
+        if (std::shared_ptr<IndexedSplitImpl> indexed_split =
+                std::dynamic_pointer_cast<IndexedSplitImpl>(split)) {
+            data_split = indexed_split->GetDataSplit();
+        }
+        std::shared_ptr<DataSplitImpl> split_impl =
+            std::dynamic_pointer_cast<DataSplitImpl>(data_split);
+        if (!split_impl) {
+            continue;
+        }
+        const std::vector<std::shared_ptr<DataFileMeta>>& split_files = split_impl->DataFiles();
+        files.insert(files.end(), split_files.begin(), split_files.end());
+    }
+    return files;
+}
+
+size_t CountIndexedSplits(const std::shared_ptr<Plan>& plan) {
+    size_t count = 0;
+    for (const std::shared_ptr<Split>& split : plan->Splits()) {
+        if (std::dynamic_pointer_cast<IndexedSplitImpl>(split)) {
+            count++;
+        }
+    }
+    return count;
+}
+
+Status EvolveSchema(const std::string& table_path, const std::shared_ptr<FileSystem>& file_system,
+                    const std::vector<DataField>& fields, int32_t highest_field_id,
+                    const std::map<std::string, std::string>& options) {
+    return TestHelper::WriteNextSchema(file_system, table_path, fields, highest_field_id, options);
+}
+
+void AssignFirstRowId(const std::vector<std::shared_ptr<CommitMessage>>& messages,
+                      int64_t first_row_id) {
+    for (const std::shared_ptr<CommitMessage>& commit_message : messages) {
+        std::shared_ptr<CommitMessageImpl> message =
+            std::dynamic_pointer_cast<CommitMessageImpl>(commit_message);
+        ASSERT_TRUE(message);
+        for (const std::shared_ptr<DataFileMeta>& file :
+             message->GetNewFilesIncrement().NewFiles()) {
+            file->AssignFirstRowId(first_row_id);
+        }
+    }
+}
+
+struct CollectedReadResult {
+    std::unique_ptr<TableRead> table_read;
+    std::unique_ptr<BatchReader> reader;
+    std::shared_ptr<arrow::ChunkedArray> data;
+};
+
+Result<CollectedReadResult> ReadRows(
+    const std::string& table_path, const std::map<std::string, std::string>& options,
+    const std::shared_ptr<MemoryPool>& pool,
+    const std::shared_ptr<RealtimeContext>& realtime_context = nullptr,
+    const std::shared_ptr<Predicate>& predicate = nullptr, bool enable_predicate_filter = true) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
+                           ScanTable(table_path, options, pool, realtime_context, predicate));
+
+    ReadContextBuilder read_builder(table_path);
+    read_builder.SetOptions(options)
+        .SetPredicate(predicate)
+        .EnablePredicateFilter(enable_predicate_filter)
+        .WithMemoryPool(pool);
+    if (realtime_context) {
+        read_builder.WithRealtimeContext(realtime_context);
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
+                           TableRead::Create(std::move(read_context)));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> batch_reader,
+                           table_read->CreateReader(plan->Splits()));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::ChunkedArray> rows,
+                           ReadResultCollector::CollectResult(batch_reader.get()));
+    return CollectedReadResult{std::move(table_read), std::move(batch_reader), std::move(rows)};
+}
+
+void AssertResultEquals(const std::shared_ptr<arrow::ChunkedArray>& actual,
+                        const arrow::FieldVector& fields, const std::string& expected_json) {
+    arrow::FieldVector fields_with_row_kind = fields;
+    fields_with_row_kind.insert(fields_with_row_kind.begin(),
+                                arrow::field("_VALUE_KIND", arrow::int8()));
+    std::shared_ptr<arrow::Array> expected_array =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_with_row_kind),
+                                                  expected_json)
+            .ValueOrDie();
+    auto expected = std::make_shared<arrow::ChunkedArray>(expected_array);
+    ASSERT_TRUE(expected->Equals(actual, arrow::EqualOptions::Defaults().diff_sink(&std::cout)))
+        << actual->ToString();
+}
+
+Status CreateTable(const std::string& warehouse, const std::shared_ptr<arrow::Schema>& schema,
+                   const std::vector<std::string>& primary_keys,
+                   const std::map<std::string, std::string>& options) {
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<Catalog> catalog, Catalog::Create(warehouse, options));
+    PAIMON_RETURN_NOT_OK(catalog->CreateDatabase("foo", options, /*ignore_if_exists=*/false));
+    ArrowSchema c_schema;
+    ArrowSchemaMarkReleased(&c_schema);
+    ScopeGuard guard([&c_schema]() { ArrowSchemaRelease(&c_schema); });
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema));
+    return catalog->CreateTable(Identifier("foo", "bar"), &c_schema,
+                                /*partition_keys=*/{}, primary_keys, options,
+                                /*ignore_if_exists=*/false);
+}
+
+Result<std::unique_ptr<FileIndexFormat::Reader>> CreateFileIndexReader(
+    const std::shared_ptr<DataFileMeta>& data_file, const std::shared_ptr<MemoryPool>& pool) {
+    if (data_file->embedded_index == nullptr) {
+        return Status::Invalid("data file does not contain an embedded file index");
+    }
+    auto input = std::make_shared<ByteArrayInputStream>(data_file->embedded_index->data(),
+                                                        data_file->embedded_index->size());
+    return FileIndexFormat::CreateReader(input, pool);
+}
+
+Result<std::vector<std::shared_ptr<FileIndexReader>>> ReadEmbeddedIndexColumn(
+    const std::shared_ptr<DataFileMeta>& data_file, const std::shared_ptr<arrow::Schema>& schema,
+    const std::string& column, const std::shared_ptr<MemoryPool>& pool) {
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileIndexFormat::Reader> reader,
+                           CreateFileIndexReader(data_file, pool));
+    auto c_schema = std::make_unique<ArrowSchema>();
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, c_schema.get()));
+    return reader->ReadColumnIndex(column, c_schema.get());
+}
+
+class SchemaEvolutionWriteVerifyTest : public ::testing::Test {
+ protected:
+    void SetUp() override {
+        pool_ = GetDefaultPool();
+        dir_ = UniqueTestDirectory::Create("local");
+        ASSERT_TRUE(dir_);
+        table_path_ = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+    }
+
+    void TearDown() override {
+        dir_.reset();
+    }
+
+    std::shared_ptr<MemoryPool> pool_;
+    std::unique_ptr<UniqueTestDirectory> dir_;
+    std::string table_path_;
+};
+
+TEST_F(SchemaEvolutionWriteVerifyTest,
+       NonRealtimeAppendOldWriterCommitsOldSchemaFileIntoNewSchemaSnapshot) {
+    std::map<std::string, std::string> options = BaseOptions();
+    options["file-index.bitmap.columns"] = "payload";
+    options[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"], [2, "skip"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
+                         old_writer->PrepareCommit(/*wait_compaction=*/false,
+                                                   /*commit_identifier=*/1));
+    ASSERT_EQ(1, messages.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
+    ASSERT_TRUE(message);
+    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+    std::shared_ptr<DataFileMeta> old_file = message->GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(0, old_file->schema_id);
+    ASSERT_TRUE(old_file->embedded_index);
+    ASSERT_TRUE(old_file->extra_files.empty());
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<std::shared_ptr<FileIndexReader>> payload_indexes,
+        ReadEmbeddedIndexColumn(old_file, arrow::schema(BaseFields()), "payload", pool_));
+    ASSERT_EQ(1, payload_indexes.size());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> payload_hit,
+                         payload_indexes[0]->VisitEqual(Literal(FieldType::STRING, "old", 3)));
+    ASSERT_OK_AND_ASSIGN(bool payload_remain, payload_hit->IsRemain());
+    ASSERT_TRUE(payload_remain);
+
+    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
+                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
+    ASSERT_TRUE(snapshot);
+    ASSERT_EQ(1, snapshot->SchemaId());
+
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult all_rows, ReadRows(table_path_, options, pool_));
+    AssertResultEquals(all_rows.data, EvolvedFields(),
+                       R"([[0, 1, "old", null], [0, 2, "skip", null]])");
+
+    auto predicate = PredicateBuilder::Equal(
+        /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
+        Literal(FieldType::STRING, "old", 3));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult filtered_rows,
+                         ReadRows(table_path_, options, pool_, /*realtime_context=*/nullptr,
+                                  predicate, /*enable_predicate_filter=*/false));
+    AssertResultEquals(filtered_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimeAppendNewWriterIndexesNewSchemaColumn) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
+
+    std::map<std::string, std::string> options_v1 = options;
+    options_v1["file-index.bitmap.columns"] = "extra";
+    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options_v1));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[1, "new", 20], [2, "skip", 30]])",
+                                   /*partition=*/{}, /*bucket=*/0));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
+                         WriteWithNewWriter(table_path_, options_v1, std::move(new_schema_batch),
+                                            /*commit_identifier=*/1));
+    ASSERT_EQ(1, messages.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
+    ASSERT_TRUE(message);
+    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(1, new_file->schema_id);
+    ASSERT_TRUE(new_file->embedded_index);
+    ASSERT_TRUE(new_file->extra_files.empty());
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<std::shared_ptr<FileIndexReader>> extra_indexes,
+        ReadEmbeddedIndexColumn(new_file, arrow::schema(EvolvedFields()), "extra", pool_));
+    ASSERT_EQ(1, extra_indexes.size());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> extra_hit,
+                         extra_indexes[0]->VisitEqual(Literal(20)));
+    ASSERT_OK_AND_ASSIGN(bool extra_remain, extra_hit->IsRemain());
+    ASSERT_TRUE(extra_remain);
+
+    ASSERT_OK(CommitMessages(table_path_, options_v1, messages, /*commit_identifier=*/1));
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::Equal(
+        /*field_index=*/2, /*field_name=*/"extra", FieldType::INT, Literal(20));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
+                         ReadRows(table_path_, options_v1, pool_, /*realtime_context=*/nullptr,
+                                  predicate, /*enable_predicate_filter=*/true));
+    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "new", 20]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimeAppendDataEvolutionWritesPartialNewColumnIndex) {
+    std::map<std::string, std::string> options = DataEvolutionOptions();
+    arrow::FieldVector fields = DataEvolutionFields();
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(fields), /*primary_keys=*/{}, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
+                         MakeUnbucketedBatch(fields, R"([[1, "old", "base"]])",
+                                             /*partition=*/{}));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> old_messages,
+                         WriteWithNewWriter(table_path_, options, std::move(old_schema_batch),
+                                            /*commit_identifier=*/1,
+                                            /*write_schema=*/{"f0", "f1", "f2"}));
+    ASSERT_OK(CommitMessages(table_path_, options, old_messages, /*commit_identifier=*/1));
+
+    std::map<std::string, std::string> options_v1 = DataEvolutionOptions();
+    options_v1["file-index.bitmap.columns"] = "f2";
+    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
+    ASSERT_OK(
+        EvolveSchema(table_path_, dir_->GetFileSystem(),
+                     {DataField(0, fields[0]), DataField(1, fields[1]), DataField(2, fields[2])},
+                     /*highest_field_id=*/2, options_v1));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> partial_batch,
+                         MakeUnbucketedBatch({fields[2]}, R"([["updated"]])",
+                                             /*partition=*/{}));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
+                         WriteWithNewWriter(table_path_, options_v1, std::move(partial_batch),
+                                            /*commit_identifier=*/2,
+                                            /*write_schema=*/{"f2"}));
+    ASSERT_EQ(1, new_messages.size());
+    std::shared_ptr<CommitMessageImpl> new_message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(new_messages[0]);
+    ASSERT_TRUE(new_message);
+    std::shared_ptr<DataFileMeta> new_file = new_message->GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(1, new_file->schema_id);
+    const std::optional<std::vector<std::string>> expected_write_cols =
+        std::vector<std::string>{"f2"};
+    ASSERT_EQ(expected_write_cols, new_file->write_cols);
+    ASSERT_TRUE(new_file->embedded_index);
+    ASSERT_TRUE(new_file->extra_files.empty());
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<std::shared_ptr<FileIndexReader>> f2_indexes,
+        ReadEmbeddedIndexColumn(new_file, arrow::schema({fields[2]}), "f2", pool_));
+    ASSERT_EQ(1, f2_indexes.size());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> f2_hit,
+                         f2_indexes[0]->VisitEqual(Literal(FieldType::STRING, "updated", 7)));
+    ASSERT_OK_AND_ASSIGN(bool f2_remain, f2_hit->IsRemain());
+    ASSERT_TRUE(f2_remain);
+
+    AssignFirstRowId(new_messages, /*first_row_id=*/0);
+    ASSERT_OK(CommitMessages(table_path_, options_v1, new_messages, /*commit_identifier=*/2));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult all_rows, ReadRows(table_path_, options_v1, pool_));
+    AssertResultEquals(all_rows.data, fields, R"([[0, 1, "old", "updated"]])");
+
+    auto predicate =
+        PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING,
+                                Literal(FieldType::STRING, "updated", 7));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult filtered_rows,
+                         ReadRows(table_path_, options_v1, pool_, /*realtime_context=*/nullptr,
+                                  predicate, /*enable_predicate_filter=*/false));
+    AssertResultEquals(filtered_rows.data, fields, R"([[0, 1, "old", "updated"]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkOldAndNewSchemaFilesReadThroughLatestSchema) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(base_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> base_messages,
+                         old_writer->PrepareCommit(/*wait_compaction=*/false,
+                                                   /*commit_identifier=*/1));
+    ASSERT_OK(CommitMessages(table_path_, options, base_messages, /*commit_identifier=*/1));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
+                         CreateWriter(table_path_, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
+                         new_writer->PrepareCommit(/*wait_compaction=*/false,
+                                                   /*commit_identifier=*/2));
+    ASSERT_EQ(1, new_messages.size());
+    std::shared_ptr<CommitMessageImpl> new_message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(new_messages[0]);
+    ASSERT_TRUE(new_message);
+    ASSERT_EQ(1, new_message->GetNewFilesIncrement().NewFiles().size());
+    ASSERT_EQ(1, new_message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
+    ASSERT_TRUE(new_message->GetNewFilesIncrement().NewFiles()[0]->extra_files.empty());
+
+    ASSERT_OK(CommitMessages(table_path_, options, new_messages, /*commit_identifier=*/2));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> stale_schema_batch,
+                         MakeBatch(BaseFields(), R"([[3, "stale"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(stale_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> stale_messages,
+                         old_writer->PrepareCommit(/*wait_compaction=*/false,
+                                                   /*commit_identifier=*/3));
+    ASSERT_EQ(1, stale_messages.size());
+    std::shared_ptr<CommitMessageImpl> stale_message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(stale_messages[0]);
+    ASSERT_TRUE(stale_message);
+    ASSERT_EQ(1, stale_message->GetNewFilesIncrement().NewFiles().size());
+    ASSERT_EQ(0, stale_message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
+    ASSERT_OK(CommitMessages(table_path_, options, stale_messages, /*commit_identifier=*/3));
+
+    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
+                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
+    ASSERT_TRUE(snapshot);
+    ASSERT_EQ(1, snapshot->SchemaId());
+
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows, ReadRows(table_path_, options, pool_));
+    AssertResultEquals(rows.data, EvolvedFields(),
+                       R"([[0, 1, "old", null], [0, 2, "new", 20], [0, 3, "stale", null]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkOldWriterCanOverwriteNewColumnWithNull) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[1, "new", 20]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
+                         WriteWithNewWriter(table_path_, options, std::move(new_schema_batch),
+                                            /*commit_identifier=*/1));
+    ASSERT_OK(CommitMessages(table_path_, options, new_messages, /*commit_identifier=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> old_messages,
+                         old_writer->PrepareCommit(/*wait_compaction=*/false,
+                                                   /*commit_identifier=*/2));
+    ASSERT_OK(CommitMessages(table_path_, options, old_messages, /*commit_identifier=*/2));
+
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows, ReadRows(table_path_, options, pool_));
+    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkNewWriterIndexesNewSchemaColumn) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> old_messages,
+                         WriteWithNewWriter(table_path_, options, std::move(old_schema_batch),
+                                            /*commit_identifier=*/1));
+    ASSERT_OK(CommitMessages(table_path_, options, old_messages, /*commit_identifier=*/1));
+
+    std::map<std::string, std::string> options_v1 = options;
+    options_v1["file-index.bitmap.columns"] = "extra";
+    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options_v1));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[2, "new", 20], [3, "skip", 30]])",
+                                   /*partition=*/{}, /*bucket=*/0));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
+                         WriteWithNewWriter(table_path_, options_v1, std::move(new_schema_batch),
+                                            /*commit_identifier=*/2));
+    ASSERT_EQ(1, new_messages.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(new_messages[0]);
+    ASSERT_TRUE(message);
+    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(1, new_file->schema_id);
+    ASSERT_FALSE(new_file->embedded_index);
+    ASSERT_EQ(1, new_file->extra_files.size());
+    ASSERT_TRUE(new_file->extra_files[0]);
+    std::string index_path =
+        PathUtil::JoinPath(table_path_, "bucket-0/" + new_file->extra_files[0].value());
+    ASSERT_OK_AND_ASSIGN(bool index_exists, dir_->GetFileSystem()->Exists(index_path));
+    ASSERT_TRUE(index_exists);
+
+    ASSERT_OK(CommitMessages(table_path_, options_v1, new_messages, /*commit_identifier=*/2));
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::Equal(
+        /*field_index=*/2, /*field_name=*/"extra", FieldType::INT, Literal(20));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
+                         ReadRows(table_path_, options_v1, pool_, /*realtime_context=*/nullptr,
+                                  predicate, /*enable_predicate_filter=*/true));
+    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 2, "new", 20]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkEmbeddedFileIndexFailsValueScan) {
+    std::map<std::string, std::string> options = BaseOptions();
+    options["file-index.bitmap.columns"] = "payload";
+    options[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch(BaseFields(), R"([[1, "a"], [2, "b"]])",
+                                   /*partition=*/{}, /*bucket=*/0));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
+                         WriteWithNewWriter(table_path_, options, std::move(batch),
+                                            /*commit_identifier=*/1));
+    ASSERT_EQ(1, messages.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
+    ASSERT_TRUE(message);
+    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+    ASSERT_TRUE(message->GetNewFilesIncrement().NewFiles()[0]->embedded_index);
+    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
+
+    std::shared_ptr<Predicate> predicate =
+        PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
+                                Literal(FieldType::STRING, "a", 1));
+    ASSERT_NOK_WITH_MSG(
+        ScanTable(table_path_, options, pool_, /*realtime_context=*/nullptr, predicate),
+        "do not support embedded index in DataFileMeta");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkSortedIndexConfigDoesNotWriteIndexOnDataWrite) {
+    std::map<std::string, std::string> options = BaseOptions();
+    options[Options::PK_BTREE_INDEX_COLUMNS] = "payload";
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch(BaseFields(), R"([[1, "a"], [2, "b"]])",
+                                   /*partition=*/{}, /*bucket=*/0));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
+                         WriteWithNewWriter(table_path_, options, std::move(batch),
+                                            /*commit_identifier=*/1));
+    ASSERT_EQ(1, messages.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
+    ASSERT_TRUE(message);
+    ASSERT_TRUE(message->GetNewFilesIncrement().NewIndexFiles().empty());
+    ASSERT_TRUE(message->GetCompactIncrement().NewIndexFiles().empty());
+    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
+                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
+    ASSERT_TRUE(snapshot);
+    ASSERT_FALSE(snapshot->IndexManifest());
+
+    std::shared_ptr<Predicate> predicate =
+        PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
+                                Literal(FieldType::STRING, "a", 1));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         ScanTable(table_path_, options, pool_,
+                                   /*realtime_context=*/nullptr, predicate));
+    ASSERT_EQ(0, CountIndexedSplits(plan));
+    std::vector<std::shared_ptr<DataFileMeta>> planned_files = DataFilesFromPlan(plan);
+    ASSERT_EQ(1, planned_files.size());
+    ASSERT_EQ(0, planned_files[0]->schema_id);
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
+                         ReadRows(table_path_, options, pool_, /*realtime_context=*/nullptr,
+                                  predicate, /*enable_predicate_filter=*/true));
+    AssertResultEquals(rows.data, BaseFields(), R"([[0, 1, "a"]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendRejectsDataEvolutionAtWriterCreation) {
+    std::map<std::string, std::string> create_options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, create_options));
+
+    std::map<std::string, std::string> write_options = BaseOptions();
+    write_options[Options::DATA_EVOLUTION_ENABLED] = "true";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_NOK_WITH_MSG(CreateWriter(table_path_, write_options, realtime_context),
+                        "real-time append write does not support data evolution");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendScanRejectsDataEvolutionTable) {
+    std::map<std::string, std::string> options = DataEvolutionOptions();
+    arrow::FieldVector fields = DataEvolutionFields();
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(fields), /*primary_keys=*/{}, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeUnbucketedBatch(fields, R"([[1, "old", "base"]])",
+                                             /*partition=*/{}));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
+                         WriteWithNewWriter(table_path_, options, std::move(batch),
+                                            /*commit_identifier=*/1,
+                                            /*write_schema=*/{"f0", "f1", "f2"}));
+    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+
+    ASSERT_NOK_WITH_MSG(ScanTable(table_path_, options, pool_, realtime_context),
+                        "real-time union read requires fixed bucket mode");
+
+    std::map<std::string, std::string> fixed_bucket_options = options;
+    fixed_bucket_options[Options::BUCKET] = "1";
+    ASSERT_NOK_WITH_MSG(ScanTable(table_path_, fixed_bucket_options, pool_, realtime_context),
+                        "real-time union read does not support data evolution");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendReuseContextKeepsOldMemorySchemaAfterAlter) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options, realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(base_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
+                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
+                         CommitRealtimeMessages(table_path_, options, base_progress,
+                                                /*commit_identifier=*/1));
+    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> reused_context_writer,
+                         CreateWriter(table_path_, options, realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_NOK_WITH_MSG(reused_context_writer->Write(std::move(new_schema_batch)),
+                        "ArrowArray struct has 3 children, expected 2");
+
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
+                         ReadRows(table_path_, options, pool_, realtime_context));
+    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendOldWriterAfterAlterCommitsOldSchemaFile) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options, realtime_context));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
+                         ReadRows(table_path_, options, pool_, realtime_context));
+    AssertResultEquals(memory_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, progress.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
+    ASSERT_TRUE(message);
+    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+    ASSERT_EQ(0, message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
+
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, CommitRealtimeMessages(table_path_, options, progress,
+                                                                     /*commit_identifier=*/1));
+    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
+    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
+                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
+    ASSERT_TRUE(snapshot);
+    ASSERT_EQ(1, snapshot->SchemaId());
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult disk_rows, ReadRows(table_path_, options, pool_));
+    AssertResultEquals(disk_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendNewContextUsesNewSchemaAfterAlter) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> old_realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options, old_realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(base_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
+                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
+                         CommitRealtimeMessages(table_path_, options, base_progress,
+                                                /*commit_identifier=*/1));
+    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    std::map<std::string, std::string> options_v1 = options;
+    options_v1["file-index.bitmap.columns"] = "extra";
+    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> new_realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
+                         CreateWriter(table_path_, options_v1, new_realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
+                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
+    AssertResultEquals(memory_rows.data, EvolvedFields(),
+                       R"([[0, 1, "old", null], [0, 2, "new", 20]])");
+
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> new_progress,
+                         new_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_EQ(1, new_progress.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(new_progress[0].commit_message);
+    ASSERT_TRUE(message);
+    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(1, new_file->schema_id);
+    ASSERT_FALSE(new_file->embedded_index);
+    ASSERT_EQ(1, new_file->extra_files.size());
+    ASSERT_TRUE(new_file->extra_files[0]);
+    std::string index_path =
+        PathUtil::JoinPath(table_path_, "bucket-0/" + new_file->extra_files[0].value());
+    ASSERT_OK_AND_ASSIGN(bool index_exists, dir_->GetFileSystem()->Exists(index_path));
+    ASSERT_TRUE(index_exists);
+
+    ASSERT_OK_AND_ASSIGN(snapshot_id, CommitRealtimeMessages(table_path_, options_v1, new_progress,
+                                                             /*commit_identifier=*/2));
+    ASSERT_OK(new_writer->RefreshCommittedSnapshot(snapshot_id));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
+                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
+    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null], [0, 2, "new", 20]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkRejectsDataEvolutionAtWriterCreation) {
+    std::map<std::string, std::string> create_options = BaseOptions();
+    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"},
+                          create_options));
+
+    std::map<std::string, std::string> write_options = BaseOptions();
+    write_options[Options::DATA_EVOLUTION_ENABLED] = "true";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_NOK_WITH_MSG(CreateWriter(table_path_, write_options, realtime_context),
+                        "PK realtime v1 does not support data evolution");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkReuseContextKeepsOldMemorySchemaAfterAlter) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options, realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(base_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
+                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
+                         CommitRealtimeMessages(table_path_, options, base_progress,
+                                                /*commit_identifier=*/1));
+    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> reused_context_writer,
+                         CreateWriter(table_path_, options, realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_NOK_WITH_MSG(reused_context_writer->Write(std::move(new_schema_batch)),
+                        "ArrowArray struct has 3 children, expected 2");
+
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
+                         ReadRows(table_path_, options, pool_, realtime_context));
+    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkOldWriterAfterAlterReadsNewColumnAsNull) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options, realtime_context));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
+                         ReadRows(table_path_, options, pool_, realtime_context));
+    AssertResultEquals(memory_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, progress.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
+    ASSERT_TRUE(message);
+    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+    ASSERT_EQ(0, message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
+
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, CommitRealtimeMessages(table_path_, options, progress,
+                                                                     /*commit_identifier=*/1));
+    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult disk_rows, ReadRows(table_path_, options, pool_));
+    AssertResultEquals(disk_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest,
+       RealtimePkOldWriterAfterAlterCannotCommitBehindNewContextOffset) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options, realtime_context));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> new_realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
+                         CreateWriter(table_path_, options, new_realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[1, "new", 20]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> new_progress,
+                         new_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
+                         CommitRealtimeMessages(table_path_, options, new_progress,
+                                                /*commit_identifier=*/1));
+    ASSERT_OK(new_writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult realtime_rows,
+                         ReadRows(table_path_, options, pool_, realtime_context));
+    AssertResultEquals(realtime_rows.data, EvolvedFields(), R"([[0, 1, "new", 20]])");
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> old_progress,
+                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_NOK_WITH_MSG(CommitRealtimeMessages(table_path_, options, old_progress,
+                                               /*commit_identifier=*/2),
+                        "real-time commit offsets for bucket 0 are not contiguous");
+
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult disk_rows, ReadRows(table_path_, options, pool_));
+    AssertResultEquals(disk_rows.data, EvolvedFields(), R"([[0, 1, "new", 20]])");
+}
+
+TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkNewContextUsesNewSchemaAfterAlter) {
+    std::map<std::string, std::string> options = BaseOptions();
+    ASSERT_OK(
+        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> old_realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
+                         CreateWriter(table_path_, options, old_realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
+                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(old_writer->Write(std::move(base_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
+                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
+                         CommitRealtimeMessages(table_path_, options, base_progress,
+                                                /*commit_identifier=*/1));
+    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
+                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
+                            DataField(2, EvolvedFields()[2])},
+                           /*highest_field_id=*/2, options));
+
+    std::map<std::string, std::string> options_v1 = options;
+    options_v1["file-index.bitmap.columns"] = "extra";
+    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> new_realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
+                         CreateWriter(table_path_, options_v1, new_realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
+                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
+                                   /*bucket=*/0));
+    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
+                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
+    AssertResultEquals(memory_rows.data, EvolvedFields(),
+                       R"([[0, 1, "old", null], [0, 2, "new", 20]])");
+
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> new_progress,
+                         new_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+    ASSERT_EQ(1, new_progress.size());
+    std::shared_ptr<CommitMessageImpl> message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(new_progress[0].commit_message);
+    ASSERT_TRUE(message);
+    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(1, new_file->schema_id);
+    ASSERT_FALSE(new_file->embedded_index);
+    ASSERT_EQ(1, new_file->extra_files.size());
+    ASSERT_TRUE(new_file->extra_files[0]);
+    std::string index_path =
+        PathUtil::JoinPath(table_path_, "bucket-0/" + new_file->extra_files[0].value());
+    ASSERT_OK_AND_ASSIGN(bool index_exists, dir_->GetFileSystem()->Exists(index_path));
+    ASSERT_TRUE(index_exists);
+
+    ASSERT_OK_AND_ASSIGN(snapshot_id, CommitRealtimeMessages(table_path_, options_v1, new_progress,
+                                                             /*commit_identifier=*/2));
+    ASSERT_OK(new_writer->RefreshCommittedSnapshot(snapshot_id));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
+                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
+    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null], [0, 2, "new", 20]])");
+}
+
+}  // namespace
+}  // namespace paimon::test

From 74d4feee023e53f9bec229de720134e8b16ce89b Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 21 Aug 2026 14:41:09 +0800
Subject: [PATCH 34/93] fix(realtime): align PK projections by field ID

---
 .../realtime/primary_key_realtime_store.cpp   |   36 +-
 .../primary_key_realtime_store_test.cpp       |   50 +-
 test/inte/CMakeLists.txt                      |    7 -
 ...chema_evolution_write_verify_inte_test.cpp | 1110 -----------------
 4 files changed, 76 insertions(+), 1127 deletions(-)
 delete mode 100644 test/inte/schema_evolution_write_verify_inte_test.cpp

diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 6565ed8d7..8f51c1b1e 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -98,6 +98,30 @@ uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
     return result;
 }
 
+int32_t FindPkQueryFieldIndex(const std::shared_ptr<arrow::Schema>& write_schema,
+                              const std::shared_ptr<arrow::Field>& read_field) {
+    Result<int32_t> read_id = NestedProjectionUtils::GetPaimonFieldId(read_field);
+    if (read_id.ok()) {
+        Result<std::shared_ptr<arrow::Field>> write_field =
+            NestedProjectionUtils::FindFieldByPaimonId(arrow::struct_(write_schema->fields()),
+                                                       read_id.value());
+        if (write_field.ok()) {
+            return write_schema->GetFieldIndex(write_field.value()->name());
+        }
+    }
+
+    const int32_t name_index = write_schema->GetFieldIndex(read_field->name());
+    if (name_index < 0) {
+        return -1;
+    }
+    Result<int32_t> write_id =
+        NestedProjectionUtils::GetPaimonFieldId(write_schema->field(name_index));
+    if (read_id.ok() && write_id.ok() && read_id.value() != write_id.value()) {
+        return -1;
+    }
+    return name_index;
+}
+
 struct StoredBatch {
     std::shared_ptr<arrow::StructArray> data;
     std::vector<RecordBatch::RowKind> row_kinds;
@@ -423,17 +447,23 @@ class PrimaryKeyRealtimeStore::Impl {
                 projection.push_back(KeyValueProjectionConsumer::kSequenceNumberProjection);
                 continue;
             }
-            int32_t index = write_schema_->GetFieldIndex(field->name());
+            int32_t index = FindPkQueryFieldIndex(write_schema_, field);
             if (index < 0) {
                 Result<int32_t> field_id = NestedProjectionUtils::GetPaimonFieldId(field);
                 if (!field_id.ok()) {
                     return Status::Invalid(
                         "PK real-time query field is missing from write schema: ", field->name());
                 }
+                std::string internal_name =
+                    "__paimon_pk_realtime_null_" + std::to_string(field_id.value());
+                while (
+                    NestedProjectionUtils::FindFieldByName(aligned_value_fields, internal_name)) {
+                    internal_name.push_back('_');
+                }
                 index = static_cast<int32_t>(aligned_value_fields.size());
-                aligned_value_fields.push_back(field);
+                aligned_value_fields.push_back(field->WithName(internal_name));
             } else {
-                aligned_value_fields[index] = field;
+                aligned_value_fields[index] = write_schema_->field(index)->WithType(field->type());
             }
             projection.push_back(index);
         }
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index ef293e54b..66901a6b1 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -392,6 +392,40 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryProjection) {
                         "query field is missing from write schema: unknown");
 }
 
+TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryProjectionMatchesRenamedFieldsById) {
+    const std::shared_ptr<arrow::Field> id =
+        DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64())));
+    const std::shared_ptr<arrow::Field> value =
+        DataField::ConvertDataFieldToArrowField(DataField(1, arrow::field("value", arrow::utf8())));
+    const std::shared_ptr<arrow::Schema> write_schema = arrow::schema({id, value});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         CreateStore(write_schema, {"id"}, /*restore_max_sequence=*/4));
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[1, "kept"]])", {}, write_schema), OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    const std::shared_ptr<arrow::Field> renamed_value = DataField::ConvertDataFieldToArrowField(
+        DataField(1, arrow::field("renamed", arrow::utf8())));
+    const std::shared_ptr<arrow::Field> renamed_id = DataField::ConvertDataFieldToArrowField(
+        DataField(0, arrow::field("renamed_id", arrow::int64())));
+    const std::shared_ptr<arrow::Field> replaced =
+        DataField::ConvertDataFieldToArrowField(DataField(2, arrow::field("value", arrow::utf8())));
+    const std::shared_ptr<arrow::Field> replaced_id =
+        DataField::ConvertDataFieldToArrowField(DataField(4, arrow::field("id", arrow::int64())));
+    const std::shared_ptr<arrow::Field> added =
+        DataField::ConvertDataFieldToArrowField(DataField(3, arrow::field("added", arrow::utf8())));
+    std::unique_ptr<ArrowSchema> read_schema =
+        MakeReadSchema({renamed_value, renamed_id, replaced, replaced_id, added});
+    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    const std::shared_ptr<arrow::DataType> result_type =
+        arrow::struct_({DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
+                        renamed_value, renamed_id, replaced, replaced_id, added});
+    AssertReaderOutput(readers, result_type, R"([[0, "kept", 1, null, null, null]])");
+}
+
 TEST_F(PrimaryKeyRealtimeStoreTest, TestNestedProjection) {
     const std::shared_ptr<arrow::Field> id =
         DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64())));
@@ -433,7 +467,10 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestCompositeKeyClipping) {
                                      {}, composite_schema),
                            OffsetRange(20, 24)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields(composite_schema));
+    const std::shared_ptr<arrow::Field> sequence =
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber());
+    std::unique_ptr<ArrowSchema> read_schema =
+        MakeReadSchema({sequence, composite_schema->field(0), composite_schema->field(2)});
     RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
@@ -445,13 +482,12 @@ TEST_F(PrimaryKeyRealtimeStoreTest, TestCompositeKeyClipping) {
     ASSERT_EQ("c", range->GetMinKey()->GetString(1).ToString());
     ASSERT_EQ(2, range->GetMaxKey()->GetLong(0));
     ASSERT_EQ("b", range->GetMaxKey()->GetString(1).ToString());
-    std::shared_ptr<arrow::DataType> query_type = arrow::struct_(
-        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
-         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()),
-         composite_schema->field(0), composite_schema->field(1), composite_schema->field(2)});
+    std::shared_ptr<arrow::DataType> query_type =
+        arrow::struct_({DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
+                        sequence, composite_schema->field(0), composite_schema->field(2)});
     AssertReaderOutput(readers, query_type,
-                       R"([[0, 7, 1, "c", "one-c"], [0, 8, 2, "a", "two-a"],
-                            [0, 6, 2, "b", "two-b"]])");
+                       R"([[0, 7, 1, "one-c"], [0, 8, 2, "two-a"],
+                            [0, 6, 2, "two-b"]])");
 }
 
 }  // namespace paimon::test
diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt
index f1b3f8ce6..75147ce60 100644
--- a/test/inte/CMakeLists.txt
+++ b/test/inte/CMakeLists.txt
@@ -43,13 +43,6 @@ if(PAIMON_BUILD_TESTS)
                     test_utils_static
                     ${GTEST_LINK_TOOLCHAIN})
 
-    add_paimon_test(schema_evolution_write_verify_inte_test
-                    STATIC_LINK_LIBS
-                    paimon_shared
-                    ${TEST_STATIC_LINK_LIBS}
-                    test_utils_static
-                    ${GTEST_LINK_TOOLCHAIN})
-
     add_paimon_test(global_index_test
                     STATIC_LINK_LIBS
                     paimon_shared
diff --git a/test/inte/schema_evolution_write_verify_inte_test.cpp b/test/inte/schema_evolution_write_verify_inte_test.cpp
deleted file mode 100644
index dcadbd9e1..000000000
--- a/test/inte/schema_evolution_write_verify_inte_test.cpp
+++ /dev/null
@@ -1,1110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <cstddef>
-#include <cstdint>
-#include <map>
-#include <memory>
-#include <optional>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "arrow/api.h"
-#include "arrow/c/bridge.h"
-#include "arrow/ipc/json_simple.h"
-#include "gtest/gtest.h"
-#include "paimon/catalog/catalog.h"
-#include "paimon/catalog/identifier.h"
-#include "paimon/commit_context.h"
-#include "paimon/common/utils/path_util.h"
-#include "paimon/common/utils/scope_guard.h"
-#include "paimon/core/core_options.h"
-#include "paimon/core/global_index/indexed_split_impl.h"
-#include "paimon/core/io/data_file_meta.h"
-#include "paimon/core/schema/schema_manager.h"
-#include "paimon/core/snapshot.h"
-#include "paimon/core/table/sink/commit_message_impl.h"
-#include "paimon/core/table/source/data_split_impl.h"
-#include "paimon/defs.h"
-#include "paimon/file_index/file_index_format.h"
-#include "paimon/file_index/file_index_reader.h"
-#include "paimon/file_index/file_index_result.h"
-#include "paimon/file_store_commit.h"
-#include "paimon/file_store_write.h"
-#include "paimon/fs/file_system.h"
-#include "paimon/io/byte_array_input_stream.h"
-#include "paimon/predicate/literal.h"
-#include "paimon/predicate/predicate_builder.h"
-#include "paimon/read_context.h"
-#include "paimon/reader/batch_reader.h"
-#include "paimon/realtime/realtime_context.h"
-#include "paimon/record_batch.h"
-#include "paimon/scan_context.h"
-#include "paimon/table/source/plan.h"
-#include "paimon/table/source/startup_mode.h"
-#include "paimon/table/source/table_read.h"
-#include "paimon/table/source/table_scan.h"
-#include "paimon/testing/utils/read_result_collector.h"
-#include "paimon/testing/utils/test_helper.h"
-#include "paimon/testing/utils/testharness.h"
-#include "paimon/write_context.h"
-
-namespace paimon::test {
-namespace {
-
-std::map<std::string, std::string> BaseOptions() {
-    return {{Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"},
-            {Options::FILE_SYSTEM, "local"},   {Options::BUCKET, "1"},
-            {Options::BUCKET_KEY, "id"},       {Options::TARGET_FILE_SIZE, "1MB"}};
-}
-
-std::map<std::string, std::string> DataEvolutionOptions() {
-    return {{Options::MANIFEST_FORMAT, "orc"},       {Options::FILE_FORMAT, "orc"},
-            {Options::FILE_SYSTEM, "local"},         {Options::TARGET_FILE_SIZE, "1MB"},
-            {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}};
-}
-
-arrow::FieldVector BaseFields() {
-    return {arrow::field("id", arrow::int64()), arrow::field("payload", arrow::utf8())};
-}
-
-arrow::FieldVector EvolvedFields() {
-    return {arrow::field("id", arrow::int64()), arrow::field("payload", arrow::utf8()),
-            arrow::field("extra", arrow::int32())};
-}
-
-arrow::FieldVector DataEvolutionFields() {
-    return {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()),
-            arrow::field("f2", arrow::utf8())};
-}
-
-Result<std::unique_ptr<RecordBatch>> MakeBatch(
-    const arrow::FieldVector& fields, const std::string& json,
-    const std::map<std::string, std::string>& partition, int32_t bucket,
-    const std::vector<RecordBatch::RowKind>& row_kinds = {}) {
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-        std::shared_ptr<arrow::Array> array,
-        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), json));
-    ArrowArray c_array;
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
-    RecordBatchBuilder builder(&c_array);
-    return builder.SetPartition(partition).SetBucket(bucket).SetRowKinds(row_kinds).Finish();
-}
-
-Result<std::unique_ptr<RecordBatch>> MakeUnbucketedBatch(
-    const arrow::FieldVector& fields, const std::string& json,
-    const std::map<std::string, std::string>& partition,
-    const std::vector<RecordBatch::RowKind>& row_kinds = {}) {
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-        std::shared_ptr<arrow::Array> array,
-        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), json));
-    ArrowArray c_array;
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
-    RecordBatchBuilder builder(&c_array);
-    return builder.SetPartition(partition).SetRowKinds(row_kinds).Finish();
-}
-
-Result<std::unique_ptr<FileStoreWrite>> CreateWriter(
-    const std::string& table_path, const std::map<std::string, std::string>& options,
-    const std::shared_ptr<RealtimeContext>& realtime_context = nullptr,
-    const std::vector<std::string>& write_schema = {}) {
-    WriteContextBuilder builder(table_path, "schema_evolution_verify");
-    builder.SetOptions(options).WithStreamingMode(true);
-    if (realtime_context) {
-        builder.WithRealtimeContext(realtime_context);
-    }
-    if (!write_schema.empty()) {
-        builder.WithWriteSchema(write_schema);
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<WriteContext> context, builder.Finish());
-    return FileStoreWrite::Create(std::move(context));
-}
-
-Result<std::vector<std::shared_ptr<CommitMessage>>> WriteWithNewWriter(
-    const std::string& table_path, const std::map<std::string, std::string>& options,
-    std::unique_ptr<RecordBatch> batch, int64_t commit_identifier,
-    const std::vector<std::string>& write_schema = {}) {
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreWrite> writer,
-                           CreateWriter(table_path, options, nullptr, write_schema));
-    PAIMON_RETURN_NOT_OK(writer->Write(std::move(batch)));
-    PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<CommitMessage>> messages,
-                           writer->PrepareCommit(/*wait_compaction=*/false, commit_identifier));
-    PAIMON_RETURN_NOT_OK(writer->Close());
-    return messages;
-}
-
-Result<std::unique_ptr<FileStoreCommit>> CreateCommit(
-    const std::string& table_path, const std::map<std::string, std::string>& options) {
-    CommitContextBuilder builder(table_path, "schema_evolution_verify");
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
-                           builder.SetOptions(options).IgnoreEmptyCommit(false).Finish());
-    return FileStoreCommit::Create(std::move(context));
-}
-
-Status CommitMessages(const std::string& table_path,
-                      const std::map<std::string, std::string>& options,
-                      const std::vector<std::shared_ptr<CommitMessage>>& messages,
-                      int64_t commit_identifier) {
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
-                           CreateCommit(table_path, options));
-    return commit->Commit(messages, commit_identifier);
-}
-
-Result<int64_t> CommitRealtimeMessages(const std::string& table_path,
-                                       const std::map<std::string, std::string>& options,
-                                       const std::vector<RealtimeCommitProgress>& messages,
-                                       int64_t commit_identifier) {
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
-                           CreateCommit(table_path, options));
-    return commit->CommitWithProgress(messages, commit_identifier, /*watermark=*/std::nullopt);
-}
-
-Result<std::optional<Snapshot>> LatestSnapshot(const std::string& table_path,
-                                               const std::map<std::string, std::string>& options,
-                                               const std::shared_ptr<FileSystem>& file_system) {
-    PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system));
-    SnapshotManager snapshot_manager(core_options.GetFileSystem(), table_path);
-    return snapshot_manager.LatestSnapshot();
-}
-
-Result<std::shared_ptr<Plan>> ScanTable(
-    const std::string& table_path, const std::map<std::string, std::string>& options,
-    const std::shared_ptr<MemoryPool>& pool,
-    const std::shared_ptr<RealtimeContext>& realtime_context = nullptr,
-    const std::shared_ptr<Predicate>& predicate = nullptr) {
-    ScanContextBuilder scan_builder(table_path);
-    scan_builder.SetOptions(options)
-        .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString())
-        .SetPredicate(predicate)
-        .WithMemoryPool(pool);
-    if (realtime_context) {
-        scan_builder.WithRealtimeContext(realtime_context);
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ScanContext> scan_context, scan_builder.Finish());
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableScan> table_scan,
-                           TableScan::Create(std::move(scan_context)));
-    return table_scan->CreatePlan();
-}
-
-std::vector<std::shared_ptr<DataFileMeta>> DataFilesFromPlan(const std::shared_ptr<Plan>& plan) {
-    std::vector<std::shared_ptr<DataFileMeta>> files;
-    for (const std::shared_ptr<Split>& split : plan->Splits()) {
-        std::shared_ptr<Split> data_split = split;
-        if (std::shared_ptr<IndexedSplitImpl> indexed_split =
-                std::dynamic_pointer_cast<IndexedSplitImpl>(split)) {
-            data_split = indexed_split->GetDataSplit();
-        }
-        std::shared_ptr<DataSplitImpl> split_impl =
-            std::dynamic_pointer_cast<DataSplitImpl>(data_split);
-        if (!split_impl) {
-            continue;
-        }
-        const std::vector<std::shared_ptr<DataFileMeta>>& split_files = split_impl->DataFiles();
-        files.insert(files.end(), split_files.begin(), split_files.end());
-    }
-    return files;
-}
-
-size_t CountIndexedSplits(const std::shared_ptr<Plan>& plan) {
-    size_t count = 0;
-    for (const std::shared_ptr<Split>& split : plan->Splits()) {
-        if (std::dynamic_pointer_cast<IndexedSplitImpl>(split)) {
-            count++;
-        }
-    }
-    return count;
-}
-
-Status EvolveSchema(const std::string& table_path, const std::shared_ptr<FileSystem>& file_system,
-                    const std::vector<DataField>& fields, int32_t highest_field_id,
-                    const std::map<std::string, std::string>& options) {
-    return TestHelper::WriteNextSchema(file_system, table_path, fields, highest_field_id, options);
-}
-
-void AssignFirstRowId(const std::vector<std::shared_ptr<CommitMessage>>& messages,
-                      int64_t first_row_id) {
-    for (const std::shared_ptr<CommitMessage>& commit_message : messages) {
-        std::shared_ptr<CommitMessageImpl> message =
-            std::dynamic_pointer_cast<CommitMessageImpl>(commit_message);
-        ASSERT_TRUE(message);
-        for (const std::shared_ptr<DataFileMeta>& file :
-             message->GetNewFilesIncrement().NewFiles()) {
-            file->AssignFirstRowId(first_row_id);
-        }
-    }
-}
-
-struct CollectedReadResult {
-    std::unique_ptr<TableRead> table_read;
-    std::unique_ptr<BatchReader> reader;
-    std::shared_ptr<arrow::ChunkedArray> data;
-};
-
-Result<CollectedReadResult> ReadRows(
-    const std::string& table_path, const std::map<std::string, std::string>& options,
-    const std::shared_ptr<MemoryPool>& pool,
-    const std::shared_ptr<RealtimeContext>& realtime_context = nullptr,
-    const std::shared_ptr<Predicate>& predicate = nullptr, bool enable_predicate_filter = true) {
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
-                           ScanTable(table_path, options, pool, realtime_context, predicate));
-
-    ReadContextBuilder read_builder(table_path);
-    read_builder.SetOptions(options)
-        .SetPredicate(predicate)
-        .EnablePredicateFilter(enable_predicate_filter)
-        .WithMemoryPool(pool);
-    if (realtime_context) {
-        read_builder.WithRealtimeContext(realtime_context);
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
-                           TableRead::Create(std::move(read_context)));
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> batch_reader,
-                           table_read->CreateReader(plan->Splits()));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::ChunkedArray> rows,
-                           ReadResultCollector::CollectResult(batch_reader.get()));
-    return CollectedReadResult{std::move(table_read), std::move(batch_reader), std::move(rows)};
-}
-
-void AssertResultEquals(const std::shared_ptr<arrow::ChunkedArray>& actual,
-                        const arrow::FieldVector& fields, const std::string& expected_json) {
-    arrow::FieldVector fields_with_row_kind = fields;
-    fields_with_row_kind.insert(fields_with_row_kind.begin(),
-                                arrow::field("_VALUE_KIND", arrow::int8()));
-    std::shared_ptr<arrow::Array> expected_array =
-        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_with_row_kind),
-                                                  expected_json)
-            .ValueOrDie();
-    auto expected = std::make_shared<arrow::ChunkedArray>(expected_array);
-    ASSERT_TRUE(expected->Equals(actual, arrow::EqualOptions::Defaults().diff_sink(&std::cout)))
-        << actual->ToString();
-}
-
-Status CreateTable(const std::string& warehouse, const std::shared_ptr<arrow::Schema>& schema,
-                   const std::vector<std::string>& primary_keys,
-                   const std::map<std::string, std::string>& options) {
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<Catalog> catalog, Catalog::Create(warehouse, options));
-    PAIMON_RETURN_NOT_OK(catalog->CreateDatabase("foo", options, /*ignore_if_exists=*/false));
-    ArrowSchema c_schema;
-    ArrowSchemaMarkReleased(&c_schema);
-    ScopeGuard guard([&c_schema]() { ArrowSchemaRelease(&c_schema); });
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema));
-    return catalog->CreateTable(Identifier("foo", "bar"), &c_schema,
-                                /*partition_keys=*/{}, primary_keys, options,
-                                /*ignore_if_exists=*/false);
-}
-
-Result<std::unique_ptr<FileIndexFormat::Reader>> CreateFileIndexReader(
-    const std::shared_ptr<DataFileMeta>& data_file, const std::shared_ptr<MemoryPool>& pool) {
-    if (data_file->embedded_index == nullptr) {
-        return Status::Invalid("data file does not contain an embedded file index");
-    }
-    auto input = std::make_shared<ByteArrayInputStream>(data_file->embedded_index->data(),
-                                                        data_file->embedded_index->size());
-    return FileIndexFormat::CreateReader(input, pool);
-}
-
-Result<std::vector<std::shared_ptr<FileIndexReader>>> ReadEmbeddedIndexColumn(
-    const std::shared_ptr<DataFileMeta>& data_file, const std::shared_ptr<arrow::Schema>& schema,
-    const std::string& column, const std::shared_ptr<MemoryPool>& pool) {
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileIndexFormat::Reader> reader,
-                           CreateFileIndexReader(data_file, pool));
-    auto c_schema = std::make_unique<ArrowSchema>();
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, c_schema.get()));
-    return reader->ReadColumnIndex(column, c_schema.get());
-}
-
-class SchemaEvolutionWriteVerifyTest : public ::testing::Test {
- protected:
-    void SetUp() override {
-        pool_ = GetDefaultPool();
-        dir_ = UniqueTestDirectory::Create("local");
-        ASSERT_TRUE(dir_);
-        table_path_ = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
-    }
-
-    void TearDown() override {
-        dir_.reset();
-    }
-
-    std::shared_ptr<MemoryPool> pool_;
-    std::unique_ptr<UniqueTestDirectory> dir_;
-    std::string table_path_;
-};
-
-TEST_F(SchemaEvolutionWriteVerifyTest,
-       NonRealtimeAppendOldWriterCommitsOldSchemaFileIntoNewSchemaSnapshot) {
-    std::map<std::string, std::string> options = BaseOptions();
-    options["file-index.bitmap.columns"] = "payload";
-    options[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"], [2, "skip"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
-                         old_writer->PrepareCommit(/*wait_compaction=*/false,
-                                                   /*commit_identifier=*/1));
-    ASSERT_EQ(1, messages.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
-    ASSERT_TRUE(message);
-    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
-    std::shared_ptr<DataFileMeta> old_file = message->GetNewFilesIncrement().NewFiles()[0];
-    ASSERT_EQ(0, old_file->schema_id);
-    ASSERT_TRUE(old_file->embedded_index);
-    ASSERT_TRUE(old_file->extra_files.empty());
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<std::shared_ptr<FileIndexReader>> payload_indexes,
-        ReadEmbeddedIndexColumn(old_file, arrow::schema(BaseFields()), "payload", pool_));
-    ASSERT_EQ(1, payload_indexes.size());
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> payload_hit,
-                         payload_indexes[0]->VisitEqual(Literal(FieldType::STRING, "old", 3)));
-    ASSERT_OK_AND_ASSIGN(bool payload_remain, payload_hit->IsRemain());
-    ASSERT_TRUE(payload_remain);
-
-    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
-                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
-    ASSERT_TRUE(snapshot);
-    ASSERT_EQ(1, snapshot->SchemaId());
-
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult all_rows, ReadRows(table_path_, options, pool_));
-    AssertResultEquals(all_rows.data, EvolvedFields(),
-                       R"([[0, 1, "old", null], [0, 2, "skip", null]])");
-
-    auto predicate = PredicateBuilder::Equal(
-        /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
-        Literal(FieldType::STRING, "old", 3));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult filtered_rows,
-                         ReadRows(table_path_, options, pool_, /*realtime_context=*/nullptr,
-                                  predicate, /*enable_predicate_filter=*/false));
-    AssertResultEquals(filtered_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimeAppendNewWriterIndexesNewSchemaColumn) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
-
-    std::map<std::string, std::string> options_v1 = options;
-    options_v1["file-index.bitmap.columns"] = "extra";
-    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options_v1));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[1, "new", 20], [2, "skip", 30]])",
-                                   /*partition=*/{}, /*bucket=*/0));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
-                         WriteWithNewWriter(table_path_, options_v1, std::move(new_schema_batch),
-                                            /*commit_identifier=*/1));
-    ASSERT_EQ(1, messages.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
-    ASSERT_TRUE(message);
-    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
-    ASSERT_EQ(1, new_file->schema_id);
-    ASSERT_TRUE(new_file->embedded_index);
-    ASSERT_TRUE(new_file->extra_files.empty());
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<std::shared_ptr<FileIndexReader>> extra_indexes,
-        ReadEmbeddedIndexColumn(new_file, arrow::schema(EvolvedFields()), "extra", pool_));
-    ASSERT_EQ(1, extra_indexes.size());
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> extra_hit,
-                         extra_indexes[0]->VisitEqual(Literal(20)));
-    ASSERT_OK_AND_ASSIGN(bool extra_remain, extra_hit->IsRemain());
-    ASSERT_TRUE(extra_remain);
-
-    ASSERT_OK(CommitMessages(table_path_, options_v1, messages, /*commit_identifier=*/1));
-    std::shared_ptr<Predicate> predicate = PredicateBuilder::Equal(
-        /*field_index=*/2, /*field_name=*/"extra", FieldType::INT, Literal(20));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
-                         ReadRows(table_path_, options_v1, pool_, /*realtime_context=*/nullptr,
-                                  predicate, /*enable_predicate_filter=*/true));
-    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "new", 20]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimeAppendDataEvolutionWritesPartialNewColumnIndex) {
-    std::map<std::string, std::string> options = DataEvolutionOptions();
-    arrow::FieldVector fields = DataEvolutionFields();
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(fields), /*primary_keys=*/{}, options));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
-                         MakeUnbucketedBatch(fields, R"([[1, "old", "base"]])",
-                                             /*partition=*/{}));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> old_messages,
-                         WriteWithNewWriter(table_path_, options, std::move(old_schema_batch),
-                                            /*commit_identifier=*/1,
-                                            /*write_schema=*/{"f0", "f1", "f2"}));
-    ASSERT_OK(CommitMessages(table_path_, options, old_messages, /*commit_identifier=*/1));
-
-    std::map<std::string, std::string> options_v1 = DataEvolutionOptions();
-    options_v1["file-index.bitmap.columns"] = "f2";
-    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
-    ASSERT_OK(
-        EvolveSchema(table_path_, dir_->GetFileSystem(),
-                     {DataField(0, fields[0]), DataField(1, fields[1]), DataField(2, fields[2])},
-                     /*highest_field_id=*/2, options_v1));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> partial_batch,
-                         MakeUnbucketedBatch({fields[2]}, R"([["updated"]])",
-                                             /*partition=*/{}));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
-                         WriteWithNewWriter(table_path_, options_v1, std::move(partial_batch),
-                                            /*commit_identifier=*/2,
-                                            /*write_schema=*/{"f2"}));
-    ASSERT_EQ(1, new_messages.size());
-    std::shared_ptr<CommitMessageImpl> new_message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(new_messages[0]);
-    ASSERT_TRUE(new_message);
-    std::shared_ptr<DataFileMeta> new_file = new_message->GetNewFilesIncrement().NewFiles()[0];
-    ASSERT_EQ(1, new_file->schema_id);
-    const std::optional<std::vector<std::string>> expected_write_cols =
-        std::vector<std::string>{"f2"};
-    ASSERT_EQ(expected_write_cols, new_file->write_cols);
-    ASSERT_TRUE(new_file->embedded_index);
-    ASSERT_TRUE(new_file->extra_files.empty());
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<std::shared_ptr<FileIndexReader>> f2_indexes,
-        ReadEmbeddedIndexColumn(new_file, arrow::schema({fields[2]}), "f2", pool_));
-    ASSERT_EQ(1, f2_indexes.size());
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> f2_hit,
-                         f2_indexes[0]->VisitEqual(Literal(FieldType::STRING, "updated", 7)));
-    ASSERT_OK_AND_ASSIGN(bool f2_remain, f2_hit->IsRemain());
-    ASSERT_TRUE(f2_remain);
-
-    AssignFirstRowId(new_messages, /*first_row_id=*/0);
-    ASSERT_OK(CommitMessages(table_path_, options_v1, new_messages, /*commit_identifier=*/2));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult all_rows, ReadRows(table_path_, options_v1, pool_));
-    AssertResultEquals(all_rows.data, fields, R"([[0, 1, "old", "updated"]])");
-
-    auto predicate =
-        PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING,
-                                Literal(FieldType::STRING, "updated", 7));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult filtered_rows,
-                         ReadRows(table_path_, options_v1, pool_, /*realtime_context=*/nullptr,
-                                  predicate, /*enable_predicate_filter=*/false));
-    AssertResultEquals(filtered_rows.data, fields, R"([[0, 1, "old", "updated"]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkOldAndNewSchemaFilesReadThroughLatestSchema) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(base_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> base_messages,
-                         old_writer->PrepareCommit(/*wait_compaction=*/false,
-                                                   /*commit_identifier=*/1));
-    ASSERT_OK(CommitMessages(table_path_, options, base_messages, /*commit_identifier=*/1));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
-                         CreateWriter(table_path_, options));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
-                         new_writer->PrepareCommit(/*wait_compaction=*/false,
-                                                   /*commit_identifier=*/2));
-    ASSERT_EQ(1, new_messages.size());
-    std::shared_ptr<CommitMessageImpl> new_message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(new_messages[0]);
-    ASSERT_TRUE(new_message);
-    ASSERT_EQ(1, new_message->GetNewFilesIncrement().NewFiles().size());
-    ASSERT_EQ(1, new_message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
-    ASSERT_TRUE(new_message->GetNewFilesIncrement().NewFiles()[0]->extra_files.empty());
-
-    ASSERT_OK(CommitMessages(table_path_, options, new_messages, /*commit_identifier=*/2));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> stale_schema_batch,
-                         MakeBatch(BaseFields(), R"([[3, "stale"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(stale_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> stale_messages,
-                         old_writer->PrepareCommit(/*wait_compaction=*/false,
-                                                   /*commit_identifier=*/3));
-    ASSERT_EQ(1, stale_messages.size());
-    std::shared_ptr<CommitMessageImpl> stale_message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(stale_messages[0]);
-    ASSERT_TRUE(stale_message);
-    ASSERT_EQ(1, stale_message->GetNewFilesIncrement().NewFiles().size());
-    ASSERT_EQ(0, stale_message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
-    ASSERT_OK(CommitMessages(table_path_, options, stale_messages, /*commit_identifier=*/3));
-
-    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
-                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
-    ASSERT_TRUE(snapshot);
-    ASSERT_EQ(1, snapshot->SchemaId());
-
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows, ReadRows(table_path_, options, pool_));
-    AssertResultEquals(rows.data, EvolvedFields(),
-                       R"([[0, 1, "old", null], [0, 2, "new", 20], [0, 3, "stale", null]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkOldWriterCanOverwriteNewColumnWithNull) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[1, "new", 20]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
-                         WriteWithNewWriter(table_path_, options, std::move(new_schema_batch),
-                                            /*commit_identifier=*/1));
-    ASSERT_OK(CommitMessages(table_path_, options, new_messages, /*commit_identifier=*/1));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> old_messages,
-                         old_writer->PrepareCommit(/*wait_compaction=*/false,
-                                                   /*commit_identifier=*/2));
-    ASSERT_OK(CommitMessages(table_path_, options, old_messages, /*commit_identifier=*/2));
-
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows, ReadRows(table_path_, options, pool_));
-    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkNewWriterIndexesNewSchemaColumn) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> old_messages,
-                         WriteWithNewWriter(table_path_, options, std::move(old_schema_batch),
-                                            /*commit_identifier=*/1));
-    ASSERT_OK(CommitMessages(table_path_, options, old_messages, /*commit_identifier=*/1));
-
-    std::map<std::string, std::string> options_v1 = options;
-    options_v1["file-index.bitmap.columns"] = "extra";
-    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options_v1));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[2, "new", 20], [3, "skip", 30]])",
-                                   /*partition=*/{}, /*bucket=*/0));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> new_messages,
-                         WriteWithNewWriter(table_path_, options_v1, std::move(new_schema_batch),
-                                            /*commit_identifier=*/2));
-    ASSERT_EQ(1, new_messages.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(new_messages[0]);
-    ASSERT_TRUE(message);
-    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
-    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
-    ASSERT_EQ(1, new_file->schema_id);
-    ASSERT_FALSE(new_file->embedded_index);
-    ASSERT_EQ(1, new_file->extra_files.size());
-    ASSERT_TRUE(new_file->extra_files[0]);
-    std::string index_path =
-        PathUtil::JoinPath(table_path_, "bucket-0/" + new_file->extra_files[0].value());
-    ASSERT_OK_AND_ASSIGN(bool index_exists, dir_->GetFileSystem()->Exists(index_path));
-    ASSERT_TRUE(index_exists);
-
-    ASSERT_OK(CommitMessages(table_path_, options_v1, new_messages, /*commit_identifier=*/2));
-    std::shared_ptr<Predicate> predicate = PredicateBuilder::Equal(
-        /*field_index=*/2, /*field_name=*/"extra", FieldType::INT, Literal(20));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
-                         ReadRows(table_path_, options_v1, pool_, /*realtime_context=*/nullptr,
-                                  predicate, /*enable_predicate_filter=*/true));
-    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 2, "new", 20]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkEmbeddedFileIndexFailsValueScan) {
-    std::map<std::string, std::string> options = BaseOptions();
-    options["file-index.bitmap.columns"] = "payload";
-    options[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch(BaseFields(), R"([[1, "a"], [2, "b"]])",
-                                   /*partition=*/{}, /*bucket=*/0));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
-                         WriteWithNewWriter(table_path_, options, std::move(batch),
-                                            /*commit_identifier=*/1));
-    ASSERT_EQ(1, messages.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
-    ASSERT_TRUE(message);
-    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
-    ASSERT_TRUE(message->GetNewFilesIncrement().NewFiles()[0]->embedded_index);
-    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
-
-    std::shared_ptr<Predicate> predicate =
-        PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
-                                Literal(FieldType::STRING, "a", 1));
-    ASSERT_NOK_WITH_MSG(
-        ScanTable(table_path_, options, pool_, /*realtime_context=*/nullptr, predicate),
-        "do not support embedded index in DataFileMeta");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, NonRealtimePkSortedIndexConfigDoesNotWriteIndexOnDataWrite) {
-    std::map<std::string, std::string> options = BaseOptions();
-    options[Options::PK_BTREE_INDEX_COLUMNS] = "payload";
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch(BaseFields(), R"([[1, "a"], [2, "b"]])",
-                                   /*partition=*/{}, /*bucket=*/0));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
-                         WriteWithNewWriter(table_path_, options, std::move(batch),
-                                            /*commit_identifier=*/1));
-    ASSERT_EQ(1, messages.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
-    ASSERT_TRUE(message);
-    ASSERT_TRUE(message->GetNewFilesIncrement().NewIndexFiles().empty());
-    ASSERT_TRUE(message->GetCompactIncrement().NewIndexFiles().empty());
-    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
-                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
-    ASSERT_TRUE(snapshot);
-    ASSERT_FALSE(snapshot->IndexManifest());
-
-    std::shared_ptr<Predicate> predicate =
-        PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
-                                Literal(FieldType::STRING, "a", 1));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
-                         ScanTable(table_path_, options, pool_,
-                                   /*realtime_context=*/nullptr, predicate));
-    ASSERT_EQ(0, CountIndexedSplits(plan));
-    std::vector<std::shared_ptr<DataFileMeta>> planned_files = DataFilesFromPlan(plan);
-    ASSERT_EQ(1, planned_files.size());
-    ASSERT_EQ(0, planned_files[0]->schema_id);
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
-                         ReadRows(table_path_, options, pool_, /*realtime_context=*/nullptr,
-                                  predicate, /*enable_predicate_filter=*/true));
-    AssertResultEquals(rows.data, BaseFields(), R"([[0, 1, "a"]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendRejectsDataEvolutionAtWriterCreation) {
-    std::map<std::string, std::string> create_options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, create_options));
-
-    std::map<std::string, std::string> write_options = BaseOptions();
-    write_options[Options::DATA_EVOLUTION_ENABLED] = "true";
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_NOK_WITH_MSG(CreateWriter(table_path_, write_options, realtime_context),
-                        "real-time append write does not support data evolution");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendScanRejectsDataEvolutionTable) {
-    std::map<std::string, std::string> options = DataEvolutionOptions();
-    arrow::FieldVector fields = DataEvolutionFields();
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(fields), /*primary_keys=*/{}, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeUnbucketedBatch(fields, R"([[1, "old", "base"]])",
-                                             /*partition=*/{}));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
-                         WriteWithNewWriter(table_path_, options, std::move(batch),
-                                            /*commit_identifier=*/1,
-                                            /*write_schema=*/{"f0", "f1", "f2"}));
-    ASSERT_OK(CommitMessages(table_path_, options, messages, /*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-
-    ASSERT_NOK_WITH_MSG(ScanTable(table_path_, options, pool_, realtime_context),
-                        "real-time union read requires fixed bucket mode");
-
-    std::map<std::string, std::string> fixed_bucket_options = options;
-    fixed_bucket_options[Options::BUCKET] = "1";
-    ASSERT_NOK_WITH_MSG(ScanTable(table_path_, fixed_bucket_options, pool_, realtime_context),
-                        "real-time union read does not support data evolution");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendReuseContextKeepsOldMemorySchemaAfterAlter) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options, realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(base_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
-                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
-                         CommitRealtimeMessages(table_path_, options, base_progress,
-                                                /*commit_identifier=*/1));
-    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> reused_context_writer,
-                         CreateWriter(table_path_, options, realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_NOK_WITH_MSG(reused_context_writer->Write(std::move(new_schema_batch)),
-                        "ArrowArray struct has 3 children, expected 2");
-
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
-                         ReadRows(table_path_, options, pool_, realtime_context));
-    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendOldWriterAfterAlterCommitsOldSchemaFile) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options, realtime_context));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
-                         ReadRows(table_path_, options, pool_, realtime_context));
-    AssertResultEquals(memory_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
-                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_EQ(1, progress.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
-    ASSERT_TRUE(message);
-    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
-    ASSERT_EQ(0, message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
-
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, CommitRealtimeMessages(table_path_, options, progress,
-                                                                     /*commit_identifier=*/1));
-    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
-    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> snapshot,
-                         LatestSnapshot(table_path_, options, dir_->GetFileSystem()));
-    ASSERT_TRUE(snapshot);
-    ASSERT_EQ(1, snapshot->SchemaId());
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult disk_rows, ReadRows(table_path_, options, pool_));
-    AssertResultEquals(disk_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimeAppendNewContextUsesNewSchemaAfterAlter) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{}, options));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> old_realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options, old_realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(base_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
-                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
-                         CommitRealtimeMessages(table_path_, options, base_progress,
-                                                /*commit_identifier=*/1));
-    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    std::map<std::string, std::string> options_v1 = options;
-    options_v1["file-index.bitmap.columns"] = "extra";
-    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> new_realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
-                         CreateWriter(table_path_, options_v1, new_realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
-                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
-    AssertResultEquals(memory_rows.data, EvolvedFields(),
-                       R"([[0, 1, "old", null], [0, 2, "new", 20]])");
-
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> new_progress,
-                         new_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
-    ASSERT_EQ(1, new_progress.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(new_progress[0].commit_message);
-    ASSERT_TRUE(message);
-    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
-    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
-    ASSERT_EQ(1, new_file->schema_id);
-    ASSERT_FALSE(new_file->embedded_index);
-    ASSERT_EQ(1, new_file->extra_files.size());
-    ASSERT_TRUE(new_file->extra_files[0]);
-    std::string index_path =
-        PathUtil::JoinPath(table_path_, "bucket-0/" + new_file->extra_files[0].value());
-    ASSERT_OK_AND_ASSIGN(bool index_exists, dir_->GetFileSystem()->Exists(index_path));
-    ASSERT_TRUE(index_exists);
-
-    ASSERT_OK_AND_ASSIGN(snapshot_id, CommitRealtimeMessages(table_path_, options_v1, new_progress,
-                                                             /*commit_identifier=*/2));
-    ASSERT_OK(new_writer->RefreshCommittedSnapshot(snapshot_id));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
-                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
-    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null], [0, 2, "new", 20]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkRejectsDataEvolutionAtWriterCreation) {
-    std::map<std::string, std::string> create_options = BaseOptions();
-    ASSERT_OK(CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"},
-                          create_options));
-
-    std::map<std::string, std::string> write_options = BaseOptions();
-    write_options[Options::DATA_EVOLUTION_ENABLED] = "true";
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_NOK_WITH_MSG(CreateWriter(table_path_, write_options, realtime_context),
-                        "PK realtime v1 does not support data evolution");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkReuseContextKeepsOldMemorySchemaAfterAlter) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options, realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(base_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
-                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
-                         CommitRealtimeMessages(table_path_, options, base_progress,
-                                                /*commit_identifier=*/1));
-    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> reused_context_writer,
-                         CreateWriter(table_path_, options, realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_NOK_WITH_MSG(reused_context_writer->Write(std::move(new_schema_batch)),
-                        "ArrowArray struct has 3 children, expected 2");
-
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
-                         ReadRows(table_path_, options, pool_, realtime_context));
-    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkOldWriterAfterAlterReadsNewColumnAsNull) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options, realtime_context));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
-                         ReadRows(table_path_, options, pool_, realtime_context));
-    AssertResultEquals(memory_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
-                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_EQ(1, progress.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
-    ASSERT_TRUE(message);
-    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
-    ASSERT_EQ(0, message->GetNewFilesIncrement().NewFiles()[0]->schema_id);
-
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, CommitRealtimeMessages(table_path_, options, progress,
-                                                                     /*commit_identifier=*/1));
-    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult disk_rows, ReadRows(table_path_, options, pool_));
-    AssertResultEquals(disk_rows.data, EvolvedFields(), R"([[0, 1, "old", null]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest,
-       RealtimePkOldWriterAfterAlterCannotCommitBehindNewContextOffset) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options, realtime_context));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> new_realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
-                         CreateWriter(table_path_, options, new_realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[1, "new", 20]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> new_progress,
-                         new_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
-                         CommitRealtimeMessages(table_path_, options, new_progress,
-                                                /*commit_identifier=*/1));
-    ASSERT_OK(new_writer->RefreshCommittedSnapshot(snapshot_id));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> old_schema_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(old_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult realtime_rows,
-                         ReadRows(table_path_, options, pool_, realtime_context));
-    AssertResultEquals(realtime_rows.data, EvolvedFields(), R"([[0, 1, "new", 20]])");
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> old_progress,
-                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
-    ASSERT_NOK_WITH_MSG(CommitRealtimeMessages(table_path_, options, old_progress,
-                                               /*commit_identifier=*/2),
-                        "real-time commit offsets for bucket 0 are not contiguous");
-
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult disk_rows, ReadRows(table_path_, options, pool_));
-    AssertResultEquals(disk_rows.data, EvolvedFields(), R"([[0, 1, "new", 20]])");
-}
-
-TEST_F(SchemaEvolutionWriteVerifyTest, RealtimePkNewContextUsesNewSchemaAfterAlter) {
-    std::map<std::string, std::string> options = BaseOptions();
-    ASSERT_OK(
-        CreateTable(dir_->Str(), arrow::schema(BaseFields()), /*primary_keys=*/{"id"}, options));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> old_realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> old_writer,
-                         CreateWriter(table_path_, options, old_realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
-                         MakeBatch(BaseFields(), R"([[1, "old"]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(old_writer->Write(std::move(base_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
-                         old_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
-                         CommitRealtimeMessages(table_path_, options, base_progress,
-                                                /*commit_identifier=*/1));
-    ASSERT_OK(old_writer->RefreshCommittedSnapshot(snapshot_id));
-
-    ASSERT_OK(EvolveSchema(table_path_, dir_->GetFileSystem(),
-                           {DataField(0, BaseFields()[0]), DataField(1, BaseFields()[1]),
-                            DataField(2, EvolvedFields()[2])},
-                           /*highest_field_id=*/2, options));
-
-    std::map<std::string, std::string> options_v1 = options;
-    options_v1["file-index.bitmap.columns"] = "extra";
-    options_v1[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> new_realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> new_writer,
-                         CreateWriter(table_path_, options_v1, new_realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> new_schema_batch,
-                         MakeBatch(EvolvedFields(), R"([[2, "new", 20]])", /*partition=*/{},
-                                   /*bucket=*/0));
-    ASSERT_OK(new_writer->Write(std::move(new_schema_batch)));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult memory_rows,
-                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
-    AssertResultEquals(memory_rows.data, EvolvedFields(),
-                       R"([[0, 1, "old", null], [0, 2, "new", 20]])");
-
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> new_progress,
-                         new_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
-    ASSERT_EQ(1, new_progress.size());
-    std::shared_ptr<CommitMessageImpl> message =
-        std::dynamic_pointer_cast<CommitMessageImpl>(new_progress[0].commit_message);
-    ASSERT_TRUE(message);
-    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
-    std::shared_ptr<DataFileMeta> new_file = message->GetNewFilesIncrement().NewFiles()[0];
-    ASSERT_EQ(1, new_file->schema_id);
-    ASSERT_FALSE(new_file->embedded_index);
-    ASSERT_EQ(1, new_file->extra_files.size());
-    ASSERT_TRUE(new_file->extra_files[0]);
-    std::string index_path =
-        PathUtil::JoinPath(table_path_, "bucket-0/" + new_file->extra_files[0].value());
-    ASSERT_OK_AND_ASSIGN(bool index_exists, dir_->GetFileSystem()->Exists(index_path));
-    ASSERT_TRUE(index_exists);
-
-    ASSERT_OK_AND_ASSIGN(snapshot_id, CommitRealtimeMessages(table_path_, options_v1, new_progress,
-                                                             /*commit_identifier=*/2));
-    ASSERT_OK(new_writer->RefreshCommittedSnapshot(snapshot_id));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult rows,
-                         ReadRows(table_path_, options_v1, pool_, new_realtime_context));
-    AssertResultEquals(rows.data, EvolvedFields(), R"([[0, 1, "old", null], [0, 2, "new", 20]])");
-}
-
-}  // namespace
-}  // namespace paimon::test

From 2f7d228ab5d1521486e13c2bfed7afaeb9bafc9c Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 11:21:45 +0800
Subject: [PATCH 35/93] refactor(mergetree): accept sorted key-value readers

---
 .../core/mergetree/merge_tree_writer.cpp      |  95 ++++---
 src/paimon/core/mergetree/merge_tree_writer.h |   3 +
 .../core/mergetree/merge_tree_writer_test.cpp | 236 ++++++++++++++++++
 3 files changed, 293 insertions(+), 41 deletions(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp
index 3b6806c73..49961536a 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer.cpp
@@ -154,6 +154,59 @@ Status MergeTreeWriter::Write(std::unique_ptr<RecordBatch>&& moved_batch) {
     return Status::OK();
 }
 
+Status MergeTreeWriter::WriteSortedReaders(
+    std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers) {
+    auto raw_readers_guard = ScopeGuard([&]() -> void {
+        for (std::unique_ptr<KeyValueRecordReader>& reader : readers) {
+            if (reader != nullptr) {
+                reader->Close();
+            }
+        }
+    });
+    if (readers.empty()) {
+        return Status::Invalid("sorted readers must not be empty");
+    }
+    for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
+        if (reader == nullptr) {
+            return Status::Invalid("sorted readers must not contain null reader");
+        }
+    }
+
+    auto sort_merge_reader = std::make_unique<SortMergeReaderWithLoserTree>(
+        std::move(readers), key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_);
+    auto create_consumer = [target_schema = write_schema_, pool = pool_]()
+        -> Result<std::unique_ptr<RowToArrowArrayConverter<KeyValue, KeyValueBatch>>> {
+        return KeyValueMetaProjectionConsumer::Create(target_schema, pool);
+    };
+    auto async_key_value_producer_consumer =
+        std::make_unique<AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>>(
+            std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(), 1, pool_);
+    raw_readers_guard.Release();
+    ScopeGuard async_readers_guard([&]() -> void { async_key_value_producer_consumer->Close(); });
+    std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>> rolling_writer;
+    PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter());
+    ScopeGuard abort_writer_guard([&]() -> void { rolling_writer->Abort(); });
+    while (true) {
+        PAIMON_ASSIGN_OR_RAISE(KeyValueBatch key_value_batch,
+                               async_key_value_producer_consumer->NextBatch());
+        if (key_value_batch.batch == nullptr) {
+            break;
+        }
+        PAIMON_RETURN_NOT_OK(rolling_writer->Write(std::move(key_value_batch)));
+    }
+    PAIMON_RETURN_NOT_OK(rolling_writer->Close());
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<DataFileMeta>> flushed_files,
+                           rolling_writer->GetResult());
+    abort_writer_guard.Release();
+
+    for (const std::shared_ptr<DataFileMeta>& flushed_file : flushed_files) {
+        new_files_.emplace_back(flushed_file);
+        PAIMON_RETURN_NOT_OK(compact_manager_->AddNewFile(flushed_file));
+    }
+    metrics_->Merge(rolling_writer->GetMetrics());
+    return Status::OK();
+}
+
 Status MergeTreeWriter::Compact(bool full_compaction) {
     return FlushWriteBuffer(/*wait_for_latest_compaction=*/true, full_compaction);
 }
@@ -256,49 +309,9 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction,
             wait_for_latest_compaction = true;
         }
         auto cleanup_guard = ScopeGuard([&]() { write_buffer_->Clear(); });
-        // 1. flush write buffer to get sorted readers
         PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
                                write_buffer_->CreateReaders());
-        // 2. prepare loser tree sort merge reader
-        auto sort_merge_reader = std::make_unique<SortMergeReaderWithLoserTree>(
-            std::move(readers), key_comparator_, user_defined_seq_comparator_,
-            merge_function_wrapper_);
-        // 3. project key value to arrow array
-        auto create_consumer = [target_schema = write_schema_, pool = pool_]()
-            -> Result<std::unique_ptr<RowToArrowArrayConverter<KeyValue, KeyValueBatch>>> {
-            return KeyValueMetaProjectionConsumer::Create(target_schema, pool);
-        };
-        // consumer batch size is WriteBatchSize
-        auto async_key_value_producer_consumer =
-            std::make_unique<AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>>(
-                std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(),
-                /*projection_thread_num=*/1, pool_);
-        std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>
-            rolling_writer;
-        PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter());
-        ScopeGuard write_guard([&]() -> void {
-            rolling_writer->Abort();
-            async_key_value_producer_consumer->Close();
-        });
-        while (true) {
-            PAIMON_ASSIGN_OR_RAISE(KeyValueBatch key_value_batch,
-                                   async_key_value_producer_consumer->NextBatch());
-            if (key_value_batch.batch == nullptr) {
-                break;
-            }
-            PAIMON_RETURN_NOT_OK(rolling_writer->Write(std::move(key_value_batch)));
-        }
-        PAIMON_RETURN_NOT_OK(rolling_writer->Close());
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<DataFileMeta>> flushed_files,
-                               rolling_writer->GetResult());
-        async_key_value_producer_consumer->Close();
-        write_guard.Release();
-
-        for (const auto& flushed_file : flushed_files) {
-            new_files_.emplace_back(flushed_file);
-            PAIMON_RETURN_NOT_OK(compact_manager_->AddNewFile(flushed_file));
-        }
-        metrics_->Merge(rolling_writer->GetMetrics());
+        PAIMON_RETURN_NOT_OK(WriteSortedReaders(std::move(readers)));
     }
     PAIMON_RETURN_NOT_OK(TrySyncLatestCompaction(wait_for_latest_compaction));
     PAIMON_RETURN_NOT_OK(compact_manager_->TriggerCompaction(forced_full_compaction));
diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h
index febce2afb..542affd81 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.h
+++ b/src/paimon/core/mergetree/merge_tree_writer.h
@@ -51,6 +51,7 @@ class IOManager;
 class FieldsComparator;
 class MemoryPool;
 class Metrics;
+class KeyValueRecordReader;
 template <typename T>
 class MergeFunctionWrapper;
 
@@ -69,6 +70,8 @@ class MergeTreeWriter : public BatchWriter {
 
     Status Write(std::unique_ptr<RecordBatch>&& batch) override;
 
+    Status WriteSortedReaders(std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers);
+
     Status Compact(bool full_compaction) override;
 
     Result<bool> CompactNotCompleted() override;
diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 2155647a1..675ce3198 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -24,6 +24,7 @@
 #include <map>
 #include <optional>
 #include <utility>
+#include <vector>
 
 #include "arrow/api.h"
 #include "arrow/array/array_base.h"
@@ -42,6 +43,7 @@
 #include "paimon/core/io/compact_increment.h"
 #include "paimon/core/io/data_file_path_factory.h"
 #include "paimon/core/io/data_increment.h"
+#include "paimon/core/io/key_value_record_reader.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
@@ -52,6 +54,8 @@
 #include "paimon/fs/file_system.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/testing/mock/mock_file_batch_reader.h"
+#include "paimon/testing/mock/mock_key_value_data_file_record_reader.h"
 #include "paimon/testing/utils/binary_row_generator.h"
 #include "paimon/testing/utils/io_exception_helper.h"
 #include "paimon/testing/utils/read_result_collector.h"
@@ -64,6 +68,60 @@ class MergeFunctionWrapper;
 }  // namespace paimon
 
 namespace paimon::test {
+namespace {
+
+class TrackingKeyValueRecordReader : public KeyValueRecordReader {
+ public:
+    TrackingKeyValueRecordReader(std::unique_ptr<KeyValueRecordReader>&& inner_reader,
+                                 bool* closed_flag)
+        : inner_reader_(std::move(inner_reader)), closed_flag_(closed_flag) {}
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
+        return inner_reader_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return inner_reader_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        if (closed_flag_ != nullptr) {
+            *closed_flag_ = true;
+        }
+        inner_reader_->Close();
+    }
+
+ private:
+    std::unique_ptr<KeyValueRecordReader> inner_reader_;
+    bool* closed_flag_;
+};
+
+class ErrorKeyValueRecordReader : public KeyValueRecordReader {
+ public:
+    ErrorKeyValueRecordReader(Status status, bool* closed_flag)
+        : status_(std::move(status)), closed_flag_(closed_flag) {}
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
+        return status_;
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return nullptr;
+    }
+
+    void Close() override {
+        if (closed_flag_ != nullptr) {
+            *closed_flag_ = true;
+        }
+    }
+
+ private:
+    Status status_;
+    bool* closed_flag_;
+};
+
+}
+
 class MergeTreeWriterTest : public ::testing::TestWithParam<bool> {
  public:
     class FakeCompactManager : public paimon::CompactManager {
@@ -211,6 +269,21 @@ class MergeTreeWriterTest : public ::testing::TestWithParam<bool> {
             writer_compact_manager, io_manager, /*enable_multi_thread_spill=*/false, pool_);
     }
 
+    std::unique_ptr<KeyValueRecordReader> CreateSingleReader(
+        const std::shared_ptr<arrow::Array>& array, int32_t batch_size = 16) const {
+        std::vector<DataField> write_fields = {SpecialFields::SequenceNumber(),
+                                               SpecialFields::ValueKind()};
+        write_fields.insert(write_fields.end(), value_fields_.begin(), value_fields_.end());
+        std::shared_ptr<arrow::Schema> write_schema =
+            DataField::ConvertDataFieldsToArrowSchema(write_fields);
+        std::shared_ptr<arrow::Schema> key_schema =
+            arrow::schema(arrow::FieldVector({write_schema->field(2)}));
+        auto file_batch_reader =
+            std::make_unique<MockFileBatchReader>(array, array->type(), batch_size);
+        return std::make_unique<MockKeyValueDataFileRecordReader>(
+            std::move(file_batch_reader), key_schema, value_schema_, 0, pool_);
+    }
+
  private:
     std::shared_ptr<MemoryPool> pool_;
     std::shared_ptr<FileSystem> file_system_;
@@ -293,6 +366,29 @@ TEST_P(MergeTreeWriterTest, TestSimple) {
     DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{},
                                           /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
+
+    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
+      [2, 0, "Alice", 10, 0, 13.1],
+      [0, 0, "Lucy", 20, 1, 14.1],
+      [1, 0, "Paul", 20, 1, null]
+    ])")
+            .ValueOrDie());
+    auto sorted_reader_path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(sorted_reader_path_factory->Init(dir->Str() + "/sorted-readers", "orc",
+                                               options.DataFilePrefix(), nullptr));
+    ASSERT_OK_AND_ASSIGN(auto sorted_reader_writer,
+                         CreateMergeWriter(-1, dir->Str(), sorted_reader_path_factory, 1, options));
+    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
+    sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
+    ASSERT_OK(sorted_reader_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK_AND_ASSIGN(CommitIncrement sorted_reader_commit_increment,
+                         sorted_reader_writer->PrepareCommit(false));
+    ASSERT_OK(sorted_reader_writer->Close());
+    ASSERT_EQ(1, sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles().size());
+    std::string sorted_reader_path = sorted_reader_path_factory->ToPath(
+        sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles()[0]);
+    CheckFileContent(sorted_reader_path, expected_array);
 }
 
 TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) {
@@ -374,6 +470,146 @@ TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) {
     DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{},
                                           /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
+
+    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
+      [16, 0, "Alice", 10, 0, 113.1],
+      [14, 0, "Lucy", 20, 1, 114.1],
+      [13, 0, "Paul", 20, 1, 15.1],
+      [15, 0, "Skye", 10, 0, 118.1]
+    ])")
+            .ValueOrDie());
+    auto sorted_reader_path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(sorted_reader_path_factory->Init(dir->Str() + "/sorted-readers", "orc",
+                                               options.DataFilePrefix(), nullptr));
+    ASSERT_OK_AND_ASSIGN(auto sorted_reader_writer,
+                         CreateMergeWriter(9, dir->Str(), sorted_reader_path_factory, 0, options));
+    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
+    sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
+    ASSERT_OK(sorted_reader_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK_AND_ASSIGN(CommitIncrement sorted_reader_commit_increment,
+                         sorted_reader_writer->PrepareCommit(false));
+    ASSERT_OK(sorted_reader_writer->Close());
+    ASSERT_EQ(1, sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles().size());
+    std::string sorted_reader_path = sorted_reader_path_factory->ToPath(
+        sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles()[0]);
+    CheckFileContent(sorted_reader_path, expected_array);
+}
+
+TEST_P(MergeTreeWriterTest, TestSortedReaders) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+    std::string uuid = path_factory->uuid_;
+
+    ASSERT_OK_AND_ASSIGN(auto merge_writer,
+                         CreateMergeWriter(4, dir->Str(), path_factory, 7, options));
+
+    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
+      [7, 0, "Alice", 20, 1, 17.1],
+      [9, 0, "Lucy", 30, 2, 19.1],
+      [8, 3, "Paul", 10, 3, null]
+    ])")
+            .ValueOrDie());
+
+    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
+    sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
+
+    ASSERT_OK(merge_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, merge_writer->PrepareCommit(false));
+    ASSERT_OK(merge_writer->Close());
+
+    std::string expected_data_file_name = "data-" + uuid + "-0.orc";
+    std::string expected_data_file_path = dir->Str() + "/" + expected_data_file_name;
+    ASSERT_OK_AND_ASSIGN(FileStatus data_file_status,
+                         options.GetFileSystem()->GetFileStatus(expected_data_file_path));
+
+    std::shared_ptr<arrow::ChunkedArray> expected_array;
+    ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(write_type_, {R"([
+      [7, 0, "Alice", 20, 1, 17.1],
+      [9, 0, "Lucy", 30, 2, 19.1],
+      [8, 3, "Paul", 10, 3, null]
+    ])"},
+                                                                 &expected_array)
+                    .ok());
+    CheckFileContent(expected_data_file_path, expected_array);
+
+    ASSERT_TRUE(commit_increment.GetCompactIncrement().IsEmpty());
+    ASSERT_EQ(1, commit_increment.GetNewFilesIncrement().NewFiles().size());
+    const std::shared_ptr<DataFileMeta>& new_file =
+        commit_increment.GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(expected_data_file_name, new_file->file_name);
+    ASSERT_EQ(data_file_status.GetLen(), new_file->file_size);
+    ASSERT_EQ(3, new_file->row_count);
+    ASSERT_EQ(7, new_file->min_sequence_number);
+    ASSERT_EQ(9, new_file->max_sequence_number);
+    ASSERT_EQ(7, new_file->schema_id);
+    ASSERT_EQ(1, new_file->delete_row_count);
+}
+
+TEST_P(MergeTreeWriterTest, TestSortedReaderOwnership) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+
+    ASSERT_OK_AND_ASSIGN(auto merge_writer,
+                         CreateMergeWriter(-1, dir->Str(), path_factory, 0, options));
+
+    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
+      [0, 0, "Alice", 10, 0, 13.1]
+    ])")
+            .ValueOrDie());
+
+    bool closed = false;
+    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
+    sorted_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
+        CreateSingleReader(sorted_reader_array), &closed));
+
+    ASSERT_OK(merge_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_TRUE(closed);
+    ASSERT_OK(merge_writer->Close());
+}
+
+TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+
+    ASSERT_OK_AND_ASSIGN(auto merge_writer,
+                         CreateMergeWriter(-1, dir->Str(), path_factory, 0, options));
+
+    std::vector<std::unique_ptr<KeyValueRecordReader>> empty_readers;
+    Status empty_status = merge_writer->WriteSortedReaders(std::move(empty_readers));
+    ASSERT_TRUE(empty_status.IsInvalid());
+
+    std::vector<std::unique_ptr<KeyValueRecordReader>> null_readers;
+    null_readers.push_back(nullptr);
+    Status null_status = merge_writer->WriteSortedReaders(std::move(null_readers));
+    ASSERT_TRUE(null_status.IsInvalid());
+
+    bool failing_reader_closed = false;
+    auto failing_reader = std::make_unique<ErrorKeyValueRecordReader>(
+        Status::IOError("sorted reader failure"), &failing_reader_closed);
+    std::vector<std::unique_ptr<KeyValueRecordReader>> failing_readers;
+    failing_readers.push_back(std::move(failing_reader));
+    Status failing_status = merge_writer->WriteSortedReaders(std::move(failing_readers));
+    ASSERT_TRUE(failing_status.IsIOError());
+    ASSERT_TRUE(failing_reader_closed);
+    ASSERT_OK(merge_writer->Close());
 }
 
 TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) {

From 3df2037efac097e0069156df0e793cb70e68897e Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 11:22:00 +0800
Subject: [PATCH 36/93] feat(realtime): adapt prepared primary-key batches

---
 src/paimon/CMakeLists.txt                     |   1 +
 .../merged_key_value_record_reader_test.cpp   | 398 ++++++++++++
 .../core/io/prepared_key_value_reader.cpp     | 565 ++++++++++++++++++
 .../core/io/prepared_key_value_reader.h       |  41 ++
 src/paimon/core/realtime/realtime_fields.h    |  37 ++
 .../core/schema/schema_validation_test.cpp    |   7 +
 6 files changed, 1049 insertions(+)
 create mode 100644 src/paimon/core/io/prepared_key_value_reader.cpp
 create mode 100644 src/paimon/core/io/prepared_key_value_reader.h
 create mode 100644 src/paimon/core/realtime/realtime_fields.h

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index d2c0a2b4f..62af55ec4 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -286,6 +286,7 @@ set(PAIMON_CORE_SRCS
     core/io/vector_file_batch_reader.cpp
     core/io/file_index_evaluator.cpp
     core/io/key_value_data_file_record_reader.cpp
+    core/io/prepared_key_value_reader.cpp
     core/io/key_value_data_file_writer_factory.cpp
     core/io/key_value_data_file_writer.cpp
     core/io/key_value_in_memory_record_reader.cpp
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 1b6b71c69..39714fa29 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -18,8 +18,13 @@
 
 #include "paimon/core/io/merged_key_value_record_reader.h"
 
+#include <cstdint>
+#include <limits>
 #include <memory>
+#include <optional>
+#include <string>
 #include <utility>
+#include <vector>
 
 #include "arrow/api.h"
 #include "arrow/array/array_nested.h"
@@ -27,10 +32,14 @@
 #include "gtest/gtest.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/fields_comparator.h"
+#include "paimon/core/io/prepared_key_value_reader.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
+#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/realtime/offset_range.h"
 #include "paimon/testing/mock/mock_file_batch_reader.h"
 #include "paimon/testing/mock/mock_key_value_data_file_record_reader.h"
 #include "paimon/testing/utils/key_value_checker.h"
@@ -38,6 +47,56 @@
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
+
+namespace {
+
+std::shared_ptr<arrow::Field> MakeField(const std::string& name,
+                                        const std::shared_ptr<arrow::DataType>& type,
+                                        int32_t field_id, bool nullable = true) {
+    return DataField::ConvertDataFieldToArrowField(
+        DataField(field_id, arrow::field(name, type, nullable)));
+}
+
+std::shared_ptr<arrow::Schema> MakePreparedSchema(const arrow::FieldVector& value_fields) {
+    arrow::FieldVector prepared_fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+            ->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+    prepared_fields.insert(prepared_fields.end(), value_fields.begin(), value_fields.end());
+    return arrow::schema(prepared_fields);
+}
+
+class TrackingBatchReader : public BatchReader {
+ public:
+    TrackingBatchReader(std::unique_ptr<BatchReader>&& delegate, int32_t* close_count)
+        : delegate_(std::move(delegate)), close_count_(close_count) {}
+
+    Result<ReadBatch> NextBatch() override {
+        return delegate_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        if (closed_) {
+            return;
+        }
+        closed_ = true;
+        ++(*close_count_);
+        delegate_->Close();
+    }
+
+ private:
+    bool closed_ = false;
+    std::unique_ptr<BatchReader> delegate_;
+    int32_t* close_count_;
+};
+
+}
+
 class MergedKeyValueRecordReaderTest : public testing::Test {
  public:
     void SetUp() override {
@@ -51,6 +110,14 @@ class MergedKeyValueRecordReaderTest : public testing::Test {
     std::shared_ptr<ReducerMergeFunctionWrapper> merge_function_wrapper_;
 };
 
+TEST_F(MergedKeyValueRecordReaderTest, TestRealtimeOffsetField) {
+    const DataField& field = RealtimeOffsetField();
+    ASSERT_EQ(std::numeric_limits<int32_t>::max() - 10002, field.Id());
+    ASSERT_EQ("_REALTIME_OFFSET", field.Name());
+    ASSERT_EQ(arrow::Type::INT64, field.Type()->id());
+    ASSERT_FALSE(field.Nullable());
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestMergeAcrossUnderlyingBatches) {
     std::vector<DataField> fields = {DataField(0, arrow::field("k0", arrow::int32())),
                                      DataField(1, arrow::field("k1", arrow::int32())),
@@ -143,4 +210,335 @@ TEST_F(MergedKeyValueRecordReaderTest, TestSkipMergedNulloptResultInHasNext) {
     }
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderOffsetFilter) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
+                                           DataField(1, arrow::field("v0", arrow::int32()))};
+    std::shared_ptr<arrow::Schema> value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+        [0, 100, 0, 1, 10],
+        [0, 101, 1, 2, 20],
+        [0, 102, 4, 3, 30],
+        [0, 103, 2, 4, 40],
+        [0, 104, 5, 5, 50],
+        [0, 105, 3, 6, 60]
+    ])")
+            .ValueOrDie());
+
+    auto batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(2, 4),
+                                 key_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> results,
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())));
+
+    std::vector<RowKind*> row_kinds = {const_cast<RowKind*>(RowKind::Insert()),
+                                       const_cast<RowKind*>(RowKind::Insert())};
+    std::vector<int64_t> levels = {KeyValue::UNKNOWN_LEVEL, KeyValue::UNKNOWN_LEVEL};
+    std::vector<KeyValue> expected = KeyValueChecker::GenerateKeyValues(
+        row_kinds, {103, 105}, levels, {{4}, {6}}, {{4, 40}, {6, 60}}, pool_);
+    KeyValueChecker::CheckResult(expected, results, 1, 2);
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRawCountBeforeDedup) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
+                                           DataField(1, arrow::field("v0", arrow::int32()))};
+    std::shared_ptr<arrow::Schema> value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+        [0, 10, 0, 1, 100],
+        [2, 11, 1, 1, 101],
+        [0, 12, 2, 2, 200]
+    ])")
+            .ValueOrDie());
+
+    int64_t raw_row_count = 0;
+    auto batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> raw_reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, std::nullopt, key_schema,
+                                 value_schema, pool_, &raw_row_count));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FieldsComparator> key_comparator,
+                         FieldsComparator::Create({value_fields[0]}, true));
+    auto merged_reader = std::make_unique<MergedKeyValueRecordReader>(
+        std::move(raw_reader), key_comparator, merge_function_wrapper_);
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> results,
+        (ReadResultCollector::CollectKeyValueResult<
+            MergedKeyValueRecordReader, KeyValueRecordReader::Iterator>(merged_reader.get())));
+
+    ASSERT_EQ(raw_row_count, 3);
+    std::vector<RowKind*> row_kinds = {const_cast<RowKind*>(RowKind::UpdateAfter()),
+                                       const_cast<RowKind*>(RowKind::Insert())};
+    std::vector<int64_t> levels = {KeyValue::UNKNOWN_LEVEL, KeyValue::UNKNOWN_LEVEL};
+    std::vector<KeyValue> expected = KeyValueChecker::GenerateKeyValues(
+        row_kinds, {11, 12}, levels, {{1}, {2}}, {{1, 101}, {2, 200}}, pool_);
+    KeyValueChecker::CheckResult(expected, results, 1, 2);
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRawCountBeforeFilter) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32()))};
+    std::shared_ptr<arrow::Schema> value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+        [0, 10, 0, 1],
+        [0, 11, 1, 2],
+        [0, 12, 2, 3],
+        [0, 13, 3, 4]
+    ])")
+            .ValueOrDie());
+
+    int64_t raw_row_count = 0;
+    auto batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(2, 4),
+                                 value_schema, value_schema, pool_, &raw_row_count));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> results,
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())));
+
+    ASSERT_EQ(results.size(), 2);
+    ASSERT_EQ(raw_row_count, 4);
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, extra});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1, 2]])")
+            .ValueOrDie());
+
+    auto query_batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> query_reader,
+        AdaptPreparedBatchReader(std::move(query_batch_reader), prepared_schema, OffsetRange(0, 1),
+                                 value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> query_results,
+        (ReadResultCollector::CollectKeyValueResult<
+            KeyValueRecordReader, KeyValueRecordReader::Iterator>(query_reader.get())));
+    ASSERT_EQ(query_results.size(), 1);
+    ASSERT_EQ(query_results[0].value->GetFieldCount(), 1);
+    ASSERT_EQ(query_results[0].value->GetInt(0), 1);
+
+    auto batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    ASSERT_NOK_WITH_MSG(AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema,
+                                                 std::nullopt, value_schema, value_schema, pool_),
+                        "exact");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+
+    arrow::FieldVector invalid_fields = prepared_schema->fields();
+    invalid_fields[0] = invalid_fields[0]->WithName("wrong_value_kind");
+    invalid_fields[3] = MakeField("wrong_key", arrow::int32(), 99);
+    std::shared_ptr<arrow::DataType> invalid_type = arrow::struct_(invalid_fields);
+    auto invalid_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(invalid_type, R"([[0, 10, 0, 1]])").ValueOrDie());
+
+    auto batch_reader =
+        std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                 value_schema, value_schema, pool_));
+    ASSERT_NOK_WITH_MSG(
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())),
+        "prepared batch field");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedProjection) {
+    std::shared_ptr<arrow::Field> id = MakeField("id", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> item_a = MakeField("a", arrow::int32(), 10);
+    std::shared_ptr<arrow::Field> item_b = MakeField("b", arrow::int32(), 11);
+    std::shared_ptr<arrow::Field> items =
+        MakeField("items", arrow::list(arrow::field("item", arrow::struct_({item_a, item_b}))), 2);
+    std::shared_ptr<arrow::Field> attr_x = MakeField("x", arrow::int32(), 20);
+    std::shared_ptr<arrow::Field> attr_y = MakeField("y", arrow::int32(), 21);
+    std::shared_ptr<arrow::Field> attrs =
+        MakeField("attrs", arrow::map(arrow::utf8(), arrow::struct_({attr_x, attr_y})), 3);
+    std::shared_ptr<arrow::Field> key_left = MakeField("left", arrow::int32(), 30);
+    std::shared_ptr<arrow::Field> key_right = MakeField("right", arrow::int32(), 31);
+    std::shared_ptr<arrow::Field> keyed_values = MakeField(
+        "keyed_values", arrow::map(arrow::struct_({key_left, key_right}), arrow::int32()), 4);
+    std::shared_ptr<arrow::Schema> full_value_schema =
+        arrow::schema({id, items, attrs, keyed_values});
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({id});
+    std::shared_ptr<arrow::Schema> prepared_schema =
+        MakePreparedSchema(full_value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+        [0, 10, 0, 1, [[100, 200], [300, 400]], [["k1", [7, 8]], ["k2", [9, 10]]], [[[11, 12], 13], [[21, 22], 23]]]
+    ])")
+            .ValueOrDie());
+
+    std::shared_ptr<arrow::Field> query_item_b = MakeField("renamed_b", arrow::int32(), 11);
+    std::shared_ptr<arrow::Field> query_item_a = MakeField("renamed_a", arrow::int32(), 10);
+    std::shared_ptr<arrow::Field> query_items = MakeField(
+        "items_renamed",
+        arrow::list(arrow::field("element", arrow::struct_({query_item_b, query_item_a}))), 2);
+    std::shared_ptr<arrow::Field> query_attr_y = MakeField("renamed_y", arrow::int32(), 21);
+    std::shared_ptr<arrow::Field> query_attr_x = MakeField("renamed_x", arrow::int32(), 20);
+    std::shared_ptr<arrow::Field> query_attrs =
+        MakeField("attrs_renamed",
+                  arrow::map(arrow::utf8(), arrow::struct_({query_attr_y, query_attr_x})), 3);
+    std::shared_ptr<arrow::Field> query_key_right = MakeField("renamed_right", arrow::int32(), 31);
+    std::shared_ptr<arrow::Field> query_key_left = MakeField("renamed_left", arrow::int32(), 30);
+    std::shared_ptr<arrow::Field> query_keyed_values =
+        MakeField("keyed_values_renamed",
+                  arrow::map(arrow::struct_({query_key_right, query_key_left}), arrow::int32()), 4);
+    std::shared_ptr<arrow::Schema> query_value_schema =
+        arrow::schema({id, query_items, query_attrs, query_keyed_values});
+
+    auto batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                 key_schema, query_value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> results,
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())));
+
+    ASSERT_EQ(results.size(), 1);
+    ASSERT_EQ(results[0].key->GetInt(0), 1);
+    ASSERT_EQ(results[0].value->GetFieldCount(), 4);
+    ASSERT_EQ(results[0].value->GetInt(0), 1);
+
+    std::shared_ptr<InternalArray> item_array = results[0].value->GetArray(1);
+    ASSERT_EQ(item_array->Size(), 2);
+    std::shared_ptr<InternalRow> first_item = item_array->GetRow(0, 2);
+    ASSERT_EQ(first_item->GetInt(0), 200);
+    ASSERT_EQ(first_item->GetInt(1), 100);
+    std::shared_ptr<InternalRow> second_item = item_array->GetRow(1, 2);
+    ASSERT_EQ(second_item->GetInt(0), 400);
+    ASSERT_EQ(second_item->GetInt(1), 300);
+
+    std::shared_ptr<InternalMap> attr_map = results[0].value->GetMap(2);
+    ASSERT_EQ(attr_map->Size(), 2);
+    std::shared_ptr<InternalArray> key_array = attr_map->KeyArray();
+    ASSERT_EQ(std::string(key_array->GetStringView(0)), "k1");
+    ASSERT_EQ(std::string(key_array->GetStringView(1)), "k2");
+    std::shared_ptr<InternalArray> value_array = attr_map->ValueArray();
+    std::shared_ptr<InternalRow> first_attr = value_array->GetRow(0, 2);
+    ASSERT_EQ(first_attr->GetInt(0), 8);
+    ASSERT_EQ(first_attr->GetInt(1), 7);
+    std::shared_ptr<InternalRow> second_attr = value_array->GetRow(1, 2);
+    ASSERT_EQ(second_attr->GetInt(0), 10);
+    ASSERT_EQ(second_attr->GetInt(1), 9);
+
+    std::shared_ptr<InternalMap> keyed_value_map = results[0].value->GetMap(3);
+    ASSERT_EQ(keyed_value_map->Size(), 2);
+    std::shared_ptr<InternalArray> struct_keys = keyed_value_map->KeyArray();
+    std::shared_ptr<InternalRow> first_key = struct_keys->GetRow(0, 2);
+    ASSERT_EQ(first_key->GetInt(0), 12);
+    ASSERT_EQ(first_key->GetInt(1), 11);
+    std::shared_ptr<InternalRow> second_key = struct_keys->GetRow(1, 2);
+    ASSERT_EQ(second_key->GetInt(0), 22);
+    ASSERT_EQ(second_key->GetInt(1), 21);
+    ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(0), 13);
+    ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(1), 23);
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderClose) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
+                                           DataField(1, arrow::field("v0", arrow::int32()))};
+    std::shared_ptr<arrow::Schema> value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+        [0, 10, 0, 1, 100]
+    ])")
+            .ValueOrDie());
+
+    int32_t explicit_close_count = 0;
+    {
+        auto tracking_reader = std::make_unique<TrackingBatchReader>(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
+            &explicit_close_count);
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr<KeyValueRecordReader> reader,
+            AdaptPreparedBatchReader(std::move(tracking_reader), prepared_schema, OffsetRange(0, 1),
+                                     key_schema, value_schema, pool_));
+        reader->Close();
+        reader->Close();
+    }
+    ASSERT_EQ(explicit_close_count, 1);
+
+    int32_t destructor_close_count = 0;
+    {
+        auto tracking_reader = std::make_unique<TrackingBatchReader>(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
+            &destructor_close_count);
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr<KeyValueRecordReader> reader,
+            AdaptPreparedBatchReader(std::move(tracking_reader), prepared_schema, OffsetRange(0, 1),
+                                     key_schema, value_schema, pool_));
+    }
+    ASSERT_EQ(destructor_close_count, 1);
+
+    int32_t factory_failure_close_count = 0;
+    {
+        std::unique_ptr<BatchReader> tracking_reader = std::make_unique<TrackingBatchReader>(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
+            &factory_failure_close_count);
+        std::shared_ptr<arrow::Schema> invalid_schema = arrow::schema(value_schema->fields());
+        ASSERT_NOK(AdaptPreparedBatchReader(std::move(tracking_reader), invalid_schema,
+                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
+        ASSERT_EQ(nullptr, tracking_reader);
+    }
+    ASSERT_EQ(factory_failure_close_count, 1);
+
+    int32_t read_failure_close_count = 0;
+    {
+        auto failing_reader =
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+        failing_reader->SetNextBatchStatus(Status::IOError("prepared reader failure"));
+        auto tracking_reader = std::make_unique<TrackingBatchReader>(std::move(failing_reader),
+                                                                     &read_failure_close_count);
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr<KeyValueRecordReader> reader,
+            AdaptPreparedBatchReader(std::move(tracking_reader), prepared_schema, OffsetRange(0, 1),
+                                     key_schema, value_schema, pool_));
+        ASSERT_NOK_WITH_MSG(reader->NextBatch(), "prepared reader failure");
+        ASSERT_EQ(read_failure_close_count, 1);
+        reader->Close();
+    }
+    ASSERT_EQ(read_failure_close_count, 1);
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/io/prepared_key_value_reader.cpp b/src/paimon/core/io/prepared_key_value_reader.cpp
new file mode 100644
index 000000000..0f4f22097
--- /dev/null
+++ b/src/paimon/core/io/prepared_key_value_reader.cpp
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/io/prepared_key_value_reader.h"
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "arrow/array/array_base.h"
+#include "arrow/array/array_nested.h"
+#include "arrow/array/array_primitive.h"
+#include "arrow/array/builder_primitive.h"
+#include "arrow/c/bridge.h"
+#include "arrow/compute/api.h"
+#include "arrow/type.h"
+#include "fmt/format.h"
+#include "paimon/common/data/columnar/columnar_batch_context.h"
+#include "paimon/common/data/columnar/columnar_row_ref.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/types/row_kind.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/realtime/realtime_fields.h"
+#include "paimon/core/utils/nested_projection_utils.h"
+#include "paimon/macros.h"
+#include "paimon/reader/batch_reader.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+namespace {
+
+constexpr int32_t kValueKindIndex = 0;
+constexpr int32_t kSequenceNumberIndex = 1;
+constexpr int32_t kRealtimeOffsetIndex = 2;
+constexpr int32_t kPreparedValueStartIndex = 3;
+
+Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
+    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type);
+
+Status CheckPreparedField(const std::shared_ptr<arrow::Schema>& schema, int32_t field_idx,
+                          const DataField& expected_field) {
+    if (schema->num_fields() <= field_idx) {
+        return Status::Invalid(fmt::format("prepared schema missing transport field {} at index {}",
+                                           expected_field.Name(), field_idx));
+    }
+    const std::shared_ptr<arrow::Field>& field = schema->field(field_idx);
+    PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(field));
+    if (field->name() != expected_field.Name() || !field->type()->Equals(*expected_field.Type()) ||
+        field->nullable() || field_id != expected_field.Id()) {
+        return Status::Invalid(fmt::format(
+            "prepared schema field {} must be non-null {}:{} with field id {}, got {}:{} "
+            "nullable={} field id {}",
+            field_idx, expected_field.Name(), expected_field.Type()->ToString(),
+            expected_field.Id(), field->name(), field->type()->ToString(), field->nullable(),
+            field_id));
+    }
+    return Status::OK();
+}
+
+Result<int32_t> FindFieldIndexByPaimonId(const arrow::FieldVector& fields, int32_t field_id) {
+    std::optional<int32_t> matching_index;
+    for (int32_t i = 0; i < static_cast<int32_t>(fields.size()); ++i) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t candidate_id,
+                               NestedProjectionUtils::GetPaimonFieldId(fields[i]));
+        if (candidate_id == field_id) {
+            if (matching_index.has_value()) {
+                return Status::Invalid(
+                    fmt::format("duplicate field id {} in prepared schema", field_id));
+            }
+            matching_index = i;
+        }
+    }
+    if (matching_index.has_value()) {
+        return matching_index.value();
+    }
+    return Status::Invalid(fmt::format("cannot find field id {} in prepared schema", field_id));
+}
+
+Status ValidateProjectionType(const std::shared_ptr<arrow::DataType>& prepared_type,
+                              const std::shared_ptr<arrow::DataType>& query_type) {
+    if (prepared_type->id() != query_type->id()) {
+        return Status::Invalid(fmt::format("prepared value type {} does not match query type {}",
+                                           prepared_type->ToString(), query_type->ToString()));
+    }
+    switch (query_type->id()) {
+        case arrow::Type::STRUCT: {
+            const arrow::FieldVector& prepared_fields = prepared_type->fields();
+            for (const std::shared_ptr<arrow::Field>& query_field : query_type->fields()) {
+                PAIMON_ASSIGN_OR_RAISE(int32_t query_id,
+                                       NestedProjectionUtils::GetPaimonFieldId(query_field));
+                PAIMON_ASSIGN_OR_RAISE(int32_t prepared_idx,
+                                       FindFieldIndexByPaimonId(prepared_fields, query_id));
+                PAIMON_RETURN_NOT_OK(ValidateProjectionType(prepared_fields[prepared_idx]->type(),
+                                                            query_field->type()));
+            }
+            return Status::OK();
+        }
+        case arrow::Type::LIST:
+            return ValidateProjectionType(prepared_type->field(0)->type(),
+                                          query_type->field(0)->type());
+        case arrow::Type::MAP: {
+            const std::shared_ptr<arrow::MapType> prepared_map =
+                checked_pointer_cast<arrow::MapType>(prepared_type);
+            const std::shared_ptr<arrow::MapType> query_map =
+                checked_pointer_cast<arrow::MapType>(query_type);
+            PAIMON_RETURN_NOT_OK(
+                ValidateProjectionType(prepared_map->key_type(), query_map->key_type()));
+            return ValidateProjectionType(prepared_map->item_type(), query_map->item_type());
+        }
+        default:
+            if (!prepared_type->Equals(*query_type)) {
+                return Status::Invalid(
+                    fmt::format("prepared leaf type {} does not match query type {}",
+                                prepared_type->ToString(), query_type->ToString()));
+            }
+            return Status::OK();
+    }
+}
+
+Status ValidateProjectionSchema(const std::shared_ptr<arrow::Schema>& prepared_schema,
+                                const std::shared_ptr<arrow::Schema>& query_schema) {
+    arrow::FieldVector prepared_value_fields(
+        prepared_schema->fields().begin() + kPreparedValueStartIndex,
+        prepared_schema->fields().end());
+    for (const std::shared_ptr<arrow::Field>& query_field : query_schema->fields()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t query_id,
+                               NestedProjectionUtils::GetPaimonFieldId(query_field));
+        PAIMON_ASSIGN_OR_RAISE(int32_t prepared_idx,
+                               FindFieldIndexByPaimonId(prepared_value_fields, query_id));
+        PAIMON_RETURN_NOT_OK(ValidateProjectionType(prepared_value_fields[prepared_idx]->type(),
+                                                    query_field->type()));
+    }
+    return Status::OK();
+}
+
+Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_schema,
+                                 const std::shared_ptr<arrow::Schema>& value_schema) {
+    if (prepared_schema->num_fields() != value_schema->num_fields() + kPreparedValueStartIndex) {
+        return Status::Invalid("commit requires the exact prepared writer schema");
+    }
+    for (int32_t i = 0; i < value_schema->num_fields(); ++i) {
+        if (!prepared_schema->field(i + kPreparedValueStartIndex)
+                 ->Equals(value_schema->field(i), true)) {
+            return Status::Invalid("commit requires the exact prepared writer schema");
+        }
+    }
+    return Status::OK();
+}
+
+Status ValidatePreparedSchema(const std::shared_ptr<arrow::Schema>& prepared_schema) {
+    if (!prepared_schema || prepared_schema->num_fields() < kPreparedValueStartIndex) {
+        return Status::Invalid("prepared schema must contain realtime transport fields");
+    }
+    PAIMON_RETURN_NOT_OK(
+        CheckPreparedField(prepared_schema, kValueKindIndex, SpecialFields::ValueKind()));
+    PAIMON_RETURN_NOT_OK(
+        CheckPreparedField(prepared_schema, kSequenceNumberIndex, SpecialFields::SequenceNumber()));
+    PAIMON_RETURN_NOT_OK(
+        CheckPreparedField(prepared_schema, kRealtimeOffsetIndex, RealtimeOffsetField()));
+    return Status::OK();
+}
+
+Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
+    const std::shared_ptr<arrow::StructArray>& array,
+    const std::shared_ptr<arrow::StructType>& read_type) {
+    const std::shared_ptr<arrow::StructType> data_type =
+        checked_pointer_cast<arrow::StructType>(array->type());
+    std::unordered_map<int32_t, int32_t> data_field_id_to_idx;
+    data_field_id_to_idx.reserve(data_type->num_fields());
+    for (int32_t i = 0; i < data_type->num_fields(); ++i) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(data_type->field(i)));
+        if (!data_field_id_to_idx.emplace(field_id, i).second) {
+            return Status::Invalid(
+                fmt::format("duplicate field id {} in prepared value struct", field_id));
+        }
+    }
+
+    arrow::ArrayVector aligned_arrays;
+    aligned_arrays.reserve(read_type->num_fields());
+    for (const std::shared_ptr<arrow::Field>& read_field : read_type->fields()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t read_field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(read_field));
+        auto data_iter = data_field_id_to_idx.find(read_field_id);
+        if (data_iter == data_field_id_to_idx.end()) {
+            return Status::Invalid(
+                fmt::format("cannot find field id {} in prepared value struct", read_field_id));
+        }
+        std::shared_ptr<arrow::Array> child = array->field(data_iter->second);
+        PAIMON_ASSIGN_OR_RAISE(child, AlignArrayByPaimonIds(child, read_field->type()));
+        aligned_arrays.push_back(std::move(child));
+    }
+
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::StructArray> aligned,
+        arrow::StructArray::Make(aligned_arrays, read_type->fields(), array->null_bitmap(),
+                                 array->null_count(), array->offset()));
+    return aligned;
+}
+
+Result<std::shared_ptr<arrow::Array>> AlignListArrayByPaimonIds(
+    const std::shared_ptr<arrow::ListArray>& array,
+    const std::shared_ptr<arrow::ListType>& read_type) {
+    std::shared_ptr<arrow::Array> values = array->values();
+    PAIMON_ASSIGN_OR_RAISE(values, AlignArrayByPaimonIds(values, read_type->value_type()));
+    std::shared_ptr<arrow::ArrayData> new_data = array->data()->Copy();
+    new_data->type = read_type;
+    new_data->child_data = {values->data()};
+    return arrow::MakeArray(new_data);
+}
+
+Result<std::shared_ptr<arrow::Array>> AlignMapArrayByPaimonIds(
+    const std::shared_ptr<arrow::MapArray>& array,
+    const std::shared_ptr<arrow::MapType>& read_type) {
+    std::shared_ptr<arrow::Array> keys = array->keys();
+    PAIMON_ASSIGN_OR_RAISE(keys, AlignArrayByPaimonIds(keys, read_type->key_type()));
+    std::shared_ptr<arrow::Array> items = array->items();
+    PAIMON_ASSIGN_OR_RAISE(items, AlignArrayByPaimonIds(items, read_type->item_type()));
+
+    const std::shared_ptr<arrow::ArrayData>& entries_data = array->data()->child_data[0];
+    std::shared_ptr<arrow::ArrayData> new_entries = entries_data->Copy();
+    new_entries->type = arrow::struct_({read_type->key_field(), read_type->item_field()});
+    new_entries->child_data = {keys->data(), items->data()};
+
+    std::shared_ptr<arrow::ArrayData> new_data = array->data()->Copy();
+    new_data->type = read_type;
+    new_data->child_data = {std::move(new_entries)};
+    return arrow::MakeArray(new_data);
+}
+
+Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
+    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type) {
+    if (array->type()->id() != read_type->id()) {
+        return Status::Invalid(fmt::format("prepared value type {} does not match query type {}",
+                                           array->type()->ToString(), read_type->ToString()));
+    }
+    switch (read_type->id()) {
+        case arrow::Type::STRUCT:
+            return AlignStructArrayByPaimonIds(checked_pointer_cast<arrow::StructArray>(array),
+                                               checked_pointer_cast<arrow::StructType>(read_type));
+        case arrow::Type::LIST:
+            return AlignListArrayByPaimonIds(checked_pointer_cast<arrow::ListArray>(array),
+                                             checked_pointer_cast<arrow::ListType>(read_type));
+        case arrow::Type::MAP:
+            return AlignMapArrayByPaimonIds(checked_pointer_cast<arrow::MapArray>(array),
+                                            checked_pointer_cast<arrow::MapType>(read_type));
+        default:
+            if (!array->type()->Equals(*read_type)) {
+                return Status::Invalid(
+                    fmt::format("prepared leaf type {} does not match query type {}",
+                                array->type()->ToString(), read_type->ToString()));
+            }
+            return array;
+    }
+}
+
+Result<arrow::ArrayVector> ProjectFieldsByPaimonIds(
+    const std::shared_ptr<arrow::StructArray>& data_batch,
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& query_schema) {
+    std::unordered_map<int32_t, int32_t> prepared_field_id_to_idx;
+    prepared_field_id_to_idx.reserve(prepared_schema->num_fields());
+    for (int32_t i = kPreparedValueStartIndex; i < prepared_schema->num_fields(); ++i) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(prepared_schema->field(i)));
+        if (!prepared_field_id_to_idx.emplace(field_id, i).second) {
+            return Status::Invalid(
+                fmt::format("duplicate field id {} in prepared schema", field_id));
+        }
+    }
+
+    arrow::ArrayVector result;
+    result.reserve(query_schema->num_fields());
+    for (const std::shared_ptr<arrow::Field>& query_field : query_schema->fields()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t query_field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(query_field));
+        auto prepared_iter = prepared_field_id_to_idx.find(query_field_id);
+        if (prepared_iter == prepared_field_id_to_idx.end()) {
+            return Status::Invalid(
+                fmt::format("cannot find field id {} in prepared schema", query_field_id));
+        }
+        std::shared_ptr<arrow::Array> field_array = data_batch->field(prepared_iter->second);
+        PAIMON_ASSIGN_OR_RAISE(field_array,
+                               AlignArrayByPaimonIds(field_array, query_field->type()));
+        result.push_back(std::move(field_array));
+    }
+    return result;
+}
+
+Result<std::shared_ptr<arrow::StructArray>> ApplyOffsetFilter(
+    const std::shared_ptr<arrow::StructArray>& data_batch,
+    const std::shared_ptr<arrow::NumericArray<arrow::Int64Type>>& offset_array,
+    const std::optional<OffsetRange>& visible_offsets, arrow::MemoryPool* arrow_pool) {
+    if (!visible_offsets.has_value()) {
+        return data_batch;
+    }
+
+    arrow::BooleanBuilder filter_builder(arrow_pool);
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(filter_builder.Reserve(offset_array->length()));
+    int64_t visible_row_count = 0;
+    for (int64_t i = 0; i < offset_array->length(); ++i) {
+        int64_t offset = offset_array->Value(i);
+        bool visible = offset >= visible_offsets->begin && offset < visible_offsets->end;
+        filter_builder.UnsafeAppend(visible);
+        visible_row_count += visible;
+    }
+    if (visible_row_count == 0) {
+        return std::shared_ptr<arrow::StructArray>();
+    }
+    if (visible_row_count == data_batch->length()) {
+        return data_batch;
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> filter,
+                                      filter_builder.Finish());
+    arrow::compute::ExecContext exec_context(arrow_pool);
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        arrow::Datum filtered,
+        arrow::compute::Filter(data_batch, filter, arrow::compute::FilterOptions::Defaults(),
+                               &exec_context));
+    return checked_pointer_cast<arrow::StructArray>(filtered.make_array());
+}
+
+class PreparedKeyValueReader final : public KeyValueRecordReader {
+ public:
+    PreparedKeyValueReader(std::unique_ptr<BatchReader>&& reader,
+                           const std::shared_ptr<arrow::Schema>& prepared_schema,
+                           const std::optional<OffsetRange>& visible_offsets,
+                           const std::shared_ptr<arrow::Schema>& key_schema,
+                           const std::shared_ptr<arrow::Schema>& value_schema,
+                           const std::shared_ptr<MemoryPool>& pool, int64_t* raw_row_count)
+        : reader_(std::move(reader)),
+          prepared_schema_(prepared_schema),
+          visible_offsets_(visible_offsets),
+          key_schema_(key_schema),
+          value_schema_(value_schema),
+          pool_(pool),
+          arrow_pool_(GetArrowPool(pool)),
+          raw_row_count_(raw_row_count) {}
+
+    ~PreparedKeyValueReader() override {
+        Close();
+    }
+
+    class Iterator final : public KeyValueRecordReader::Iterator {
+     public:
+        explicit Iterator(PreparedKeyValueReader* reader) : reader_(reader) {}
+
+        Result<bool> HasNext() const override {
+            return cursor_ < reader_->row_kind_array_->length();
+        }
+
+        Result<KeyValue> Next() override {
+            if (cursor_ >= reader_->row_kind_array_->length()) {
+                return Status::Invalid("No more prepared key values in current iterator");
+            }
+            std::shared_ptr<InternalRow> key =
+                std::make_shared<ColumnarRowRef>(reader_->key_ctx_, cursor_);
+            auto value = std::make_unique<ColumnarRowRef>(reader_->value_ctx_, cursor_);
+            PAIMON_ASSIGN_OR_RAISE(
+                const RowKind* row_kind,
+                RowKind::FromByteValue(reader_->row_kind_array_->Value(cursor_)));
+            int64_t sequence_number = reader_->sequence_number_array_->Value(cursor_);
+            ++cursor_;
+            return KeyValue(row_kind, sequence_number, KeyValue::UNKNOWN_LEVEL, std::move(key),
+                            std::move(value));
+        }
+
+     private:
+        PreparedKeyValueReader* reader_;
+        int64_t cursor_ = 0;
+    };
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
+        Result<std::unique_ptr<KeyValueRecordReader::Iterator>> result = NextBatchImpl();
+        if (!result.ok()) {
+            Close();
+        }
+        return result;
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return reader_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        if (closed_) {
+            return;
+        }
+        closed_ = true;
+        ResetBatchState();
+        reader_->Close();
+    }
+
+ private:
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatchImpl() {
+        if (closed_) {
+            return std::unique_ptr<KeyValueRecordReader::Iterator>();
+        }
+
+        while (true) {
+            ResetBatchState();
+            PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
+            if (BatchReader::IsEofBatch(batch)) {
+                return std::unique_ptr<KeyValueRecordReader::Iterator>();
+            }
+            auto& [c_array, c_schema] = batch;
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> arrow_array,
+                                              arrow::ImportArray(c_array.get(), c_schema.get()));
+            if (!arrow_array || arrow_array->type_id() != arrow::Type::STRUCT) {
+                return Status::Invalid("cannot cast prepared batch to StructArray");
+            }
+            std::shared_ptr<arrow::StructArray> data_batch =
+                checked_pointer_cast<arrow::StructArray>(arrow_array);
+            PAIMON_RETURN_NOT_OK(ValidatePreparedBatch(data_batch));
+            if (raw_row_count_ != nullptr) {
+                int64_t updated_count = 0;
+                if (__builtin_add_overflow(*raw_row_count_, data_batch->length(), &updated_count)) {
+                    return Status::Invalid("prepared raw row count overflow");
+                }
+                *raw_row_count_ = updated_count;
+            }
+
+            std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> offset_array =
+                checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
+                    data_batch->field(kRealtimeOffsetIndex));
+            PAIMON_ASSIGN_OR_RAISE(
+                data_batch,
+                ApplyOffsetFilter(data_batch, offset_array, visible_offsets_, arrow_pool_.get()));
+            if (!data_batch) {
+                continue;
+            }
+
+            row_kind_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int8Type>>(
+                data_batch->field(kValueKindIndex));
+            sequence_number_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
+                data_batch->field(kSequenceNumberIndex));
+            PAIMON_ASSIGN_OR_RAISE(
+                arrow::ArrayVector key_fields,
+                ProjectFieldsByPaimonIds(data_batch, prepared_schema_, key_schema_));
+            PAIMON_ASSIGN_OR_RAISE(
+                arrow::ArrayVector value_fields,
+                ProjectFieldsByPaimonIds(data_batch, prepared_schema_, value_schema_));
+            key_ctx_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
+            value_ctx_ = std::make_shared<ColumnarBatchContext>(value_fields, pool_);
+            ArrowUtils::TraverseArray(data_batch);
+            return std::make_unique<Iterator>(this);
+        }
+    }
+
+    Status ValidatePreparedBatch(const std::shared_ptr<arrow::StructArray>& data_batch) const {
+        if (data_batch->num_fields() != prepared_schema_->num_fields()) {
+            return Status::Invalid(fmt::format(
+                "prepared batch field count {} does not match prepared schema field count {}",
+                data_batch->num_fields(), prepared_schema_->num_fields()));
+        }
+        const arrow::FieldVector& batch_fields = data_batch->type()->fields();
+        for (int32_t i = 0; i < data_batch->num_fields(); ++i) {
+            if (!batch_fields[i]->Equals(prepared_schema_->field(i), true)) {
+                return Status::Invalid(fmt::format(
+                    "prepared batch field {} does not match declared prepared schema", i));
+            }
+        }
+        if (!data_batch->field(kValueKindIndex) ||
+            data_batch->field(kValueKindIndex)->type_id() != arrow::Type::INT8) {
+            return Status::Invalid("cannot cast VALUE_KIND column to int8 arrow array");
+        }
+        if (!data_batch->field(kSequenceNumberIndex) ||
+            data_batch->field(kSequenceNumberIndex)->type_id() != arrow::Type::INT64) {
+            return Status::Invalid("cannot cast SEQUENCE_NUMBER column to int64 arrow array");
+        }
+        if (!data_batch->field(kRealtimeOffsetIndex) ||
+            data_batch->field(kRealtimeOffsetIndex)->type_id() != arrow::Type::INT64) {
+            return Status::Invalid("cannot cast REALTIME_OFFSET column to int64 arrow array");
+        }
+        if (data_batch->field(kValueKindIndex)->null_count() != 0 ||
+            data_batch->field(kSequenceNumberIndex)->null_count() != 0 ||
+            data_batch->field(kRealtimeOffsetIndex)->null_count() != 0) {
+            return Status::Invalid("prepared transport columns must not contain nulls");
+        }
+        return Status::OK();
+    }
+
+    void ResetBatchState() {
+        key_ctx_.reset();
+        value_ctx_.reset();
+        row_kind_array_.reset();
+        sequence_number_array_.reset();
+    }
+
+ private:
+    bool closed_ = false;
+    std::unique_ptr<BatchReader> reader_;
+    std::shared_ptr<arrow::Schema> prepared_schema_;
+    std::optional<OffsetRange> visible_offsets_;
+    std::shared_ptr<arrow::Schema> key_schema_;
+    std::shared_ptr<arrow::Schema> value_schema_;
+    std::shared_ptr<MemoryPool> pool_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    int64_t* raw_row_count_;
+    std::shared_ptr<ColumnarBatchContext> key_ctx_;
+    std::shared_ptr<ColumnarBatchContext> value_ctx_;
+    std::shared_ptr<arrow::NumericArray<arrow::Int8Type>> row_kind_array_;
+    std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> sequence_number_array_;
+};
+
+}
+
+Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::optional<OffsetRange>& visible_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool, int64_t* raw_row_count) {
+    std::unique_ptr<BatchReader> owned_reader = std::move(reader);
+    if (!owned_reader) {
+        return Status::Invalid("prepared batch reader cannot be null");
+    }
+    ScopeGuard close_guard([&owned_reader]() -> void { owned_reader->Close(); });
+    PAIMON_RETURN_NOT_OK(ValidatePreparedSchema(prepared_schema));
+    if (!key_schema) {
+        return Status::Invalid("prepared key schema cannot be null");
+    }
+    if (!value_schema) {
+        return Status::Invalid("prepared value schema cannot be null");
+    }
+    if (!memory_pool) {
+        return Status::Invalid("prepared reader memory pool cannot be null");
+    }
+    PAIMON_RETURN_NOT_OK(ValidateProjectionSchema(prepared_schema, key_schema));
+    PAIMON_RETURN_NOT_OK(ValidateProjectionSchema(prepared_schema, value_schema));
+    if (!visible_offsets.has_value()) {
+        PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
+    }
+    std::unique_ptr<KeyValueRecordReader> result(
+        new PreparedKeyValueReader(std::move(owned_reader), prepared_schema, visible_offsets,
+                                   key_schema, value_schema, memory_pool, raw_row_count));
+    close_guard.Release();
+    return result;
+}
+
+}
diff --git a/src/paimon/core/io/prepared_key_value_reader.h b/src/paimon/core/io/prepared_key_value_reader.h
new file mode 100644
index 000000000..e7a6f9651
--- /dev/null
+++ b/src/paimon/core/io/prepared_key_value_reader.h
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+
+#include "arrow/type_fwd.h"
+#include "paimon/core/io/key_value_record_reader.h"
+#include "paimon/realtime/offset_range.h"
+#include "paimon/result.h"
+
+namespace paimon {
+class BatchReader;
+class MemoryPool;
+
+Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::optional<OffsetRange>& visible_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool, int64_t* raw_row_count = nullptr);
+
+}
diff --git a/src/paimon/core/realtime/realtime_fields.h b/src/paimon/core/realtime/realtime_fields.h
new file mode 100644
index 000000000..6ed04b38a
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_fields.h
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <limits>
+
+#include "arrow/type.h"
+#include "paimon/common/types/data_field.h"
+
+namespace paimon {
+
+inline const DataField& RealtimeOffsetField() {
+    static const DataField data_field =
+        DataField(std::numeric_limits<int32_t>::max() - 10002,
+                  arrow::field("_REALTIME_OFFSET", arrow::int64(), /*nullable=*/false));
+    return data_field;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp
index 47603497b..050f09701 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -46,6 +46,13 @@ TEST(SchemaValidationTest, TestSimple) {
     ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
 }
 
+TEST(SchemaValidationTest, TestRealtimeOffsetIsNotGloballyReserved) {
+    auto schema = arrow::schema({arrow::field("_REALTIME_OFFSET", arrow::int64())});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
+                         TableSchema::Create(0, schema, {}, {}, {}));
+    ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+}
+
 TEST(SchemaValidationTest, TestVectorType) {
     auto vector_field = arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3));
     auto schema = arrow::schema({arrow::field("id", arrow::int64()), vector_field});

From 232d587093d89c6320d4e107a0b9772040cd3d72 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 11:26:17 +0800
Subject: [PATCH 37/93] refactor(realtime): prepare primary-key batches in
 framework

---
 include/paimon/realtime/realtime_context.h    |   4 +
 include/paimon/realtime/realtime_store.h      |  52 +-
 .../core/mergetree/merge_tree_writer_test.cpp |  15 +
 src/paimon/core/mergetree/write_buffer.cpp    |   4 +
 .../operation/key_value_file_store_write.cpp  |  40 +-
 .../key_value_file_store_write_test.cpp       | 374 +++++++++++-
 .../core/operation/merge_file_split_read.cpp  |  10 +-
 .../core/operation/merge_file_split_read.h    |   1 -
 .../realtime/arrow_realtime_store_factory.cpp |  32 +-
 .../realtime/primary_key_realtime_store.cpp   | 548 ++++--------------
 .../realtime/primary_key_realtime_store.h     |  27 +-
 .../primary_key_realtime_store_test.cpp       | 502 +++-------------
 .../core/realtime/realtime_context_impl.cpp   |  41 +-
 .../core/realtime/realtime_context_impl.h     |   5 -
 .../core/realtime/realtime_context_test.cpp   | 133 ++---
 .../realtime/realtime_primary_key_writer.cpp  | 332 +++++++----
 .../realtime/realtime_primary_key_writer.h    |  35 +-
 .../table/source/key_value_table_read.cpp     | 193 ++----
 test/inte/realtime_write_inte_test.cpp        | 239 ++++----
 19 files changed, 1106 insertions(+), 1481 deletions(-)

diff --git a/include/paimon/realtime/realtime_context.h b/include/paimon/realtime/realtime_context.h
index 200e4ba4c..8f2967b32 100644
--- a/include/paimon/realtime/realtime_context.h
+++ b/include/paimon/realtime/realtime_context.h
@@ -78,6 +78,10 @@ using RealtimeOffsetMap = std::map<RealtimePartitionBucket, int64_t>;
 /// partition drop, and rollback operations do not automatically clear process-local real-time
 /// state. Applications must coordinate these operations with active real-time writers and recreate
 /// the `RealtimeContext` and writers before continuing.
+///
+/// A primary-key writer and its context form one lifecycle. After a primary-key write or prepare
+/// returns an error, discard both, create fresh instances from the latest committed snapshot, and
+/// replay the caller-owned WAL. Reusing the failed context or retrying prepare is unsupported.
 class PAIMON_EXPORT RealtimeContext {
  public:
     /// Creates a context backed by Paimon's default in-memory Arrow `RealtimeStore`.
diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 1e53c173e..dc5d543ac 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -47,19 +47,15 @@ struct PAIMON_EXPORT AppendRealtimeStoreCreateConfig {
     StatisticsMode statistics_mode;
 };
 
-struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {
-    std::vector<std::string> primary_keys;
-    /// Largest sequence restored from the committed snapshot. A PK store assigns one contiguous
-    /// sequence to every mutation in `Write` order, starting at the next value, and rejects
-    /// `Write` before the assigned sequence would exceed `INT64_MAX - 1`.
-    int64_t restore_max_sequence_number;
-};
+struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {};
 
 using RealtimeStoreCreateConfig =
     std::variant<AppendRealtimeStoreCreateConfig, PrimaryKeyRealtimeStoreCreateConfig>;
 
 struct PAIMON_EXPORT RealtimeStoreCreateRequest {
-    /// Complete table write schema whose ownership is transferred to the factory.
+    /// Schema whose ownership is transferred to the factory. Append mode receives the complete
+    /// table write schema. Primary-key mode receives the prepared transport schema:
+    /// [_VALUE_KIND, _SEQUENCE_NUMBER, _REALTIME_OFFSET, table write fields].
     std::unique_ptr<::ArrowSchema> write_schema;
     std::map<std::string, std::string> options;
     std::shared_ptr<MemoryPool> memory_pool;
@@ -68,10 +64,12 @@ struct PAIMON_EXPORT RealtimeStoreCreateRequest {
     RealtimeStoreCreateConfig mode_config;
 };
 
-/// A table record batch and its framework-assigned contiguous offset range.
+/// A record batch and its framework-assigned contiguous offset range.
 ///
-/// The batch contains only table write fields. Row `i` is associated with
-/// `offset_range.begin + i`; the offset is progress metadata and is not a table field.
+/// Append-mode batches contain table write fields, and row `i` is associated with
+/// `offset_range.begin + i`. Primary-key batches contain the prepared transport schema supplied
+/// to the factory and are physically sorted by full primary key then sequence number; their
+/// per-row `_REALTIME_OFFSET` field preserves the original write-order offset after sorting.
 struct PAIMON_EXPORT RealtimeWriteBatch {
     /// Input batch whose ownership is transferred to `RealtimeStore::Write`.
     std::unique_ptr<RecordBatch> batch;
@@ -106,7 +104,8 @@ class PAIMON_EXPORT RealtimeReadView {
 
 /// Parameters used by a `RealtimeStore` to create readers for a query.
 struct PAIMON_EXPORT RealtimeQueryContext {
-    /// Requested output fields before the mandatory leading `_VALUE_KIND` field is added.
+    /// Append mode receives the requested output fields before the mandatory leading
+    /// `_VALUE_KIND` field is added. Primary-key mode receives the complete prepared schema.
     ::ArrowSchema* read_schema;
     /// Predicate using field indexes from `read_schema`.
     std::shared_ptr<Predicate> predicate;
@@ -143,9 +142,13 @@ class PAIMON_EXPORT RealtimeStore {
 
     /// Creates readers that expose all rows in a sealed segment for Paimon file writing.
     ///
-    /// Concatenating the returned readers must produce every sealed row exactly once and in write
-    /// order. Each output batch contains `_VALUE_KIND` followed by all fields from the factory's
-    /// `write_schema`.
+    /// Concatenating the returned readers must produce every sealed row exactly once. Append-mode
+    /// readers preserve write order and contain `_VALUE_KIND` followed by table write fields.
+    /// Primary-key readers expose raw prepared rows. Each returned reader's complete stream,
+    /// including across `NextBatch` boundaries, is sorted by full primary key then sequence
+    /// number; all readers collectively cover sealed mutations exactly once. Reader cardinality is
+    /// independent of the number of writes. Paimon adapts and merges those rows before writing
+    /// files.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) = 0;
 
@@ -155,16 +158,17 @@ class PAIMON_EXPORT RealtimeStore {
     /// also provide a consistent snapshot when a write or seal is in progress.
     virtual Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() = 0;
 
-    /// Creates readers over rows in `view` whose offsets are greater than or equal to
-    /// `offset_begin`.
+    /// Creates readers over rows in `view`. Append mode returns rows whose offsets are greater
+    /// than or equal to `offset_begin`. Primary-key mode ignores `offset_begin` and returns raw
+    /// prepared rows; Paimon applies offset filtering, projection, and merge-on-read adaptation.
     ///
-    /// Each output batch contains `_VALUE_KIND` first, followed by the fields requested by
-    /// `context.read_schema` except a duplicate `_VALUE_KIND`. Concatenating all returned readers
-    /// must produce every matching row once. Primary-key readers additionally provide a non-null
-    /// `_SEQUENCE_NUMBER` when requested, are individually sorted by primary key, and contain at
-    /// most one mutation per key. Assigned sequences remain stable across views and queries;
-    /// readers need not be globally sorted with one another. Paimon retains `view` for the lifetime
-    /// of the resulting framework reader.
+    /// Append-mode output batches contain `_VALUE_KIND` first, followed by requested fields except
+    /// a duplicate `_VALUE_KIND`; all returned append readers collectively cover every matching
+    /// row exactly once. Primary-key output batches use the prepared transport schema and may
+    /// contain multiple mutations per key. Each returned primary-key reader's complete stream is
+    /// sorted by full primary key then sequence number, and all readers collectively cover raw
+    /// mutations exactly once. Reader cardinality is independent of the number of writes. Paimon
+    /// retains `view` for the lifetime of the resulting framework reader.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) = 0;
diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 675ce3198..aa2d0c959 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -21,6 +21,7 @@
 #include <cassert>
 #include <cstddef>
 #include <functional>
+#include <limits>
 #include <map>
 #include <optional>
 #include <utility>
@@ -612,6 +613,20 @@ TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
     ASSERT_OK(merge_writer->Close());
 }
 
+TEST_P(MergeTreeWriterTest, TestRejectsExhaustedSequence) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+
+    ASSERT_NOK_WITH_MSG(CreateMergeWriter(std::numeric_limits<int64_t>::max(), dir->Str(),
+                                          path_factory, 0, options),
+                        "sequence number has reached INT64_MAX");
+}
+
 TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
                          CoreOptions::FromMap({
diff --git a/src/paimon/core/mergetree/write_buffer.cpp b/src/paimon/core/mergetree/write_buffer.cpp
index 549975a33..3d3fdc196 100644
--- a/src/paimon/core/mergetree/write_buffer.cpp
+++ b/src/paimon/core/mergetree/write_buffer.cpp
@@ -18,6 +18,7 @@
 
 #include "paimon/core/mergetree/write_buffer.h"
 
+#include <limits>
 #include <memory>
 #include <utility>
 
@@ -39,6 +40,9 @@ Result<std::unique_ptr<WriteBuffer>> WriteBuffer::Create(
     const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper,
     const CoreOptions& options, const std::shared_ptr<IOManager>& io_manager,
     bool enable_multi_thread_spill, const std::shared_ptr<MemoryPool>& pool) {
+    if (last_sequence_number == std::numeric_limits<int64_t>::max()) {
+        return Status::Invalid("sequence number has reached INT64_MAX");
+    }
     auto value_type = arrow::struct_(value_schema->fields());
     auto in_memory_buffer = std::make_unique<InMemorySortBuffer>(
         last_sequence_number, value_type, trimmed_primary_keys, user_defined_sequence_fields,
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 492161cf8..d2c97abcf 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -18,12 +18,13 @@
 
 #include "paimon/core/operation/key_value_file_store_write.h"
 
-#include <limits>
 #include <optional>
 #include <vector>
 
 #include "arrow/c/bridge.h"
 #include "paimon/common/data/binary_row.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
 #include "paimon/core/compact/noop_compact_manager.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/io/data_file_meta.h"
@@ -35,6 +36,7 @@
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/realtime/realtime_primary_key_writer.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/utils/file_store_path_factory.h"
@@ -124,7 +126,6 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         std::shared_ptr<Levels> levels,
         Levels::Create(key_comparator_, restore_data_files, options_.GetNumLevels()));
     std::map<std::string, std::string> partition_map;
-    int64_t initial_max_sequence_number = restore_max_seq_number;
     std::shared_ptr<CompactManager> compact_manager;
     std::shared_ptr<RealtimeContextImpl> realtime_context_impl;
     std::optional<RealtimeStoreState> realtime_store_state;
@@ -135,19 +136,27 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         partition_map =
             std::map<std::string, std::string>(partition_values.begin(), partition_values.end());
         PAIMON_ASSIGN_OR_RAISE(realtime_context_impl, RealtimeContextImpl::Cast(realtime_context_));
+        if (schema_->GetFieldByName(RealtimeOffsetField().Name())) {
+            return Status::Invalid("PK real-time write schema contains reserved transport field " +
+                                   RealtimeOffsetField().Name());
+        }
+        arrow::FieldVector prepared_fields = {
+            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
+                ->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+                ->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+        prepared_fields.insert(prepared_fields.end(), schema_->fields().begin(),
+                               schema_->fields().end());
         auto c_write_schema = std::make_unique<ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, c_write_schema.get()));
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(
+            arrow::ExportSchema(*arrow::schema(std::move(prepared_fields)), c_write_schema.get()));
         PAIMON_ASSIGN_OR_RAISE(
             RealtimeStoreState store_state,
             realtime_context_impl->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
                 std::move(c_write_schema), options_.ToMap(), pool_, partition_map, bucket,
-                PrimaryKeyRealtimeStoreCreateConfig{trimmed_primary_keys,
-                                                    restore_max_seq_number}}));
+                PrimaryKeyRealtimeStoreCreateConfig{}}));
         realtime_store_state = std::move(store_state);
-        initial_max_sequence_number = realtime_store_state->initial_max_sequence_number.value();
-        if (initial_max_sequence_number == std::numeric_limits<int64_t>::max()) {
-            return Status::Invalid("PK sequence number has reached INT64_MAX");
-        }
         compact_manager = std::make_shared<NoopCompactManager>();
     } else {
         auto compact_strategy = compact_manager_factory_->CreateCompactStrategy();
@@ -159,15 +168,16 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr<MergeTreeWriter> writer,
         MergeTreeWriter::Create(
-            initial_max_sequence_number, trimmed_primary_keys, data_file_path_factory,
-            key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_,
-            table_schema_->Id(), schema_, options_, compact_manager,
-            realtime_context_ ? nullptr : io_manager_, enable_multi_thread_spill_, pool_));
+            restore_max_seq_number, trimmed_primary_keys, data_file_path_factory, key_comparator_,
+            user_defined_seq_comparator_, merge_function_wrapper_, table_schema_->Id(), schema_,
+            options_, compact_manager, realtime_context_ ? nullptr : io_manager_,
+            enable_multi_thread_spill_, pool_));
     if (!realtime_context_) {
         return std::shared_ptr<BatchWriter>(std::move(writer));
     }
-    return RealtimePrimaryKeyWriter::Create(partition_map, bucket, schema_, realtime_context_impl,
-                                            writer, pool_, realtime_store_state.value());
+    return RealtimePrimaryKeyWriter::Create(schema_, trimmed_primary_keys, key_comparator_,
+                                            realtime_store_state.value(), restore_max_seq_number,
+                                            writer, pool_);
 }
 
 Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index 45462ea6e..cbd2189fc 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -19,9 +19,13 @@
 #include "paimon/core/operation/key_value_file_store_write.h"
 
 #include <cstddef>
+#include <limits>
 #include <map>
 #include <memory>
+#include <new>
+#include <optional>
 #include <string>
+#include <tuple>
 #include <utility>
 #include <vector>
 
@@ -44,6 +48,7 @@
 #include "paimon/common/utils/path_util.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/operation/restore_files.h"
+#include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/stats/simple_stats.h"
 #include "paimon/core/table/sink/commit_message_impl.h"
 #include "paimon/file_store_commit.h"
@@ -52,7 +57,9 @@
 #include "paimon/format/file_format_factory.h"
 #include "paimon/format/reader_builder.h"
 #include "paimon/fs/local/local_file_system.h"
+#include "paimon/memory/memory_pool.h"
 #include "paimon/reader/file_batch_reader.h"
+#include "paimon/realtime/arrow_realtime_store_factory.h"
 #include "paimon/realtime/realtime_context.h"
 #include "paimon/record_batch.h"
 #include "paimon/status.h"
@@ -61,6 +68,113 @@
 #include "paimon/write_context.h"
 
 namespace paimon::test {
+namespace {
+
+class TestingMemoryPool final : public MemoryPool {
+ public:
+    void* Malloc(uint64_t size, uint64_t alignment) override {
+        ++allocation_count;
+        if (reject_allocations) {
+            throw std::bad_alloc();
+        }
+        return delegate_->Malloc(size, alignment);
+    }
+
+    void* Realloc(void* pointer, size_t old_size, size_t new_size, uint64_t alignment) override {
+        ++allocation_count;
+        if (reject_allocations) {
+            throw std::bad_alloc();
+        }
+        return delegate_->Realloc(pointer, old_size, new_size, alignment);
+    }
+
+    void Free(void* pointer, uint64_t size) override {
+        delegate_->Free(pointer, size);
+    }
+
+    void Free(void* pointer, uint64_t size, uint64_t alignment) override {
+        delegate_->Free(pointer, size, alignment);
+    }
+
+    uint64_t CurrentUsage() const override {
+        return delegate_->CurrentUsage();
+    }
+
+    uint64_t MaxMemoryUsage() const override {
+        return delegate_->MaxMemoryUsage();
+    }
+
+    bool reject_allocations = false;
+    int64_t allocation_count = 0;
+
+ private:
+    std::unique_ptr<MemoryPool> delegate_ = GetMemoryPool();
+};
+
+class FailOnceRealtimeStore final : public RealtimeStore {
+ public:
+    FailOnceRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
+                          const std::shared_ptr<bool>& fail_next_write)
+        : delegate_(delegate), fail_next_write_(fail_next_write) {}
+
+    Status Write(RealtimeWriteBatch&& batch) override {
+        if (*fail_next_write_) {
+            *fail_next_write_ = false;
+            return Status::Invalid("injected real-time store write failure");
+        }
+        return delegate_->Write(std::move(batch));
+    }
+
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
+        return delegate_->SealForCommit();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
+        return delegate_->CreateCommitReaders(segment);
+    }
+
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
+        return delegate_->AcquireReadView();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        return delegate_->CreateQueryReaders(view, offset_begin, context);
+    }
+
+    Status AdvanceCommittedOffset(int64_t committed_offset) override {
+        return delegate_->AdvanceCommittedOffset(committed_offset);
+    }
+
+    uint64_t GetMemoryUsage() const override {
+        return delegate_->GetMemoryUsage();
+    }
+
+ private:
+    std::shared_ptr<RealtimeStore> delegate_;
+    std::shared_ptr<bool> fail_next_write_;
+};
+
+class FailOnceRealtimeStoreFactory final : public RealtimeStoreFactory {
+ public:
+    explicit FailOnceRealtimeStoreFactory(const std::shared_ptr<bool>& fail_next_write)
+        : fail_next_write_(fail_next_write) {}
+
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
+                               delegate_.Create(std::move(request)));
+        return std::shared_ptr<RealtimeStore>(
+            std::make_shared<FailOnceRealtimeStore>(delegate, fail_next_write_));
+    }
+
+ private:
+    ArrowRealtimeStoreFactory delegate_;
+    std::shared_ptr<bool> fail_next_write_;
+};
+
+}
 
 class KeyValueFileStoreWriteTest : public ::testing::Test {
  protected:
@@ -128,14 +242,15 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
                                        /*ignore_if_exists=*/false));
     }
 
-    std::unique_ptr<RecordBatch> MakeBatch(const std::shared_ptr<arrow::Schema>& schema,
-                                           const std::string& json) const {
+    std::unique_ptr<RecordBatch> MakeBatch(
+        const std::shared_ptr<arrow::Schema>& schema, const std::string& json,
+        const std::vector<RecordBatch::RowKind>& row_kinds = {}) const {
         auto struct_type = arrow::struct_(schema->fields());
         auto array = arrow::ipc::internal::json::ArrayFromJSON(struct_type, json).ValueOrDie();
         ::ArrowArray arrow_array;
         EXPECT_TRUE(arrow::ExportArray(*array, &arrow_array).ok());
         RecordBatchBuilder batch_builder(&arrow_array);
-        return batch_builder.SetBucket(0).Finish().value();
+        return batch_builder.SetRowKinds(row_kinds).SetBucket(0).Finish().value();
     }
 
     std::vector<std::shared_ptr<CommitMessage>> WriteAndPrepare(
@@ -194,6 +309,58 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
         EXPECT_NE(nullptr, metadata);
         return MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy()).value();
     }
+
+    Result<std::vector<std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>>>
+    ReadPreparedRows(const std::shared_ptr<RealtimeContext>& realtime_context) const {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> context,
+                               RealtimeContextImpl::Cast(realtime_context));
+        PAIMON_ASSIGN_OR_RAISE(std::vector<RealtimePartitionBucketView> views,
+                               context->AcquireReadViews());
+        if (views.size() != 1) {
+            return Status::Invalid("expected exactly one real-time store");
+        }
+        RealtimeQueryContext query_context{nullptr, nullptr, false};
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               views[0].store->CreateQueryReaders(
+                                   views[0].read_view, 0, query_context));
+        std::vector<std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>> rows;
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            while (true) {
+                PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
+                if (BatchReader::IsEofBatch(batch)) {
+                    break;
+                }
+                PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                    std::shared_ptr<arrow::Array> array,
+                    arrow::ImportArray(batch.first.get(), batch.second.get()));
+                std::shared_ptr<arrow::StructArray> values =
+                    std::dynamic_pointer_cast<arrow::StructArray>(array);
+                if (!values || values->num_fields() != 5) {
+                    return Status::Invalid("unexpected prepared real-time batch");
+                }
+                std::shared_ptr<arrow::Int8Array> row_kinds =
+                    std::dynamic_pointer_cast<arrow::Int8Array>(values->field(0));
+                std::shared_ptr<arrow::Int64Array> sequences =
+                    std::dynamic_pointer_cast<arrow::Int64Array>(values->field(1));
+                std::shared_ptr<arrow::Int64Array> offsets =
+                    std::dynamic_pointer_cast<arrow::Int64Array>(values->field(2));
+                std::shared_ptr<arrow::Int64Array> ids =
+                    std::dynamic_pointer_cast<arrow::Int64Array>(values->field(3));
+                std::shared_ptr<arrow::StringArray> payloads =
+                    std::dynamic_pointer_cast<arrow::StringArray>(values->field(4));
+                if (!row_kinds || !sequences || !offsets || !ids || !payloads) {
+                    return Status::Invalid("unexpected prepared real-time column type");
+                }
+                for (int64_t row = 0; row < values->length(); ++row) {
+                    rows.emplace_back(row_kinds->Value(row), ids->Value(row),
+                                      payloads->GetString(row), sequences->Value(row),
+                                      offsets->Value(row));
+                }
+            }
+            reader->Close();
+        }
+        return rows;
+    }
 };
 
 TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) {
@@ -310,7 +477,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
         {Options::WRITE_BUFFER_SIZE, "1"},
     };
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
-        arrow::field("id", arrow::int64(), /*nullable=*/false),
+        arrow::field("id", arrow::int64(), false),
         arrow::field("value", arrow::utf8()),
     });
     std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
@@ -329,13 +496,23 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                          FileStoreWrite::Create(std::move(write_context)));
 
-    ASSERT_OK(writer->Write(MakeBatch(schema, R"([
+    std::unique_ptr<RecordBatch> batch =
+        MakeBatch(schema, R"([
         [1, "old"],
         [2, "two"],
         [1, "new"]
-    ])")));
+    ])",
+                  {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE,
+                   RecordBatch::RowKind::UPDATE_AFTER});
+    ASSERT_OK(writer->Write(std::move(batch)));
+    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
+    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> prepared_rows,
+                         ReadPreparedRows(realtime_context));
+    ASSERT_EQ(
+        (std::vector<PreparedRow>{{0, 1, "old", 0, 0}, {2, 1, "new", 2, 2}, {3, 2, "two", 1, 1}}),
+        prepared_rows);
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progresses,
-                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+                         writer->PrepareCommitWithProgress(0));
     ASSERT_EQ(1, progresses.size());
     ASSERT_EQ(OffsetRange(0, 3), progresses[0].offset_range);
     std::shared_ptr<CommitMessageImpl> commit_message =
@@ -351,6 +528,189 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
+    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"}};
+    const std::shared_ptr<arrow::Schema> schema = arrow::schema({
+        arrow::field("id", arrow::int64(), false),
+        arrow::field("_REALTIME_OFFSET", arrow::int64()),
+    });
+    std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), schema, options);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    WriteContextBuilder builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test");
+    builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         FileStoreWrite::Create(std::move(write_context)));
+
+    ASSERT_NOK_WITH_MSG(writer->Write(MakeBatch(schema, R"([[1, 10]])")),
+                        "PK real-time write schema contains reserved transport field");
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(KeyValueFileStoreWriteTest, TestWriteFailureKeepsCursors) {
+    const std::map<std::string, std::string> options = {
+        {Options::BUCKET, "1"},
+        {Options::WRITE_BUFFER_SIZE, "1"},
+    };
+    const std::shared_ptr<arrow::Schema> schema = arrow::schema({
+        arrow::field("id", arrow::int64(), false),
+        arrow::field("value", arrow::utf8()),
+    });
+    std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), schema, options);
+    const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    auto fail_next_write = std::make_shared<bool>(true);
+    auto factory = std::make_shared<FailOnceRealtimeStoreFactory>(fail_next_write);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    WriteContextBuilder builder(table_path, "test");
+    builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         FileStoreWrite::Create(std::move(write_context)));
+
+    ASSERT_NOK_WITH_MSG(writer->Write(MakeBatch(schema, R"([[9, "rejected"]])")),
+                        "injected real-time store write failure");
+    ASSERT_OK(writer->Write(MakeBatch(schema, R"([[1, "kept"]])")));
+    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
+    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> prepared_rows,
+                         ReadPreparedRows(realtime_context));
+    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "kept", 0, 0}}), prepared_rows);
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
+    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"}};
+    const std::shared_ptr<arrow::Schema> schema = arrow::schema({
+        arrow::field("id", arrow::int64(), false),
+        arrow::field("value", arrow::utf8()),
+    });
+    std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), schema, options);
+    const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    WriteContextBuilder builder(table_path, "test");
+    builder.SetOptions(options)
+        .WithStreamingMode(true)
+        .WithRealtimeContext(realtime_context)
+        .WithMemoryPool(pool);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         FileStoreWrite::Create(std::move(write_context)));
+
+    const int64_t allocations_before_write = pool->allocation_count;
+    ASSERT_OK(writer->Write(MakeBatch(schema, R"([[1, "one"]])")));
+    ASSERT_GT(pool->allocation_count, allocations_before_write);
+    ASSERT_OK(writer->Close());
+    writer.reset();
+    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
+    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> retained_rows,
+                         ReadPreparedRows(realtime_context));
+    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "one", 0, 0}}), retained_rows);
+
+    std::shared_ptr<TestingMemoryPool> rejecting_pool = std::make_shared<TestingMemoryPool>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> rejecting_context,
+                         RealtimeContext::Create());
+    WriteContextBuilder rejecting_builder(table_path, "rejecting");
+    rejecting_builder.SetOptions(options)
+        .WithStreamingMode(true)
+        .WithRealtimeContext(rejecting_context)
+        .WithMemoryPool(rejecting_pool);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> rejecting_write_context,
+                         rejecting_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> rejecting_writer,
+                         FileStoreWrite::Create(std::move(rejecting_write_context)));
+    ASSERT_OK(rejecting_writer->Write(MakeBatch(schema, "[]")));
+    const int64_t rejecting_allocations_before_write = rejecting_pool->allocation_count;
+    rejecting_pool->reject_allocations = true;
+    ASSERT_NOK_WITH_MSG(rejecting_writer->Write(MakeBatch(schema, R"([[2, "two"]])")),
+                        "Out of memory");
+    ASSERT_GT(rejecting_pool->allocation_count, rejecting_allocations_before_write);
+    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> rejected_rows,
+                         ReadPreparedRows(rejecting_context));
+    ASSERT_TRUE(rejected_rows.empty());
+    ASSERT_OK(rejecting_writer->Close());
+}
+
+TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) {
+    const int64_t max = std::numeric_limits<int64_t>::max();
+    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"}};
+    const std::shared_ptr<arrow::Schema> schema = arrow::schema({
+        arrow::field("id", arrow::int64(), false),
+        arrow::field("value", arrow::utf8()),
+    });
+    std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), schema, options);
+    const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> initial_context,
+                         RealtimeContext::Create());
+    WriteContextBuilder initial_builder(table_path, "initial");
+    initial_builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(
+        initial_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> initial_write_context,
+                         initial_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> initial_writer,
+                         FileStoreWrite::Create(std::move(initial_write_context)));
+    ASSERT_OK(initial_writer->Write(MakeBatch(schema, R"([[0, "initial"]])")));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> initial_progress,
+                         initial_writer->PrepareCommitWithProgress(0));
+    ASSERT_EQ(1, initial_progress.size());
+    std::shared_ptr<CommitMessageImpl> initial_message =
+        std::dynamic_pointer_cast<CommitMessageImpl>(initial_progress[0].commit_message);
+    ASSERT_NE(nullptr, initial_message);
+    ASSERT_EQ(1, initial_message->GetNewFilesIncrement().NewFiles().size());
+    initial_message->GetNewFilesIncrement().NewFiles()[0]->AssignSequenceNumber(max - 2, max - 2);
+    initial_progress[0].offset_range = OffsetRange(0, max - 1);
+
+    CommitContextBuilder commit_builder(table_path, "initial");
+    commit_builder.SetOptions(options);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context, commit_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> committer,
+                         FileStoreCommit::Create(std::move(commit_context)));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id,
+                         committer->CommitWithProgress(initial_progress, 0, std::nullopt));
+    ASSERT_OK(initial_writer->Close());
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    WriteContextBuilder builder(table_path, "boundary");
+    builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         FileStoreWrite::Create(std::move(write_context)));
+    ASSERT_OK(writer->Write(MakeBatch(schema, R"([[1, "legal"]])")));
+    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
+    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> prepared_rows,
+                         ReadPreparedRows(realtime_context));
+    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "legal", max - 1, max - 1}}), prepared_rows);
+
+    ASSERT_NOK_WITH_MSG(writer->Write(MakeBatch(schema, R"([[2, "overflow"]])")),
+                        "real-time offset range exceeds INT64_MAX");
+    ASSERT_OK_AND_ASSIGN(prepared_rows, ReadPreparedRows(realtime_context));
+    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "legal", max - 1, max - 1}}), prepared_rows);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context_impl,
+                         RealtimeContextImpl::Cast(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimePartitionBucketView> views,
+                         context_impl->AcquireReadViews());
+    ASSERT_EQ(1, views.size());
+    ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(max - 1, max)),
+              views[0].read_view->GetOffsetRange());
+    ASSERT_OK(writer->Close());
+    ASSERT_GE(snapshot_id, 1);
+}
+
 TEST_F(KeyValueFileStoreWriteTest,
        TestPrepareCommitShouldSucceedWhenDefaultCompactRewriterPathEnabled) {
     ASSERT_OK_AND_ASSIGN(
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 8d8367e39..2f64f6df8 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -81,8 +81,6 @@ class MergeFunctionWrapper;
 
 namespace {
 
-/// Concatenates merge readers whose key ranges are ordered and non-overlapping, preserving one
-/// projection pipeline without merging independent disk-only components.
 class ConcatNonOverlappingMergeReaders final : public SortMergeReader {
  public:
     explicit ConcatNonOverlappingMergeReaders(
@@ -117,7 +115,7 @@ class ConcatNonOverlappingMergeReaders final : public SortMergeReader {
     size_t current_ = 0;
 };
 
-}  // namespace
+}
 
 class MergeFileSplitRead::RealtimeReaderBuilder {
  public:
@@ -219,8 +217,8 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
         inputs_.reserve(inputs_.size() + additional_readers.size());
         for (AdditionalKeyValueReader& additional : additional_readers) {
             has_unknown_range_ |= !additional.min_key || !additional.max_key;
-            inputs_.push_back(RangeInput{additional.min_key, additional.max_key,
-                                         /*disk_runs=*/{}, std::move(additional.reader)});
+            inputs_.push_back(RangeInput{additional.min_key, additional.max_key, {},
+                                         std::move(additional.reader)});
         }
     }
 
@@ -310,7 +308,7 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
                         component.front().disk_runs, first_split_->Partition(), dv_factory_,
                         component.front().disk_runs.size() == 1 ? owner_->context_->GetPredicate()
                                                                 : owner_->predicate_for_keys_,
-                        data_file_path_factory_, /*drop_delete=*/false));
+                        data_file_path_factory_, false));
                 component_readers.push_back(std::move(disk_component));
                 continue;
             }
diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h
index 0824254b7..85b5b2a28 100644
--- a/src/paimon/core/operation/merge_file_split_read.h
+++ b/src/paimon/core/operation/merge_file_split_read.h
@@ -127,7 +127,6 @@ class MergeFileSplitRead : public AbstractSplitRead {
         return key_schema_;
     }
 
-    /// Merges ordinary disk splits with generic additional sorted KeyValue readers.
     Result<std::unique_ptr<BatchReader>> CreateRealtimeReader(
         const std::vector<std::shared_ptr<Split>>& disk_splits,
         std::vector<AdditionalKeyValueReader>&& additional_readers);
diff --git a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
index e6e22edfd..4cfdb4c3d 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
@@ -21,14 +21,9 @@
 
 #include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
-#include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
-#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/common/utils/scope_guard.h"
-#include "paimon/core/core_options.h"
-#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
-#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/realtime/arrow_realtime_store.h"
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 #include "paimon/macros.h"
@@ -55,31 +50,8 @@ Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
                                                     request.memory_pool, arrow_pool);
     }
 
-    const PrimaryKeyRealtimeStoreCreateConfig& primary_key_config =
-        std::get<PrimaryKeyRealtimeStoreCreateConfig>(request.mode_config);
-    std::vector<DataField> key_fields;
-    key_fields.reserve(primary_key_config.primary_keys.size());
-    for (const std::string& primary_key : primary_key_config.primary_keys) {
-        const int32_t field_index = imported_schema->GetFieldIndex(primary_key);
-        if (field_index < 0) {
-            return Status::Invalid("primary key ", primary_key, " is missing from write schema");
-        }
-        key_fields.emplace_back(field_index, imported_schema->field(field_index));
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
-                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
-    auto merge_function_wrapper_factory = []() {
-        auto merge_function = std::make_unique<DeduplicateMergeFunction>(
-            /*ignore_delete=*/false);
-        return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
-    };
-    PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(request.options));
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<PrimaryKeyRealtimeStore> store,
-        PrimaryKeyRealtimeStore::Create(imported_schema, primary_key_config.primary_keys,
-                                        key_comparator, merge_function_wrapper_factory,
-                                        primary_key_config.restore_max_sequence_number,
-                                        core_options.GetReadBatchSize(), request.memory_pool));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                           PrimaryKeyRealtimeStore::Create(imported_schema, request.memory_pool));
     return std::shared_ptr<RealtimeStore>(std::move(store));
 }
 
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 8f51c1b1e..0d6de9f5f 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -9,41 +9,24 @@
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
-#include <algorithm>
-#include <limits>
 #include <mutex>
 #include <utility>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
-#include "paimon/common/data/binary_row_writer.h"
-#include "paimon/common/data/columnar/columnar_row_ref.h"
 #include "paimon/common/metrics/metrics_impl.h"
-#include "paimon/common/table/special_fields.h"
-#include "paimon/common/types/data_field.h"
-#include "paimon/common/types/row_kind.h"
-#include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
-#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/core_options.h"
-#include "paimon/core/io/key_value_in_memory_record_reader.h"
-#include "paimon/core/io/key_value_projection_consumer.h"
-#include "paimon/core/io/key_value_projection_reader.h"
-#include "paimon/core/io/merged_key_value_record_reader.h"
-#include "paimon/core/key_value.h"
-#include "paimon/core/mergetree/compact/sort_merge_reader_with_loser_tree.h"
-#include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
 
 namespace paimon {
@@ -83,562 +66,255 @@ Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options) {
 namespace {
 
 uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
-    uint64_t result = 0;
+    uint64_t total = 0;
     for (const std::shared_ptr<arrow::Buffer>& buffer : data->buffers) {
         if (buffer) {
-            result += static_cast<uint64_t>(buffer->size());
+            total += static_cast<uint64_t>(buffer->size());
         }
     }
     for (const std::shared_ptr<arrow::ArrayData>& child : data->child_data) {
-        result += GetArrayMemoryUsage(child);
+        total += GetArrayMemoryUsage(child);
     }
     if (data->dictionary) {
-        result += GetArrayMemoryUsage(data->dictionary);
+        total += GetArrayMemoryUsage(data->dictionary);
     }
-    return result;
-}
-
-int32_t FindPkQueryFieldIndex(const std::shared_ptr<arrow::Schema>& write_schema,
-                              const std::shared_ptr<arrow::Field>& read_field) {
-    Result<int32_t> read_id = NestedProjectionUtils::GetPaimonFieldId(read_field);
-    if (read_id.ok()) {
-        Result<std::shared_ptr<arrow::Field>> write_field =
-            NestedProjectionUtils::FindFieldByPaimonId(arrow::struct_(write_schema->fields()),
-                                                       read_id.value());
-        if (write_field.ok()) {
-            return write_schema->GetFieldIndex(write_field.value()->name());
-        }
-    }
-
-    const int32_t name_index = write_schema->GetFieldIndex(read_field->name());
-    if (name_index < 0) {
-        return -1;
-    }
-    Result<int32_t> write_id =
-        NestedProjectionUtils::GetPaimonFieldId(write_schema->field(name_index));
-    if (read_id.ok() && write_id.ok() && read_id.value() != write_id.value()) {
-        return -1;
-    }
-    return name_index;
+    return total;
 }
 
 struct StoredBatch {
     std::shared_ptr<arrow::StructArray> data;
-    std::vector<RecordBatch::RowKind> row_kinds;
     OffsetRange offset_range;
-    int64_t first_sequence_number;
     uint64_t memory_usage;
 };
-using BatchGroup = std::vector<std::shared_ptr<const StoredBatch>>;
 
 class Segment final : public RealtimeSegmentHandle {
  public:
-    Segment(const OffsetRange& offset_range,
-            std::vector<std::shared_ptr<const StoredBatch>>&& batches)
-        : offset_range_(offset_range), batches_(std::move(batches)) {}
+    Segment(const OffsetRange& range, std::vector<StoredBatch>&& batches)
+        : range_(range), batches_(std::move(batches)) {}
 
     OffsetRange GetOffsetRange() const override {
-        return offset_range_;
+        return range_;
     }
-
-    const std::vector<std::shared_ptr<const StoredBatch>>& Batches() const {
+    const std::vector<StoredBatch>& Batches() const {
         return batches_;
     }
 
-    uint64_t GetMemoryUsage() const {
-        uint64_t result = 0;
-        for (const std::shared_ptr<const StoredBatch>& batch : batches_) {
-            result += batch->memory_usage;
-        }
-        return result;
-    }
-
  private:
-    OffsetRange offset_range_;
-    std::vector<std::shared_ptr<const StoredBatch>> batches_;
+    OffsetRange range_;
+    std::vector<StoredBatch> batches_;
 };
 
-class PrimaryKeyRealtimeReadView final : public RealtimeReadView {
+class ReadView final : public RealtimeReadView {
  public:
-    explicit PrimaryKeyRealtimeReadView(std::vector<BatchGroup>&& groups)
-        : groups_(std::move(groups)) {
-        if (!groups_.empty()) {
-            offset_range_ = OffsetRange(groups_.front().front()->offset_range.begin,
-                                        groups_.back().back()->offset_range.end);
+    explicit ReadView(std::vector<std::shared_ptr<Segment>>&& segments)
+        : segments_(std::move(segments)) {
+        if (!segments_.empty()) {
+            range_ = OffsetRange(segments_.front()->GetOffsetRange().begin,
+                                 segments_.back()->GetOffsetRange().end);
         }
     }
 
     std::optional<OffsetRange> GetOffsetRange() const override {
-        return offset_range_;
+        return range_;
     }
-
-    const std::vector<BatchGroup>& Groups() const {
-        return groups_;
+    const std::vector<std::shared_ptr<Segment>>& Segments() const {
+        return segments_;
     }
 
  private:
-    std::vector<BatchGroup> groups_;
-    std::optional<OffsetRange> offset_range_;
+    std::vector<std::shared_ptr<Segment>> segments_;
+    std::optional<OffsetRange> range_;
 };
 
-class CommitBatchReader final : public BatchReader {
+class RawBatchReader final : public BatchReader {
  public:
-    CommitBatchReader(const std::shared_ptr<Segment>& segment,
-                      const std::shared_ptr<arrow::MemoryPool>& arrow_pool)
-        : segment_(segment), arrow_pool_(arrow_pool), metrics_(std::make_shared<MetricsImpl>()) {}
+    RawBatchReader(std::vector<StoredBatch> batches)
+        : batches_(std::move(batches)), metrics_(std::make_shared<MetricsImpl>()) {}
 
     Result<ReadBatch> NextBatch() override {
-        if (!segment_ || next_batch_ >= static_cast<int32_t>(segment_->Batches().size())) {
+        if (next_ == batches_.size()) {
             return MakeEofBatch();
         }
-        const std::shared_ptr<const StoredBatch>& stored = segment_->Batches()[next_batch_++];
-        const int64_t row_count = stored->data->length();
-        arrow::Int8Builder row_kind_builder(arrow_pool_.get());
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(row_kind_builder.Reserve(row_count));
-        if (stored->row_kinds.empty()) {
-            for (int64_t i = 0; i < row_count; ++i) {
-                row_kind_builder.UnsafeAppend(static_cast<int8_t>(RecordBatch::RowKind::INSERT));
-            }
-        } else {
-            for (RecordBatch::RowKind row_kind : stored->row_kinds) {
-                row_kind_builder.UnsafeAppend(static_cast<int8_t>(row_kind));
-            }
-        }
-        std::shared_ptr<arrow::Array> row_kind_array;
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(row_kind_builder.Finish(&row_kind_array));
-        arrow::ArrayVector arrays = {std::move(row_kind_array)};
-        arrays.insert(arrays.end(), stored->data->fields().begin(), stored->data->fields().end());
-        arrow::FieldVector fields = {
-            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())};
-        const arrow::FieldVector& value_fields = stored->data->struct_type()->fields();
-        fields.insert(fields.end(), value_fields.begin(), value_fields.end());
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> output,
-                                          arrow::StructArray::Make(arrays, fields));
-        auto c_array = std::make_unique<ArrowArray>();
-        auto c_schema = std::make_unique<ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*output, c_array.get(), c_schema.get()));
-        return ReadBatch(std::move(c_array), std::move(c_schema));
+        const std::shared_ptr<arrow::StructArray>& batch = batches_[next_++].data;
+        auto array = std::make_unique<ArrowArray>();
+        auto schema = std::make_unique<ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*batch, array.get(), schema.get()));
+        return ReadBatch(std::move(array), std::move(schema));
     }
 
     std::shared_ptr<Metrics> GetReaderMetrics() const override {
         return metrics_;
     }
-
     void Close() override {
-        segment_.reset();
+        batches_.clear();
     }
 
  private:
-    std::shared_ptr<Segment> segment_;
-    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    std::vector<StoredBatch> batches_;
+    size_t next_ = 0;
     std::shared_ptr<Metrics> metrics_;
-    int32_t next_batch_ = 0;
-};
-
-class KeyRangeBatchReader final : public BatchReader, public PrimaryKeyRangeProvider {
- public:
-    KeyRangeBatchReader(std::unique_ptr<BatchReader>&& reader,
-                        const std::shared_ptr<InternalRow>& min_key,
-                        const std::shared_ptr<InternalRow>& max_key)
-        : reader_(std::move(reader)), min_key_(min_key), max_key_(max_key) {}
-
-    Result<ReadBatch> NextBatch() override {
-        return reader_->NextBatch();
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return reader_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        reader_->Close();
-    }
-
-    std::shared_ptr<InternalRow> GetMinKey() const override {
-        return min_key_;
-    }
-
-    std::shared_ptr<InternalRow> GetMaxKey() const override {
-        return max_key_;
-    }
-
- private:
-    std::unique_ptr<BatchReader> reader_;
-    std::shared_ptr<InternalRow> min_key_;
-    std::shared_ptr<InternalRow> max_key_;
 };
 
 }  // namespace
 
 class PrimaryKeyRealtimeStore::Impl {
  public:
-    Impl(const std::shared_ptr<arrow::Schema>& write_schema, std::vector<std::string> primary_keys,
-         const std::shared_ptr<FieldsComparator>& key_comparator,
-         const std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>&
-             merge_function_wrapper_factory,
-         int64_t next_sequence_number, int32_t read_batch_size,
-         const std::shared_ptr<MemoryPool>& memory_pool)
-        : write_schema_(write_schema),
-          primary_keys_(std::move(primary_keys)),
-          key_comparator_(key_comparator),
-          merge_function_wrapper_factory_(merge_function_wrapper_factory),
-          next_sequence_number_(next_sequence_number),
-          read_batch_size_(read_batch_size),
-          memory_pool_(memory_pool),
-          arrow_pool_(GetArrowPool(memory_pool)) {}
-
-    Result<std::shared_ptr<InternalRow>> CopyKey(const InternalRow& key) const {
-        auto result = std::make_shared<BinaryRow>(static_cast<int32_t>(primary_keys_.size()));
-        BinaryRowWriter writer(result.get(), /*initial_size=*/128, memory_pool_.get());
-        writer.Reset();
-        for (int32_t index = 0; index < static_cast<int32_t>(primary_keys_.size()); ++index) {
-            std::shared_ptr<arrow::Field> field =
-                write_schema_->GetFieldByName(primary_keys_[index]);
-            PAIMON_ASSIGN_OR_RAISE(InternalRow::FieldGetterFunc getter,
-                                   InternalRow::CreateFieldGetter(index, field->type(),
-                                                                  /*use_view=*/true));
-            PAIMON_ASSIGN_OR_RAISE(BinaryRowWriter::FieldSetterFunc setter,
-                                   BinaryRowWriter::CreateFieldSetter(index, field->type()));
-            setter(getter(key), &writer);
-        }
-        writer.Complete();
-        return std::static_pointer_cast<InternalRow>(result);
-    }
-
-    Result<std::pair<std::shared_ptr<InternalRow>, std::shared_ptr<InternalRow>>> GetKeyRange(
-        const std::shared_ptr<arrow::StructArray>& values) const {
-        arrow::ArrayVector key_arrays;
-        key_arrays.reserve(primary_keys_.size());
-        for (const std::string& primary_key : primary_keys_) {
-            std::shared_ptr<arrow::Array> key_array = values->GetFieldByName(primary_key);
-            if (!key_array) {
-                return Status::Invalid("primary key is missing from PK query batch: ", primary_key);
-            }
-            key_arrays.push_back(std::move(key_array));
-        }
-        auto context = std::make_shared<ColumnarBatchContext>(key_arrays, memory_pool_);
-        int64_t min_row = 0;
-        int64_t max_row = 0;
-        for (int64_t row = 1; row < values->length(); ++row) {
-            ColumnarRowRef current(context, row);
-            ColumnarRowRef min_key(context, min_row);
-            ColumnarRowRef max_key(context, max_row);
-            if (key_comparator_->CompareTo(current, min_key) < 0) {
-                min_row = row;
-            }
-            if (key_comparator_->CompareTo(current, max_key) > 0) {
-                max_row = row;
-            }
-        }
-        ColumnarRowRef min_key(context, min_row);
-        ColumnarRowRef max_key(context, max_row);
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalRow> copied_min, CopyKey(min_key));
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalRow> copied_max, CopyKey(max_key));
-        return std::make_pair(std::move(copied_min), std::move(copied_max));
-    }
+    explicit Impl(std::shared_ptr<arrow::Schema> prepared_schema)
+        : prepared_schema_(std::move(prepared_schema)) {}
 
     Status Write(RealtimeWriteBatch&& write_batch) {
         if (!write_batch.batch || !write_batch.batch->GetData()) {
             return Status::Invalid("PK real-time write batch is null");
         }
         const int64_t row_count = write_batch.batch->GetData()->length;
-        if (row_count <= 0 || write_batch.offset_range.begin < 0 ||
-            write_batch.offset_range.Count() != row_count) {
+        if (write_batch.offset_range.begin < 0 || write_batch.offset_range.Count() != row_count ||
+            row_count <= 0) {
             return Status::Invalid("PK real-time offset range does not match batch row count");
         }
-        const std::vector<RecordBatch::RowKind>& row_kinds = write_batch.batch->GetRowKind();
-        if (!row_kinds.empty() && static_cast<int64_t>(row_kinds.size()) != row_count) {
-            return Status::Invalid("PK real-time row-kind count does not match batch row count");
-        }
-        for (RecordBatch::RowKind row_kind : row_kinds) {
-            PAIMON_ASSIGN_OR_RAISE(const RowKind* validated,
-                                   RowKind::FromByteValue(static_cast<int8_t>(row_kind)));
-            static_cast<void>(validated);
-        }
         PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-            std::shared_ptr<arrow::Array> imported,
+            std::shared_ptr<arrow::Array> array,
             arrow::ImportArray(write_batch.batch->GetData(),
-                               arrow::struct_(write_schema_->fields())));
-        if (!imported || imported->type_id() != arrow::Type::STRUCT) {
-            return Status::Invalid("PK real-time write data is not a StructArray");
+                               arrow::struct_(prepared_schema_->fields())));
+        if (!array || array->type_id() != arrow::Type::STRUCT) {
+            return Status::Invalid("PK real-time prepared batch is not a StructArray");
         }
-        std::shared_ptr<arrow::StructArray> values =
-            checked_pointer_cast<arrow::StructArray>(imported);
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(values->ValidateFull());
-
+        std::shared_ptr<arrow::StructArray> prepared =
+            checked_pointer_cast<arrow::StructArray>(array);
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(prepared->ValidateFull());
         std::lock_guard<std::mutex> lock(mutex_);
         if (last_offset_ && write_batch.offset_range.begin != last_offset_.value()) {
             return Status::Invalid("PK real-time offset ranges must be contiguous");
         }
-        if (row_count > std::numeric_limits<int64_t>::max() - next_sequence_number_) {
-            return Status::Invalid("PK sequence range exceeds INT64_MAX");
-        }
-        auto stored = std::make_shared<const StoredBatch>(
-            StoredBatch{std::move(values), row_kinds, write_batch.offset_range,
-                        next_sequence_number_, GetArrayMemoryUsage(imported->data())});
-        building_batches_.push_back(std::move(stored));
-        building_memory_usage_ += building_batches_.back()->memory_usage;
+        building_.push_back(
+            StoredBatch{prepared, write_batch.offset_range, GetArrayMemoryUsage(prepared->data())});
+        building_memory_usage_ += building_.back().memory_usage;
         last_offset_ = write_batch.offset_range.end;
-        next_sequence_number_ += row_count;
         return Status::OK();
     }
 
     Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() {
         std::lock_guard<std::mutex> lock(mutex_);
-        if (building_batches_.empty()) {
+        if (building_.empty()) {
             return std::optional<std::shared_ptr<RealtimeSegmentHandle>>();
         }
-        const OffsetRange range(building_batches_.front()->offset_range.begin,
-                                building_batches_.back()->offset_range.end);
-        auto segment = std::make_shared<Segment>(range, std::move(building_batches_));
-        sealed_segments_.push_back(segment);
-        building_batches_.clear();
+        OffsetRange range(building_.front().offset_range.begin, building_.back().offset_range.end);
+        std::shared_ptr<Segment> segment = std::make_shared<Segment>(range, std::move(building_));
+        sealed_.push_back(segment);
+        building_.clear();
         building_memory_usage_ = 0;
         return std::optional<std::shared_ptr<RealtimeSegmentHandle>>(std::move(segment));
     }
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>& segment) {
-        std::shared_ptr<Segment> typed = std::dynamic_pointer_cast<Segment>(segment);
-        if (!typed) {
+        const std::shared_ptr<RealtimeSegmentHandle>& handle) {
+        std::shared_ptr<Segment> segment = std::dynamic_pointer_cast<Segment>(handle);
+        if (!segment) {
             return Status::Invalid("segment was not created by the PK real-time store");
         }
-        std::vector<std::unique_ptr<BatchReader>> result;
-        result.push_back(std::make_unique<CommitBatchReader>(typed, arrow_pool_));
-        return result;
+        std::vector<std::unique_ptr<BatchReader>> readers;
+        readers.reserve(segment->Batches().size());
+        for (const StoredBatch& batch : segment->Batches()) {
+            readers.push_back(std::make_unique<RawBatchReader>(std::vector<StoredBatch>{batch}));
+        }
+        return readers;
     }
 
     Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() {
         std::lock_guard<std::mutex> lock(mutex_);
-        std::vector<BatchGroup> groups;
-        groups.reserve(sealed_segments_.size() + (building_batches_.empty() ? 0 : 1));
-        for (const std::shared_ptr<Segment>& segment : sealed_segments_) {
-            groups.push_back(segment->Batches());
+        std::vector<std::shared_ptr<Segment>> segments = sealed_;
+        if (!building_.empty()) {
+            OffsetRange range(building_.front().offset_range.begin,
+                              building_.back().offset_range.end);
+            segments.push_back(
+                std::make_shared<Segment>(range, std::vector<StoredBatch>(building_)));
         }
-        if (!building_batches_.empty()) {
-            groups.push_back(building_batches_);
-        }
-        return std::shared_ptr<RealtimeReadView>(new PrimaryKeyRealtimeReadView(std::move(groups)));
+        return std::shared_ptr<RealtimeReadView>(new ReadView(std::move(segments)));
     }
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t lower,
-        const RealtimeQueryContext& context) {
-        std::shared_ptr<PrimaryKeyRealtimeReadView> typed =
-            std::dynamic_pointer_cast<PrimaryKeyRealtimeReadView>(view);
+        const std::shared_ptr<RealtimeReadView>& view, int64_t, const RealtimeQueryContext&) {
+        std::shared_ptr<ReadView> typed = std::dynamic_pointer_cast<ReadView>(view);
         if (!typed) {
             return Status::Invalid("read view was not created by the PK real-time store");
         }
-        if (!context.read_schema || !context.read_schema->release) {
-            return Status::Invalid("PK real-time query read schema is null");
-        }
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> requested,
-                                          arrow::ImportSchema(context.read_schema));
-        arrow::FieldVector output_fields = {
-            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())};
-        arrow::FieldVector aligned_value_fields = write_schema_->fields();
-        std::vector<int32_t> projection = {KeyValueProjectionConsumer::kValueKindProjection};
-        for (const std::shared_ptr<arrow::Field>& field : requested->fields()) {
-            if (field->name() == SpecialFields::ValueKind().Name()) {
-                continue;
-            }
-            output_fields.push_back(field);
-            if (field->name() == SpecialFields::SequenceNumber().Name()) {
-                projection.push_back(KeyValueProjectionConsumer::kSequenceNumberProjection);
-                continue;
-            }
-            int32_t index = FindPkQueryFieldIndex(write_schema_, field);
-            if (index < 0) {
-                Result<int32_t> field_id = NestedProjectionUtils::GetPaimonFieldId(field);
-                if (!field_id.ok()) {
-                    return Status::Invalid(
-                        "PK real-time query field is missing from write schema: ", field->name());
-                }
-                std::string internal_name =
-                    "__paimon_pk_realtime_null_" + std::to_string(field_id.value());
-                while (
-                    NestedProjectionUtils::FindFieldByName(aligned_value_fields, internal_name)) {
-                    internal_name.push_back('_');
-                }
-                index = static_cast<int32_t>(aligned_value_fields.size());
-                aligned_value_fields.push_back(field->WithName(internal_name));
-            } else {
-                aligned_value_fields[index] = write_schema_->field(index)->WithType(field->type());
-            }
-            projection.push_back(index);
+        std::vector<std::unique_ptr<BatchReader>> readers;
+        size_t batch_count = 0;
+        for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
+            batch_count += segment->Batches().size();
         }
-        const std::shared_ptr<arrow::DataType> aligned_value_type =
-            arrow::struct_(aligned_value_fields);
-
-        std::vector<std::unique_ptr<BatchReader>> result;
-        for (const BatchGroup& group : typed->Groups()) {
-            std::vector<std::unique_ptr<KeyValueRecordReader>> batch_readers;
-            std::shared_ptr<InternalRow> min_key;
-            std::shared_ptr<InternalRow> max_key;
-            for (const std::shared_ptr<const StoredBatch>& batch : group) {
-                if (batch->offset_range.end <= lower) {
-                    continue;
-                }
-                const int64_t offset = std::max<int64_t>(0, lower - batch->offset_range.begin);
-                const int64_t length = batch->data->length() - offset;
-                std::shared_ptr<arrow::Array> sliced = batch->data->Slice(offset, length);
-                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> aligned,
-                                       NestedProjectionUtils::AlignArrayToReadType(
-                                           sliced, aligned_value_type, arrow_pool_.get()));
-                if (!aligned || aligned->type_id() != arrow::Type::STRUCT) {
-                    return Status::Invalid(
-                        "PK real-time query projection did not produce a "
-                        "StructArray");
-                }
-                std::shared_ptr<arrow::StructArray> selected =
-                    checked_pointer_cast<arrow::StructArray>(aligned);
-                using KeyRange =
-                    std::pair<std::shared_ptr<InternalRow>, std::shared_ptr<InternalRow>>;
-                PAIMON_ASSIGN_OR_RAISE(KeyRange key_range, GetKeyRange(selected));
-                if (!min_key || key_comparator_->CompareTo(*key_range.first, *min_key) < 0) {
-                    min_key = key_range.first;
-                }
-                if (!max_key || key_comparator_->CompareTo(*key_range.second, *max_key) > 0) {
-                    max_key = key_range.second;
-                }
-                std::vector<RecordBatch::RowKind> selected_kinds;
-                if (!batch->row_kinds.empty()) {
-                    selected_kinds.assign(batch->row_kinds.begin() + offset,
-                                          batch->row_kinds.end());
-                }
-                std::unique_ptr<KeyValueRecordReader> reader =
-                    std::make_unique<KeyValueInMemoryRecordReader>(
-                        batch->first_sequence_number + offset, selected, selected_kinds,
-                        primary_keys_, /*user_defined_sequence_fields=*/std::vector<std::string>(),
-                        /*sequence_fields_ascending=*/true, key_comparator_, memory_pool_);
-                std::shared_ptr<MergeFunctionWrapper<KeyValue>> batch_merge =
-                    merge_function_wrapper_factory_();
-                if (!batch_merge) {
-                    return Status::Invalid("merge function wrapper factory returned null");
-                }
-                batch_readers.push_back(std::make_unique<MergedKeyValueRecordReader>(
-                    std::move(reader), key_comparator_, batch_merge));
-            }
-            if (batch_readers.empty()) {
-                continue;
-            }
-            std::shared_ptr<MergeFunctionWrapper<KeyValue>> group_merge =
-                merge_function_wrapper_factory_();
-            if (!group_merge) {
-                return Status::Invalid("merge function wrapper factory returned null");
+        readers.reserve(batch_count);
+        for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
+            for (const StoredBatch& batch : segment->Batches()) {
+                readers.push_back(
+                    std::make_unique<RawBatchReader>(std::vector<StoredBatch>{batch}));
             }
-            auto merged = std::make_unique<SortMergeReaderWithLoserTree>(
-                std::move(batch_readers), key_comparator_,
-                /*user_defined_seq_comparator=*/nullptr, group_merge);
-            PAIMON_ASSIGN_OR_RAISE(
-                std::unique_ptr<KeyValueProjectionReader> projected,
-                KeyValueProjectionReader::Create(std::move(merged), arrow::schema(output_fields),
-                                                 projection, read_batch_size_, memory_pool_));
-            result.push_back(
-                std::make_unique<KeyRangeBatchReader>(std::move(projected), min_key, max_key));
         }
-        return result;
+        return readers;
     }
 
-    Status AdvanceCommittedOffset(int64_t committed_end_offset) {
+    Status AdvanceCommittedOffset(int64_t committed_end) {
         std::lock_guard<std::mutex> lock(mutex_);
-        sealed_segments_.erase(
-            std::remove_if(sealed_segments_.begin(), sealed_segments_.end(),
-                           [committed_end_offset](const std::shared_ptr<Segment>& segment) {
-                               return segment->GetOffsetRange().end <= committed_end_offset;
-                           }),
-            sealed_segments_.end());
+        while (!sealed_.empty() && sealed_.front()->GetOffsetRange().end <= committed_end) {
+            sealed_.erase(sealed_.begin());
+        }
         return Status::OK();
     }
 
     uint64_t GetMemoryUsage() const {
         std::lock_guard<std::mutex> lock(mutex_);
-        uint64_t result = building_memory_usage_;
-        for (const std::shared_ptr<Segment>& segment : sealed_segments_) {
-            result += segment->GetMemoryUsage();
+        uint64_t total = building_memory_usage_;
+        for (const std::shared_ptr<Segment>& segment : sealed_) {
+            for (const StoredBatch& batch : segment->Batches()) {
+                total += batch.memory_usage;
+            }
         }
-        return result;
+        return total;
     }
 
  private:
-    std::shared_ptr<arrow::Schema> write_schema_;
-    std::vector<std::string> primary_keys_;
-    std::shared_ptr<FieldsComparator> key_comparator_;
-    std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>
-        merge_function_wrapper_factory_;
-    int64_t next_sequence_number_;
-    int32_t read_batch_size_;
-    std::shared_ptr<MemoryPool> memory_pool_;
-    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    std::shared_ptr<arrow::Schema> prepared_schema_;
     mutable std::mutex mutex_;
-    std::vector<std::shared_ptr<const StoredBatch>> building_batches_;
-    std::vector<std::shared_ptr<Segment>> sealed_segments_;
+    std::vector<StoredBatch> building_;
+    std::vector<std::shared_ptr<Segment>> sealed_;
     uint64_t building_memory_usage_ = 0;
     std::optional<int64_t> last_offset_;
 };
 
-Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
-    const std::shared_ptr<arrow::Schema>& write_schema,
-    const std::vector<std::string>& primary_keys,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
-    const std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>&
-        merge_function_wrapper_factory,
-    int64_t restore_max_sequence_number, int32_t read_batch_size,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
-    if (!write_schema || primary_keys.empty() || !key_comparator ||
-        !merge_function_wrapper_factory || !memory_pool || read_batch_size <= 0) {
-        return Status::Invalid("PK real-time store requires schema, keys, merge helpers, and pool");
-    }
-    if (restore_max_sequence_number < -1) {
-        return Status::Invalid("PK restore max sequence number must be at least -1");
-    }
-    if (restore_max_sequence_number == std::numeric_limits<int64_t>::max()) {
-        return Status::Invalid("PK sequence number has reached INT64_MAX");
-    }
-    for (const std::string& key : primary_keys) {
-        if (write_schema->GetFieldIndex(key) < 0) {
-            return Status::Invalid("primary key ", key, " is missing from write schema");
-        }
-    }
-    auto impl = std::make_unique<Impl>(
-        write_schema, primary_keys, key_comparator, merge_function_wrapper_factory,
-        restore_max_sequence_number + 1, read_batch_size, memory_pool);
-    return std::shared_ptr<PrimaryKeyRealtimeStore>(new PrimaryKeyRealtimeStore(std::move(impl)));
-}
-
 PrimaryKeyRealtimeStore::PrimaryKeyRealtimeStore(std::unique_ptr<Impl>&& impl)
     : impl_(std::move(impl)) {}
-
 PrimaryKeyRealtimeStore::~PrimaryKeyRealtimeStore() = default;
 
+Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    if (!prepared_schema || !memory_pool) {
+        return Status::Invalid("PK prepared schema or memory pool is null");
+    }
+    return std::shared_ptr<PrimaryKeyRealtimeStore>(
+        new PrimaryKeyRealtimeStore(std::make_unique<Impl>(prepared_schema)));
+}
 Status PrimaryKeyRealtimeStore::Write(RealtimeWriteBatch&& batch) {
     return impl_->Write(std::move(batch));
 }
-
 Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>>
 PrimaryKeyRealtimeStore::SealForCommit() {
     return impl_->SealForCommit();
 }
-
 Result<std::vector<std::unique_ptr<BatchReader>>> PrimaryKeyRealtimeStore::CreateCommitReaders(
     const std::shared_ptr<RealtimeSegmentHandle>& segment) {
     return impl_->CreateCommitReaders(segment);
 }
-
 Result<std::shared_ptr<RealtimeReadView>> PrimaryKeyRealtimeStore::AcquireReadView() {
     return impl_->AcquireReadView();
 }
-
 Result<std::vector<std::unique_ptr<BatchReader>>> PrimaryKeyRealtimeStore::CreateQueryReaders(
-    const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+    const std::shared_ptr<RealtimeReadView>& view, int64_t offset,
     const RealtimeQueryContext& context) {
-    return impl_->CreateQueryReaders(view, offset_begin, context);
+    return impl_->CreateQueryReaders(view, offset, context);
 }
-
-Status PrimaryKeyRealtimeStore::AdvanceCommittedOffset(int64_t committed_offset) {
-    return impl_->AdvanceCommittedOffset(committed_offset);
+Status PrimaryKeyRealtimeStore::AdvanceCommittedOffset(int64_t offset) {
+    return impl_->AdvanceCommittedOffset(offset);
 }
-
 uint64_t PrimaryKeyRealtimeStore::GetMemoryUsage() const {
     return impl_->GetMemoryUsage();
 }
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index 017864c04..5e18dd74f 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -19,11 +19,7 @@
 
 #pragma once
 
-#include <cstdint>
-#include <functional>
 #include <memory>
-#include <string>
-#include <vector>
 
 #include "paimon/realtime/realtime_store.h"
 
@@ -34,34 +30,15 @@ class Schema;
 namespace paimon {
 
 class CoreOptions;
-class FieldsComparator;
-struct KeyValue;
 class MemoryPool;
-class InternalRow;
-template <typename T>
-class MergeFunctionWrapper;
 
 Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options);
 
-/// Optional metadata exposed by PK query readers with a known inclusive key range.
-class PrimaryKeyRangeProvider {
- public:
-    virtual ~PrimaryKeyRangeProvider() = default;
-
-    virtual std::shared_ptr<InternalRow> GetMinKey() const = 0;
-    virtual std::shared_ptr<InternalRow> GetMaxKey() const = 0;
-};
-
-/// In-memory store for primary-key real-time writes.
+/// In-memory store for prepared primary-key real-time batches.
 class PrimaryKeyRealtimeStore final : public RealtimeStore {
  public:
     static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
-        const std::shared_ptr<arrow::Schema>& write_schema,
-        const std::vector<std::string>& primary_keys,
-        const std::shared_ptr<FieldsComparator>& key_comparator,
-        const std::function<std::shared_ptr<MergeFunctionWrapper<KeyValue>>()>&
-            merge_function_wrapper_factory,
-        int64_t restore_max_sequence_number, int32_t read_batch_size,
+        const std::shared_ptr<arrow::Schema>& prepared_schema,
         const std::shared_ptr<MemoryPool>& memory_pool);
 
     ~PrimaryKeyRealtimeStore() override;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 66901a6b1..43831d7be 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -9,23 +9,18 @@
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
-#include <cstdint>
-#include <limits>
-#include <map>
 #include <memory>
 #include <optional>
 #include <string>
-#include <utility>
 #include <vector>
 
 #include "arrow/api.h"
@@ -33,14 +28,52 @@
 #include "arrow/ipc/json_simple.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
-#include "paimon/common/utils/fields_comparator.h"
+#include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/core/core_options.h"
-#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
-#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
+#include "paimon/core/realtime/realtime_fields.h"
+#include "paimon/macros.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
+namespace {
+
+std::shared_ptr<arrow::Schema> PreparedSchema() {
+    return arrow::schema(
+        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+             ->WithNullable(false),
+         DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false),
+         arrow::field("id", arrow::int64()), arrow::field("value", arrow::utf8())});
+}
+
+std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) {
+    std::shared_ptr<arrow::Array> array =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(PreparedSchema()->fields()), json)
+            .ValueOrDie();
+    auto c_array = std::make_unique<ArrowArray>();
+    EXPECT_TRUE(arrow::ExportArray(*array, c_array.get()).ok());
+    return RecordBatchBuilder(c_array.get()).Finish().value();
+}
+
+Result<std::string> ReadJson(const std::vector<std::unique_ptr<BatchReader>>& readers) {
+    std::vector<std::shared_ptr<arrow::Array>> batches;
+    for (const std::unique_ptr<BatchReader>& reader : readers) {
+        while (true) {
+            PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
+            if (BatchReader::IsEofBatch(batch)) {
+                break;
+            }
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> array,
+                arrow::ImportArray(batch.first.get(), batch.second.get()));
+            batches.push_back(std::move(array));
+        }
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> result,
+                                      arrow::Concatenate(batches));
+    return result->ToString();
+}
 
 TEST(PrimaryKeyRealtimeStoreOptionsTest, TestSupportedOptions) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
@@ -66,428 +99,69 @@ TEST(PrimaryKeyRealtimeStoreOptionsTest, TestUnsupportedOptions) {
     }
 }
 
-class PrimaryKeyRealtimeStoreTest : public testing::Test {
- public:
-    void SetUp() override {
-        pool_ = std::shared_ptr<MemoryPool>(GetMemoryPool());
-        schema_ = arrow::schema(
-            {arrow::field("id", arrow::int64()), arrow::field("value", arrow::utf8())});
-        ASSERT_OK_AND_ASSIGN(store_, CreateStore(schema_, {"id"}, /*restore_max_sequence=*/4));
-    }
-
-    Result<std::shared_ptr<PrimaryKeyRealtimeStore>> CreateStore(
-        const std::shared_ptr<arrow::Schema>& schema, const std::vector<std::string>& primary_keys,
-        int64_t restore_max_sequence) const {
-        std::vector<DataField> key_fields;
-        key_fields.reserve(primary_keys.size());
-        for (const std::string& primary_key : primary_keys) {
-            const int32_t index = schema->GetFieldIndex(primary_key);
-            key_fields.emplace_back(index, schema->field(index));
-        }
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
-                               FieldsComparator::Create(key_fields,
-                                                        /*is_ascending_order=*/true));
-        auto merge_factory = []() {
-            auto merge_function =
-                std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
-            return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
-        };
-        return PrimaryKeyRealtimeStore::Create(schema, primary_keys, key_comparator, merge_factory,
-                                               restore_max_sequence,
-                                               /*read_batch_size=*/2, pool_);
-    }
-
-    std::unique_ptr<RecordBatch> MakeBatch(
-        const std::string& json, const std::vector<RecordBatch::RowKind>& row_kinds = {},
-        const std::shared_ptr<arrow::Schema>& schema = nullptr) const {
-        const std::shared_ptr<arrow::Schema>& batch_schema = schema ? schema : schema_;
-        std::shared_ptr<arrow::Array> array =
-            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(batch_schema->fields()), json)
-                .ValueOrDie();
-        ArrowArray c_array;
-        EXPECT_TRUE(arrow::ExportArray(*array, &c_array).ok());
-        RecordBatchBuilder builder(&c_array);
-        builder.SetRowKinds(row_kinds);
-        return builder.Finish().value();
-    }
-
-    std::unique_ptr<ArrowSchema> MakeReadSchema(const arrow::FieldVector& fields) const {
-        auto c_schema = std::make_unique<ArrowSchema>();
-        EXPECT_TRUE(arrow::ExportSchema(*arrow::schema(fields), c_schema.get()).ok());
-        return c_schema;
-    }
-
-    void AssertReaderOutput(const std::vector<std::unique_ptr<BatchReader>>& readers,
-                            const std::shared_ptr<arrow::DataType>& type,
-                            const std::string& json) const {
-        std::vector<std::shared_ptr<arrow::Array>> batches;
-        for (const std::unique_ptr<BatchReader>& reader : readers) {
-            while (true) {
-                ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
-                if (BatchReader::IsEofBatch(batch)) {
-                    break;
-                }
-                arrow::Result<std::shared_ptr<arrow::Array>> imported =
-                    arrow::ImportArray(batch.first.get(), batch.second.get());
-                ASSERT_TRUE(imported.ok()) << imported.status().ToString();
-                batches.push_back(std::move(imported).ValueOrDie());
-            }
-        }
-        ASSERT_FALSE(batches.empty());
-        arrow::Result<std::shared_ptr<arrow::Array>> concatenated = arrow::Concatenate(batches);
-        ASSERT_TRUE(concatenated.ok()) << concatenated.status().ToString();
-        std::shared_ptr<arrow::Array> actual = std::move(concatenated).ValueOrDie();
-        std::shared_ptr<arrow::Array> expected =
-            arrow::ipc::internal::json::ArrayFromJSON(type, json).ValueOrDie();
-        ASSERT_TRUE(actual->Equals(*expected))
-            << "expected: " << expected->ToString() << ", actual: " << actual->ToString();
-        for (const std::unique_ptr<BatchReader>& reader : readers) {
-            reader->Close();
-        }
-    }
-
-    std::shared_ptr<arrow::DataType> CommitType() const {
-        return arrow::struct_({
-            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
-            schema_->field(0),
-            schema_->field(1),
-        });
-    }
-
-    std::shared_ptr<arrow::DataType> QueryType() const {
-        return arrow::struct_({
-            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
-            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()),
-            schema_->field(0),
-            schema_->field(1),
-        });
-    }
-
-    arrow::FieldVector FullQueryFields(
-        const std::shared_ptr<arrow::Schema>& schema = nullptr) const {
-        const std::shared_ptr<arrow::Schema>& query_schema = schema ? schema : schema_;
-        arrow::FieldVector fields = {
-            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())};
-        fields.insert(fields.end(), query_schema->fields().begin(), query_schema->fields().end());
-        return fields;
-    }
-
- protected:
-    std::shared_ptr<MemoryPool> pool_;
-    std::shared_ptr<arrow::Schema> schema_;
-    std::shared_ptr<PrimaryKeyRealtimeStore> store_;
-};
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestWriteAndSeal) {
+TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
-                         store_->SealForCommit());
+                         store->SealForCommit());
     ASSERT_FALSE(segment.has_value());
-    ASSERT_NOK_WITH_MSG(store_->Write(RealtimeWriteBatch{nullptr, OffsetRange(0, 0)}),
+    ASSERT_NOK_WITH_MSG(store->Write(RealtimeWriteBatch{nullptr, OffsetRange(0, 0)}),
                         "write batch is null");
     ASSERT_NOK_WITH_MSG(
-        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[1, "a"], [2, "b"]])"), OffsetRange(0, 0)}),
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 1, "one"]])"), OffsetRange(0, 0)}),
         "offset range does not match batch row count");
 
-    ASSERT_OK(
-        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[1, "a"], [2, "b"]])"), OffsetRange(0, 2)}));
-    ASSERT_NOK_WITH_MSG(
-        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[4, "d"]])"), OffsetRange(3, 4)}),
-        "offset ranges must be contiguous");
-    ASSERT_OK(store_->Write(RealtimeWriteBatch{MakeBatch(R"([[3, "c"]])"), OffsetRange(2, 3)}));
+    ASSERT_OK(store->Write(RealtimeWriteBatch{
+        MakeBatch(R"([[0, 1, 0, 1, "one"], [0, 2, 1, 2, "two"]])"), OffsetRange(0, 2)}));
+    ASSERT_NOK_WITH_MSG(store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 3, 3, 3, "three"]])"),
+                                                        OffsetRange(3, 4)}),
+                        "offset ranges must be contiguous");
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[0, 3, 2, 3, "three"]])"), OffsetRange(2, 3)}));
 
-    ASSERT_OK_AND_ASSIGN(segment, store_->SealForCommit());
+    ASSERT_OK_AND_ASSIGN(segment, store->SealForCommit());
     ASSERT_TRUE(segment.has_value());
     ASSERT_EQ(OffsetRange(0, 3), segment.value()->GetOffsetRange());
-    ASSERT_GT(store_->GetMemoryUsage(), 0);
-
-    struct ValidationCase {
-        int64_t restore_max_sequence;
-        std::string error;
-    };
-    const std::vector<ValidationCase> cases = {
-        {-2, "restore max sequence number must be at least -1"},
-        {std::numeric_limits<int64_t>::max(), "sequence number has reached INT64_MAX"},
-    };
-    for (const ValidationCase& test_case : cases) {
-        ASSERT_NOK_WITH_MSG(CreateStore(schema_, {"id"}, test_case.restore_max_sequence),
-                            test_case.error);
-    }
-}
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
-    ASSERT_OK(store_->Write(RealtimeWriteBatch{
-        MakeBatch(R"([[3, "three"], [1, "before"]])",
-                  {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_BEFORE}),
-        OffsetRange(0, 2)}));
-    ASSERT_OK(store_->Write(RealtimeWriteBatch{
-        MakeBatch(R"([[2, "after"]])", {RecordBatch::RowKind::UPDATE_AFTER}), OffsetRange(2, 3)}));
-    ASSERT_OK(store_->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[4, "deleted"], [0, "zero"]])",
-                                     {RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT}),
-                           OffsetRange(3, 5)}));
-    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
-                         store_->SealForCommit());
-    ASSERT_TRUE(segment.has_value());
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store_->CreateCommitReaders(segment.value()));
-    AssertReaderOutput(readers, CommitType(),
-                       R"([[0, 3, "three"], [1, 1, "before"], [2, 2, "after"],
-                            [3, 4, "deleted"], [0, 0, "zero"]])");
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(readers, store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    AssertReaderOutput(readers, QueryType(),
-                       R"([[0, 9, 0, "zero"], [1, 6, 1, "before"], [2, 7, 2, "after"],
-                            [0, 5, 3, "three"], [3, 8, 4, "deleted"]])");
-}
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestMutationMerge) {
-    ASSERT_OK(store_->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[1, "old"], [2, "two"]])"), OffsetRange(0, 2)}));
-    ASSERT_OK(store_->Write(RealtimeWriteBatch{
-        MakeBatch(R"([[1, "new"], [2, "gone"]])",
-                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE}),
-        OffsetRange(2, 4)}));
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    AssertReaderOutput(readers, QueryType(), R"([[2, 7, 1, "new"], [3, 8, 2, "gone"]])");
+    ASSERT_GT(store->GetMemoryUsage(), 0);
+    ASSERT_NOK_WITH_MSG(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 4, 4, 4, "four"]])"), OffsetRange(4, 5)}),
+        "offset ranges must be contiguous");
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[0, 4, 3, 4, "four"]])"), OffsetRange(3, 4)}));
 }
 
-TEST_F(PrimaryKeyRealtimeStoreTest, TestReadViewLifecycle) {
-    ASSERT_OK(store_->Write(RealtimeWriteBatch{MakeBatch(R"([[10, "a"], [11, "b"], [12, "c"]])"),
-                                               OffsetRange(10, 13)}));
+TEST(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+    ASSERT_OK(store->Write(RealtimeWriteBatch{
+        MakeBatch(R"([[0, 5, 0, 3, "three"], [1, 6, 1, 1, "before"]])"), OffsetRange(0, 2)}));
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[2, 7, 2, 2, "after"]])"), OffsetRange(2, 3)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
-                         store_->SealForCommit());
+                         store->SealForCommit());
     ASSERT_TRUE(segment.has_value());
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
-    ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(10, 13)), view->GetOffsetRange());
-
-    ASSERT_OK(store_->AdvanceCommittedOffset(13));
-    ASSERT_EQ(0, store_->GetMemoryUsage());
-    ASSERT_OK(
-        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[13, "later"]])"), OffsetRange(13, 14)}));
-
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store_->CreateQueryReaders(view, /*offset_begin=*/11, context));
-    AssertReaderOutput(readers, QueryType(), R"([[0, 6, 11, "b"], [0, 7, 12, "c"]])");
-
-    std::unique_ptr<ArrowSchema> empty_schema = MakeReadSchema(FullQueryFields());
-    context.read_schema = empty_schema.get();
-    ASSERT_OK_AND_ASSIGN(readers, store_->CreateQueryReaders(view, /*offset_begin=*/13, context));
-    ASSERT_TRUE(readers.empty());
-}
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryKeyRange) {
-    ASSERT_OK(store_->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[5, "five"], [1, "one"]])"), OffsetRange(0, 2)}));
-    ASSERT_OK(store_->SealForCommit());
-    ASSERT_OK(store_->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[9, "nine"], [7, "seven"]])"), OffsetRange(2, 4)}));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    ASSERT_EQ(2, readers.size());
-    const std::vector<std::pair<int64_t, int64_t>> key_ranges = {{1, 5}, {7, 9}};
-    for (size_t i = 0; i < readers.size(); ++i) {
-        auto* range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[i].get());
-        ASSERT_NE(nullptr, range);
-        ASSERT_EQ(key_ranges[i].first, range->GetMinKey()->GetLong(0));
-        ASSERT_EQ(key_ranges[i].second, range->GetMaxKey()->GetLong(0));
-    }
-    AssertReaderOutput(readers, QueryType(),
-                       R"([[0, 6, 1, "one"], [0, 5, 5, "five"], [0, 8, 7, "seven"],
-                            [0, 7, 9, "nine"]])");
-
-    ASSERT_OK(store_->AdvanceCommittedOffset(2));
-    ASSERT_OK_AND_ASSIGN(view, store_->AcquireReadView());
-    read_schema = MakeReadSchema(FullQueryFields());
-    context.read_schema = read_schema.get();
-    ASSERT_OK_AND_ASSIGN(readers, store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    ASSERT_EQ(1, readers.size());
-    auto* range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[0].get());
-    ASSERT_NE(nullptr, range);
-    ASSERT_EQ(7, range->GetMinKey()->GetLong(0));
-    ASSERT_EQ(9, range->GetMaxKey()->GetLong(0));
-    AssertReaderOutput(readers, QueryType(), R"([[0, 8, 7, "seven"], [0, 7, 9, "nine"]])");
+                         store->CreateCommitReaders(segment.value()));
+    ASSERT_OK_AND_ASSIGN(std::string actual, ReadJson(readers));
+    ASSERT_EQ(
+        "-- is_valid: all not null\n-- child 0 type: int8\n  [\n    0,\n    1,\n    2\n  ]\n-- "
+        "child 1 type: int64\n  [\n    5,\n    6,\n    7\n  ]\n-- child 2 type: int64\n  [\n    "
+        "0,\n    1,\n    2\n  ]\n-- child 3 type: int64\n  [\n    3,\n    1,\n    2\n  ]\n-- child "
+        "4 type: string\n  [\n    \"three\",\n    \"before\",\n    \"after\"\n  ]",
+        actual);
 }
 
-TEST_F(PrimaryKeyRealtimeStoreTest, TestSequenceExhaustion) {
-    const int64_t max_sequence = std::numeric_limits<int64_t>::max();
+TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         CreateStore(schema_, {"id"}, max_sequence - 3));
-    ASSERT_OK(store->Write(RealtimeWriteBatch{MakeBatch(R"([[1, "kept"]])"), OffsetRange(10, 11)}));
-    ASSERT_NOK_WITH_MSG(
-        store->Write(RealtimeWriteBatch{
-            MakeBatch(R"([[7, "rejected-a"], [8, "rejected-b"], [9, "rejected-c"]])"),
-            OffsetRange(11, 14)}),
-        "sequence range exceeds INT64_MAX");
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
     ASSERT_OK(
-        store->Write(RealtimeWriteBatch{MakeBatch(R"([[2, "also-kept"]])"), OffsetRange(11, 12)}));
-
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 0, 4, 1, "one"]])"), OffsetRange(4, 5)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
-    ASSERT_TRUE(segment.has_value());
-    ASSERT_EQ(OffsetRange(10, 12), segment.value()->GetOffsetRange());
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-    ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(10, 12)), view->GetOffsetRange());
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(FullQueryFields());
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateQueryReaders(view, /*offset_begin=*/10, context));
-    AssertReaderOutput(readers, QueryType(),
-                       R"([[0, 9223372036854775805, 1, "kept"],
-                            [0, 9223372036854775806, 2, "also-kept"]])");
-}
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryProjection) {
-    ASSERT_OK(
-        store_->Write(RealtimeWriteBatch{MakeBatch(R"([[2, "b"], [1, "a"]])"), OffsetRange(0, 2)}));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store_->AcquireReadView());
-    const std::shared_ptr<arrow::Field> value_kind =
-        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind());
-    const std::shared_ptr<arrow::Field> sequence =
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber());
-    struct ProjectionCase {
-        arrow::FieldVector requested;
-        std::shared_ptr<arrow::DataType> expected_type;
-        std::string expected_json;
-    };
-    const std::vector<ProjectionCase> cases = {
-        {{schema_->field(1), value_kind, sequence, schema_->field(0)},
-         arrow::struct_({value_kind, schema_->field(1), sequence, schema_->field(0)}),
-         R"([[0, "a", 6, 1], [0, "b", 5, 2]])"},
-        {{schema_->field(0), value_kind},
-         arrow::struct_({value_kind, schema_->field(0)}),
-         R"([[0, 1], [0, 2]])"},
-    };
-    for (const ProjectionCase& test_case : cases) {
-        std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema(test_case.requested);
-        RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                     /*enable_predicate_pushdown=*/false};
-        ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                             store_->CreateQueryReaders(view, /*offset_begin=*/0, context));
-        AssertReaderOutput(readers, test_case.expected_type, test_case.expected_json);
-    }
-
-    std::unique_ptr<ArrowSchema> read_schema =
-        MakeReadSchema({arrow::field("unknown", arrow::int64())});
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_NOK_WITH_MSG(store_->CreateQueryReaders(view, /*offset_begin=*/0, context),
-                        "query field is missing from write schema: unknown");
-}
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestQueryProjectionMatchesRenamedFieldsById) {
-    const std::shared_ptr<arrow::Field> id =
-        DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64())));
-    const std::shared_ptr<arrow::Field> value =
-        DataField::ConvertDataFieldToArrowField(DataField(1, arrow::field("value", arrow::utf8())));
-    const std::shared_ptr<arrow::Schema> write_schema = arrow::schema({id, value});
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         CreateStore(write_schema, {"id"}, /*restore_max_sequence=*/4));
-    ASSERT_OK(store->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[1, "kept"]])", {}, write_schema), OffsetRange(0, 1)}));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-
-    const std::shared_ptr<arrow::Field> renamed_value = DataField::ConvertDataFieldToArrowField(
-        DataField(1, arrow::field("renamed", arrow::utf8())));
-    const std::shared_ptr<arrow::Field> renamed_id = DataField::ConvertDataFieldToArrowField(
-        DataField(0, arrow::field("renamed_id", arrow::int64())));
-    const std::shared_ptr<arrow::Field> replaced =
-        DataField::ConvertDataFieldToArrowField(DataField(2, arrow::field("value", arrow::utf8())));
-    const std::shared_ptr<arrow::Field> replaced_id =
-        DataField::ConvertDataFieldToArrowField(DataField(4, arrow::field("id", arrow::int64())));
-    const std::shared_ptr<arrow::Field> added =
-        DataField::ConvertDataFieldToArrowField(DataField(3, arrow::field("added", arrow::utf8())));
-    std::unique_ptr<ArrowSchema> read_schema =
-        MakeReadSchema({renamed_value, renamed_id, replaced, replaced_id, added});
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    const std::shared_ptr<arrow::DataType> result_type =
-        arrow::struct_({DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
-                        renamed_value, renamed_id, replaced, replaced_id, added});
-    AssertReaderOutput(readers, result_type, R"([[0, "kept", 1, null, null, null]])");
-}
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestNestedProjection) {
-    const std::shared_ptr<arrow::Field> id =
-        DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64())));
-    const std::shared_ptr<arrow::Field> a =
-        DataField::ConvertDataFieldToArrowField(DataField(10, arrow::field("a", arrow::int64())));
-    const std::shared_ptr<arrow::Field> b =
-        DataField::ConvertDataFieldToArrowField(DataField(11, arrow::field("b", arrow::int64())));
-    const std::shared_ptr<arrow::Field> payload = DataField::ConvertDataFieldToArrowField(
-        DataField(1, arrow::field("payload", arrow::struct_({a, b}))));
-    const std::shared_ptr<arrow::Schema> nested_schema = arrow::schema({id, payload});
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         CreateStore(nested_schema, {"id"}, /*restore_max_sequence=*/4));
-    ASSERT_OK(store->Write(RealtimeWriteBatch{
-        MakeBatch(R"([[2, [200, 2000]], [1, [100, null]], [3, [300, 3000]]])", {}, nested_schema),
-        OffsetRange(0, 3)}));
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-    const std::shared_ptr<arrow::Field> projected_payload = payload->WithType(arrow::struct_({b}));
-    std::unique_ptr<ArrowSchema> read_schema = MakeReadSchema({projected_payload});
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    const std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
-        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()), projected_payload});
-    AssertReaderOutput(readers, result_type, R"([[0, [null]], [0, [2000]], [0, [3000]]])");
-}
-
-TEST_F(PrimaryKeyRealtimeStoreTest, TestCompositeKeyClipping) {
-    std::shared_ptr<arrow::Schema> composite_schema =
-        arrow::schema({arrow::field("id", arrow::int64()), arrow::field("region", arrow::utf8()),
-                       arrow::field("value", arrow::utf8())});
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         CreateStore(composite_schema, {"id", "region"},
-                                     /*restore_max_sequence=*/4));
-    ASSERT_OK(store->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[9, "z", "clipped"], [2, "b", "two-b"], [1, "c", "one-c"],
-                       [2, "a", "two-a"]])",
-                                     {}, composite_schema),
-                           OffsetRange(20, 24)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-    const std::shared_ptr<arrow::Field> sequence =
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber());
-    std::unique_ptr<ArrowSchema> read_schema =
-        MakeReadSchema({sequence, composite_schema->field(0), composite_schema->field(2)});
-    RealtimeQueryContext context{read_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateQueryReaders(view, /*offset_begin=*/21, context));
-    ASSERT_EQ(1, readers.size());
-    auto* range = dynamic_cast<PrimaryKeyRangeProvider*>(readers[0].get());
-    ASSERT_NE(nullptr, range);
-    ASSERT_EQ(1, range->GetMinKey()->GetLong(0));
-    ASSERT_EQ("c", range->GetMinKey()->GetString(1).ToString());
-    ASSERT_EQ(2, range->GetMaxKey()->GetLong(0));
-    ASSERT_EQ("b", range->GetMaxKey()->GetString(1).ToString());
-    std::shared_ptr<arrow::DataType> query_type =
-        arrow::struct_({DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()),
-                        sequence, composite_schema->field(0), composite_schema->field(2)});
-    AssertReaderOutput(readers, query_type,
-                       R"([[0, 7, 1, "one-c"], [0, 8, 2, "two-a"],
-                            [0, 6, 2, "two-b"]])");
+    ASSERT_OK(store->AdvanceCommittedOffset(5));
+    ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(4, 5)), view->GetOffsetRange());
 }
 
+}  // namespace
 }  // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 066e54e8a..b73cfdb8a 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -42,7 +42,6 @@
 #include "paimon/status.h"
 
 namespace paimon {
-
 RealtimeContextImpl::RealtimeContextImpl(const std::shared_ptr<RealtimeStoreFactory>& factory)
     : factory_(factory) {}
 
@@ -83,26 +82,6 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     std::lock_guard<std::mutex> registry_lock(mutex_);
     const RealtimePartitionBucket key(request.partition, request.bucket);
     auto iter = stores_.find(key);
-    std::optional<int64_t> initial_max_sequence_number;
-    PrimaryKeyRealtimeStoreCreateConfig* primary_key_config =
-        std::get_if<PrimaryKeyRealtimeStoreCreateConfig>(&request.mode_config);
-    if (primary_key_config) {
-        auto [sequence_iter, inserted] = materialized_max_sequence_numbers_.emplace(
-            key, primary_key_config->restore_max_sequence_number);
-        if (!inserted && primary_key_config->restore_max_sequence_number > sequence_iter->second) {
-            if (iter != stores_.end()) {
-                if (request.write_schema) {
-                    ArrowSchemaRelease(request.write_schema.get());
-                }
-                return Status::Invalid(
-                    "restore max sequence number exceeds the materialized watermark of an "
-                    "existing PK real-time store");
-            }
-            sequence_iter->second = primary_key_config->restore_max_sequence_number;
-        }
-        initial_max_sequence_number = sequence_iter->second;
-        primary_key_config->restore_max_sequence_number = sequence_iter->second;
-    }
     int64_t initial_offset = 0;
     auto offset_iter = committed_offsets_.find(key);
     if (offset_iter != committed_offsets_.end()) {
@@ -134,7 +113,13 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
                 initial_offset = memory_range->end;
             }
         }
-        return RealtimeStoreState{iter->second, initial_offset, initial_max_sequence_number};
+        return RealtimeStoreState{iter->second, initial_offset};
+    }
+    if (!request.memory_pool) {
+        if (request.write_schema) {
+            ArrowSchemaRelease(request.write_schema.get());
+        }
+        return Status::Invalid("real-time store memory pool is null");
     }
     Result<std::shared_ptr<RealtimeStore>> store_result = factory_->Create(std::move(request));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
@@ -142,17 +127,7 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
     }
-    return RealtimeStoreState{std::move(store), initial_offset, initial_max_sequence_number};
-}
-
-void RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
-    const RealtimePartitionBucket& partition_bucket, int64_t max_sequence_number) {
-    std::lock_guard<std::mutex> lock(mutex_);
-    auto [iter, inserted] =
-        materialized_max_sequence_numbers_.emplace(partition_bucket, max_sequence_number);
-    if (!inserted && max_sequence_number > iter->second) {
-        iter->second = max_sequence_number;
-    }
+    return RealtimeStoreState{std::move(store), initial_offset};
 }
 
 Result<std::vector<RealtimePartitionBucketView>> RealtimeContextImpl::AcquireReadViews() {
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index f4cd3866e..4f62cf1ee 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -47,7 +47,6 @@ class MemoryPool;
 struct RealtimeStoreState {
     std::shared_ptr<RealtimeStore> store;
     int64_t initial_offset;
-    std::optional<int64_t> initial_max_sequence_number;
 };
 
 struct RealtimePartitionBucketView {
@@ -68,9 +67,6 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
 
     Result<RealtimeStoreState> GetOrCreateRealtimeStore(RealtimeStoreCreateRequest&& request);
 
-    void AdvanceMaterializedMaxSequenceNumber(const RealtimePartitionBucket& partition_bucket,
-                                              int64_t max_sequence_number);
-
     Result<std::vector<RealtimePartitionBucketView>> AcquireReadViews();
 
     Result<std::string> PinReadView(const RealtimePartitionBucketView& view, int64_t ttl_millis);
@@ -100,7 +96,6 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     std::mutex mutex_;
     std::mutex progress_mutex_;
     std::map<RealtimePartitionBucket, std::shared_ptr<RealtimeStore>> stores_;
-    std::map<RealtimePartitionBucket, int64_t> materialized_max_sequence_numbers_;
     // Full-table progress used as the initial offset when a store is created lazily.
     RealtimeOffsetMap committed_offsets_;
     // Progress already reflected in stores owned by this context.
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index ab0abe4a7..07bbf555b 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -9,12 +9,11 @@
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 #include <chrono>
@@ -31,7 +30,6 @@
 #include "arrow/c/helpers.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/memory/memory_pool.h"
-#include "paimon/realtime/realtime_store.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
@@ -49,26 +47,21 @@ class TestingRealtimeStore : public RealtimeStore {
     Status Write(RealtimeWriteBatch&&) override {
         return Status::OK();
     }
-
     Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
         return std::optional<std::shared_ptr<RealtimeSegmentHandle>>();
     }
-
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>&) override {
         return std::vector<std::unique_ptr<BatchReader>>();
     }
-
     Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
         ++acquire_count;
         return std::make_shared<TestingReadView>();
     }
-
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>&, int64_t, const RealtimeQueryContext&) override {
         return std::vector<std::unique_ptr<BatchReader>>();
     }
-
     Status AdvanceCommittedOffset(int64_t committed_offset) override {
         ++advance_count;
         if (fail_next_advance) {
@@ -78,7 +71,6 @@ class TestingRealtimeStore : public RealtimeStore {
         committed_offsets.push_back(committed_offset);
         return Status::OK();
     }
-
     uint64_t GetMemoryUsage() const override {
         return 0;
     }
@@ -105,11 +97,11 @@ class TestingRealtimeStoreFactory : public RealtimeStoreFactory {
 };
 
 std::unique_ptr<ArrowSchema> MakeWriteSchema() {
-    auto c_schema = std::make_unique<ArrowSchema>();
+    auto schema = std::make_unique<ArrowSchema>();
     EXPECT_TRUE(
-        arrow::ExportSchema(*arrow::schema({arrow::field("id", arrow::int64())}), c_schema.get())
+        arrow::ExportSchema(*arrow::schema({arrow::field("id", arrow::int64())}), schema.get())
             .ok());
-    return c_schema;
+    return schema;
 }
 
 Result<std::shared_ptr<RealtimeContextImpl>> CreateContext(
@@ -129,41 +121,29 @@ Result<RealtimeStoreState> GetOrCreateAppendStore(
                                    AppendRealtimeStoreCreateConfig{StatisticsMode::NONE}});
 }
 
-Result<RealtimeStoreState> GetOrCreatePrimaryKeyStore(
-    const std::shared_ptr<RealtimeContextImpl>& context,
-    const std::map<std::string, std::string>& partition, int32_t bucket,
-    int64_t restore_max_sequence_number, const std::shared_ptr<MemoryPool>& memory_pool) {
-    return context->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
-        MakeWriteSchema(), /*options=*/{}, memory_pool, partition, bucket,
-        PrimaryKeyRealtimeStoreCreateConfig{{"id"}, restore_max_sequence_number}});
-}
-
-TEST(RealtimeContextTest, TestReusesIndexerAndCapturesRegisteredViews) {
+TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
-
-    ASSERT_OK_AND_ASSIGN(RealtimeStoreState first_state,
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState first,
                          GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0,
-                                                MakeWriteSchema(), {{"k", "v"}}, pool));
-    ASSERT_EQ(0, first_state.initial_offset);
-    ASSERT_FALSE(first_state.initial_max_sequence_number);
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState first_again_state,
-        GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0, MakeWriteSchema(), {}, pool));
-    ASSERT_EQ(first_state.store, first_again_state.store);
-    ASSERT_EQ(0, first_again_state.initial_offset);
+                                                MakeWriteSchema(), {{"k", "v"}}, GetDefaultPool()));
+    ASSERT_EQ(0, first.initial_offset);
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState second,
+                         GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0,
+                                                MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_EQ(first.store, second.store);
+    ASSERT_EQ(0, second.initial_offset);
     ASSERT_EQ(1, factory->stores.size());
     ASSERT_EQ(1, factory->stores[0]->acquire_count);
 
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState second_state,
-        GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 1, MakeWriteSchema(), {}, pool));
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState third_state,
-        GetOrCreateAppendStore(context, {{"dt", "2026-08-03"}}, 0, MakeWriteSchema(), {}, pool));
-    ASSERT_NE(first_state.store, second_state.store);
-    ASSERT_NE(first_state.store, third_state.store);
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState third,
+                         GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 1,
+                                                MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState fourth,
+                         GetOrCreateAppendStore(context, {{"dt", "2026-08-03"}}, 0,
+                                                MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_NE(first.store, third.store);
+    ASSERT_NE(first.store, fourth.store);
     ASSERT_EQ(3, factory->stores.size());
 
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimePartitionBucketView> views,
@@ -171,57 +151,22 @@ TEST(RealtimeContextTest, TestReusesIndexerAndCapturesRegisteredViews) {
     ASSERT_EQ(3, views.size());
     const RealtimePartitionBucket expected_partition_bucket({{"dt", "2026-08-02"}}, 0);
     ASSERT_EQ(expected_partition_bucket, views[0].partition_bucket);
-    ASSERT_EQ(first_state.store, views[0].store);
+    ASSERT_EQ(first.store, views[0].store);
     ASSERT_TRUE(views[0].read_view);
     ASSERT_EQ(2, factory->stores[0]->acquire_count);
     ASSERT_EQ(1, factory->stores[1]->acquire_count);
     ASSERT_EQ(1, factory->stores[2]->acquire_count);
 }
 
-TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
-    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
-
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState first_state,
-        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/0,
-                                   /*restore_max_sequence_number=*/4, GetDefaultPool()));
-    ASSERT_EQ(4, first_state.initial_max_sequence_number);
-
-    const RealtimePartitionBucket partition_bucket(partition, /*bucket=*/0);
-    context->AdvanceMaterializedMaxSequenceNumber(partition_bucket, /*max_sequence_number=*/8);
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState retained_state,
-        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/0,
-                                   /*restore_max_sequence_number=*/6, GetDefaultPool()));
-    ASSERT_EQ(first_state.store, retained_state.store);
-    ASSERT_EQ(8, retained_state.initial_max_sequence_number);
-
-    ASSERT_NOK_WITH_MSG(
-        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/0,
-                                   /*restore_max_sequence_number=*/10, GetDefaultPool()),
-        "restore max sequence number exceeds the materialized watermark of an "
-        "existing PK real-time store");
-
-    const RealtimePartitionBucket new_partition_bucket(partition, /*bucket=*/1);
-    context->AdvanceMaterializedMaxSequenceNumber(new_partition_bucket,
-                                                  /*max_sequence_number=*/8);
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState new_state,
-        GetOrCreatePrimaryKeyStore(context, partition, /*bucket=*/1,
-                                   /*restore_max_sequence_number=*/10, GetDefaultPool()));
-    ASSERT_EQ(10, new_state.initial_max_sequence_number);
-}
-
 TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
     const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
 
-    ASSERT_OK(GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, pool));
-    ASSERT_OK(GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, pool));
+    ASSERT_OK(
+        GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_OK(
+        GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, GetDefaultPool()));
     ASSERT_EQ(2, factory->stores.size());
 
     ASSERT_NOK_WITH_MSG(context->AdvanceCommittedProgress(-1, {}),
@@ -238,9 +183,9 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
     ASSERT_TRUE(factory->stores[1]->committed_offsets.empty());
 
-    ASSERT_OK_AND_ASSIGN(
-        RealtimeStoreState restored_state,
-        GetOrCreateAppendStore(context, {{"dt", "unknown"}}, 0, MakeWriteSchema(), {}, pool));
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState restored_state,
+                         GetOrCreateAppendStore(context, {{"dt", "unknown"}}, 0, MakeWriteSchema(),
+                                                {}, GetDefaultPool()));
     ASSERT_EQ(9, restored_state.initial_offset);
 
     ASSERT_OK(context->AdvanceCommittedProgress(
@@ -260,12 +205,14 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
 TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
     const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
 
-    ASSERT_OK(GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, pool));
-    ASSERT_OK(GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, pool));
-    ASSERT_OK(GetOrCreateAppendStore(context, partition, 2, MakeWriteSchema(), {}, pool));
+    ASSERT_OK(
+        GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_OK(
+        GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_OK(
+        GetOrCreateAppendStore(context, partition, 2, MakeWriteSchema(), {}, GetDefaultPool()));
     ASSERT_EQ(3, factory->stores.size());
     factory->stores[1]->fail_next_advance = true;
 
@@ -281,7 +228,7 @@ TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
 
     ASSERT_OK_AND_ASSIGN(
         RealtimeStoreState failed_store_state,
-        GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, pool));
+        GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, GetDefaultPool()));
     ASSERT_EQ(8, failed_store_state.initial_offset);
 
     ASSERT_OK(context->AdvanceCommittedProgress(5, committed_offsets));
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 65bcebcad..c85ff6322 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -9,12 +9,11 @@
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 #include "paimon/core/realtime/realtime_primary_key_writer.h"
@@ -26,95 +25,254 @@
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
-#include "paimon/common/reader/concat_batch_reader.h"
+#include "arrow/compute/api.h"
 #include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
 #include "paimon/common/types/row_kind.h"
-#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/scope_guard.h"
-#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/merged_key_value_record_reader.h"
+#include "paimon/core/io/prepared_key_value_reader.h"
+#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
+#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/mergetree/merge_tree_writer.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/utils/commit_increment.h"
 #include "paimon/macros.h"
-#include "paimon/realtime/realtime_context.h"
 
 namespace paimon {
 
+namespace {
+
+struct PreparedArrayPrivateData {
+    void (*release)(ArrowArray*);
+    void* private_data;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool;
+};
+
+void ReleasePreparedArray(ArrowArray* array) {
+    auto* data = static_cast<PreparedArrayPrivateData*>(array->private_data);
+    array->release = data->release;
+    array->private_data = data->private_data;
+    array->release(array);
+    delete data;
+}
+
+Status RetainPreparedArrayPool(ArrowArray* array,
+                               const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+    if (!array || !array->release || !arrow_pool) {
+        return Status::Invalid("cannot retain prepared batch memory pool");
+    }
+    array->private_data =
+        new PreparedArrayPrivateData{array->release, array->private_data, arrow_pool};
+    array->release = ReleasePreparedArray;
+    return Status::OK();
+}
+
+Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
+    std::unique_ptr<RecordBatch>&& batch, const std::shared_ptr<arrow::Schema>& write_schema,
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::vector<std::string>& trimmed_primary_keys, int64_t first_sequence_number,
+    int64_t first_offset, arrow::MemoryPool* arrow_pool) {
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::Array> input,
+        arrow::ImportArray(batch->GetData(), arrow::struct_(write_schema->fields())));
+    if (!input || input->type_id() != arrow::Type::STRUCT) {
+        return Status::Invalid("PK real-time write data is not a StructArray");
+    }
+    std::shared_ptr<arrow::StructArray> values = checked_pointer_cast<arrow::StructArray>(input);
+    const int64_t count = values->length();
+    arrow::Int8Builder kinds(arrow_pool);
+    arrow::Int64Builder sequences(arrow_pool);
+    arrow::Int64Builder offsets(arrow_pool);
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(kinds.Reserve(count));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(sequences.Reserve(count));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets.Reserve(count));
+    const std::vector<RecordBatch::RowKind>& row_kinds = batch->GetRowKind();
+    for (int64_t row = 0; row < count; ++row) {
+        const RecordBatch::RowKind kind =
+            row_kinds.empty() ? RecordBatch::RowKind::INSERT : row_kinds[row];
+        kinds.UnsafeAppend(static_cast<int8_t>(kind));
+        sequences.UnsafeAppend(first_sequence_number + row);
+        offsets.UnsafeAppend(first_offset + row);
+    }
+    std::shared_ptr<arrow::Array> kind_array;
+    std::shared_ptr<arrow::Array> sequence_array;
+    std::shared_ptr<arrow::Array> offset_array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(kinds.Finish(&kind_array));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(sequences.Finish(&sequence_array));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets.Finish(&offset_array));
+    arrow::ArrayVector columns = {std::move(kind_array), std::move(sequence_array),
+                                  std::move(offset_array)};
+    columns.insert(columns.end(), values->fields().begin(), values->fields().end());
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::shared_ptr<arrow::StructArray> prepared,
+        arrow::StructArray::Make(std::move(columns), prepared_schema->fields()));
+
+    std::vector<arrow::compute::SortKey> sort_keys;
+    sort_keys.reserve(trimmed_primary_keys.size() + 1);
+    for (const std::string& key : trimmed_primary_keys) {
+        sort_keys.emplace_back(key, arrow::compute::SortOrder::Ascending);
+    }
+    sort_keys.emplace_back(SpecialFields::SequenceNumber().Name(),
+                           arrow::compute::SortOrder::Ascending);
+    arrow::compute::ExecContext context(arrow_pool);
+    arrow::compute::SortOptions options(sort_keys, arrow::compute::NullPlacement::AtStart);
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        arrow::Datum indices,
+        arrow::compute::SortIndices(arrow::Datum(prepared), options, &context));
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        arrow::Datum sorted,
+        arrow::compute::Take(arrow::Datum(prepared), indices,
+                             arrow::compute::TakeOptions::NoBoundsCheck(), &context));
+    std::shared_ptr<arrow::Array> sorted_array = sorted.make_array();
+    if (!sorted_array || sorted_array->type_id() != arrow::Type::STRUCT) {
+        return Status::Invalid("PK real-time sorted batch is not a StructArray");
+    }
+    return checked_pointer_cast<arrow::StructArray>(std::move(sorted_array));
+}
+
+}  // namespace
+
 Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Create(
-    const std::map<std::string, std::string>& partition, int32_t bucket,
     const std::shared_ptr<arrow::Schema>& write_schema,
-    const std::shared_ptr<RealtimeContextImpl>& realtime_context,
-    const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
-    const std::shared_ptr<MemoryPool>& memory_pool, const RealtimeStoreState& store_state) {
-    return std::shared_ptr<RealtimePrimaryKeyWriter>(
-        new RealtimePrimaryKeyWriter(store_state.store, merge_tree_writer, realtime_context,
-                                     RealtimePartitionBucket(partition, bucket), write_schema,
-                                     store_state.initial_offset, memory_pool));
+    const std::vector<std::string>& trimmed_primary_keys,
+    const std::shared_ptr<FieldsComparator>& key_comparator, const RealtimeStoreState& store_state,
+    int64_t restored_max_sequence_number, const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    if (!store_state.store || !merge_tree_writer || !write_schema || !key_comparator ||
+        !memory_pool) {
+        return Status::Invalid("PK real-time writer received a null dependency");
+    }
+    if (trimmed_primary_keys.empty()) {
+        return Status::Invalid("PK real-time writer requires at least one primary key");
+    }
+    if (restored_max_sequence_number < -1 ||
+        restored_max_sequence_number == std::numeric_limits<int64_t>::max()) {
+        return Status::Invalid("PK restored sequence number is invalid");
+    }
+    arrow::FieldVector key_fields;
+    key_fields.reserve(trimmed_primary_keys.size());
+    for (const std::string& key : trimmed_primary_keys) {
+        std::shared_ptr<arrow::Field> field = write_schema->GetFieldByName(key);
+        if (!field) {
+            return Status::Invalid("PK field is missing from write schema: ", key);
+        }
+        key_fields.push_back(std::move(field));
+    }
+    arrow::FieldVector prepared_fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+            ->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+    prepared_fields.insert(prepared_fields.end(), write_schema->fields().begin(),
+                           write_schema->fields().end());
+    return std::shared_ptr<RealtimePrimaryKeyWriter>(new RealtimePrimaryKeyWriter(
+        store_state.store, merge_tree_writer, write_schema,
+        arrow::schema(std::move(prepared_fields)), arrow::schema(std::move(key_fields)),
+        trimmed_primary_keys, key_comparator, store_state.initial_offset,
+        restored_max_sequence_number, memory_pool));
 }
 
 RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
     const std::shared_ptr<RealtimeStore>& realtime_store,
     const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
-    const std::shared_ptr<RealtimeContextImpl>& realtime_context,
-    const RealtimePartitionBucket& partition_bucket,
-    const std::shared_ptr<arrow::Schema>& write_schema, int64_t next_offset,
-    const std::shared_ptr<MemoryPool>& memory_pool)
+    const std::shared_ptr<arrow::Schema>& write_schema,
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::vector<std::string>& trimmed_primary_keys,
+    const std::shared_ptr<FieldsComparator>& key_comparator, int64_t next_offset,
+    int64_t last_sequence_number, const std::shared_ptr<MemoryPool>& memory_pool)
     : memory_pool_(memory_pool),
+      arrow_pool_(GetArrowPool(memory_pool)),
       realtime_store_(realtime_store),
       merge_tree_writer_(merge_tree_writer),
-      realtime_context_(realtime_context),
-      partition_bucket_(partition_bucket),
       write_schema_(write_schema),
-      next_offset_(next_offset) {}
+      prepared_schema_(prepared_schema),
+      key_schema_(key_schema),
+      trimmed_primary_keys_(trimmed_primary_keys),
+      key_comparator_(key_comparator),
+      next_offset_(next_offset),
+      last_sequence_number_(last_sequence_number) {}
 
 Status RealtimePrimaryKeyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
     if (!batch || !batch->GetData()) {
         return Status::Invalid("PK real-time write batch is null");
     }
-    const int64_t row_count = batch->GetData()->length;
-    if (row_count == 0) {
+    const int64_t count = batch->GetData()->length;
+    if (count == 0) {
         return Status::OK();
     }
+    const std::vector<RecordBatch::RowKind>& row_kinds = batch->GetRowKind();
+    if (!row_kinds.empty() && static_cast<int64_t>(row_kinds.size()) != count) {
+        return Status::Invalid("PK real-time row-kind count does not match batch row count");
+    }
+    for (RecordBatch::RowKind row_kind : row_kinds) {
+        PAIMON_ASSIGN_OR_RAISE(const RowKind* validated,
+                               RowKind::FromByteValue(static_cast<int8_t>(row_kind)));
+        static_cast<void>(validated);
+    }
     std::lock_guard<std::mutex> lock(realtime_store_mutex_);
-    if (row_count > std::numeric_limits<int64_t>::max() - next_offset_) {
+    if (count > std::numeric_limits<int64_t>::max() - next_offset_) {
         return Status::Invalid("real-time offset range exceeds INT64_MAX");
     }
-    const OffsetRange range(next_offset_, next_offset_ + row_count);
-    PAIMON_RETURN_NOT_OK(realtime_store_->Write(RealtimeWriteBatch{std::move(batch), range}));
-    next_offset_ += row_count;
+    // Reserve INT64_MAX as the exhausted sequence-number sentinel.
+    if (last_sequence_number_ >= std::numeric_limits<int64_t>::max() - count) {
+        return Status::Invalid("PK sequence range exceeds INT64_MAX");
+    }
+    const int64_t first_sequence = last_sequence_number_ + 1;
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<arrow::StructArray> prepared,
+        PrepareBatch(std::move(batch), write_schema_, prepared_schema_, trimmed_primary_keys_,
+                     first_sequence, next_offset_, arrow_pool_.get()));
+    auto output = std::make_unique<ArrowArray>();
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*prepared, output.get()));
+    PAIMON_RETURN_NOT_OK(RetainPreparedArrayPool(output.get(), arrow_pool_));
+    RecordBatchBuilder builder(output.get());
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> prepared_batch, builder.Finish());
+    PAIMON_RETURN_NOT_OK(realtime_store_->Write(RealtimeWriteBatch{
+        std::move(prepared_batch), OffsetRange(next_offset_, next_offset_ + count)}));
+    next_offset_ += count;
+    last_sequence_number_ += count;
     return Status::OK();
 }
 
 Result<CommitIncrement> RealtimePrimaryKeyWriter::PrepareCommit(bool wait_compaction) {
-    std::lock_guard<std::mutex> lock(prepare_mutex_);
+    std::lock_guard<std::mutex> prepare_lock(prepare_mutex_);
     std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment;
     {
-        std::lock_guard<std::mutex> realtime_store_lock(realtime_store_mutex_);
-        PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<RealtimeSegmentHandle>> sealed_segment,
+        std::lock_guard<std::mutex> store_lock(realtime_store_mutex_);
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<RealtimeSegmentHandle>> sealed,
                                realtime_store_->SealForCommit());
-        segment = std::move(sealed_segment);
+        segment = std::move(sealed);
+    }
+    if (segment && !segment.value()) {
+        return Status::Invalid("PK real-time store sealed a null segment");
     }
+    std::optional<OffsetRange> sealed_range;
+    int64_t expected_raw_row_count = 0;
     if (segment) {
-        PAIMON_RETURN_NOT_OK(FlushSegment(segment.value()));
+        sealed_range = segment.value()->GetOffsetRange();
+        if (sealed_range->begin < 0 || sealed_range->end < sealed_range->begin ||
+            __builtin_sub_overflow(sealed_range->end, sealed_range->begin,
+                                   &expected_raw_row_count)) {
+            return Status::Invalid("PK real-time store returned an invalid sealed offset range");
+        }
+        PAIMON_RETURN_NOT_OK(FlushSegment(segment.value(), expected_raw_row_count));
     }
     PAIMON_ASSIGN_OR_RAISE(CommitIncrement increment,
                            merge_tree_writer_->PrepareCommit(wait_compaction));
     if (segment) {
-        const std::vector<std::shared_ptr<DataFileMeta>>& new_files =
-            increment.GetNewFilesIncrement().NewFiles();
-        if (!new_files.empty()) {
-            realtime_context_->AdvanceMaterializedMaxSequenceNumber(
-                partition_bucket_, DataFileMeta::GetMaxSequenceNumber(new_files));
-        }
-        increment.SetRealtimeOffsetRange(segment.value()->GetOffsetRange());
+        increment.SetRealtimeOffsetRange(sealed_range.value());
     }
     return increment;
 }
 
-Status RealtimePrimaryKeyWriter::FlushSegment(
-    const std::shared_ptr<RealtimeSegmentHandle>& segment) {
+Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment,
+                                              int64_t expected_raw_row_count) {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                            realtime_store_->CreateCommitReaders(segment));
     ScopeGuard readers_guard([&readers]() {
@@ -124,72 +282,26 @@ Status RealtimePrimaryKeyWriter::FlushSegment(
             }
         }
     });
-    for (const std::unique_ptr<BatchReader>& reader : readers) {
+    int64_t raw_row_count = 0;
+    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
+    sorted_readers.reserve(readers.size());
+    for (std::unique_ptr<BatchReader>& reader : readers) {
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null commit reader");
         }
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<KeyValueRecordReader> prepared_reader,
+            AdaptPreparedBatchReader(std::move(reader), prepared_schema_, std::nullopt, key_schema_,
+                                     write_schema_, memory_pool_, &raw_row_count));
+        auto merge_function = std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
+        sorted_readers.push_back(std::make_unique<MergedKeyValueRecordReader>(
+            std::move(prepared_reader), key_comparator_,
+            std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function))));
     }
-    ConcatBatchReader reader(std::move(readers), memory_pool_);
-    ScopeGuard reader_guard([&reader]() { reader.Close(); });
-    const OffsetRange offset_range = segment->GetOffsetRange();
-    int64_t emitted_rows = 0;
-    while (true) {
-        PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader.NextBatch());
-        if (BatchReader::IsEofBatch(batch)) {
-            break;
-        }
-        auto& [c_array, c_schema] = batch;
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> imported,
-                                          arrow::ImportArray(c_array.get(), c_schema.get()));
-        if (!imported || imported->type_id() != arrow::Type::STRUCT) {
-            return Status::Invalid("PK real-time store commit reader returned a non-StructArray");
-        }
-        std::shared_ptr<arrow::StructArray> struct_array =
-            checked_pointer_cast<arrow::StructArray>(imported);
-        std::shared_ptr<arrow::Array> value_kind =
-            struct_array->GetFieldByName(SpecialFields::ValueKind().Name());
-        if (!value_kind || value_kind->type_id() != arrow::Type::INT8) {
-            return Status::Invalid(
-                "PK real-time store commit reader must return an INT8 _VALUE_KIND field");
-        }
-        std::shared_ptr<arrow::Int8Array> encoded_row_kinds =
-            checked_pointer_cast<arrow::Int8Array>(value_kind);
-        std::vector<RecordBatch::RowKind> row_kinds;
-        row_kinds.reserve(static_cast<size_t>(encoded_row_kinds->length()));
-        for (int64_t i = 0; i < encoded_row_kinds->length(); ++i) {
-            if (encoded_row_kinds->IsNull(i)) {
-                return Status::Invalid("PK real-time store commit reader returned a null row kind");
-            }
-            PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind,
-                                   RowKind::FromByteValue(encoded_row_kinds->Value(i)));
-            row_kinds.push_back(static_cast<RecordBatch::RowKind>(row_kind->ToByteValue()));
-        }
-        PAIMON_ASSIGN_OR_RAISE(struct_array, ArrowUtils::RemoveFieldFromStructArray(
-                                                 struct_array, SpecialFields::ValueKind().Name()));
-        if (!struct_array->type()->Equals(arrow::struct_(write_schema_->fields()))) {
-            return Status::Invalid(
-                "PK real-time store commit reader schema does not match table write schema");
-        }
-        const int64_t row_count = struct_array->length();
-        if (row_count > offset_range.Count() - emitted_rows) {
-            return Status::Invalid(
-                "PK real-time store commit readers returned more rows than the sealed offset "
-                "range");
-        }
-        emitted_rows += row_count;
-        if (row_count == 0) {
-            continue;
-        }
-        auto output = std::make_unique<ArrowArray>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, output.get()));
-        RecordBatchBuilder builder(output.get());
-        builder.SetRowKinds(row_kinds);
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> record_batch, builder.Finish());
-        PAIMON_RETURN_NOT_OK(merge_tree_writer_->Write(std::move(record_batch)));
-    }
-    if (emitted_rows != offset_range.Count()) {
-        return Status::Invalid(
-            "PK real-time store commit readers returned fewer rows than the sealed offset range");
+    readers_guard.Release();
+    PAIMON_RETURN_NOT_OK(merge_tree_writer_->WriteSortedReaders(std::move(sorted_readers)));
+    if (raw_row_count != expected_raw_row_count) {
+        return Status::Invalid("PK real-time store commit readers did not cover the sealed range");
     }
     return Status::OK();
 }
@@ -197,27 +309,21 @@ Status RealtimePrimaryKeyWriter::FlushSegment(
 Status RealtimePrimaryKeyWriter::Compact(bool) {
     return Status::Invalid("PK real-time write does not support explicit compaction");
 }
-
 uint64_t RealtimePrimaryKeyWriter::GetMemoryUsage() const {
     return realtime_store_->GetMemoryUsage();
 }
-
 Status RealtimePrimaryKeyWriter::FlushMemory() {
     return Status::OK();
 }
-
 Result<bool> RealtimePrimaryKeyWriter::CompactNotCompleted() {
     return merge_tree_writer_->CompactNotCompleted();
 }
-
 Status RealtimePrimaryKeyWriter::Sync() {
     return merge_tree_writer_->Sync();
 }
-
 Status RealtimePrimaryKeyWriter::Close() {
     return merge_tree_writer_->Close();
 }
-
 std::shared_ptr<Metrics> RealtimePrimaryKeyWriter::GetMetrics() const {
     return merge_tree_writer_->GetMetrics();
 }
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index c1e893c85..6abb1ccd0 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -20,16 +20,16 @@
 #pragma once
 
 #include <cstdint>
-#include <map>
 #include <memory>
 #include <mutex>
 #include <string>
+#include <vector>
 
 #include "paimon/core/utils/batch_writer.h"
-#include "paimon/realtime/realtime_context.h"
 #include "paimon/realtime/realtime_store.h"
 
 namespace arrow {
+class MemoryPool;
 class Schema;
 }  // namespace arrow
 
@@ -37,18 +37,19 @@ namespace paimon {
 
 class MemoryPool;
 class MergeTreeWriter;
-class RealtimeContextImpl;
+class FieldsComparator;
 struct RealtimeStoreState;
 
-/// Primary-key real-time writer backed by an in-memory mutation indexer.
+/// Coordinates framework-prepared primary-key real-time writes.
 class RealtimePrimaryKeyWriter final : public BatchWriter {
  public:
     static Result<std::shared_ptr<RealtimePrimaryKeyWriter>> Create(
-        const std::map<std::string, std::string>& partition, int32_t bucket,
         const std::shared_ptr<arrow::Schema>& write_schema,
-        const std::shared_ptr<RealtimeContextImpl>& realtime_context,
+        const std::vector<std::string>& trimmed_primary_keys,
+        const std::shared_ptr<FieldsComparator>& key_comparator,
+        const RealtimeStoreState& store_state, int64_t restore_max_sequence_number,
         const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
-        const std::shared_ptr<MemoryPool>& memory_pool, const RealtimeStoreState& store_state);
+        const std::shared_ptr<MemoryPool>& memory_pool);
 
     Status Write(std::unique_ptr<RecordBatch>&& batch) override;
     Result<CommitIncrement> PrepareCommit(bool wait_compaction) override;
@@ -63,20 +64,28 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
  private:
     RealtimePrimaryKeyWriter(const std::shared_ptr<RealtimeStore>& realtime_store,
                              const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
-                             const std::shared_ptr<RealtimeContextImpl>& realtime_context,
-                             const RealtimePartitionBucket& partition_bucket,
                              const std::shared_ptr<arrow::Schema>& write_schema,
-                             int64_t next_offset, const std::shared_ptr<MemoryPool>& memory_pool);
+                             const std::shared_ptr<arrow::Schema>& prepared_schema,
+                             const std::shared_ptr<arrow::Schema>& key_schema,
+                             const std::vector<std::string>& trimmed_primary_keys,
+                             const std::shared_ptr<FieldsComparator>& key_comparator,
+                             int64_t next_offset, int64_t last_sequence_number,
+                             const std::shared_ptr<MemoryPool>& memory_pool);
 
-    Status FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment);
+    Status FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment,
+                        int64_t expected_raw_row_count);
 
     std::shared_ptr<MemoryPool> memory_pool_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     std::shared_ptr<RealtimeStore> realtime_store_;
     std::shared_ptr<MergeTreeWriter> merge_tree_writer_;
-    std::shared_ptr<RealtimeContextImpl> realtime_context_;
-    RealtimePartitionBucket partition_bucket_;
     std::shared_ptr<arrow::Schema> write_schema_;
+    std::shared_ptr<arrow::Schema> prepared_schema_;
+    std::shared_ptr<arrow::Schema> key_schema_;
+    std::vector<std::string> trimmed_primary_keys_;
+    std::shared_ptr<FieldsComparator> key_comparator_;
     int64_t next_offset_;
+    int64_t last_sequence_number_;
     std::mutex realtime_store_mutex_;
     std::mutex prepare_mutex_;
 };
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 76160ac93..f510c987e 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -24,20 +24,21 @@
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
-#include "paimon/common/data/columnar/columnar_row_ref.h"
 #include "paimon/common/reader/concat_batch_reader.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
-#include "paimon/common/types/row_kind.h"
-#include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/global_index/indexed_split_impl.h"
+#include "paimon/core/io/merged_key_value_record_reader.h"
+#include "paimon/core/io/prepared_key_value_reader.h"
 #include "paimon/core/key_value.h"
+#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
+#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/operation/merge_file_split_read.h"
 #include "paimon/core/operation/raw_file_split_read.h"
-#include "paimon/core/realtime/primary_key_realtime_store.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/realtime/realtime_reader.h"
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/table/source/pk_count_reader.h"
@@ -54,177 +55,60 @@ struct ColumnarBatchContext;
 
 namespace {
 
-class QueryBatchKeyValueReader final : public KeyValueRecordReader {
- public:
-    QueryBatchKeyValueReader(std::unique_ptr<BatchReader>&& reader,
-                             const std::shared_ptr<arrow::Schema>& key_schema,
-                             const std::shared_ptr<arrow::Schema>& value_schema,
-                             const std::shared_ptr<MemoryPool>& pool)
-        : reader_(std::move(reader)),
-          key_schema_(key_schema),
-          value_schema_(value_schema),
-          pool_(pool) {}
-
-    ~QueryBatchKeyValueReader() override {
-        Close();
-    }
-
-    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override;
-    std::shared_ptr<Metrics> GetReaderMetrics() const override;
-    void Close() override;
-
- private:
-    class Iterator;
-
-    std::unique_ptr<BatchReader> reader_;
-    std::shared_ptr<arrow::Schema> key_schema_;
-    std::shared_ptr<arrow::Schema> value_schema_;
-    std::shared_ptr<MemoryPool> pool_;
-    std::shared_ptr<arrow::StructArray> values_;
-    std::shared_ptr<arrow::Int64Array> sequences_;
-    std::shared_ptr<arrow::Int8Array> row_kinds_;
-    std::shared_ptr<ColumnarBatchContext> key_context_;
-    std::shared_ptr<ColumnarBatchContext> value_context_;
-    bool closed_ = false;
-};
-
-class QueryBatchKeyValueReader::Iterator final : public KeyValueRecordReader::Iterator {
- public:
-    explicit Iterator(QueryBatchKeyValueReader* reader) : reader_(reader) {}
-
-    Result<bool> HasNext() const override {
-        return cursor_ < reader_->values_->length();
-    }
-
-    Result<KeyValue> Next() override {
-        if (reader_->sequences_->IsNull(cursor_) || reader_->row_kinds_->IsNull(cursor_)) {
-            return Status::Invalid("PK merge metadata must not be null");
-        }
-        PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind,
-                               RowKind::FromByteValue(reader_->row_kinds_->Value(cursor_)));
-        const int64_t sequence = reader_->sequences_->Value(cursor_);
-        std::shared_ptr<InternalRow> key =
-            std::make_shared<ColumnarRowRef>(reader_->key_context_, cursor_);
-        auto value = std::make_unique<ColumnarRowRef>(reader_->value_context_, cursor_++);
-        return KeyValue(row_kind, sequence, KeyValue::UNKNOWN_LEVEL, std::move(key),
-                        std::move(value));
-    }
-
- private:
-    QueryBatchKeyValueReader* reader_;
-    int64_t cursor_ = 0;
-};
-
-Result<std::unique_ptr<KeyValueRecordReader::Iterator>> QueryBatchKeyValueReader::NextBatch() {
-    PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
-    if (BatchReader::IsEofBatch(batch)) {
-        return std::unique_ptr<KeyValueRecordReader::Iterator>();
-    }
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> imported,
-                                      arrow::ImportArray(batch.first.get(), batch.second.get()));
-    std::shared_ptr<arrow::StructArray> input =
-        std::dynamic_pointer_cast<arrow::StructArray>(imported);
-    if (!input) {
-        return Status::Invalid("PK merge input is not a StructArray");
-    }
-    sequences_ = std::dynamic_pointer_cast<arrow::Int64Array>(
-        input->GetFieldByName(SpecialFields::SequenceNumber().Name()));
-    row_kinds_ = std::dynamic_pointer_cast<arrow::Int8Array>(
-        input->GetFieldByName(SpecialFields::ValueKind().Name()));
-    if (!sequences_ || !row_kinds_) {
-        return Status::Invalid("PK merge input is missing sequence or value-kind metadata");
-    }
-    PAIMON_ASSIGN_OR_RAISE(input, ArrowUtils::RemoveFieldFromStructArray(
-                                      input, SpecialFields::SequenceNumber().Name()));
-    PAIMON_ASSIGN_OR_RAISE(
-        values_, ArrowUtils::RemoveFieldFromStructArray(input, SpecialFields::ValueKind().Name()));
-    if (!ArrowUtils::EqualsIgnoreNullable(values_->type(),
-                                          arrow::struct_(value_schema_->fields()))) {
-        return Status::Invalid("PK merge input value schema does not match the table read schema");
-    }
-    arrow::ArrayVector key_fields;
-    key_fields.reserve(key_schema_->num_fields());
-    for (const std::shared_ptr<arrow::Field>& field : key_schema_->fields()) {
-        std::shared_ptr<arrow::Array> key = values_->GetFieldByName(field->name());
-        if (!key) {
-            return Status::Invalid("PK merge input is missing key field ", field->name());
-        }
-        key_fields.push_back(std::move(key));
-    }
-    key_context_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
-    value_context_ = std::make_shared<ColumnarBatchContext>(values_->fields(), pool_);
-    return std::make_unique<Iterator>(this);
-}
-
-std::shared_ptr<Metrics> QueryBatchKeyValueReader::GetReaderMetrics() const {
-    return reader_->GetReaderMetrics();
-}
-
-void QueryBatchKeyValueReader::Close() {
-    if (closed_) {
-        return;
-    }
-    closed_ = true;
-    values_.reset();
-    sequences_.reset();
-    row_kinds_.reset();
-    key_context_.reset();
-    value_context_.reset();
-    if (reader_) {
-        reader_->Close();
-    }
-}
-
 Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
     const std::shared_ptr<RealtimeSplit>& split, const RealtimePartitionBucketView& memory,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<InternalReadContext>& context,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    arrow::FieldVector requested_fields = {
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())};
-    requested_fields.insert(requested_fields.end(), value_schema->fields().begin(),
-                            value_schema->fields().end());
+    std::shared_ptr<arrow::Schema> full_value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(context->GetTableSchema()->Fields());
+    arrow::FieldVector prepared_fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+            ->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+    prepared_fields.insert(prepared_fields.end(), full_value_schema->fields().begin(),
+                           full_value_schema->fields().end());
+    std::shared_ptr<arrow::Schema> prepared_schema = arrow::schema(std::move(prepared_fields));
     auto c_schema = std::make_unique<ArrowSchema>();
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(
-        arrow::ExportSchema(*arrow::schema(requested_fields), c_schema.get()));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
     ScopeGuard schema_guard([schema = c_schema.get()]() { ArrowSchemaRelease(schema); });
-    RealtimeQueryContext query_context{c_schema.get(), /*predicate=*/nullptr,
-                                       /*enable_predicate_pushdown=*/false};
-    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> batch_readers,
-                           memory.store->CreateQueryReaders(
-                               memory.read_view, split->CommittedEndOffset(), query_context));
-    ScopeGuard reader_guard([&batch_readers]() {
+    RealtimeQueryContext query_context{c_schema.get(), nullptr, false};
+    PAIMON_ASSIGN_OR_RAISE(
+        std::vector<std::unique_ptr<BatchReader>> batch_readers,
+        memory.store->CreateQueryReaders(memory.read_view, 0, query_context));
+    ScopeGuard batch_readers_guard([&batch_readers]() {
         for (const std::unique_ptr<BatchReader>& reader : batch_readers) {
             if (reader) {
                 reader->Close();
             }
         }
     });
-    if (batch_readers.empty()) {
-        return Status::Invalid("PK real-time store returned no query readers for active memory");
-    }
     std::vector<AdditionalKeyValueReader> result;
     result.reserve(batch_readers.size());
     for (std::unique_ptr<BatchReader>& reader : batch_readers) {
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null query reader");
         }
-        std::shared_ptr<InternalRow> min_key;
-        std::shared_ptr<InternalRow> max_key;
-        if (auto* provider = dynamic_cast<PrimaryKeyRangeProvider*>(reader.get())) {
-            min_key = provider->GetMinKey();
-            max_key = provider->GetMaxKey();
-        }
-        result.push_back(
-            AdditionalKeyValueReader{std::make_unique<QueryBatchKeyValueReader>(
-                                         std::move(reader), key_schema, value_schema, memory_pool),
-                                     std::move(min_key), std::move(max_key)});
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> prepared_reader,
+                               AdaptPreparedBatchReader(std::move(reader), prepared_schema,
+                                                        OffsetRange(split->CommittedEndOffset(),
+                                                                    split->MemoryEndOffset()),
+                                                        key_schema, value_schema, memory_pool));
+        auto merge = std::make_unique<DeduplicateMergeFunction>(false);
+        result.push_back(AdditionalKeyValueReader{
+            std::make_unique<MergedKeyValueRecordReader>(
+                std::move(prepared_reader), key_comparator,
+                std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge))),
+            nullptr, nullptr});
     }
+    batch_readers_guard.Release();
     return result;
 }
 
-}  // namespace
+}
 
 KeyValueTableRead::KeyValueTableRead(std::vector<std::unique_ptr<SplitRead>>&& split_reads,
                                      const std::shared_ptr<FileStorePathFactory>& path_factory,
@@ -268,7 +152,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
     const std::shared_ptr<Split>& split) {
     std::shared_ptr<RealtimeSplit> realtime_split = std::dynamic_pointer_cast<RealtimeSplit>(split);
     if (realtime_split) {
-        return CreateRealtimeReader(realtime_split, /*release_ticket=*/true);
+        return CreateRealtimeReader(realtime_split, true);
     }
 
     std::shared_ptr<Split> dispatch_split = split;
@@ -332,7 +216,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
             std::dynamic_pointer_cast<RealtimeSplit>(split);
         if (realtime_split) {
             PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
-                                   CreateRealtimeReader(realtime_split, /*release_ticket=*/false));
+                                   CreateRealtimeReader(realtime_split, false));
             readers.push_back(std::move(reader));
             realtime_splits.push_back(std::move(realtime_split));
         } else {
@@ -386,7 +270,8 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
             PAIMON_ASSIGN_OR_RAISE(
                 std::vector<AdditionalKeyValueReader> memory_readers,
                 CreateMemoryReaders(realtime_split, memory, merge_read->GetKeySchema(),
-                                    merge_read->GetValueSchema(), context_, GetMemoryPool()));
+                                    merge_read->GetValueSchema(), merge_read->GetKeyComparator(),
+                                    context_, GetMemoryPool()));
             PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
                                    merge_read->CreateRealtimeReader(realtime_split->DiskSplits(),
                                                                     std::move(memory_readers)));
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 9f302eb37..be7595381 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -443,9 +443,52 @@ class CloseTrackingRealtimeStoreFactory final : public RealtimeStoreFactory {
     std::shared_ptr<CloseTrackingReaderState> state_;
 };
 
-class InvalidReaderRealtimeStore final : public RealtimeStore {
+class SplitBatchReader final : public BatchReader {
  public:
-    explicit InvalidReaderRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
+    explicit SplitBatchReader(std::unique_ptr<BatchReader> delegate)
+        : delegate_(std::move(delegate)) {}
+
+    Result<ReadBatch> NextBatch() override {
+        while (!current_batch_ || next_row_ == current_batch_->length()) {
+            PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
+            if (BatchReader::IsEofBatch(batch)) {
+                return MakeEofBatch();
+            }
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> array,
+                arrow::ImportArray(batch.first.get(), batch.second.get()));
+            if (!array || array->type_id() != arrow::Type::STRUCT) {
+                return Status::Invalid("split batch reader received a non-struct batch");
+            }
+            current_batch_ = std::dynamic_pointer_cast<arrow::StructArray>(array);
+            next_row_ = 0;
+        }
+        std::shared_ptr<arrow::Array> slice = current_batch_->Slice(next_row_, /*length=*/1);
+        ++next_row_;
+        auto output = std::make_unique<ArrowArray>();
+        auto schema = std::make_unique<ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*slice, output.get(), schema.get()));
+        return ReadBatch(std::move(output), std::move(schema));
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        current_batch_.reset();
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    std::shared_ptr<arrow::StructArray> current_batch_;
+    int64_t next_row_ = 0;
+};
+
+class SplitCommitReaderRealtimeStore final : public RealtimeStore {
+ public:
+    explicit SplitCommitReaderRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
         : delegate_(delegate) {}
 
     Status Write(RealtimeWriteBatch&& batch) override {
@@ -457,9 +500,12 @@ class InvalidReaderRealtimeStore final : public RealtimeStore {
     }
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>&) override {
-        std::vector<std::unique_ptr<BatchReader>> readers;
-        readers.push_back(nullptr);
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               delegate_->CreateCommitReaders(segment));
+        for (std::unique_ptr<BatchReader>& reader : readers) {
+            reader = std::make_unique<SplitBatchReader>(std::move(reader));
+        }
         return readers;
     }
 
@@ -468,8 +514,9 @@ class InvalidReaderRealtimeStore final : public RealtimeStore {
     }
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>&, int64_t, const RealtimeQueryContext&) override {
-        return std::vector<std::unique_ptr<BatchReader>>();
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        return delegate_->CreateQueryReaders(view, offset_begin, context);
     }
 
     Status AdvanceCommittedOffset(int64_t committed_offset) override {
@@ -484,13 +531,13 @@ class InvalidReaderRealtimeStore final : public RealtimeStore {
     std::shared_ptr<RealtimeStore> delegate_;
 };
 
-class InvalidReaderRealtimeStoreFactory final : public RealtimeStoreFactory {
+class SplitCommitReaderRealtimeStoreFactory final : public RealtimeStoreFactory {
  public:
     Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
                                delegate_.Create(std::move(request)));
         return std::shared_ptr<RealtimeStore>(
-            std::make_shared<InvalidReaderRealtimeStore>(delegate));
+            std::make_shared<SplitCommitReaderRealtimeStore>(delegate));
     }
 
  private:
@@ -1311,10 +1358,11 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) {
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                          CreateRealtimeWriter(realtime_context));
 
-    std::vector<Row> first_rows = {{1, "old", "p0"}, {2, "two", "p0"}};
+    std::vector<Row> first_rows = {{1, "old", "p0"}, {2, "two", "p0"}, {1, "new-in-run", "p0"}};
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                          MakeBatch(first_rows, /*partitioned=*/false, /*bucket=*/0,
-                                   {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT}));
+                                   {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT,
+                                    RecordBatch::RowKind::UPDATE_AFTER}));
     ASSERT_OK(writer->Write(std::move(first_batch)));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> update_batch,
                          MakeBatch({Row{1, "new", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
@@ -1756,13 +1804,13 @@ TEST_F(RealtimeWriteInteTest, TestPkRecovery) {
                          MakeBatch(mutations, /*partitioned=*/false, /*bucket=*/0, mutation_kinds));
     ASSERT_OK(first_writer->Write(std::move(batch)));
     ASSERT_OK_AND_ASSIGN(std::vector<int64_t> memory_sequences, ReadPkSequences(first_context));
-    ASSERT_EQ((std::vector<int64_t>{2, 3, 4}), memory_sequences);
+    ASSERT_EQ((std::vector<int64_t>{1, 2, 3, 4}), memory_sequences);
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                          first_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
     ASSERT_EQ(1, progress.size());
     ASSERT_EQ(OffsetRange(0, 4), progress[0].offset_range);
     ASSERT_EQ(1, NewFiles(progress).size());
-    ASSERT_EQ(memory_sequences.front(), NewFiles(progress)[0]->min_sequence_number);
+    ASSERT_EQ(2, NewFiles(progress)[0]->min_sequence_number);
     ASSERT_EQ(memory_sequences.back(), NewFiles(progress)[0]->max_sequence_number);
     ASSERT_OK(Commit(progress, /*commit_identifier=*/1));
     ASSERT_OK(first_writer->Close());
@@ -1801,9 +1849,16 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
     constexpr int64_t kCommitRoundsBeforeCompaction = 4;
     std::set<std::string> committed_file_names;
     for (int64_t round = 0; round < kCommitRoundsBeforeCompaction; ++round) {
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                             MakeBatch({Row{round, "value-" + std::to_string(round), "p0"}},
-                                       /*partitioned=*/false));
+        const bool delete_latest_live_row = round == kCommitRoundsBeforeCompaction - 1;
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr<RecordBatch> batch,
+            MakeBatch(
+                {Row{delete_latest_live_row ? round - 1 : round,
+                     delete_latest_live_row ? "deleted" : "value-" + std::to_string(round), "p0"}},
+                /*partitioned=*/false, /*bucket=*/0,
+                delete_latest_live_row
+                    ? std::vector<RecordBatch::RowKind>{RecordBatch::RowKind::DELETE}
+                    : std::vector<RecordBatch::RowKind>{}));
         ASSERT_OK(writer->Write(std::move(batch)));
         ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                              writer->PrepareCommitWithProgress(round));
@@ -1819,11 +1874,6 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
         ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
         ASSERT_EQ(0, memory_usage);
     }
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> next_batch,
-                         MakeBatch({Row{4, "value-4", "p0"}},
-                                   /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(next_batch)));
-
     WriteContextBuilder compact_builder(table_path_, commit_user_);
     compact_builder.SetOptions(options_).WithStreamingMode(true);
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> compact_context, compact_builder.Finish());
@@ -1848,6 +1898,14 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
     }
     ASSERT_EQ(committed_file_names, compacted_file_names);
     ASSERT_FALSE(compact_message->GetCompactIncrement().CompactAfter().empty());
+    constexpr int64_t kHistoricalMaxSequenceNumber = kCommitRoundsBeforeCompaction - 1;
+    int64_t compacted_live_max_sequence_number = -1;
+    for (const std::shared_ptr<DataFileMeta>& file :
+         compact_message->GetCompactIncrement().CompactAfter()) {
+        compacted_live_max_sequence_number =
+            std::max(compacted_live_max_sequence_number, file->max_sequence_number);
+    }
+    ASSERT_LT(compacted_live_max_sequence_number, kHistoricalMaxSequenceNumber);
     ASSERT_OK(CommitMessages(compact_messages, /*commit_identifier=*/4));
     ASSERT_OK(compact_writer->Close());
 
@@ -1859,45 +1917,38 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
     ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot->GetCommitKind());
     ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets, ReadCommittedOffsets());
     ASSERT_EQ(4, offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
-    ASSERT_OK(writer->RefreshCommittedSnapshot(compact_snapshot->Id()));
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> compacted_rows, ReadRows(realtime_context));
-    ASSERT_EQ((std::vector<Row>{{0, "value-0", "p0"},
-                                {1, "value-1", "p0"},
-                                {2, "value-2", "p0"},
-                                {3, "value-3", "p0"},
-                                {4, "value-4", "p0"}}),
-              compacted_rows);
-
-    constexpr int64_t kCommitRoundsAfterCompaction = 2;
-    for (int64_t round = 0; round < kCommitRoundsAfterCompaction; ++round) {
-        if (round > 0) {
-            ASSERT_OK_AND_ASSIGN(
-                std::unique_ptr<RecordBatch> batch,
-                MakeBatch({Row{4 + round, "value-" + std::to_string(4 + round), "p0"}},
-                          /*partitioned=*/false));
-            ASSERT_OK(writer->Write(std::move(batch)));
-        }
-        const int64_t commit_identifier = 5 + round;
-        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
-                             writer->PrepareCommitWithProgress(commit_identifier));
-        ASSERT_EQ(1, progress.size());
-        ASSERT_EQ(OffsetRange(4 + round, 5 + round), progress[0].offset_range);
-        ASSERT_OK_AND_ASSIGN(latest_snapshot_id, Commit(progress, commit_identifier));
-        ASSERT_OK(writer->RefreshCommittedSnapshot(latest_snapshot_id));
-        ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
-        ASSERT_EQ(0, memory_usage);
-    }
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> compacted_rows, ReadRows());
+    ASSERT_EQ((std::vector<Row>{{0, "value-0", "p0"}, {1, "value-1", "p0"}}), compacted_rows);
+    ASSERT_OK(writer->Close());
+    writer.reset();
+    realtime_context.reset();
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> fresh_context, RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> fresh_writer,
+                         CreateRealtimeWriter(fresh_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> fresh_batch,
+                         MakeBatch({Row{4, "value-4", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(fresh_writer->Write(std::move(fresh_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<int64_t> fresh_sequences, ReadPkSequences(fresh_context));
+    ASSERT_EQ((std::vector<int64_t>{compacted_live_max_sequence_number + 1}), fresh_sequences);
+    ASSERT_LT(fresh_sequences.front(), kHistoricalMaxSequenceNumber);
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> fresh_progress,
+                         fresh_writer->PrepareCommitWithProgress(/*commit_identifier=*/5));
+    ASSERT_EQ(1, fresh_progress.size());
+    ASSERT_EQ(OffsetRange(4, 5), fresh_progress[0].offset_range);
+    ASSERT_EQ(compacted_live_max_sequence_number + 1,
+              NewFiles(fresh_progress)[0]->min_sequence_number);
+    ASSERT_EQ(compacted_live_max_sequence_number + 1,
+              NewFiles(fresh_progress)[0]->max_sequence_number);
+    ASSERT_OK_AND_ASSIGN(latest_snapshot_id, Commit(fresh_progress, /*commit_identifier=*/5));
+    ASSERT_OK(fresh_writer->Close());
+
     ASSERT_OK_AND_ASSIGN(offsets, ReadCommittedOffsets());
-    ASSERT_EQ(6, offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows(realtime_context));
-    ASSERT_EQ((std::vector<Row>{{0, "value-0", "p0"},
-                                {1, "value-1", "p0"},
-                                {2, "value-2", "p0"},
-                                {3, "value-3", "p0"},
-                                {4, "value-4", "p0"},
-                                {5, "value-5", "p0"}}),
+    ASSERT_EQ(5, offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows());
+    ASSERT_EQ((std::vector<Row>{{0, "value-0", "p0"}, {1, "value-1", "p0"}, {4, "value-4", "p0"}}),
               final_rows);
-    ASSERT_OK(writer->Close());
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkConcurrency) {
@@ -2054,19 +2105,29 @@ TEST_F(RealtimeWriteInteTest, TestPkWriteDuringPrepare) {
 
 TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
     CreatePkTable();
-    auto factory = std::make_shared<InvalidReaderRealtimeStoreFactory>();
+    auto factory = std::make_shared<SplitCommitReaderRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                          CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+                         MakeBatch({Row{4, "four", "p0"}, Row{3, "three", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(first_batch)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+                         MakeBatch({Row{2, "two", "p0"}, Row{1, "one", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(second_batch)));
 
-    ASSERT_NOK_WITH_MSG(ReadRows(realtime_context),
-                        "PK real-time store returned no query readers for active memory");
-    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
-                        "PK real-time store returned a null commit reader");
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, progress.size());
+    ASSERT_EQ(OffsetRange(0, 4), progress[0].offset_range);
+    ASSERT_OK(Commit(progress, /*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, ReadRows());
+    ASSERT_EQ((std::vector<Row>{
+                  {1, "one", "p0"}, {2, "two", "p0"}, {3, "three", "p0"}, {4, "four", "p0"}}),
+              rows);
     ASSERT_OK(writer->Close());
 }
 
@@ -2757,52 +2818,6 @@ TEST_F(RealtimeWriteInteTest, TestCloseWriterAllowsContextReuseByLaterWriter) {
     ASSERT_OK(second_writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkWriterHandoff) {
-    CreatePkTable();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
-                         CreateRealtimeWriter(realtime_context));
-    std::vector<Row> first_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
-                         MakeBatch(first_rows, /*partitioned=*/false));
-    ASSERT_OK(first_writer->Write(std::move(first_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
-                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
-    ASSERT_EQ(1, commits.size());
-    ASSERT_EQ(OffsetRange(0, 3), commits[0].offset_range);
-    ASSERT_EQ(1, NewFiles(commits).size());
-    ASSERT_EQ(0, NewFiles(commits)[0]->min_sequence_number);
-    ASSERT_EQ(2, NewFiles(commits)[0]->max_sequence_number);
-    ASSERT_OK(first_writer->Close());
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
-                         CreateRealtimeWriter(realtime_context));
-    std::vector<Row> second_rows = {
-        Row{0, "updated-0", "p0"},
-        Row{3, "value-3", "p0"},
-    };
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
-                         MakeBatch(second_rows, /*partitioned=*/false));
-    ASSERT_OK(second_writer->Write(std::move(second_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
-                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_EQ(1, second_commits.size());
-    ASSERT_EQ(OffsetRange(3, 5), second_commits[0].offset_range);
-    ASSERT_EQ(1, NewFiles(second_commits).size());
-    ASSERT_EQ(3, NewFiles(second_commits)[0]->min_sequence_number);
-    ASSERT_EQ(4, NewFiles(second_commits)[0]->max_sequence_number);
-
-    commits.push_back(std::move(second_commits[0]));
-    ASSERT_OK(Commit(commits, /*commit_identifier=*/1));
-    std::vector<Row> expected_rows = first_rows;
-    expected_rows[0] = second_rows[0];
-    expected_rows.push_back(second_rows[1]);
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
-    ASSERT_EQ(expected_rows, actual_rows);
-    ASSERT_OK(second_writer->Close());
-}
-
 TEST_F(RealtimeWriteInteTest, TestReadCommittedDiskAndBuildingMemory) {
     CreateTable(/*partition_keys=*/{});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,

From 62108e2456407e0bbb2012d5aa5343018378e9b0 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 12:35:37 +0800
Subject: [PATCH 38/93] refactor(realtime): simplify primary-key write
 preparation

---
 include/paimon/realtime/realtime_context.h    |   4 -
 src/paimon/CMakeLists.txt                     |   2 +-
 .../merged_key_value_record_reader_test.cpp   |   2 +-
 .../core/mergetree/merge_tree_writer_test.cpp |  15 -
 src/paimon/core/mergetree/write_buffer.cpp    |   4 -
 .../key_value_file_store_write_test.cpp       |  98 -------
 .../prepared_key_value_reader.cpp             |   2 +-
 .../prepared_key_value_reader.h               |   0
 .../realtime/realtime_primary_key_writer.cpp  |   2 +-
 .../table/source/key_value_table_read.cpp     |   2 +-
 .../core/utils/primary_key_table_utils.h      |   1 -
 test/inte/realtime_write_inte_test.cpp        | 273 ------------------
 12 files changed, 5 insertions(+), 400 deletions(-)
 rename src/paimon/core/{io => realtime}/prepared_key_value_reader.cpp (99%)
 rename src/paimon/core/{io => realtime}/prepared_key_value_reader.h (100%)

diff --git a/include/paimon/realtime/realtime_context.h b/include/paimon/realtime/realtime_context.h
index 8f2967b32..200e4ba4c 100644
--- a/include/paimon/realtime/realtime_context.h
+++ b/include/paimon/realtime/realtime_context.h
@@ -78,10 +78,6 @@ using RealtimeOffsetMap = std::map<RealtimePartitionBucket, int64_t>;
 /// partition drop, and rollback operations do not automatically clear process-local real-time
 /// state. Applications must coordinate these operations with active real-time writers and recreate
 /// the `RealtimeContext` and writers before continuing.
-///
-/// A primary-key writer and its context form one lifecycle. After a primary-key write or prepare
-/// returns an error, discard both, create fresh instances from the latest committed snapshot, and
-/// replay the caller-owned WAL. Reusing the failed context or retrying prepare is unsupported.
 class PAIMON_EXPORT RealtimeContext {
  public:
     /// Creates a context backed by Paimon's default in-memory Arrow `RealtimeStore`.
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 62af55ec4..0a78b0902 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -286,7 +286,6 @@ set(PAIMON_CORE_SRCS
     core/io/vector_file_batch_reader.cpp
     core/io/file_index_evaluator.cpp
     core/io/key_value_data_file_record_reader.cpp
-    core/io/prepared_key_value_reader.cpp
     core/io/key_value_data_file_writer_factory.cpp
     core/io/key_value_data_file_writer.cpp
     core/io/key_value_in_memory_record_reader.cpp
@@ -383,6 +382,7 @@ set(PAIMON_CORE_SRCS
     core/operation/write_restore.cpp
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
+    core/realtime/prepared_key_value_reader.cpp
     core/realtime/primary_key_realtime_store.cpp
     core/realtime/realtime_append_only_writer.cpp
     core/realtime/realtime_context.cpp
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 39714fa29..21b0a16b1 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -34,9 +34,9 @@
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/fields_comparator.h"
-#include "paimon/core/io/prepared_key_value_reader.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
+#include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/realtime/offset_range.h"
diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index aa2d0c959..675ce3198 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -21,7 +21,6 @@
 #include <cassert>
 #include <cstddef>
 #include <functional>
-#include <limits>
 #include <map>
 #include <optional>
 #include <utility>
@@ -613,20 +612,6 @@ TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
     ASSERT_OK(merge_writer->Close());
 }
 
-TEST_P(MergeTreeWriterTest, TestRejectsExhaustedSequence) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options,
-                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
-
-    auto dir = UniqueTestDirectory::Create();
-    ASSERT_TRUE(dir);
-    auto path_factory = std::make_shared<DataFilePathFactory>();
-    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
-
-    ASSERT_NOK_WITH_MSG(CreateMergeWriter(std::numeric_limits<int64_t>::max(), dir->Str(),
-                                          path_factory, 0, options),
-                        "sequence number has reached INT64_MAX");
-}
-
 TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
                          CoreOptions::FromMap({
diff --git a/src/paimon/core/mergetree/write_buffer.cpp b/src/paimon/core/mergetree/write_buffer.cpp
index 3d3fdc196..549975a33 100644
--- a/src/paimon/core/mergetree/write_buffer.cpp
+++ b/src/paimon/core/mergetree/write_buffer.cpp
@@ -18,7 +18,6 @@
 
 #include "paimon/core/mergetree/write_buffer.h"
 
-#include <limits>
 #include <memory>
 #include <utility>
 
@@ -40,9 +39,6 @@ Result<std::unique_ptr<WriteBuffer>> WriteBuffer::Create(
     const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper,
     const CoreOptions& options, const std::shared_ptr<IOManager>& io_manager,
     bool enable_multi_thread_spill, const std::shared_ptr<MemoryPool>& pool) {
-    if (last_sequence_number == std::numeric_limits<int64_t>::max()) {
-        return Status::Invalid("sequence number has reached INT64_MAX");
-    }
     auto value_type = arrow::struct_(value_schema->fields());
     auto in_memory_buffer = std::make_unique<InMemorySortBuffer>(
         last_sequence_number, value_type, trimmed_primary_keys, user_defined_sequence_fields,
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index cbd2189fc..733c19d6e 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -59,7 +59,6 @@
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/reader/file_batch_reader.h"
-#include "paimon/realtime/arrow_realtime_store_factory.h"
 #include "paimon/realtime/realtime_context.h"
 #include "paimon/record_batch.h"
 #include "paimon/status.h"
@@ -111,69 +110,6 @@ class TestingMemoryPool final : public MemoryPool {
     std::unique_ptr<MemoryPool> delegate_ = GetMemoryPool();
 };
 
-class FailOnceRealtimeStore final : public RealtimeStore {
- public:
-    FailOnceRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                          const std::shared_ptr<bool>& fail_next_write)
-        : delegate_(delegate), fail_next_write_(fail_next_write) {}
-
-    Status Write(RealtimeWriteBatch&& batch) override {
-        if (*fail_next_write_) {
-            *fail_next_write_ = false;
-            return Status::Invalid("injected real-time store write failure");
-        }
-        return delegate_->Write(std::move(batch));
-    }
-
-    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
-        return delegate_->SealForCommit();
-    }
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
-        return delegate_->CreateCommitReaders(segment);
-    }
-
-    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
-        return delegate_->AcquireReadView();
-    }
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
-        const RealtimeQueryContext& context) override {
-        return delegate_->CreateQueryReaders(view, offset_begin, context);
-    }
-
-    Status AdvanceCommittedOffset(int64_t committed_offset) override {
-        return delegate_->AdvanceCommittedOffset(committed_offset);
-    }
-
-    uint64_t GetMemoryUsage() const override {
-        return delegate_->GetMemoryUsage();
-    }
-
- private:
-    std::shared_ptr<RealtimeStore> delegate_;
-    std::shared_ptr<bool> fail_next_write_;
-};
-
-class FailOnceRealtimeStoreFactory final : public RealtimeStoreFactory {
- public:
-    explicit FailOnceRealtimeStoreFactory(const std::shared_ptr<bool>& fail_next_write)
-        : fail_next_write_(fail_next_write) {}
-
-    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
-                               delegate_.Create(std::move(request)));
-        return std::shared_ptr<RealtimeStore>(
-            std::make_shared<FailOnceRealtimeStore>(delegate, fail_next_write_));
-    }
-
- private:
-    ArrowRealtimeStoreFactory delegate_;
-    std::shared_ptr<bool> fail_next_write_;
-};
-
 }
 
 class KeyValueFileStoreWriteTest : public ::testing::Test {
@@ -551,40 +487,6 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(KeyValueFileStoreWriteTest, TestWriteFailureKeepsCursors) {
-    const std::map<std::string, std::string> options = {
-        {Options::BUCKET, "1"},
-        {Options::WRITE_BUFFER_SIZE, "1"},
-    };
-    const std::shared_ptr<arrow::Schema> schema = arrow::schema({
-        arrow::field("id", arrow::int64(), false),
-        arrow::field("value", arrow::utf8()),
-    });
-    std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
-    ASSERT_TRUE(dir);
-    CreateTable(dir->Str(), schema, options);
-    const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
-
-    auto fail_next_write = std::make_shared<bool>(true);
-    auto factory = std::make_shared<FailOnceRealtimeStoreFactory>(fail_next_write);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    WriteContextBuilder builder(table_path, "test");
-    builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         FileStoreWrite::Create(std::move(write_context)));
-
-    ASSERT_NOK_WITH_MSG(writer->Write(MakeBatch(schema, R"([[9, "rejected"]])")),
-                        "injected real-time store write failure");
-    ASSERT_OK(writer->Write(MakeBatch(schema, R"([[1, "kept"]])")));
-    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
-    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> prepared_rows,
-                         ReadPreparedRows(realtime_context));
-    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "kept", 0, 0}}), prepared_rows);
-    ASSERT_OK(writer->Close());
-}
-
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
     const std::map<std::string, std::string> options = {{Options::BUCKET, "1"}};
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
diff --git a/src/paimon/core/io/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
similarity index 99%
rename from src/paimon/core/io/prepared_key_value_reader.cpp
rename to src/paimon/core/realtime/prepared_key_value_reader.cpp
index 0f4f22097..b99f67dd9 100644
--- a/src/paimon/core/io/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-#include "paimon/core/io/prepared_key_value_reader.h"
+#include "paimon/core/realtime/prepared_key_value_reader.h"
 
 #include <cstdint>
 #include <memory>
diff --git a/src/paimon/core/io/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
similarity index 100%
rename from src/paimon/core/io/prepared_key_value_reader.h
rename to src/paimon/core/realtime/prepared_key_value_reader.h
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index c85ff6322..2dc5a71b4 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -34,10 +34,10 @@
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/io/merged_key_value_record_reader.h"
-#include "paimon/core/io/prepared_key_value_reader.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/mergetree/merge_tree_writer.h"
+#include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/utils/commit_increment.h"
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index f510c987e..31779e049 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -31,12 +31,12 @@
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/global_index/indexed_split_impl.h"
 #include "paimon/core/io/merged_key_value_record_reader.h"
-#include "paimon/core/io/prepared_key_value_reader.h"
 #include "paimon/core/key_value.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/operation/merge_file_split_read.h"
 #include "paimon/core/operation/raw_file_split_read.h"
+#include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/realtime/realtime_reader.h"
diff --git a/src/paimon/core/utils/primary_key_table_utils.h b/src/paimon/core/utils/primary_key_table_utils.h
index c40e92cda..82a108ab7 100644
--- a/src/paimon/core/utils/primary_key_table_utils.h
+++ b/src/paimon/core/utils/primary_key_table_utils.h
@@ -24,7 +24,6 @@
 
 #include "arrow/type.h"
 #include "paimon/result.h"
-#include "paimon/status.h"
 
 namespace arrow {
 class Schema;
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index be7595381..c61b04b0d 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -23,7 +23,6 @@
 #include <condition_variable>
 #include <cstdint>
 #include <deque>
-#include <future>
 #include <iterator>
 #include <limits>
 #include <map>
@@ -79,126 +78,6 @@
 namespace paimon::test {
 namespace {
 
-class BlockingState {
- public:
-    void Block() {
-        std::unique_lock<std::mutex> lock(mutex_);
-        entered_ = true;
-        entered_cv_.notify_all();
-        release_cv_.wait(lock, [this]() { return released_; });
-    }
-
-    bool WaitUntilBlocked() {
-        std::unique_lock<std::mutex> lock(mutex_);
-        return entered_cv_.wait_for(lock, std::chrono::seconds(30), [this]() { return entered_; });
-    }
-
-    void Release() {
-        std::lock_guard<std::mutex> lock(mutex_);
-        released_ = true;
-        release_cv_.notify_all();
-    }
-
- private:
-    std::mutex mutex_;
-    std::condition_variable entered_cv_;
-    std::condition_variable release_cv_;
-    bool entered_ = false;
-    bool released_ = false;
-};
-
-class BlockingBatchReader final : public BatchReader {
- public:
-    BlockingBatchReader(std::unique_ptr<BatchReader>&& reader,
-                        const std::shared_ptr<BlockingState>& state)
-        : reader_(std::move(reader)), state_(state) {}
-
-    Result<ReadBatch> NextBatch() override {
-        if (!blocked_) {
-            blocked_ = true;
-            state_->Block();
-        }
-        return reader_->NextBatch();
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return reader_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        reader_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> reader_;
-    std::shared_ptr<BlockingState> state_;
-    bool blocked_ = false;
-};
-
-class BlockingRealtimeStore final : public RealtimeStore {
- public:
-    BlockingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                          const std::shared_ptr<BlockingState>& state)
-        : delegate_(delegate), state_(state) {}
-
-    Status Write(RealtimeWriteBatch&& batch) override {
-        return delegate_->Write(std::move(batch));
-    }
-
-    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
-        return delegate_->SealForCommit();
-    }
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               delegate_->CreateCommitReaders(segment));
-        if (!readers.empty()) {
-            readers[0] = std::make_unique<BlockingBatchReader>(std::move(readers[0]), state_);
-        }
-        return readers;
-    }
-
-    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
-        return delegate_->AcquireReadView();
-    }
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
-        const RealtimeQueryContext& context) override {
-        return delegate_->CreateQueryReaders(view, offset_begin, context);
-    }
-
-    Status AdvanceCommittedOffset(int64_t committed_offset) override {
-        return delegate_->AdvanceCommittedOffset(committed_offset);
-    }
-
-    uint64_t GetMemoryUsage() const override {
-        return delegate_->GetMemoryUsage();
-    }
-
- private:
-    std::shared_ptr<RealtimeStore> delegate_;
-    std::shared_ptr<BlockingState> state_;
-};
-
-class BlockingRealtimeStoreFactory final : public RealtimeStoreFactory {
- public:
-    explicit BlockingRealtimeStoreFactory(const std::shared_ptr<BlockingState>& state)
-        : state_(state) {}
-
-    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
-                               delegate_.Create(std::move(request)));
-        return std::shared_ptr<RealtimeStore>(
-            std::make_shared<BlockingRealtimeStore>(delegate, state_));
-    }
-
- private:
-    ArrowRealtimeStoreFactory delegate_;
-    std::shared_ptr<BlockingState> state_;
-};
-
 class TrackingRealtimeReadView final : public RealtimeReadView {
  public:
     explicit TrackingRealtimeReadView(std::shared_ptr<RealtimeReadView> delegate)
@@ -1951,158 +1830,6 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
               final_rows);
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkConcurrency) {
-    CreatePkTable();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-
-    constexpr int64_t kRowCount = 20;
-    constexpr int32_t kReaderCount = 2;
-    std::atomic<bool> writer_done{false};
-    std::atomic<bool> control_done{false};
-    std::atomic<int32_t> commit_count{0};
-    ConcurrentTestState state;
-    std::vector<int32_t> read_counts(kReaderCount, 0);
-
-    std::thread write_thread([&]() {
-        state.WaitForStart();
-        for (int64_t id = 0; id < kRowCount && !state.ShouldStop(); ++id) {
-            Result<std::unique_ptr<RecordBatch>> batch =
-                MakeBatch(MakeRows(id, /*count=*/1, /*partition=*/"p0"),
-                          /*partitioned=*/false);
-            if (state.RecordErrorIfNotOk(batch) ||
-                state.RecordErrorIfNotOk(writer->Write(std::move(batch).value()))) {
-                break;
-            }
-            std::this_thread::sleep_for(std::chrono::milliseconds(2));
-        }
-        writer_done.store(true, std::memory_order_release);
-    });
-
-    std::thread control_thread([&]() {
-        state.WaitForStart();
-        int64_t commit_identifier = 0;
-        do {
-            Result<std::vector<RealtimeCommitProgress>> progress =
-                writer->PrepareCommitWithProgress(commit_identifier);
-            if (state.RecordErrorIfNotOk(progress)) {
-                break;
-            }
-            if (!progress.value().empty()) {
-                Result<int64_t> snapshot = Commit(progress.value(), commit_identifier++);
-                if (state.RecordErrorIfNotOk(snapshot) ||
-                    state.RecordErrorIfNotOk(writer->RefreshCommittedSnapshot(snapshot.value()))) {
-                    break;
-                }
-                commit_count.fetch_add(1, std::memory_order_relaxed);
-            }
-            std::this_thread::sleep_for(std::chrono::milliseconds(1));
-        } while (!writer_done.load(std::memory_order_acquire) && !state.ShouldStop());
-        if (!state.ShouldStop()) {
-            Result<std::vector<RealtimeCommitProgress>> progress =
-                writer->PrepareCommitWithProgress(commit_identifier);
-            if (!state.RecordErrorIfNotOk(progress) && !progress.value().empty()) {
-                Result<int64_t> snapshot = Commit(progress.value(), commit_identifier);
-                if (!state.RecordErrorIfNotOk(snapshot) &&
-                    !state.RecordErrorIfNotOk(writer->RefreshCommittedSnapshot(snapshot.value()))) {
-                    commit_count.fetch_add(1, std::memory_order_relaxed);
-                }
-            }
-        }
-        control_done.store(true, std::memory_order_release);
-    });
-
-    std::vector<std::thread> read_threads;
-    read_threads.reserve(kReaderCount);
-    for (int32_t reader_index = 0; reader_index < kReaderCount; ++reader_index) {
-        read_threads.emplace_back([&, reader_index]() {
-            state.WaitForStart();
-            while (!control_done.load(std::memory_order_acquire) && !state.ShouldStop()) {
-                Result<std::vector<Row>> rows = ReadRows(realtime_context);
-                ++read_counts[reader_index];
-                if (state.RecordErrorIfNotOk(rows) ||
-                    state.RecordErrorIfNotOk(ValidateReadPrefix(rows.value(), kRowCount))) {
-                    break;
-                }
-                std::this_thread::sleep_for(std::chrono::milliseconds(1));
-            }
-        });
-    }
-
-    state.StartWhenReady(/*worker_count=*/2 + kReaderCount);
-    write_thread.join();
-    control_thread.join();
-    for (std::thread& read_thread : read_threads) {
-        read_thread.join();
-    }
-
-    ASSERT_TRUE(state.Errors().empty()) << (state.Errors().empty() ? "" : state.Errors().front());
-    ASSERT_GT(commit_count.load(), 0);
-    for (int32_t read_count : read_counts) {
-        ASSERT_GT(read_count, 0);
-    }
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows(realtime_context));
-    ASSERT_EQ(kRowCount, static_cast<int64_t>(final_rows.size()));
-    ASSERT_OK(ValidateReadPrefix(final_rows, kRowCount));
-    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
-    ASSERT_EQ(0, memory_usage);
-    ASSERT_OK(writer->Close());
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkWriteDuringPrepare) {
-    CreatePkTable();
-    auto state = std::make_shared<BlockingState>();
-    auto factory = std::make_shared<BlockingRealtimeStoreFactory>(state);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
-                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(first_batch)));
-
-    Result<std::vector<RealtimeCommitProgress>> prepare_result =
-        Status::Invalid("prepare did not run");
-    std::thread prepare_thread(
-        [&]() { prepare_result = writer->PrepareCommitWithProgress(/*commit_identifier=*/0); });
-    const bool prepare_blocked = state->WaitUntilBlocked();
-    if (!prepare_blocked) {
-        state->Release();
-        prepare_thread.join();
-        ASSERT_TRUE(prepare_blocked);
-    }
-
-    std::promise<Status> write_promise;
-    std::future<Status> write_future = write_promise.get_future();
-    std::thread write_thread([&]() {
-        Result<std::unique_ptr<RecordBatch>> batch =
-            MakeBatch({Row{2, "two", "p0"}}, /*partitioned=*/false);
-        if (!batch.ok()) {
-            write_promise.set_value(batch.status());
-            return;
-        }
-        write_promise.set_value(writer->Write(std::move(batch).value()));
-    });
-    const bool write_completed =
-        write_future.wait_for(std::chrono::seconds(5)) == std::future_status::ready;
-    state->Release();
-    prepare_thread.join();
-    write_thread.join();
-
-    ASSERT_TRUE(write_completed);
-    ASSERT_OK(write_future.get());
-    ASSERT_OK(prepare_result);
-    ASSERT_EQ(1, prepare_result.value().size());
-    ASSERT_EQ(OffsetRange(0, 1), prepare_result.value()[0].offset_range);
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
-                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_EQ(1, second_progress.size());
-    ASSERT_EQ(OffsetRange(1, 2), second_progress[0].offset_range);
-    ASSERT_OK(writer->Close());
-}
-
 TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
     CreatePkTable();
     auto factory = std::make_shared<SplitCommitReaderRealtimeStoreFactory>();

From b6bf461806cb09f495bb6d62eb14b833a7453457 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 14:49:07 +0800
Subject: [PATCH 39/93] test(mergetree): reuse reader failure mock

---
 .../core/mergetree/merge_tree_writer_test.cpp | 44 ++++++-------------
 1 file changed, 14 insertions(+), 30 deletions(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 675ce3198..63e896573 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -96,31 +96,7 @@ class TrackingKeyValueRecordReader : public KeyValueRecordReader {
     bool* closed_flag_;
 };
 
-class ErrorKeyValueRecordReader : public KeyValueRecordReader {
- public:
-    ErrorKeyValueRecordReader(Status status, bool* closed_flag)
-        : status_(std::move(status)), closed_flag_(closed_flag) {}
-
-    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
-        return status_;
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return nullptr;
-    }
-
-    void Close() override {
-        if (closed_flag_ != nullptr) {
-            *closed_flag_ = true;
-        }
-    }
-
- private:
-    Status status_;
-    bool* closed_flag_;
-};
-
-}
+}  // namespace
 
 class MergeTreeWriterTest : public ::testing::TestWithParam<bool> {
  public:
@@ -270,7 +246,8 @@ class MergeTreeWriterTest : public ::testing::TestWithParam<bool> {
     }
 
     std::unique_ptr<KeyValueRecordReader> CreateSingleReader(
-        const std::shared_ptr<arrow::Array>& array, int32_t batch_size = 16) const {
+        const std::shared_ptr<arrow::Array>& array, int32_t batch_size = 16,
+        const Status& next_batch_status = Status::OK()) const {
         std::vector<DataField> write_fields = {SpecialFields::SequenceNumber(),
                                                SpecialFields::ValueKind()};
         write_fields.insert(write_fields.end(), value_fields_.begin(), value_fields_.end());
@@ -280,6 +257,7 @@ class MergeTreeWriterTest : public ::testing::TestWithParam<bool> {
             arrow::schema(arrow::FieldVector({write_schema->field(2)}));
         auto file_batch_reader =
             std::make_unique<MockFileBatchReader>(array, array->type(), batch_size);
+        file_batch_reader->SetNextBatchStatus(next_batch_status);
         return std::make_unique<MockKeyValueDataFileRecordReader>(
             std::move(file_batch_reader), key_schema, value_schema_, 0, pool_);
     }
@@ -601,13 +579,19 @@ TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
     Status null_status = merge_writer->WriteSortedReaders(std::move(null_readers));
     ASSERT_TRUE(null_status.IsInvalid());
 
+    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
+      [0, 0, "Alice", 10, 0, 13.1]
+    ])")
+            .ValueOrDie());
+    Status expected_status = Status::IOError("sorted reader failure");
     bool failing_reader_closed = false;
-    auto failing_reader = std::make_unique<ErrorKeyValueRecordReader>(
-        Status::IOError("sorted reader failure"), &failing_reader_closed);
     std::vector<std::unique_ptr<KeyValueRecordReader>> failing_readers;
-    failing_readers.push_back(std::move(failing_reader));
+    failing_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
+        CreateSingleReader(sorted_reader_array, /*batch_size=*/16, expected_status),
+        &failing_reader_closed));
     Status failing_status = merge_writer->WriteSortedReaders(std::move(failing_readers));
-    ASSERT_TRUE(failing_status.IsIOError());
+    ASSERT_EQ(expected_status, failing_status);
     ASSERT_TRUE(failing_reader_closed);
     ASSERT_OK(merge_writer->Close());
 }

From 8c4c4c8b20754882eb2b2bc0749dcc9464b62aca Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 14:49:38 +0800
Subject: [PATCH 40/93] fix(realtime): preserve PK sequence across writer
 handoff

---
 .../operation/key_value_file_store_write.cpp  |  6 +--
 .../core/realtime/realtime_context_impl.cpp   | 11 +++++
 .../core/realtime/realtime_context_impl.h     |  4 ++
 .../core/realtime/realtime_context_test.cpp   | 16 +++++++
 .../realtime/realtime_primary_key_writer.cpp  | 23 +++++++---
 .../realtime/realtime_primary_key_writer.h    | 10 ++++-
 test/inte/realtime_write_inte_test.cpp        | 45 +++++++++++++++++++
 7 files changed, 106 insertions(+), 9 deletions(-)

diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index d2c97abcf..de7217ec3 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -175,9 +175,9 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
     if (!realtime_context_) {
         return std::shared_ptr<BatchWriter>(std::move(writer));
     }
-    return RealtimePrimaryKeyWriter::Create(schema_, trimmed_primary_keys, key_comparator_,
-                                            realtime_store_state.value(), restore_max_seq_number,
-                                            writer, pool_);
+    return RealtimePrimaryKeyWriter::Create(
+        partition_map, bucket, schema_, trimmed_primary_keys, key_comparator_,
+        realtime_context_impl, realtime_store_state.value(), restore_max_seq_number, writer, pool_);
 }
 
 Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index b73cfdb8a..415052a69 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -130,6 +130,17 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     return RealtimeStoreState{std::move(store), initial_offset};
 }
 
+int64_t RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
+    const RealtimePartitionBucket& partition_bucket, int64_t max_sequence_number) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    auto [iter, inserted] =
+        materialized_max_sequence_numbers_.emplace(partition_bucket, max_sequence_number);
+    if (!inserted && max_sequence_number > iter->second) {
+        iter->second = max_sequence_number;
+    }
+    return iter->second;
+}
+
 Result<std::vector<RealtimePartitionBucketView>> RealtimeContextImpl::AcquireReadViews() {
     std::lock_guard<std::mutex> lock(mutex_);
     std::vector<RealtimePartitionBucketView> result;
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index 4f62cf1ee..aa4d263c6 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -67,6 +67,9 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
 
     Result<RealtimeStoreState> GetOrCreateRealtimeStore(RealtimeStoreCreateRequest&& request);
 
+    int64_t AdvanceMaterializedMaxSequenceNumber(const RealtimePartitionBucket& partition_bucket,
+                                                 int64_t max_sequence_number);
+
     Result<std::vector<RealtimePartitionBucketView>> AcquireReadViews();
 
     Result<std::string> PinReadView(const RealtimePartitionBucketView& view, int64_t ttl_millis);
@@ -96,6 +99,7 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     std::mutex mutex_;
     std::mutex progress_mutex_;
     std::map<RealtimePartitionBucket, std::shared_ptr<RealtimeStore>> stores_;
+    std::map<RealtimePartitionBucket, int64_t> materialized_max_sequence_numbers_;
     // Full-table progress used as the initial offset when a store is created lazily.
     RealtimeOffsetMap committed_offsets_;
     // Progress already reflected in stores owned by this context.
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 07bbf555b..15066ca1d 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -158,6 +158,22 @@ TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
     ASSERT_EQ(1, factory->stores[2]->acquire_count);
 }
 
+TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
+    const RealtimePartitionBucket partition_bucket(partition, /*bucket=*/0);
+
+    ASSERT_EQ(4, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
+                                                               /*max_sequence_number=*/4));
+    ASSERT_EQ(8, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
+                                                               /*max_sequence_number=*/8));
+    ASSERT_EQ(8, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
+                                                               /*max_sequence_number=*/6));
+    ASSERT_EQ(10, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
+                                                                /*max_sequence_number=*/10));
+}
+
 TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 2dc5a71b4..b53831f0a 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -138,13 +138,16 @@ Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
 }  // namespace
 
 Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Create(
+    const std::map<std::string, std::string>& partition, int32_t bucket,
     const std::shared_ptr<arrow::Schema>& write_schema,
     const std::vector<std::string>& trimmed_primary_keys,
-    const std::shared_ptr<FieldsComparator>& key_comparator, const RealtimeStoreState& store_state,
-    int64_t restored_max_sequence_number, const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<RealtimeContextImpl>& realtime_context,
+    const RealtimeStoreState& store_state, int64_t restored_max_sequence_number,
+    const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     if (!store_state.store || !merge_tree_writer || !write_schema || !key_comparator ||
-        !memory_pool) {
+        !realtime_context || !memory_pool) {
         return Status::Invalid("PK real-time writer received a null dependency");
     }
     if (trimmed_primary_keys.empty()) {
@@ -170,16 +173,22 @@ Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Crea
         DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
     prepared_fields.insert(prepared_fields.end(), write_schema->fields().begin(),
                            write_schema->fields().end());
+    const RealtimePartitionBucket partition_bucket(partition, bucket);
+    const int64_t initial_max_sequence_number =
+        realtime_context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
+                                                               restored_max_sequence_number);
     return std::shared_ptr<RealtimePrimaryKeyWriter>(new RealtimePrimaryKeyWriter(
-        store_state.store, merge_tree_writer, write_schema,
+        store_state.store, merge_tree_writer, realtime_context, partition_bucket, write_schema,
         arrow::schema(std::move(prepared_fields)), arrow::schema(std::move(key_fields)),
         trimmed_primary_keys, key_comparator, store_state.initial_offset,
-        restored_max_sequence_number, memory_pool));
+        initial_max_sequence_number, memory_pool));
 }
 
 RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
     const std::shared_ptr<RealtimeStore>& realtime_store,
     const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+    const std::shared_ptr<RealtimeContextImpl>& realtime_context,
+    const RealtimePartitionBucket& partition_bucket,
     const std::shared_ptr<arrow::Schema>& write_schema,
     const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::shared_ptr<arrow::Schema>& key_schema,
@@ -190,6 +199,8 @@ RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
       arrow_pool_(GetArrowPool(memory_pool)),
       realtime_store_(realtime_store),
       merge_tree_writer_(merge_tree_writer),
+      realtime_context_(realtime_context),
+      partition_bucket_(partition_bucket),
       write_schema_(write_schema),
       prepared_schema_(prepared_schema),
       key_schema_(key_schema),
@@ -237,6 +248,8 @@ Status RealtimePrimaryKeyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
         std::move(prepared_batch), OffsetRange(next_offset_, next_offset_ + count)}));
     next_offset_ += count;
     last_sequence_number_ += count;
+    realtime_context_->AdvanceMaterializedMaxSequenceNumber(partition_bucket_,
+                                                            last_sequence_number_);
     return Status::OK();
 }
 
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index 6abb1ccd0..9a5aa4c68 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -26,6 +26,7 @@
 #include <vector>
 
 #include "paimon/core/utils/batch_writer.h"
+#include "paimon/realtime/realtime_context.h"
 #include "paimon/realtime/realtime_store.h"
 
 namespace arrow {
@@ -38,16 +39,19 @@ namespace paimon {
 class MemoryPool;
 class MergeTreeWriter;
 class FieldsComparator;
+class RealtimeContextImpl;
 struct RealtimeStoreState;
 
 /// Coordinates framework-prepared primary-key real-time writes.
 class RealtimePrimaryKeyWriter final : public BatchWriter {
  public:
     static Result<std::shared_ptr<RealtimePrimaryKeyWriter>> Create(
+        const std::map<std::string, std::string>& partition, int32_t bucket,
         const std::shared_ptr<arrow::Schema>& write_schema,
         const std::vector<std::string>& trimmed_primary_keys,
         const std::shared_ptr<FieldsComparator>& key_comparator,
-        const RealtimeStoreState& store_state, int64_t restore_max_sequence_number,
+        const std::shared_ptr<RealtimeContextImpl>& realtime_context,
+        const RealtimeStoreState& store_state, int64_t restored_max_sequence_number,
         const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
         const std::shared_ptr<MemoryPool>& memory_pool);
 
@@ -64,6 +68,8 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
  private:
     RealtimePrimaryKeyWriter(const std::shared_ptr<RealtimeStore>& realtime_store,
                              const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
+                             const std::shared_ptr<RealtimeContextImpl>& realtime_context,
+                             const RealtimePartitionBucket& partition_bucket,
                              const std::shared_ptr<arrow::Schema>& write_schema,
                              const std::shared_ptr<arrow::Schema>& prepared_schema,
                              const std::shared_ptr<arrow::Schema>& key_schema,
@@ -79,6 +85,8 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     std::shared_ptr<RealtimeStore> realtime_store_;
     std::shared_ptr<MergeTreeWriter> merge_tree_writer_;
+    std::shared_ptr<RealtimeContextImpl> realtime_context_;
+    RealtimePartitionBucket partition_bucket_;
     std::shared_ptr<arrow::Schema> write_schema_;
     std::shared_ptr<arrow::Schema> prepared_schema_;
     std::shared_ptr<arrow::Schema> key_schema_;
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index c61b04b0d..e68ff670d 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -1526,6 +1526,51 @@ TEST_F(RealtimeWriteInteTest, TestPkCompositeMerge) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkWriterHandoff) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
+                         CreateRealtimeWriter(realtime_context));
+    const std::vector<Row> first_rows = {
+        {0, "value-0", "p0"}, {1, "value-1", "p0"}, {2, "value-2", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+                         MakeBatch(first_rows, /*partitioned=*/false));
+    ASSERT_OK(first_writer->Write(std::move(first_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_progress,
+                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, first_progress.size());
+    ASSERT_EQ(OffsetRange(0, 3), first_progress[0].offset_range);
+    ASSERT_EQ(1, NewFiles(first_progress).size());
+    ASSERT_EQ(0, NewFiles(first_progress)[0]->min_sequence_number);
+    ASSERT_EQ(2, NewFiles(first_progress)[0]->max_sequence_number);
+    ASSERT_OK(first_writer->Close());
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
+                         CreateRealtimeWriter(realtime_context));
+    const std::vector<Row> second_rows = {{0, "updated-0", "p0"}, {3, "value-3", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+                         MakeBatch(second_rows, /*partitioned=*/false));
+    ASSERT_OK(second_writer->Write(std::move(second_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
+                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, second_progress.size());
+    ASSERT_EQ(OffsetRange(3, 5), second_progress[0].offset_range);
+    ASSERT_EQ(1, NewFiles(second_progress).size());
+    ASSERT_EQ(3, NewFiles(second_progress)[0]->min_sequence_number);
+    ASSERT_EQ(4, NewFiles(second_progress)[0]->max_sequence_number);
+
+    first_progress.push_back(std::move(second_progress[0]));
+    ASSERT_OK(Commit(first_progress, /*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
+    ASSERT_EQ((std::vector<Row>{{0, "updated-0", "p0"},
+                                {1, "value-1", "p0"},
+                                {2, "value-2", "p0"},
+                                {3, "value-3", "p0"}}),
+              actual_rows);
+    ASSERT_OK(second_writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkPartitionBucketRecovery) {
     options_[Options::BUCKET] = "2";
     CreatePkTable(/*partition_keys=*/{"pt"});

From aec0a2ec2dc3015cfc5f6168a22f1adcd060c2c5 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 15:56:00 +0800
Subject: [PATCH 41/93] refactor(realtime): simplify primary key merge readers

---
 .../core/operation/merge_file_split_read.cpp  | 196 +++---------------
 .../core/operation/merge_file_split_read.h    |   9 +-
 .../table/source/key_value_table_read.cpp     |  21 +-
 3 files changed, 35 insertions(+), 191 deletions(-)

diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 2f64f6df8..c85e75ee0 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -30,7 +30,6 @@
 #include "arrow/c/bridge.h"
 #include "arrow/type.h"
 #include "fmt/format.h"
-#include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/reader/complete_row_kind_batch_reader.h"
 #include "paimon/common/reader/concat_batch_reader.h"
 #include "paimon/common/table/special_fields.h"
@@ -79,82 +78,36 @@ struct KeyValue;
 template <typename T>
 class MergeFunctionWrapper;
 
-namespace {
-
-class ConcatNonOverlappingMergeReaders final : public SortMergeReader {
- public:
-    explicit ConcatNonOverlappingMergeReaders(
-        std::vector<std::unique_ptr<SortMergeReader>>&& readers)
-        : readers_(std::move(readers)) {}
-
-    Result<std::unique_ptr<SortMergeReader::Iterator>> NextBatch() override {
-        while (current_ < readers_.size()) {
-            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader::Iterator> iterator,
-                                   readers_[current_]->NextBatch());
-            if (iterator) {
-                return iterator;
-            }
-            readers_[current_]->Close();
-            ++current_;
-        }
-        return std::unique_ptr<SortMergeReader::Iterator>();
-    }
-
-    void Close() override {
-        while (current_ < readers_.size()) {
-            readers_[current_++]->Close();
-        }
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return MetricsImpl::CollectReadMetrics(readers_);
-    }
-
- private:
-    std::vector<std::unique_ptr<SortMergeReader>> readers_;
-    size_t current_ = 0;
-};
-
-}
-
 class MergeFileSplitRead::RealtimeReaderBuilder {
  public:
     static Result<std::unique_ptr<BatchReader>> Create(
         MergeFileSplitRead* owner, const std::vector<std::shared_ptr<Split>>& disk_splits,
-        std::vector<AdditionalKeyValueReader>&& additional_readers) {
+        std::vector<std::unique_ptr<KeyValueRecordReader>>&& additional_readers) {
         RealtimeReaderBuilder builder(owner);
-        if (disk_splits.empty()) {
-            std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
-            readers.reserve(additional_readers.size());
-            for (AdditionalKeyValueReader& additional : additional_readers) {
-                readers.push_back(std::move(additional.reader));
-            }
-            return builder.CreateMergedReader(std::move(readers));
+        std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
+        if (!disk_splits.empty()) {
+            PAIMON_RETURN_NOT_OK(builder.CollectDiskReaders(disk_splits, &readers));
         }
-
-        PAIMON_RETURN_NOT_OK(builder.CollectDiskInputs(disk_splits));
-        builder.AddRangeInputs(std::move(additional_readers));
-        return builder.CreateReader();
+        readers.reserve(readers.size() + additional_readers.size());
+        for (std::unique_ptr<KeyValueRecordReader>& additional_reader : additional_readers) {
+            readers.push_back(std::move(additional_reader));
+        }
+        return builder.CreateMergedReader(std::move(readers));
     }
 
  private:
-    struct RangeInput {
-        std::shared_ptr<InternalRow> min_key;
-        std::shared_ptr<InternalRow> max_key;
-        std::vector<SortedRun> disk_runs;
-        std::unique_ptr<KeyValueRecordReader> additional_reader;
-    };
-
     explicit RealtimeReaderBuilder(MergeFileSplitRead* owner) : owner_(owner) {}
 
-    Status CollectDiskInputs(const std::vector<std::shared_ptr<Split>>& disk_splits) {
-        first_split_ = std::dynamic_pointer_cast<DataSplitImpl>(disk_splits.front());
-        if (!first_split_) {
+    Status CollectDiskReaders(const std::vector<std::shared_ptr<Split>>& disk_splits,
+                              std::vector<std::unique_ptr<KeyValueRecordReader>>* readers) {
+        std::shared_ptr<DataSplitImpl> first_split =
+            std::dynamic_pointer_cast<DataSplitImpl>(disk_splits.front());
+        if (!first_split) {
             return Status::Invalid("merge input disk split is not a data split");
         }
-        const BinaryRow& partition = first_split_->Partition();
-        const int32_t bucket = first_split_->Bucket();
-        PAIMON_ASSIGN_OR_RAISE(data_file_path_factory_,
+        const BinaryRow& partition = first_split->Partition();
+        const int32_t bucket = first_split->Bucket();
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_file_path_factory,
                                owner_->path_factory_->CreateDataFilePathFactory(partition, bucket));
 
         std::vector<std::shared_ptr<DataFileMeta>> data_files;
@@ -187,46 +140,23 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
             }
         }
 
-        dv_factory_ = DeletionVector::CreateFactory(
+        DeletionVector::Factory dv_factory = DeletionVector::CreateFactory(
             owner_->options_.GetFileSystem(),
             DeletionVector::CreateDeletionFileMap(data_files, deletion_files), owner_->pool_);
         std::vector<std::vector<SortedRun>> disk_sections =
             IntervalPartition(data_files, owner_->key_comparator_).Partition();
-        inputs_.reserve(disk_sections.size());
-        for (std::vector<SortedRun>& section : disk_sections) {
-            std::shared_ptr<DataFileMeta> min_file = section.front().Files().front();
-            std::shared_ptr<DataFileMeta> max_file = min_file;
+        for (const std::vector<SortedRun>& section : disk_sections) {
             for (const SortedRun& run : section) {
-                for (const std::shared_ptr<DataFileMeta>& file : run.Files()) {
-                    if (owner_->key_comparator_->CompareTo(file->min_key, min_file->min_key) < 0) {
-                        min_file = file;
-                    }
-                    if (owner_->key_comparator_->CompareTo(file->max_key, max_file->max_key) > 0) {
-                        max_file = file;
-                    }
-                }
+                PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> disk_reader,
+                                       owner_->CreateReaderForRun(partition, run, dv_factory,
+                                                                  owner_->predicate_for_keys_,
+                                                                  data_file_path_factory));
+                readers->push_back(std::move(disk_reader));
             }
-            inputs_.push_back(RangeInput{std::shared_ptr<InternalRow>(min_file, &min_file->min_key),
-                                         std::shared_ptr<InternalRow>(max_file, &max_file->max_key),
-                                         std::move(section), nullptr});
         }
         return Status::OK();
     }
 
-    void AddRangeInputs(std::vector<AdditionalKeyValueReader>&& additional_readers) {
-        inputs_.reserve(inputs_.size() + additional_readers.size());
-        for (AdditionalKeyValueReader& additional : additional_readers) {
-            has_unknown_range_ |= !additional.min_key || !additional.max_key;
-            inputs_.push_back(RangeInput{additional.min_key, additional.max_key, {},
-                                         std::move(additional.reader)});
-        }
-    }
-
-    Result<std::unique_ptr<KeyValueRecordReader>> CreateDiskReader(const SortedRun& run) {
-        return owner_->CreateReaderForRun(first_split_->Partition(), run, dv_factory_,
-                                          owner_->predicate_for_keys_, data_file_path_factory_);
-    }
-
     Result<std::unique_ptr<BatchReader>> CreateMergedReader(
         std::vector<std::unique_ptr<KeyValueRecordReader>>&& record_readers) {
         if (record_readers.empty()) {
@@ -265,83 +195,7 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
                                                             owner_->pool_);
     }
 
-    Result<std::unique_ptr<BatchReader>> CreateUnknownRangeReader() {
-        std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
-        for (RangeInput& input : inputs_) {
-            for (const SortedRun& run : input.disk_runs) {
-                PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> disk_reader,
-                                       CreateDiskReader(run));
-                readers.push_back(std::move(disk_reader));
-            }
-            if (input.additional_reader) {
-                readers.push_back(std::move(input.additional_reader));
-            }
-        }
-        return CreateMergedReader(std::move(readers));
-    }
-
-    Result<std::unique_ptr<BatchReader>> CreateKnownRangeReader() {
-        std::sort(inputs_.begin(), inputs_.end(),
-                  [this](const RangeInput& lhs, const RangeInput& rhs) {
-                      return owner_->key_comparator_->CompareTo(*lhs.min_key, *rhs.min_key) < 0;
-                  });
-        std::vector<std::vector<RangeInput>> components;
-        std::shared_ptr<InternalRow> component_max_key;
-        for (RangeInput& input : inputs_) {
-            if (components.empty() ||
-                owner_->key_comparator_->CompareTo(*input.min_key, *component_max_key) > 0) {
-                components.emplace_back();
-                component_max_key = input.max_key;
-            } else if (owner_->key_comparator_->CompareTo(*input.max_key, *component_max_key) > 0) {
-                component_max_key = input.max_key;
-            }
-            components.back().push_back(std::move(input));
-        }
-
-        std::vector<std::unique_ptr<SortMergeReader>> component_readers;
-        component_readers.reserve(components.size());
-        for (std::vector<RangeInput>& component : components) {
-            if (component.size() == 1 && !component.front().additional_reader) {
-                PAIMON_ASSIGN_OR_RAISE(
-                    std::unique_ptr<SortMergeReader> disk_component,
-                    owner_->CreateSortMergeReaderForSection(
-                        component.front().disk_runs, first_split_->Partition(), dv_factory_,
-                        component.front().disk_runs.size() == 1 ? owner_->context_->GetPredicate()
-                                                                : owner_->predicate_for_keys_,
-                        data_file_path_factory_, false));
-                component_readers.push_back(std::move(disk_component));
-                continue;
-            }
-
-            std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
-            for (RangeInput& input : component) {
-                for (const SortedRun& run : input.disk_runs) {
-                    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> disk_reader,
-                                           CreateDiskReader(run));
-                    readers.push_back(std::move(disk_reader));
-                }
-                if (input.additional_reader) {
-                    readers.push_back(std::move(input.additional_reader));
-                }
-            }
-            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> component_reader,
-                                   owner_->CreateSortMergeReader(std::move(readers)));
-            component_readers.push_back(std::move(component_reader));
-        }
-        return CreateProjectedReader(
-            std::make_unique<ConcatNonOverlappingMergeReaders>(std::move(component_readers)));
-    }
-
-    Result<std::unique_ptr<BatchReader>> CreateReader() {
-        return has_unknown_range_ ? CreateUnknownRangeReader() : CreateKnownRangeReader();
-    }
-
     MergeFileSplitRead* owner_;
-    std::shared_ptr<DataSplitImpl> first_split_;
-    std::shared_ptr<DataFilePathFactory> data_file_path_factory_;
-    DeletionVector::Factory dv_factory_;
-    std::vector<RangeInput> inputs_;
-    bool has_unknown_range_ = false;
 };
 
 Result<std::unique_ptr<MergeFileSplitRead>> MergeFileSplitRead::Create(
@@ -426,7 +280,7 @@ Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateReader(
 
 Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateRealtimeReader(
     const std::vector<std::shared_ptr<Split>>& disk_splits,
-    std::vector<AdditionalKeyValueReader>&& additional_readers) {
+    std::vector<std::unique_ptr<KeyValueRecordReader>>&& additional_readers) {
     return RealtimeReaderBuilder::Create(this, disk_splits, std::move(additional_readers));
 }
 
diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h
index 85b5b2a28..f01b252be 100644
--- a/src/paimon/core/operation/merge_file_split_read.h
+++ b/src/paimon/core/operation/merge_file_split_read.h
@@ -55,7 +55,6 @@ class FieldsComparator;
 class FileBatchReader;
 class FileStorePathFactory;
 class InternalReadContext;
-class InternalRow;
 class MemoryPool;
 class SchemaManager;
 class SortedRun;
@@ -66,12 +65,6 @@ struct KeyValue;
 template <typename T>
 class MergeFunctionWrapper;
 
-struct AdditionalKeyValueReader {
-    std::unique_ptr<KeyValueRecordReader> reader;
-    std::shared_ptr<InternalRow> min_key;
-    std::shared_ptr<InternalRow> max_key;
-};
-
 /// If the class name below is enclosed in parentheses, it might be present in the read path;
 /// otherwise, it must be present in the read path.
 ///
@@ -129,7 +122,7 @@ class MergeFileSplitRead : public AbstractSplitRead {
 
     Result<std::unique_ptr<BatchReader>> CreateRealtimeReader(
         const std::vector<std::shared_ptr<Split>>& disk_splits,
-        std::vector<AdditionalKeyValueReader>&& additional_readers);
+        std::vector<std::unique_ptr<KeyValueRecordReader>>&& additional_readers);
 
     void SetMergeFunctionWrapper(
         const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper);
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 31779e049..dc69ebb55 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -55,7 +55,7 @@ struct ColumnarBatchContext;
 
 namespace {
 
-Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
     const std::shared_ptr<RealtimeSplit>& split, const RealtimePartitionBucketView& memory,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
@@ -76,9 +76,8 @@ Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
     ScopeGuard schema_guard([schema = c_schema.get()]() { ArrowSchemaRelease(schema); });
     RealtimeQueryContext query_context{c_schema.get(), nullptr, false};
-    PAIMON_ASSIGN_OR_RAISE(
-        std::vector<std::unique_ptr<BatchReader>> batch_readers,
-        memory.store->CreateQueryReaders(memory.read_view, 0, query_context));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> batch_readers,
+                           memory.store->CreateQueryReaders(memory.read_view, 0, query_context));
     ScopeGuard batch_readers_guard([&batch_readers]() {
         for (const std::unique_ptr<BatchReader>& reader : batch_readers) {
             if (reader) {
@@ -86,7 +85,7 @@ Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
             }
         }
     });
-    std::vector<AdditionalKeyValueReader> result;
+    std::vector<std::unique_ptr<KeyValueRecordReader>> result;
     result.reserve(batch_readers.size());
     for (std::unique_ptr<BatchReader>& reader : batch_readers) {
         if (!reader) {
@@ -98,17 +97,15 @@ Result<std::vector<AdditionalKeyValueReader>> CreateMemoryReaders(
                                                                     split->MemoryEndOffset()),
                                                         key_schema, value_schema, memory_pool));
         auto merge = std::make_unique<DeduplicateMergeFunction>(false);
-        result.push_back(AdditionalKeyValueReader{
-            std::make_unique<MergedKeyValueRecordReader>(
-                std::move(prepared_reader), key_comparator,
-                std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge))),
-            nullptr, nullptr});
+        result.push_back(std::make_unique<MergedKeyValueRecordReader>(
+            std::move(prepared_reader), key_comparator,
+            std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge))));
     }
     batch_readers_guard.Release();
     return result;
 }
 
-}
+}  // namespace
 
 KeyValueTableRead::KeyValueTableRead(std::vector<std::unique_ptr<SplitRead>>&& split_reads,
                                      const std::shared_ptr<FileStorePathFactory>& path_factory,
@@ -268,7 +265,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
         auto* merge_read = dynamic_cast<MergeFileSplitRead*>(read.get());
         if (merge_read) {
             PAIMON_ASSIGN_OR_RAISE(
-                std::vector<AdditionalKeyValueReader> memory_readers,
+                std::vector<std::unique_ptr<KeyValueRecordReader>> memory_readers,
                 CreateMemoryReaders(realtime_split, memory, merge_read->GetKeySchema(),
                                     merge_read->GetValueSchema(), merge_read->GetKeyComparator(),
                                     context_, GetMemoryPool()));

From c248373d918afce1f9df9b8338bb69d9789ba17c Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 17:20:13 +0800
Subject: [PATCH 42/93] fix(realtime): validate PK reader contracts

---
 include/paimon/realtime/realtime_store.h      |  11 +-
 .../merged_key_value_record_reader_test.cpp   |  98 ++----
 .../core/operation/file_store_write.cpp       |   2 +-
 .../operation/key_value_file_store_write.cpp  |   2 +-
 .../realtime/arrow_realtime_store_factory.cpp |   5 +-
 .../realtime/prepared_key_value_reader.cpp    | 231 ++++++++++++--
 .../core/realtime/prepared_key_value_reader.h |  22 +-
 .../realtime/primary_key_realtime_store.cpp   | 160 ++++++++--
 .../realtime/primary_key_realtime_store.h     |   4 +-
 .../primary_key_realtime_store_test.cpp       | 144 ++++++++-
 .../core/realtime/realtime_context_impl.cpp   |  55 +++-
 .../core/realtime/realtime_context_impl.h     |  12 +-
 .../core/realtime/realtime_context_test.cpp   |  27 +-
 .../realtime/realtime_primary_key_writer.cpp  |  32 +-
 .../realtime/realtime_primary_key_writer.h    |   2 +-
 src/paimon/core/realtime/realtime_reader.h    |  11 +
 .../core/realtime/realtime_reader_test.cpp    |  27 +-
 .../table/source/key_value_table_read.cpp     |  11 +-
 src/paimon/core/table/source/table_scan.cpp   |   2 +-
 test/inte/realtime_write_inte_test.cpp        | 300 +++++++++++++++++-
 20 files changed, 971 insertions(+), 187 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index dc5d543ac..792bb1c56 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -47,7 +47,10 @@ struct PAIMON_EXPORT AppendRealtimeStoreCreateConfig {
     StatisticsMode statistics_mode;
 };
 
-struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {};
+struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {
+    /// Primary-key fields after removing partition fields, in comparison order.
+    std::vector<std::string> trimmed_primary_keys;
+};
 
 using RealtimeStoreCreateConfig =
     std::variant<AppendRealtimeStoreCreateConfig, PrimaryKeyRealtimeStoreCreateConfig>;
@@ -148,7 +151,8 @@ class PAIMON_EXPORT RealtimeStore {
     /// including across `NextBatch` boundaries, is sorted by full primary key then sequence
     /// number; all readers collectively cover sealed mutations exactly once. Reader cardinality is
     /// independent of the number of writes. Paimon adapts and merges those rows before writing
-    /// files.
+    /// files. Paimon validates the complete ordering and coverage before publishing generated file
+    /// state; a violation fails the prepare operation.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) = 0;
 
@@ -168,7 +172,8 @@ class PAIMON_EXPORT RealtimeStore {
     /// contain multiple mutations per key. Each returned primary-key reader's complete stream is
     /// sorted by full primary key then sequence number, and all readers collectively cover raw
     /// mutations exactly once. Reader cardinality is independent of the number of writes. Paimon
-    /// retains `view` for the lifetime of the resulting framework reader.
+    /// validates ordering while adapting each complete reader stream and retains `view` for the
+    /// lifetime of the resulting framework reader.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) = 0;
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 21b0a16b1..775f271e3 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -95,7 +95,7 @@ class TrackingBatchReader : public BatchReader {
     int32_t* close_count_;
 };
 
-}
+}  // namespace
 
 class MergedKeyValueRecordReaderTest : public testing::Test {
  public:
@@ -229,8 +229,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderOffsetFilter) {
     ])")
             .ValueOrDie());
 
-    auto batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
+    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
         AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(2, 4),
@@ -248,77 +247,28 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderOffsetFilter) {
     KeyValueChecker::CheckResult(expected, results, 1, 2);
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRawCountBeforeDedup) {
-    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
-                                           DataField(1, arrow::field("v0", arrow::int32()))};
+TEST_F(MergedKeyValueRecordReaderTest, TestRejectsUnsortedPluginRowsAcrossBatches) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("id", arrow::int32()))};
     std::shared_ptr<arrow::Schema> value_schema =
         DataField::ConvertDataFieldsToArrowSchema(value_fields);
     std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
     std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
     std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
-        [0, 10, 0, 1, 100],
-        [2, 11, 1, 1, 101],
-        [0, 12, 2, 2, 200]
-    ])")
-            .ValueOrDie());
-
-    int64_t raw_row_count = 0;
-    auto batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> raw_reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, std::nullopt, key_schema,
-                                 value_schema, pool_, &raw_row_count));
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FieldsComparator> key_comparator,
-                         FieldsComparator::Create({value_fields[0]}, true));
-    auto merged_reader = std::make_unique<MergedKeyValueRecordReader>(
-        std::move(raw_reader), key_comparator, merge_function_wrapper_);
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<KeyValue> results,
-        (ReadResultCollector::CollectKeyValueResult<
-            MergedKeyValueRecordReader, KeyValueRecordReader::Iterator>(merged_reader.get())));
-
-    ASSERT_EQ(raw_row_count, 3);
-    std::vector<RowKind*> row_kinds = {const_cast<RowKind*>(RowKind::UpdateAfter()),
-                                       const_cast<RowKind*>(RowKind::Insert())};
-    std::vector<int64_t> levels = {KeyValue::UNKNOWN_LEVEL, KeyValue::UNKNOWN_LEVEL};
-    std::vector<KeyValue> expected = KeyValueChecker::GenerateKeyValues(
-        row_kinds, {11, 12}, levels, {{1}, {2}}, {{1, 101}, {2, 200}}, pool_);
-    KeyValueChecker::CheckResult(expected, results, 1, 2);
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRawCountBeforeFilter) {
-    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32()))};
-    std::shared_ptr<arrow::Schema> value_schema =
-        DataField::ConvertDataFieldsToArrowSchema(value_fields);
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+    std::shared_ptr<arrow::Array> prepared_array =
         arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
-        [0, 10, 0, 1],
-        [0, 11, 1, 2],
-        [0, 12, 2, 3],
-        [0, 13, 3, 4]
-    ])")
-            .ValueOrDie());
-
-    int64_t raw_row_count = 0;
+            [0, 10, 0, 2],
+            [0, 11, 1, 1]
+        ])")
+            .ValueOrDie();
     auto batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(2, 4),
-                                 value_schema, value_schema, pool_, &raw_row_count));
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<KeyValue> results,
-        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                    KeyValueRecordReader::Iterator>(reader.get())));
-
-    ASSERT_EQ(results.size(), 2);
-    ASSERT_EQ(raw_row_count, 4);
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema,
+                                                  std::nullopt, key_schema, value_schema, pool_));
+    Result<std::vector<KeyValue>> result =
+        ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                   KeyValueRecordReader::Iterator>(reader.get());
+    ASSERT_NOK_WITH_MSG(result, "not globally sorted by primary key and sequence number");
 }
 
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
@@ -345,8 +295,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
     ASSERT_EQ(query_results[0].value->GetFieldCount(), 1);
     ASSERT_EQ(query_results[0].value->GetInt(0), 1);
 
-    auto batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
     ASSERT_NOK_WITH_MSG(AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema,
                                                  std::nullopt, value_schema, value_schema, pool_),
                         "exact");
@@ -364,8 +313,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
     auto invalid_array = std::dynamic_pointer_cast<arrow::StructArray>(
         arrow::ipc::internal::json::ArrayFromJSON(invalid_type, R"([[0, 10, 0, 1]])").ValueOrDie());
 
-    auto batch_reader =
-        std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
+    auto batch_reader = std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
         AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
@@ -398,9 +346,12 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedProjection) {
     std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
     auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
         arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
-        [0, 10, 0, 1, [[100, 200], [300, 400]], [["k1", [7, 8]], ["k2", [9, 10]]], [[[11, 12], 13], [[21, 22], 23]]]
+        [0, 9, 9, 0, [[1, 2]], [["prefix", [3, 4]]], [[[5, 6], 7]]],
+        [0, 10, 0, 1, [[100, 200], [300, 400]], [["k1", [7, 8]], ["k2", [9, 10]]], [[[11, 12], 13], [[21, 22], 23]]],
+        [0, 11, 11, 2, [[8, 9]], [["suffix", [10, 11]]], [[[12, 13], 14]]]
     ])")
             .ValueOrDie());
+    prepared_array = checked_pointer_cast<arrow::StructArray>(prepared_array->Slice(1, 1));
 
     std::shared_ptr<arrow::Field> query_item_b = MakeField("renamed_b", arrow::int32(), 11);
     std::shared_ptr<arrow::Field> query_item_a = MakeField("renamed_a", arrow::int32(), 10);
@@ -420,8 +371,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedProjection) {
     std::shared_ptr<arrow::Schema> query_value_schema =
         arrow::schema({id, query_items, query_attrs, query_keyed_values});
 
-    auto batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
         AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 4d4f45156..84a324762 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -198,7 +198,7 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     } else {
         // pk table
         if (ctx->GetRealtimeContext()) {
-            PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(options));
+            PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(options, *schema));
             if (ignore_previous_files) {
                 return Status::NotImplemented(
                     "PK realtime v1 requires restore from the latest snapshot");
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index de7217ec3..ee6445057 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -155,7 +155,7 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
             RealtimeStoreState store_state,
             realtime_context_impl->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
                 std::move(c_write_schema), options_.ToMap(), pool_, partition_map, bucket,
-                PrimaryKeyRealtimeStoreCreateConfig{}}));
+                PrimaryKeyRealtimeStoreCreateConfig{trimmed_primary_keys}}));
         realtime_store_state = std::move(store_state);
         compact_manager = std::make_shared<NoopCompactManager>();
     } else {
diff --git a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
index 4cfdb4c3d..babc55a3d 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
@@ -50,8 +50,11 @@ Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
                                                     request.memory_pool, arrow_pool);
     }
 
+    const PrimaryKeyRealtimeStoreCreateConfig& config =
+        std::get<PrimaryKeyRealtimeStoreCreateConfig>(request.mode_config);
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                           PrimaryKeyRealtimeStore::Create(imported_schema, request.memory_pool));
+                           PrimaryKeyRealtimeStore::Create(
+                               imported_schema, config.trimmed_primary_keys, request.memory_pool));
     return std::shared_ptr<RealtimeStore>(std::move(store));
 }
 
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index b99f67dd9..6b3afcd19 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -20,6 +20,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <mutex>
 #include <optional>
 #include <unordered_map>
 #include <utility>
@@ -29,9 +30,11 @@
 #include "arrow/array/array_nested.h"
 #include "arrow/array/array_primitive.h"
 #include "arrow/array/builder_primitive.h"
+#include "arrow/buffer.h"
 #include "arrow/c/bridge.h"
 #include "arrow/compute/api.h"
 #include "arrow/type.h"
+#include "arrow/util/bit_util.h"
 #include "fmt/format.h"
 #include "paimon/common/data/columnar/columnar_batch_context.h"
 #include "paimon/common/data/columnar/columnar_row_ref.h"
@@ -42,6 +45,7 @@
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/utils/nested_projection_utils.h"
@@ -61,6 +65,68 @@ constexpr int32_t kPreparedValueStartIndex = 3;
 Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
     const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type);
 
+class RealtimeOffsetCoverage {
+ public:
+    static Result<std::shared_ptr<RealtimeOffsetCoverage>> Create(
+        const OffsetRange& sealed_offsets, size_t reader_count,
+        const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+        if (sealed_offsets.begin < 0 || sealed_offsets.end < sealed_offsets.begin) {
+            return Status::Invalid("PK real-time store returned an invalid sealed offset range");
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Buffer> seen_offsets,
+            arrow::AllocateEmptyBitmap(sealed_offsets.Count(), arrow_pool.get()));
+        return std::shared_ptr<RealtimeOffsetCoverage>(new RealtimeOffsetCoverage(
+            sealed_offsets, reader_count, std::move(seen_offsets), arrow_pool));
+    }
+
+    Status Add(const arrow::Int64Array& offsets) {
+        std::lock_guard<std::mutex> lock(mutex_);
+        for (int64_t row = 0; row < offsets.length(); ++row) {
+            const int64_t offset = offsets.Value(row);
+            if (offset < sealed_offsets_.begin || offset >= sealed_offsets_.end) {
+                return Status::Invalid(
+                    "PK real-time store commit reader offset is outside the sealed range");
+            }
+            const int64_t index = offset - sealed_offsets_.begin;
+            if (arrow::bit_util::GetBit(seen_offsets_->data(), index)) {
+                return Status::Invalid(
+                    "PK real-time store commit readers contain duplicate REALTIME_OFFSET");
+            }
+            arrow::bit_util::SetBit(seen_offsets_->mutable_data(), index);
+            ++seen_count_;
+        }
+        return Status::OK();
+    }
+
+    Status FinishReader() {
+        std::lock_guard<std::mutex> lock(mutex_);
+        ++finished_reader_count_;
+        if (finished_reader_count_ == reader_count_ && seen_count_ != sealed_offsets_.Count()) {
+            return Status::Invalid(
+                "PK real-time store commit readers did not cover the sealed range");
+        }
+        return Status::OK();
+    }
+
+ private:
+    RealtimeOffsetCoverage(const OffsetRange& sealed_offsets, size_t reader_count,
+                           std::shared_ptr<arrow::Buffer> seen_offsets,
+                           const std::shared_ptr<arrow::MemoryPool>& arrow_pool)
+        : sealed_offsets_(sealed_offsets),
+          reader_count_(reader_count),
+          arrow_pool_(arrow_pool),
+          seen_offsets_(std::move(seen_offsets)) {}
+
+    OffsetRange sealed_offsets_;
+    size_t reader_count_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    std::shared_ptr<arrow::Buffer> seen_offsets_;
+    int64_t seen_count_ = 0;
+    size_t finished_reader_count_ = 0;
+    std::mutex mutex_;
+};
+
 Status CheckPreparedField(const std::shared_ptr<arrow::Schema>& schema, int32_t field_idx,
                           const DataField& expected_field) {
     if (schema->num_fields() <= field_idx) {
@@ -210,16 +276,20 @@ Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
             return Status::Invalid(
                 fmt::format("cannot find field id {} in prepared value struct", read_field_id));
         }
-        std::shared_ptr<arrow::Array> child = array->field(data_iter->second);
+        std::shared_ptr<arrow::Array> child =
+            arrow::MakeArray(array->data()->child_data[data_iter->second]);
         PAIMON_ASSIGN_OR_RAISE(child, AlignArrayByPaimonIds(child, read_field->type()));
         aligned_arrays.push_back(std::move(child));
     }
 
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-        std::shared_ptr<arrow::StructArray> aligned,
-        arrow::StructArray::Make(aligned_arrays, read_type->fields(), array->null_bitmap(),
-                                 array->null_count(), array->offset()));
-    return aligned;
+    std::shared_ptr<arrow::ArrayData> aligned_data = array->data()->Copy();
+    aligned_data->type = read_type;
+    aligned_data->child_data.clear();
+    aligned_data->child_data.reserve(aligned_arrays.size());
+    for (const std::shared_ptr<arrow::Array>& aligned_array : aligned_arrays) {
+        aligned_data->child_data.push_back(aligned_array->data());
+    }
+    return arrow::MakeArray(std::move(aligned_data));
 }
 
 Result<std::shared_ptr<arrow::Array>> AlignListArrayByPaimonIds(
@@ -351,15 +421,18 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                            const std::optional<OffsetRange>& visible_offsets,
                            const std::shared_ptr<arrow::Schema>& key_schema,
                            const std::shared_ptr<arrow::Schema>& value_schema,
-                           const std::shared_ptr<MemoryPool>& pool, int64_t* raw_row_count)
+                           const std::shared_ptr<FieldsComparator>& key_comparator,
+                           const std::shared_ptr<MemoryPool>& pool,
+                           const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage)
         : reader_(std::move(reader)),
           prepared_schema_(prepared_schema),
           visible_offsets_(visible_offsets),
           key_schema_(key_schema),
           value_schema_(value_schema),
+          key_comparator_(key_comparator),
           pool_(pool),
           arrow_pool_(GetArrowPool(pool)),
-          raw_row_count_(raw_row_count) {}
+          offset_coverage_(offset_coverage) {}
 
     ~PreparedKeyValueReader() override {
         Close();
@@ -425,6 +498,10 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             ResetBatchState();
             PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
             if (BatchReader::IsEofBatch(batch)) {
+                if (offset_coverage_ && !offset_coverage_finished_) {
+                    offset_coverage_finished_ = true;
+                    PAIMON_RETURN_NOT_OK(offset_coverage_->FinishReader());
+                }
                 return std::unique_ptr<KeyValueRecordReader::Iterator>();
             }
             auto& [c_array, c_schema] = batch;
@@ -436,17 +513,14 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             std::shared_ptr<arrow::StructArray> data_batch =
                 checked_pointer_cast<arrow::StructArray>(arrow_array);
             PAIMON_RETURN_NOT_OK(ValidatePreparedBatch(data_batch));
-            if (raw_row_count_ != nullptr) {
-                int64_t updated_count = 0;
-                if (__builtin_add_overflow(*raw_row_count_, data_batch->length(), &updated_count)) {
-                    return Status::Invalid("prepared raw row count overflow");
-                }
-                *raw_row_count_ = updated_count;
-            }
+            PAIMON_RETURN_NOT_OK(ValidateOrdering(data_batch));
 
             std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> offset_array =
                 checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
                     data_batch->field(kRealtimeOffsetIndex));
+            if (offset_coverage_) {
+                PAIMON_RETURN_NOT_OK(offset_coverage_->Add(*offset_array));
+            }
             PAIMON_ASSIGN_OR_RAISE(
                 data_batch,
                 ApplyOffsetFilter(data_batch, offset_array, visible_offsets_, arrow_pool_.get()));
@@ -504,6 +578,36 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         return Status::OK();
     }
 
+    Status ValidateOrdering(const std::shared_ptr<arrow::StructArray>& data_batch) {
+        if (data_batch->length() == 0) {
+            return Status::OK();
+        }
+        PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector key_fields,
+                               ProjectFieldsByPaimonIds(data_batch, prepared_schema_, key_schema_));
+        std::shared_ptr<ColumnarBatchContext> key_context =
+            std::make_shared<ColumnarBatchContext>(key_fields, pool_);
+        std::shared_ptr<arrow::Int64Array> sequences =
+            checked_pointer_cast<arrow::Int64Array>(data_batch->field(kSequenceNumberIndex));
+        for (int64_t row = 0; row < data_batch->length(); ++row) {
+            ColumnarRowRef current_key(key_context, row);
+            if (previous_key_context_) {
+                ColumnarRowRef previous_key(previous_key_context_, previous_key_row_);
+                const int32_t key_comparison =
+                    key_comparator_->CompareTo(previous_key, current_key);
+                if (key_comparison > 0 ||
+                    (key_comparison == 0 && previous_sequence_ > sequences->Value(row))) {
+                    return Status::Invalid(
+                        "PK real-time plugin reader is not globally sorted by primary key and "
+                        "sequence number");
+                }
+            }
+            previous_key_context_ = key_context;
+            previous_key_row_ = row;
+            previous_sequence_ = sequences->Value(row);
+        }
+        return Status::OK();
+    }
+
     void ResetBatchState() {
         key_ctx_.reset();
         value_ctx_.reset();
@@ -518,23 +622,32 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     std::optional<OffsetRange> visible_offsets_;
     std::shared_ptr<arrow::Schema> key_schema_;
     std::shared_ptr<arrow::Schema> value_schema_;
+    std::shared_ptr<FieldsComparator> key_comparator_;
     std::shared_ptr<MemoryPool> pool_;
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
-    int64_t* raw_row_count_;
+    std::shared_ptr<RealtimeOffsetCoverage> offset_coverage_;
+    bool offset_coverage_finished_ = false;
     std::shared_ptr<ColumnarBatchContext> key_ctx_;
     std::shared_ptr<ColumnarBatchContext> value_ctx_;
     std::shared_ptr<arrow::NumericArray<arrow::Int8Type>> row_kind_array_;
     std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> sequence_number_array_;
+    std::shared_ptr<ColumnarBatchContext> previous_key_context_;
+    int64_t previous_key_row_ = 0;
+    int64_t previous_sequence_ = 0;
 };
 
-}
+}  // namespace
 
-Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
+namespace {
+
+Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
     std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::optional<OffsetRange>& visible_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool, int64_t* raw_row_count) {
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<MemoryPool>& memory_pool,
+    const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage) {
     std::unique_ptr<BatchReader> owned_reader = std::move(reader);
     if (!owned_reader) {
         return Status::Invalid("prepared batch reader cannot be null");
@@ -547,6 +660,9 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     if (!value_schema) {
         return Status::Invalid("prepared value schema cannot be null");
     }
+    if (!key_comparator) {
+        return Status::Invalid("prepared key comparator cannot be null");
+    }
     if (!memory_pool) {
         return Status::Invalid("prepared reader memory pool cannot be null");
     }
@@ -555,11 +671,82 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     if (!visible_offsets.has_value()) {
         PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
     }
-    std::unique_ptr<KeyValueRecordReader> result(
-        new PreparedKeyValueReader(std::move(owned_reader), prepared_schema, visible_offsets,
-                                   key_schema, value_schema, memory_pool, raw_row_count));
+    std::unique_ptr<KeyValueRecordReader> result(new PreparedKeyValueReader(
+        std::move(owned_reader), prepared_schema, visible_offsets, key_schema, value_schema,
+        key_comparator, memory_pool, offset_coverage));
     close_guard.Release();
     return result;
 }
 
+}  // namespace
+
+Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::optional<OffsetRange>& visible_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    return AdaptPreparedBatchReaderImpl(std::move(reader), prepared_schema, visible_offsets,
+                                        key_schema, value_schema, key_comparator, memory_pool,
+                                        /*offset_coverage=*/nullptr);
+}
+
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBatchReaders(
+    std::vector<std::unique_ptr<BatchReader>>&& readers,
+    const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    if (!memory_pool) {
+        return Status::Invalid("prepared reader memory pool cannot be null");
+    }
+    std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
+        RealtimeOffsetCoverage::Create(sealed_offsets, readers.size(), arrow_pool));
+    ScopeGuard readers_guard([&readers]() {
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            if (reader) {
+                reader->Close();
+            }
+        }
+    });
+    std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
+    ScopeGuard adapted_readers_guard([&adapted_readers]() {
+        for (const std::unique_ptr<KeyValueRecordReader>& reader : adapted_readers) {
+            reader->Close();
+        }
+    });
+    adapted_readers.reserve(readers.size());
+    for (std::unique_ptr<BatchReader>& reader : readers) {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> adapted_reader,
+                               AdaptPreparedBatchReaderImpl(
+                                   std::move(reader), prepared_schema, std::nullopt, key_schema,
+                                   value_schema, key_comparator, memory_pool, offset_coverage));
+        adapted_readers.push_back(std::move(adapted_reader));
+    }
+    readers_guard.Release();
+    adapted_readers_guard.Release();
+    return adapted_readers;
+}
+
+Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::optional<OffsetRange>& visible_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    if (!key_schema) {
+        return Status::Invalid("prepared key schema cannot be null");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> key_fields,
+                           DataField::ConvertArrowSchemaToDataFields(key_schema));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
+                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
+    return AdaptPreparedBatchReader(std::move(reader), prepared_schema, visible_offsets, key_schema,
+                                    value_schema, key_comparator, memory_pool);
 }
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
index e7a6f9651..064a62958 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/prepared_key_value_reader.h
@@ -21,6 +21,7 @@
 #include <cstdint>
 #include <memory>
 #include <optional>
+#include <vector>
 
 #include "arrow/type_fwd.h"
 #include "paimon/core/io/key_value_record_reader.h"
@@ -29,6 +30,7 @@
 
 namespace paimon {
 class BatchReader;
+class FieldsComparator;
 class MemoryPool;
 
 Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
@@ -36,6 +38,22 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     const std::optional<OffsetRange>& visible_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool, int64_t* raw_row_count = nullptr);
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<MemoryPool>& memory_pool);
 
-}
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBatchReaders(
+    std::vector<std::unique_ptr<BatchReader>>&& readers,
+    const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<MemoryPool>& memory_pool);
+
+Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::optional<OffsetRange>& visible_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool);
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 0d6de9f5f..f43f60472 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -18,20 +18,31 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
+#include <cstddef>
 #include <mutex>
+#include <optional>
 #include <utility>
+#include <vector>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
+#include "arrow/compute/api.h"
+#include "paimon/common/data/columnar/columnar_batch_context.h"
+#include "paimon/common/data/columnar/columnar_row_ref.h"
 #include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/core_options.h"
+#include "paimon/core/index/pk/primary_key_index_definitions.h"
+#include "paimon/core/schema/table_schema.h"
 #include "paimon/macros.h"
 
 namespace paimon {
 
-Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options) {
+Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const TableSchema& schema) {
     if (options.GetBucket() <= 0) {
         return Status::NotImplemented("PK realtime v1 requires fixed buckets");
     }
@@ -60,6 +71,21 @@ Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options) {
         options.GetChangelogProducer() != ChangelogProducer::NONE) {
         return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
     }
+    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> primary_key_fields,
+                           schema.TrimmedPrimaryKeyFields());
+    for (const DataField& field : primary_key_fields) {
+        if (field.Type()->id() == arrow::Type::FLOAT || field.Type()->id() == arrow::Type::DOUBLE) {
+            return Status::NotImplemented(
+                "PK realtime v1 does not support FLOAT or DOUBLE primary keys");
+        }
+    }
+    if (options.GlobalIndexEnabled()) {
+        PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions,
+                               PrimaryKeyIndexDefinitions::Create(schema));
+        if (!definitions.Definitions().empty()) {
+            return Status::NotImplemented("PK realtime v1 does not support global indexes");
+        }
+    }
     return Status::OK();
 }
 
@@ -128,14 +154,56 @@ class ReadView final : public RealtimeReadView {
 
 class RawBatchReader final : public BatchReader {
  public:
-    RawBatchReader(std::vector<StoredBatch> batches)
-        : batches_(std::move(batches)), metrics_(std::make_shared<MetricsImpl>()) {}
+    RawBatchReader(std::vector<StoredBatch> batches, std::vector<int32_t> key_field_indexes,
+                   const std::shared_ptr<FieldsComparator>& key_comparator,
+                   const std::shared_ptr<MemoryPool>& memory_pool)
+        : batches_(std::move(batches)),
+          positions_(batches_.size(), 0),
+          key_field_indexes_(std::move(key_field_indexes)),
+          key_comparator_(key_comparator),
+          memory_pool_(memory_pool),
+          arrow_pool_(GetArrowPool(memory_pool)),
+          metrics_(std::make_shared<MetricsImpl>()) {
+        key_contexts_.reserve(batches_.size());
+        for (const StoredBatch& batch : batches_) {
+            arrow::ArrayVector key_arrays;
+            key_arrays.reserve(key_field_indexes_.size());
+            for (int32_t field_index : key_field_indexes_) {
+                key_arrays.push_back(batch.data->field(field_index));
+            }
+            key_contexts_.push_back(
+                std::make_shared<ColumnarBatchContext>(key_arrays, memory_pool_));
+        }
+    }
 
     Result<ReadBatch> NextBatch() override {
-        if (next_ == batches_.size()) {
+        if (closed_) {
+            return MakeEofBatch();
+        }
+        std::optional<size_t> selected;
+        for (size_t i = 0; i < batches_.size(); ++i) {
+            if (positions_[i] >= batches_[i].data->length()) {
+                continue;
+            }
+            if (!selected.has_value() || Less(i, selected.value())) {
+                selected = i;
+            }
+        }
+        if (!selected.has_value()) {
             return MakeEofBatch();
         }
-        const std::shared_ptr<arrow::StructArray>& batch = batches_[next_++].data;
+        const size_t batch_index = selected.value();
+        arrow::Int64Builder index_builder(arrow_pool_.get());
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(index_builder.Append(positions_[batch_index]));
+        std::shared_ptr<arrow::Array> index;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(index_builder.Finish(&index));
+        arrow::compute::ExecContext context(arrow_pool_.get());
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            arrow::Datum taken,
+            arrow::compute::Take(arrow::Datum(batches_[batch_index].data), arrow::Datum(index),
+                                 arrow::compute::TakeOptions::NoBoundsCheck(), &context));
+        std::shared_ptr<arrow::Array> batch = taken.make_array();
+        ++positions_[batch_index];
         auto array = std::make_unique<ArrowArray>();
         auto schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*batch, array.get(), schema.get()));
@@ -146,12 +214,38 @@ class RawBatchReader final : public BatchReader {
         return metrics_;
     }
     void Close() override {
+        if (closed_) {
+            return;
+        }
+        closed_ = true;
         batches_.clear();
+        positions_.clear();
+        key_contexts_.clear();
     }
 
  private:
+    bool Less(size_t left, size_t right) const {
+        ColumnarRowRef left_key(key_contexts_[left], positions_[left]);
+        ColumnarRowRef right_key(key_contexts_[right], positions_[right]);
+        const int32_t key_comparison = key_comparator_->CompareTo(left_key, right_key);
+        if (key_comparison != 0) {
+            return key_comparison < 0;
+        }
+        const std::shared_ptr<arrow::Int64Array> left_sequences =
+            checked_pointer_cast<arrow::Int64Array>(batches_[left].data->field(1));
+        const std::shared_ptr<arrow::Int64Array> right_sequences =
+            checked_pointer_cast<arrow::Int64Array>(batches_[right].data->field(1));
+        return left_sequences->Value(positions_[left]) < right_sequences->Value(positions_[right]);
+    }
+
+    bool closed_ = false;
     std::vector<StoredBatch> batches_;
-    size_t next_ = 0;
+    std::vector<int64_t> positions_;
+    std::vector<int32_t> key_field_indexes_;
+    std::shared_ptr<FieldsComparator> key_comparator_;
+    std::shared_ptr<MemoryPool> memory_pool_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    std::vector<std::shared_ptr<ColumnarBatchContext>> key_contexts_;
     std::shared_ptr<Metrics> metrics_;
 };
 
@@ -159,8 +253,13 @@ class RawBatchReader final : public BatchReader {
 
 class PrimaryKeyRealtimeStore::Impl {
  public:
-    explicit Impl(std::shared_ptr<arrow::Schema> prepared_schema)
-        : prepared_schema_(std::move(prepared_schema)) {}
+    Impl(std::shared_ptr<arrow::Schema> prepared_schema, std::vector<int32_t> key_field_indexes,
+         const std::shared_ptr<FieldsComparator>& key_comparator,
+         const std::shared_ptr<MemoryPool>& memory_pool)
+        : prepared_schema_(std::move(prepared_schema)),
+          key_field_indexes_(std::move(key_field_indexes)),
+          key_comparator_(key_comparator),
+          memory_pool_(memory_pool) {}
 
     Status Write(RealtimeWriteBatch&& write_batch) {
         if (!write_batch.batch || !write_batch.batch->GetData()) {
@@ -212,9 +311,9 @@ class PrimaryKeyRealtimeStore::Impl {
             return Status::Invalid("segment was not created by the PK real-time store");
         }
         std::vector<std::unique_ptr<BatchReader>> readers;
-        readers.reserve(segment->Batches().size());
-        for (const StoredBatch& batch : segment->Batches()) {
-            readers.push_back(std::make_unique<RawBatchReader>(std::vector<StoredBatch>{batch}));
+        if (!segment->Batches().empty()) {
+            readers.push_back(std::make_unique<RawBatchReader>(
+                segment->Batches(), key_field_indexes_, key_comparator_, memory_pool_));
         }
         return readers;
     }
@@ -238,16 +337,13 @@ class PrimaryKeyRealtimeStore::Impl {
             return Status::Invalid("read view was not created by the PK real-time store");
         }
         std::vector<std::unique_ptr<BatchReader>> readers;
-        size_t batch_count = 0;
+        std::vector<StoredBatch> batches;
         for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
-            batch_count += segment->Batches().size();
+            batches.insert(batches.end(), segment->Batches().begin(), segment->Batches().end());
         }
-        readers.reserve(batch_count);
-        for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
-            for (const StoredBatch& batch : segment->Batches()) {
-                readers.push_back(
-                    std::make_unique<RawBatchReader>(std::vector<StoredBatch>{batch}));
-            }
+        if (!batches.empty()) {
+            readers.push_back(std::make_unique<RawBatchReader>(
+                std::move(batches), key_field_indexes_, key_comparator_, memory_pool_));
         }
         return readers;
     }
@@ -273,6 +369,9 @@ class PrimaryKeyRealtimeStore::Impl {
 
  private:
     std::shared_ptr<arrow::Schema> prepared_schema_;
+    std::vector<int32_t> key_field_indexes_;
+    std::shared_ptr<FieldsComparator> key_comparator_;
+    std::shared_ptr<MemoryPool> memory_pool_;
     mutable std::mutex mutex_;
     std::vector<StoredBatch> building_;
     std::vector<std::shared_ptr<Segment>> sealed_;
@@ -286,12 +385,29 @@ PrimaryKeyRealtimeStore::~PrimaryKeyRealtimeStore() = default;
 
 Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
     const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::vector<std::string>& trimmed_primary_keys,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    if (!prepared_schema || !memory_pool) {
+    if (!prepared_schema || trimmed_primary_keys.empty() || !memory_pool) {
         return Status::Invalid("PK prepared schema or memory pool is null");
     }
-    return std::shared_ptr<PrimaryKeyRealtimeStore>(
-        new PrimaryKeyRealtimeStore(std::make_unique<Impl>(prepared_schema)));
+    std::vector<int32_t> key_field_indexes;
+    std::vector<DataField> key_fields;
+    key_field_indexes.reserve(trimmed_primary_keys.size());
+    key_fields.reserve(trimmed_primary_keys.size());
+    for (const std::string& key : trimmed_primary_keys) {
+        const int32_t field_index = prepared_schema->GetFieldIndex(key);
+        if (field_index < 3) {
+            return Status::Invalid("PK field is missing from prepared schema: ", key);
+        }
+        key_field_indexes.push_back(field_index);
+        PAIMON_ASSIGN_OR_RAISE(DataField field, DataField::ConvertArrowFieldToDataField(
+                                                    prepared_schema->field(field_index)));
+        key_fields.push_back(std::move(field));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
+                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
+    return std::shared_ptr<PrimaryKeyRealtimeStore>(new PrimaryKeyRealtimeStore(
+        std::make_unique<Impl>(prepared_schema, key_field_indexes, key_comparator, memory_pool)));
 }
 Status PrimaryKeyRealtimeStore::Write(RealtimeWriteBatch&& batch) {
     return impl_->Write(std::move(batch));
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index 5e18dd74f..d6a23ccf9 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -31,14 +31,16 @@ namespace paimon {
 
 class CoreOptions;
 class MemoryPool;
+class TableSchema;
 
-Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options);
+Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const TableSchema& schema);
 
 /// In-memory store for prepared primary-key real-time batches.
 class PrimaryKeyRealtimeStore final : public RealtimeStore {
  public:
     static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
         const std::shared_ptr<arrow::Schema>& prepared_schema,
+        const std::vector<std::string>& trimmed_primary_keys,
         const std::shared_ptr<MemoryPool>& memory_pool);
 
     ~PrimaryKeyRealtimeStore() override;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 43831d7be..cafe3682e 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -31,6 +31,7 @@
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/realtime/realtime_fields.h"
+#include "paimon/core/schema/table_schema.h"
 #include "paimon/macros.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/testing/utils/testharness.h"
@@ -44,7 +45,33 @@ std::shared_ptr<arrow::Schema> PreparedSchema() {
          DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
              ->WithNullable(false),
          DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false),
-         arrow::field("id", arrow::int64()), arrow::field("value", arrow::utf8())});
+         DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64()))),
+         DataField::ConvertDataFieldToArrowField(
+             DataField(1, arrow::field("value", arrow::utf8())))});
+}
+
+std::shared_ptr<arrow::Schema> NestedPreparedSchema() {
+    return arrow::schema(
+        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+             ->WithNullable(false),
+         DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false),
+         DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64()))),
+         DataField::ConvertDataFieldToArrowField(DataField(
+             1,
+             arrow::field("value",
+                          arrow::struct_({arrow::field("name", arrow::utf8()),
+                                          arrow::field("items", arrow::list(arrow::int32()))}))))});
+}
+
+std::shared_ptr<TableSchema> PkSchema(
+    const std::shared_ptr<arrow::DataType>& key_type = arrow::int64(),
+    const std::map<std::string, std::string>& options = {}) {
+    return TableSchema::Create(
+               /*schema_id=*/0,
+               arrow::schema({arrow::field("id", key_type), arrow::field("value", arrow::utf8())}),
+               /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options)
+        .value();
 }
 
 std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) {
@@ -56,6 +83,27 @@ std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) {
     return RecordBatchBuilder(c_array.get()).Finish().value();
 }
 
+std::unique_ptr<RecordBatch> MakeBatch(const std::shared_ptr<arrow::Schema>& schema,
+                                       const std::string& json) {
+    std::shared_ptr<arrow::Array> array =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), json)
+            .ValueOrDie();
+    auto c_array = std::make_unique<ArrowArray>();
+    EXPECT_TRUE(arrow::ExportArray(*array, c_array.get()).ok());
+    return RecordBatchBuilder(c_array.get()).Finish().value();
+}
+
+void AssertOffsetsZero(const ArrowArray* array) {
+    ASSERT_NE(nullptr, array);
+    ASSERT_EQ(0, array->offset);
+    for (int64_t child = 0; child < array->n_children; ++child) {
+        AssertOffsetsZero(array->children[child]);
+    }
+    if (array->dictionary) {
+        AssertOffsetsZero(array->dictionary);
+    }
+}
+
 Result<std::string> ReadJson(const std::vector<std::unique_ptr<BatchReader>>& readers) {
     std::vector<std::shared_ptr<arrow::Array>> batches;
     for (const std::unique_ptr<BatchReader>& reader : readers) {
@@ -77,7 +125,7 @@ Result<std::string> ReadJson(const std::vector<std::unique_ptr<BatchReader>>& re
 
 TEST(PrimaryKeyRealtimeStoreOptionsTest, TestSupportedOptions) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_OK(ValidatePrimaryKeyRealtimeOptions(options));
+    ASSERT_OK(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema()));
 }
 
 TEST(PrimaryKeyRealtimeStoreOptionsTest, TestUnsupportedOptions) {
@@ -95,13 +143,31 @@ TEST(PrimaryKeyRealtimeStoreOptionsTest, TestUnsupportedOptions) {
     };
     for (const std::map<std::string, std::string>& option_map : unsupported_options) {
         ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-        ASSERT_NOK(ValidatePrimaryKeyRealtimeOptions(options));
+        ASSERT_NOK(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema()));
     }
 }
 
+TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsFloatingPrimaryKeys) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_NOK_WITH_MSG(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema(arrow::float32())),
+                        "FLOAT or DOUBLE primary keys");
+    ASSERT_NOK_WITH_MSG(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema(arrow::float64())),
+                        "FLOAT or DOUBLE primary keys");
+}
+
+TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsEnabledGlobalIndex) {
+    const std::map<std::string, std::string> option_map = {{Options::BUCKET, "1"},
+                                                           {Options::PK_BTREE_INDEX_COLUMNS, "id"}};
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+    ASSERT_NOK_WITH_MSG(
+        ValidatePrimaryKeyRealtimeOptions(options, *PkSchema(arrow::int64(), option_map)),
+        "does not support global indexes");
+}
+
 TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<PrimaryKeyRealtimeStore> store,
+        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
     ASSERT_FALSE(segment.has_value());
@@ -131,10 +197,11 @@ TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<PrimaryKeyRealtimeStore> store,
+        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
-        MakeBatch(R"([[0, 5, 0, 3, "three"], [1, 6, 1, 1, "before"]])"), OffsetRange(0, 2)}));
+        MakeBatch(R"([[1, 6, 1, 1, "before"], [0, 5, 0, 3, "three"]])"), OffsetRange(0, 2)}));
     ASSERT_OK(store->Write(
         RealtimeWriteBatch{MakeBatch(R"([[2, 7, 2, 2, "after"]])"), OffsetRange(2, 3)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -142,18 +209,45 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
     ASSERT_TRUE(segment.has_value());
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateCommitReaders(segment.value()));
+    ASSERT_EQ(1, readers.size());
     ASSERT_OK_AND_ASSIGN(std::string actual, ReadJson(readers));
     ASSERT_EQ(
-        "-- is_valid: all not null\n-- child 0 type: int8\n  [\n    0,\n    1,\n    2\n  ]\n-- "
-        "child 1 type: int64\n  [\n    5,\n    6,\n    7\n  ]\n-- child 2 type: int64\n  [\n    "
-        "0,\n    1,\n    2\n  ]\n-- child 3 type: int64\n  [\n    3,\n    1,\n    2\n  ]\n-- child "
-        "4 type: string\n  [\n    \"three\",\n    \"before\",\n    \"after\"\n  ]",
+        "-- is_valid: all not null\n-- child 0 type: int8\n  [\n    1,\n    2,\n    0\n  ]\n-- "
+        "child 1 type: int64\n  [\n    6,\n    7,\n    5\n  ]\n-- child 2 type: int64\n  [\n    "
+        "1,\n    2,\n    0\n  ]\n-- child 3 type: int64\n  [\n    1,\n    2,\n    3\n  ]\n-- child "
+        "4 type: string\n  [\n    \"before\",\n    \"after\",\n    \"three\"\n  ]",
         actual);
+    readers[0]->Close();
+    readers[0]->Close();
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof, readers[0]->NextBatch());
+    ASSERT_TRUE(BatchReader::IsEofBatch(eof));
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
+TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
+    std::shared_ptr<arrow::Schema> schema = NestedPreparedSchema();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+                         PrimaryKeyRealtimeStore::Create(schema, {"id"}, GetDefaultPool()));
+    ASSERT_OK(store->Write(RealtimeWriteBatch{
+        MakeBatch(schema, R"([[0, 1, 0, 1, ["one", [1, 2]]], [0, 2, 1, 2, ["two", [3, 4]]]])"),
+        OffsetRange(0, 2)}));
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateCommitReaders(segment.value()));
+    ASSERT_EQ(1, readers.size());
+    for (int32_t row = 0; row < 2; ++row) {
+        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
+        ASSERT_FALSE(BatchReader::IsEofBatch(batch));
+        AssertOffsetsZero(batch.first.get());
+        ASSERT_TRUE(arrow::ImportArray(batch.first.get(), batch.second.get()).ok());
+    }
+}
+
+TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<PrimaryKeyRealtimeStore> store,
+        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 0, 4, 1, "one"]])"), OffsetRange(4, 5)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -163,5 +257,27 @@ TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(4, 5)), view->GetOffsetRange());
 }
 
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderCardinalityIsConstant) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<PrimaryKeyRealtimeStore> store,
+        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 2, "two"]])"), OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 2, 1, 1, "one"]])"), OffsetRange(1, 2)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+    RealtimeQueryContext context{/*read_schema=*/nullptr, /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(1, readers.size());
+    ASSERT_OK_AND_ASSIGN(std::string actual, ReadJson(readers));
+    ASSERT_NE(std::string::npos, actual.find("\"one\""));
+    ASSERT_NE(std::string::npos, actual.find("\"two\""));
+}
+
 }  // namespace
 }  // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 415052a69..215e066ee 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -34,14 +34,33 @@
 #include <utility>
 #include <vector>
 
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
 #include "paimon/arrow/abi.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/scope_guard.h"
 #include "paimon/common/utils/uuid.h"
 #include "paimon/macros.h"
 #include "paimon/realtime/realtime_store.h"
 #include "paimon/status.h"
 
 namespace paimon {
+namespace {
+
+bool SameMode(const RealtimeStoreCreateConfig& left, const RealtimeStoreCreateConfig& right) {
+    if (left.index() != right.index()) {
+        return false;
+    }
+    if (const auto* left_pk = std::get_if<PrimaryKeyRealtimeStoreCreateConfig>(&left)) {
+        const auto& right_pk = std::get<PrimaryKeyRealtimeStoreCreateConfig>(right);
+        return left_pk->trimmed_primary_keys == right_pk.trimmed_primary_keys;
+    }
+    return true;
+}
+
+}  // namespace
+
 RealtimeContextImpl::RealtimeContextImpl(const std::shared_ptr<RealtimeStoreFactory>& factory)
     : factory_(factory) {}
 
@@ -78,6 +97,14 @@ Status RealtimeContextImpl::Start() {
 
 Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     RealtimeStoreCreateRequest&& request) {
+    if (!request.write_schema || !request.write_schema->release) {
+        return Status::Invalid("real-time store write schema is null");
+    }
+    ScopeGuard schema_guard(
+        [schema = request.write_schema.get()]() { ArrowSchemaRelease(schema); });
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> requested_schema,
+                                      arrow::ImportSchema(request.write_schema.get()));
+    schema_guard.Release();
     std::lock_guard<std::mutex> progress_lock(progress_mutex_);
     std::lock_guard<std::mutex> registry_lock(mutex_);
     const RealtimePartitionBucket key(request.partition, request.bucket);
@@ -86,19 +113,18 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     auto offset_iter = committed_offsets_.find(key);
     if (offset_iter != committed_offsets_.end()) {
         if (offset_iter->second == std::numeric_limits<int64_t>::max()) {
-            if (request.write_schema) {
-                ArrowSchemaRelease(request.write_schema.get());
-            }
             return Status::Invalid("real-time offset has reached INT64_MAX");
         }
         initial_offset = offset_iter->second;
     }
     if (iter != stores_.end()) {
-        if (request.write_schema) {
-            ArrowSchemaRelease(request.write_schema.get());
+        if (!SameMode(iter->second.mode_config, request.mode_config) ||
+            !iter->second.write_schema->Equals(*requested_schema, /*check_metadata=*/true)) {
+            return Status::Invalid(
+                "real-time store schema or mode does not match the registered store");
         }
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
-                               iter->second->AcquireReadView());
+                               iter->second.store->AcquireReadView());
         if (!read_view) {
             return Status::Invalid("real-time store returned a null read view");
         }
@@ -113,17 +139,18 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
                 initial_offset = memory_range->end;
             }
         }
-        return RealtimeStoreState{iter->second, initial_offset};
+        return RealtimeStoreState{iter->second.store, initial_offset};
     }
     if (!request.memory_pool) {
-        if (request.write_schema) {
-            ArrowSchemaRelease(request.write_schema.get());
-        }
         return Status::Invalid("real-time store memory pool is null");
     }
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(
+        arrow::ExportSchema(*requested_schema, request.write_schema.get()));
+    RealtimeStoreCreateConfig mode_config = request.mode_config;
     Result<std::shared_ptr<RealtimeStore>> store_result = factory_->Create(std::move(request));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
-    stores_.emplace(key, store);
+    stores_.emplace(key,
+                    RealtimeStoreRegistryEntry{store, requested_schema, std::move(mode_config)});
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
     }
@@ -147,9 +174,9 @@ Result<std::vector<RealtimePartitionBucketView>> RealtimeContextImpl::AcquireRea
     result.reserve(stores_.size());
     for (const auto& [partition_bucket, store] : stores_) {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
-                               store->AcquireReadView());
+                               store.store->AcquireReadView());
         result.push_back(
-            RealtimePartitionBucketView{partition_bucket, store, std::move(read_view)});
+            RealtimePartitionBucketView{partition_bucket, store.store, std::move(read_view)});
     }
     return result;
 }
@@ -266,7 +293,7 @@ Status RealtimeContextImpl::AdvanceCommittedProgress(int64_t snapshot_id,
             }
             auto store_iter = stores_.find(partition_bucket);
             if (store_iter != stores_.end()) {
-                notifications.emplace_back(partition_bucket, store_iter->second,
+                notifications.emplace_back(partition_bucket, store_iter->second.store,
                                            committed_end_offset);
             }
         }
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index aa4d263c6..9fa145e99 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -38,6 +38,10 @@
 
 struct ArrowSchema;
 
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
 namespace paimon {
 
 class RealtimeStore;
@@ -55,6 +59,12 @@ struct RealtimePartitionBucketView {
     std::shared_ptr<RealtimeReadView> read_view;
 };
 
+struct RealtimeStoreRegistryEntry {
+    std::shared_ptr<RealtimeStore> store;
+    std::shared_ptr<arrow::Schema> write_schema;
+    RealtimeStoreCreateConfig mode_config;
+};
+
 class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
  public:
     static Result<std::shared_ptr<RealtimeContextImpl>> Create(
@@ -98,7 +108,7 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     std::shared_ptr<RealtimeStoreFactory> factory_;
     std::mutex mutex_;
     std::mutex progress_mutex_;
-    std::map<RealtimePartitionBucket, std::shared_ptr<RealtimeStore>> stores_;
+    std::map<RealtimePartitionBucket, RealtimeStoreRegistryEntry> stores_;
     std::map<RealtimePartitionBucket, int64_t> materialized_max_sequence_numbers_;
     // Full-table progress used as the initial offset when a store is created lazily.
     RealtimeOffsetMap committed_offsets_;
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 15066ca1d..2b47e9dc9 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -96,10 +96,12 @@ class TestingRealtimeStoreFactory : public RealtimeStoreFactory {
     std::vector<std::shared_ptr<TestingRealtimeStore>> stores;
 };
 
-std::unique_ptr<ArrowSchema> MakeWriteSchema() {
+std::unique_ptr<ArrowSchema> MakeWriteSchema(
+    const std::shared_ptr<arrow::DataType>& id_type = arrow::int64(),
+    const std::shared_ptr<arrow::KeyValueMetadata>& metadata = nullptr) {
     auto schema = std::make_unique<ArrowSchema>();
     EXPECT_TRUE(
-        arrow::ExportSchema(*arrow::schema({arrow::field("id", arrow::int64())}), schema.get())
+        arrow::ExportSchema(*arrow::schema({arrow::field("id", id_type)}, metadata), schema.get())
             .ok());
     return schema;
 }
@@ -158,6 +160,27 @@ TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
     ASSERT_EQ(1, factory->stores[2]->acquire_count);
 }
 
+TEST(RealtimeContextTest, TestRejectsMismatchedSchemaOnStoreReuse) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
+    std::shared_ptr<arrow::KeyValueMetadata> metadata =
+        arrow::key_value_metadata({"identity"}, {"v1"});
+    ASSERT_OK(GetOrCreateAppendStore(
+        context, partition, 0, MakeWriteSchema(arrow::int64(), metadata), {}, GetDefaultPool()));
+    ASSERT_NOK_WITH_MSG(
+        GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(arrow::int32(), metadata), {},
+                               GetDefaultPool()),
+        "schema or mode does not match");
+    ASSERT_NOK_WITH_MSG(
+        GetOrCreateAppendStore(
+            context, partition, 0,
+            MakeWriteSchema(arrow::int64(), arrow::key_value_metadata({"identity"}, {"v2"})), {},
+            GetDefaultPool()),
+        "schema or mode does not match");
+    ASSERT_EQ(1, factory->stores.size());
+}
+
 TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index b53831f0a..82318eadb 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -266,15 +266,12 @@ Result<CommitIncrement> RealtimePrimaryKeyWriter::PrepareCommit(bool wait_compac
         return Status::Invalid("PK real-time store sealed a null segment");
     }
     std::optional<OffsetRange> sealed_range;
-    int64_t expected_raw_row_count = 0;
     if (segment) {
         sealed_range = segment.value()->GetOffsetRange();
-        if (sealed_range->begin < 0 || sealed_range->end < sealed_range->begin ||
-            __builtin_sub_overflow(sealed_range->end, sealed_range->begin,
-                                   &expected_raw_row_count)) {
+        if (sealed_range->begin < 0 || sealed_range->end < sealed_range->begin) {
             return Status::Invalid("PK real-time store returned an invalid sealed offset range");
         }
-        PAIMON_RETURN_NOT_OK(FlushSegment(segment.value(), expected_raw_row_count));
+        PAIMON_RETURN_NOT_OK(FlushSegment(segment.value(), sealed_range.value()));
     }
     PAIMON_ASSIGN_OR_RAISE(CommitIncrement increment,
                            merge_tree_writer_->PrepareCommit(wait_compaction));
@@ -285,7 +282,7 @@ Result<CommitIncrement> RealtimePrimaryKeyWriter::PrepareCommit(bool wait_compac
 }
 
 Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment,
-                                              int64_t expected_raw_row_count) {
+                                              const OffsetRange& sealed_offsets) {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                            realtime_store_->CreateCommitReaders(segment));
     ScopeGuard readers_guard([&readers]() {
@@ -295,28 +292,25 @@ Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegm
             }
         }
     });
-    int64_t raw_row_count = 0;
-    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
-    sorted_readers.reserve(readers.size());
-    for (std::unique_ptr<BatchReader>& reader : readers) {
+    for (const std::unique_ptr<BatchReader>& reader : readers) {
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null commit reader");
         }
-        PAIMON_ASSIGN_OR_RAISE(
-            std::unique_ptr<KeyValueRecordReader> prepared_reader,
-            AdaptPreparedBatchReader(std::move(reader), prepared_schema_, std::nullopt, key_schema_,
-                                     write_schema_, memory_pool_, &raw_row_count));
+    }
+    PAIMON_ASSIGN_OR_RAISE(
+        std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
+        AdaptPreparedCommitBatchReaders(std::move(readers), prepared_schema_, sealed_offsets,
+                                        key_schema_, write_schema_, key_comparator_, memory_pool_));
+    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
+    sorted_readers.reserve(prepared_readers.size());
+    for (std::unique_ptr<KeyValueRecordReader>& prepared_reader : prepared_readers) {
         auto merge_function = std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
         sorted_readers.push_back(std::make_unique<MergedKeyValueRecordReader>(
             std::move(prepared_reader), key_comparator_,
             std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function))));
     }
     readers_guard.Release();
-    PAIMON_RETURN_NOT_OK(merge_tree_writer_->WriteSortedReaders(std::move(sorted_readers)));
-    if (raw_row_count != expected_raw_row_count) {
-        return Status::Invalid("PK real-time store commit readers did not cover the sealed range");
-    }
-    return Status::OK();
+    return merge_tree_writer_->WriteSortedReaders(std::move(sorted_readers));
 }
 
 Status RealtimePrimaryKeyWriter::Compact(bool) {
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index 9a5aa4c68..2eaf7ce24 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -79,7 +79,7 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
                              const std::shared_ptr<MemoryPool>& memory_pool);
 
     Status FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment,
-                        int64_t expected_raw_row_count);
+                        const OffsetRange& sealed_offsets);
 
     std::shared_ptr<MemoryPool> memory_pool_;
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
diff --git a/src/paimon/core/realtime/realtime_reader.h b/src/paimon/core/realtime/realtime_reader.h
index 6c25fd853..a041e0caa 100644
--- a/src/paimon/core/realtime/realtime_reader.h
+++ b/src/paimon/core/realtime/realtime_reader.h
@@ -44,10 +44,16 @@ class RealtimeReader final : public BatchReader {
     }
 
     Result<ReadBatch> NextBatch() override {
+        if (closed_) {
+            return MakeEofBatch();
+        }
         return reader_->NextBatch();
     }
 
     Result<ReadBatchWithBitmap> NextBatchWithBitmap() override {
+        if (closed_) {
+            return MakeEofBatchWithBitmap();
+        }
         return reader_->NextBatchWithBitmap();
     }
 
@@ -56,6 +62,10 @@ class RealtimeReader final : public BatchReader {
     }
 
     void Close() override {
+        if (closed_) {
+            return;
+        }
+        closed_ = true;
         reader_->Close();
         read_view_.reset();
     }
@@ -68,6 +78,7 @@ class RealtimeReader final : public BatchReader {
     // before releasing the data it references.
     std::shared_ptr<RealtimeReadView> read_view_;
     std::unique_ptr<BatchReader> reader_;
+    bool closed_ = false;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_reader_test.cpp b/src/paimon/core/realtime/realtime_reader_test.cpp
index ec37cfed4..10f6ce5be 100644
--- a/src/paimon/core/realtime/realtime_reader_test.cpp
+++ b/src/paimon/core/realtime/realtime_reader_test.cpp
@@ -37,6 +37,8 @@ class TestingReadView : public RealtimeReadView {
 
 class TestingBatchReader : public BatchReader {
  public:
+    explicit TestingBatchReader(int32_t* close_count = nullptr) : close_count_(close_count) {}
+
     Result<ReadBatch> NextBatch() override {
         return MakeEofBatch();
     }
@@ -45,7 +47,14 @@ class TestingBatchReader : public BatchReader {
         return nullptr;
     }
 
-    void Close() override {}
+    void Close() override {
+        if (close_count_) {
+            ++(*close_count_);
+        }
+    }
+
+ private:
+    int32_t* close_count_;
 };
 
 TEST(RealtimeReaderTest, TestRejectsIncompleteReader) {
@@ -57,5 +66,21 @@ TEST(RealtimeReaderTest, TestRejectsIncompleteReader) {
                         "inner reader is null");
 }
 
+TEST(RealtimeReaderTest, TestCloseIsIdempotentAndReturnsEof) {
+    int32_t close_count = 0;
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RealtimeReader> reader,
+        RealtimeReader::Create(std::make_shared<TestingReadView>(),
+                               std::make_unique<TestingBatchReader>(&close_count)));
+    reader->Close();
+    reader->Close();
+    ASSERT_EQ(1, close_count);
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
+    ASSERT_TRUE(BatchReader::IsEofBatch(batch));
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch_with_bitmap,
+                         reader->NextBatchWithBitmap());
+    ASSERT_TRUE(BatchReader::IsEofBatch(batch_with_bitmap));
+}
+
 }  // namespace
 }  // namespace paimon::test
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index dc69ebb55..64d722097 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -91,11 +91,12 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null query reader");
         }
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> prepared_reader,
-                               AdaptPreparedBatchReader(std::move(reader), prepared_schema,
-                                                        OffsetRange(split->CommittedEndOffset(),
-                                                                    split->MemoryEndOffset()),
-                                                        key_schema, value_schema, memory_pool));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<KeyValueRecordReader> prepared_reader,
+            AdaptPreparedBatchReader(
+                std::move(reader), prepared_schema,
+                OffsetRange(split->CommittedEndOffset(), split->MemoryEndOffset()), key_schema,
+                value_schema, key_comparator, memory_pool));
         auto merge = std::make_unique<DeduplicateMergeFunction>(false);
         result.push_back(std::make_unique<MergedKeyValueRecordReader>(
             std::move(prepared_reader), key_comparator,
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index dcf10e90c..0bcd79f61 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -233,7 +233,7 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
         return Status::Invalid("real-time union read does not support data evolution");
     }
     if (!table_schema.PrimaryKeys().empty()) {
-        PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(core_options));
+        PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(core_options, table_schema));
     }
     if (context.IsStreamingMode()) {
         return Status::Invalid("real-time union read currently supports batch scans only");
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index e68ff670d..e27b36903 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -423,6 +423,208 @@ class SplitCommitReaderRealtimeStoreFactory final : public RealtimeStoreFactory
     ArrowRealtimeStoreFactory delegate_;
 };
 
+class DropLastBatchReader final : public BatchReader {
+ public:
+    explicit DropLastBatchReader(std::unique_ptr<BatchReader> delegate)
+        : delegate_(std::move(delegate)) {}
+
+    Result<ReadBatch> NextBatch() override {
+        if (!buffered_.has_value()) {
+            PAIMON_ASSIGN_OR_RAISE(ReadBatch first, delegate_->NextBatch());
+            if (BatchReader::IsEofBatch(first)) {
+                return MakeEofBatch();
+            }
+            buffered_ = std::move(first);
+        }
+        PAIMON_ASSIGN_OR_RAISE(ReadBatch next, delegate_->NextBatch());
+        if (BatchReader::IsEofBatch(next)) {
+            buffered_.reset();
+            return MakeEofBatch();
+        }
+        ReadBatch result = std::move(buffered_.value());
+        buffered_ = std::move(next);
+        return result;
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        buffered_.reset();
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    std::optional<ReadBatch> buffered_;
+};
+
+class SwapFirstTwoBatchReader final : public BatchReader {
+ public:
+    explicit SwapFirstTwoBatchReader(std::unique_ptr<BatchReader> delegate)
+        : delegate_(std::move(delegate)) {}
+
+    Result<ReadBatch> NextBatch() override {
+        if (!initialized_) {
+            initialized_ = true;
+            PAIMON_ASSIGN_OR_RAISE(ReadBatch first, delegate_->NextBatch());
+            if (BatchReader::IsEofBatch(first)) {
+                return MakeEofBatch();
+            }
+            PAIMON_ASSIGN_OR_RAISE(ReadBatch second, delegate_->NextBatch());
+            if (BatchReader::IsEofBatch(second)) {
+                return first;
+            }
+            first_ = std::move(first);
+            return second;
+        }
+        if (first_.has_value()) {
+            ReadBatch first = std::move(first_.value());
+            first_.reset();
+            return first;
+        }
+        return delegate_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        first_.reset();
+        delegate_->Close();
+    }
+
+ private:
+    bool initialized_ = false;
+    std::unique_ptr<BatchReader> delegate_;
+    std::optional<ReadBatch> first_;
+};
+
+class SubstituteOffsetBatchReader final : public BatchReader {
+ public:
+    explicit SubstituteOffsetBatchReader(std::unique_ptr<BatchReader> delegate)
+        : delegate_(std::move(delegate)) {}
+
+    Result<ReadBatch> NextBatch() override {
+        PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
+        if (BatchReader::IsEofBatch(batch)) {
+            return batch;
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> array,
+            arrow::ImportArray(batch.first.get(), batch.second.get()));
+        if (!array || array->type_id() != arrow::Type::STRUCT || array->length() == 0) {
+            return Status::Invalid("offset substitution requires a non-empty struct batch");
+        }
+        std::shared_ptr<arrow::StructArray> struct_array =
+            std::dynamic_pointer_cast<arrow::StructArray>(array);
+        std::shared_ptr<arrow::Int64Array> offsets =
+            std::dynamic_pointer_cast<arrow::Int64Array>(struct_array->field(2));
+        if (!offsets) {
+            return Status::Invalid("offset substitution requires an int64 REALTIME_OFFSET");
+        }
+        arrow::Int64Builder builder;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(offsets->length()));
+        for (int64_t row = 0; row < offsets->length(); ++row) {
+            builder.UnsafeAppend(0);
+        }
+        std::shared_ptr<arrow::Array> substituted_offsets;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&substituted_offsets));
+        std::shared_ptr<arrow::ArrayData> substituted_data = struct_array->data()->Copy();
+        substituted_data->child_data[2] = substituted_offsets->data();
+        std::shared_ptr<arrow::Array> substituted = arrow::MakeArray(std::move(substituted_data));
+        auto output = std::make_unique<ArrowArray>();
+        auto schema = std::make_unique<ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(
+            arrow::ExportArray(*substituted, output.get(), schema.get()));
+        return ReadBatch(std::move(output), std::move(schema));
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+};
+
+enum class CommitReaderMalformation { DROP_LAST, UNSORTED, SUBSTITUTE_OFFSET };
+
+class MalformedCoverageRealtimeStore final : public RealtimeStore {
+ public:
+    MalformedCoverageRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
+                                   CommitReaderMalformation malformation)
+        : delegate_(delegate), malformation_(malformation) {}
+
+    Status Write(RealtimeWriteBatch&& batch) override {
+        return delegate_->Write(std::move(batch));
+    }
+    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
+        return delegate_->SealForCommit();
+    }
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               delegate_->CreateCommitReaders(segment));
+        for (std::unique_ptr<BatchReader>& reader : readers) {
+            switch (malformation_) {
+                case CommitReaderMalformation::DROP_LAST:
+                    reader = std::make_unique<DropLastBatchReader>(std::move(reader));
+                    break;
+                case CommitReaderMalformation::UNSORTED:
+                    reader = std::make_unique<SwapFirstTwoBatchReader>(std::move(reader));
+                    break;
+                case CommitReaderMalformation::SUBSTITUTE_OFFSET:
+                    reader = std::make_unique<SubstituteOffsetBatchReader>(std::move(reader));
+                    break;
+            }
+        }
+        return readers;
+    }
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
+        return delegate_->AcquireReadView();
+    }
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        return delegate_->CreateQueryReaders(view, offset_begin, context);
+    }
+    Status AdvanceCommittedOffset(int64_t committed_offset) override {
+        return delegate_->AdvanceCommittedOffset(committed_offset);
+    }
+    uint64_t GetMemoryUsage() const override {
+        return delegate_->GetMemoryUsage();
+    }
+
+ private:
+    std::shared_ptr<RealtimeStore> delegate_;
+    CommitReaderMalformation malformation_;
+};
+
+class MalformedCoverageRealtimeStoreFactory final : public RealtimeStoreFactory {
+ public:
+    explicit MalformedCoverageRealtimeStoreFactory(
+        CommitReaderMalformation malformation = CommitReaderMalformation::DROP_LAST)
+        : malformation_(malformation) {}
+
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
+                               delegate_.Create(std::move(request)));
+        return std::shared_ptr<RealtimeStore>(
+            std::make_shared<MalformedCoverageRealtimeStore>(delegate, malformation_));
+    }
+
+ private:
+    ArrowRealtimeStoreFactory delegate_;
+    CommitReaderMalformation malformation_;
+};
+
 }  // namespace
 
 namespace {
@@ -1312,6 +1514,50 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) {
     ASSERT_TRUE(query_view->expired());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkDeleteInsertAndPinnedReadsAcrossRefresh) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> delete_batch,
+                         MakeBatch({Row{1, "deleted", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
+                                   {RecordBatch::RowKind::DELETE}));
+    ASSERT_OK(writer->Write(std::move(delete_batch)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> insert_batch,
+                         MakeBatch({Row{1, "inserted", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
+                                   {RecordBatch::RowKind::INSERT}));
+    ASSERT_OK(writer->Write(std::move(insert_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_EQ(1, progress.size());
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> pinned_plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> reader_plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ReadContextBuilder read_builder(table_path_);
+    read_builder.SetOptions(options_)
+        .SetReadFieldNames({"id", "payload", "pt"})
+        .WithRealtimeContext(realtime_context)
+        .WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+                         TableRead::Create(std::move(read_context)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> pinned_reader,
+                         table_read->CreateReader(reader_plan->Splits()));
+
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> plan_rows, ReadRows(pinned_plan, realtime_context));
+    ASSERT_EQ((std::vector<Row>{{1, "inserted", "p0"}}), plan_rows);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> reader_rows,
+                         ReadResultCollector::CollectResult(pinned_reader.get()));
+    ASSERT_EQ(1, reader_rows->length());
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkMergeDiskSealedAndActive) {
     options_[Options::READ_BATCH_SIZE] = "2";
     CreatePkTable();
@@ -1903,6 +2149,56 @@ TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkRejectsMalformedCoverage) {
+    CreatePkTable();
+    auto factory = std::make_shared<MalformedCoverageRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
+                        "commit readers did not cover the sealed range");
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkRejectsEqualCardinalityOffsetSubstitution) {
+    CreatePkTable();
+    auto factory = std::make_shared<MalformedCoverageRealtimeStoreFactory>(
+        CommitReaderMalformation::SUBSTITUTE_OFFSET);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
+                        "duplicate REALTIME_OFFSET");
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkRejectsUnsortedPluginRows) {
+    CreatePkTable();
+    auto factory =
+        std::make_shared<MalformedCoverageRealtimeStoreFactory>(CommitReaderMalformation::UNSORTED);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
+                        "not globally sorted by primary key and sequence number");
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
     CreatePkTable();
     auto state = std::make_shared<CloseTrackingReaderState>();
@@ -1973,10 +2269,10 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderCloseFailure) {
         return table_read->CreateReader(plan->Splits());
     };
 
-    for (int32_t null_index = 0; null_index <= 2; ++null_index) {
+    for (int32_t null_index = 0; null_index <= 1; ++null_index) {
         state->query_null_index = null_index;
         ASSERT_NOK_WITH_MSG(create_reader(), "PK real-time store returned a null query reader");
-        ASSERT_EQ(2 * (null_index + 1), state->query_close_count->load(std::memory_order_acquire));
+        ASSERT_EQ(null_index + 1, state->query_close_count->load(std::memory_order_acquire));
     }
     ASSERT_OK(writer->Close());
 }

From 1e1c7d2660dc0c21d42a8bda0aecca64c2f8868a Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 18:27:48 +0800
Subject: [PATCH 43/93] refactor(realtime): simplify reader lifecycle cleanup

---
 .../merged_key_value_record_reader_test.cpp   |  9 +------
 .../key_value_file_store_write_test.cpp       | 10 ++++---
 .../realtime/arrow_realtime_store_test.cpp    |  8 +++++-
 .../realtime/prepared_key_value_reader.cpp    |  4 ---
 .../realtime/primary_key_realtime_store.cpp   |  8 ------
 .../primary_key_realtime_store_test.cpp       |  3 ---
 .../realtime/realtime_append_only_writer.cpp  |  2 +-
 .../core/realtime/realtime_context_impl.cpp   | 26 +++++++++++++++----
 .../core/realtime/realtime_context_impl.h     |  3 +++
 src/paimon/core/realtime/realtime_reader.h    | 11 --------
 .../core/realtime/realtime_reader_test.cpp    | 15 +++++------
 test/inte/realtime_write_inte_test.cpp        |  2 +-
 12 files changed, 48 insertions(+), 53 deletions(-)

diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 775f271e3..a83c9bbd6 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -81,16 +81,11 @@ class TrackingBatchReader : public BatchReader {
     }
 
     void Close() override {
-        if (closed_) {
-            return;
-        }
-        closed_ = true;
         ++(*close_count_);
         delegate_->Close();
     }
 
  private:
-    bool closed_ = false;
     std::unique_ptr<BatchReader> delegate_;
     int32_t* close_count_;
 };
@@ -421,7 +416,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedProjection) {
     ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(1), 23);
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderClose) {
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
     std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
                                            DataField(1, arrow::field("v0", arrow::int32()))};
     std::shared_ptr<arrow::Schema> value_schema =
@@ -445,7 +440,6 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderClose) {
             AdaptPreparedBatchReader(std::move(tracking_reader), prepared_schema, OffsetRange(0, 1),
                                      key_schema, value_schema, pool_));
         reader->Close();
-        reader->Close();
     }
     ASSERT_EQ(explicit_close_count, 1);
 
@@ -486,7 +480,6 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderClose) {
                                      key_schema, value_schema, pool_));
         ASSERT_NOK_WITH_MSG(reader->NextBatch(), "prepared reader failure");
         ASSERT_EQ(read_failure_close_count, 1);
-        reader->Close();
     }
     ASSERT_EQ(read_failure_close_count, 1);
 }
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index 733c19d6e..a2344e803 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -411,6 +411,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
     const std::map<std::string, std::string> options = {
         {Options::BUCKET, "1"},
         {Options::WRITE_BUFFER_SIZE, "1"},
+        {Options::REALTIME_ENABLED, "true"},
     };
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
         arrow::field("id", arrow::int64(), false),
@@ -465,7 +466,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
 }
 
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
-    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"}};
+    const std::map<std::string, std::string> options = {
+        {Options::BUCKET, "1"}, {Options::REALTIME_ENABLED, "true"}};
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
         arrow::field("id", arrow::int64(), false),
         arrow::field("_REALTIME_OFFSET", arrow::int64()),
@@ -488,7 +490,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
 }
 
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
-    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"}};
+    const std::map<std::string, std::string> options = {
+        {Options::BUCKET, "1"}, {Options::REALTIME_ENABLED, "true"}};
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
         arrow::field("id", arrow::int64(), false),
         arrow::field("value", arrow::utf8()),
@@ -546,7 +549,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
 
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) {
     const int64_t max = std::numeric_limits<int64_t>::max();
-    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"}};
+    const std::map<std::string, std::string> options = {
+        {Options::BUCKET, "1"}, {Options::REALTIME_ENABLED, "true"}};
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
         arrow::field("id", arrow::int64(), false),
         arrow::field("value", arrow::utf8()),
diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
index 9aae99332..f186a8161 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
@@ -232,8 +232,14 @@ TEST_F(ArrowRealtimeStoreTest, TestCommitReaderPreservesSlicedBatch) {
 TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) {
     ArrowRealtimeStoreFactory factory;
     std::unique_ptr<ArrowSchema> write_schema = MakeReadSchema(schema_);
+    RealtimeStoreCreateRequest request{std::move(write_schema),
+                                       /*options=*/{},
+                                       pool_,
+                                       /*partition=*/{},
+                                       /*bucket=*/0,
+                                       AppendRealtimeStoreCreateConfig{StatisticsMode::FULL}};
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> realtime_store,
-                         factory.Create(std::move(write_schema), StatisticsMode::FULL, {}, pool_));
+                         factory.Create(std::move(request)));
     std::shared_ptr<ArrowRealtimeStore> store =
         std::dynamic_pointer_cast<ArrowRealtimeStore>(realtime_store);
     ASSERT_NE(nullptr, store);
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 6b3afcd19..5b0375ad1 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -490,10 +490,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
  private:
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatchImpl() {
-        if (closed_) {
-            return std::unique_ptr<KeyValueRecordReader::Iterator>();
-        }
-
         while (true) {
             ResetBatchState();
             PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index f43f60472..2f04aae79 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -177,9 +177,6 @@ class RawBatchReader final : public BatchReader {
     }
 
     Result<ReadBatch> NextBatch() override {
-        if (closed_) {
-            return MakeEofBatch();
-        }
         std::optional<size_t> selected;
         for (size_t i = 0; i < batches_.size(); ++i) {
             if (positions_[i] >= batches_[i].data->length()) {
@@ -214,10 +211,6 @@ class RawBatchReader final : public BatchReader {
         return metrics_;
     }
     void Close() override {
-        if (closed_) {
-            return;
-        }
-        closed_ = true;
         batches_.clear();
         positions_.clear();
         key_contexts_.clear();
@@ -238,7 +231,6 @@ class RawBatchReader final : public BatchReader {
         return left_sequences->Value(positions_[left]) < right_sequences->Value(positions_[right]);
     }
 
-    bool closed_ = false;
     std::vector<StoredBatch> batches_;
     std::vector<int64_t> positions_;
     std::vector<int32_t> key_field_indexes_;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index cafe3682e..116c6e389 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -218,9 +218,6 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
         "4 type: string\n  [\n    \"before\",\n    \"after\",\n    \"three\"\n  ]",
         actual);
     readers[0]->Close();
-    readers[0]->Close();
-    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof, readers[0]->NextBatch());
-    ASSERT_TRUE(BatchReader::IsEofBatch(eof));
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
diff --git a/src/paimon/core/realtime/realtime_append_only_writer.cpp b/src/paimon/core/realtime/realtime_append_only_writer.cpp
index 21d6cfb74..ea5feecce 100644
--- a/src/paimon/core/realtime/realtime_append_only_writer.cpp
+++ b/src/paimon/core/realtime/realtime_append_only_writer.cpp
@@ -47,7 +47,7 @@ Result<std::shared_ptr<RealtimeAppendOnlyWriter>> RealtimeAppendOnlyWriter::Crea
     std::unique_ptr<::ArrowSchema> write_schema,
     const std::shared_ptr<RealtimeContext>& realtime_context,
     const std::shared_ptr<AppendOnlyWriter>& file_writer,
-    const std::shared_ptr<arrow::Schema>& input_schema,
+    const std::shared_ptr<arrow::Schema>& input_schema, StatisticsMode statistics_mode,
     const std::map<std::string, std::string>& options,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     if (!realtime_context) {
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 215e066ee..ba4c8b7a6 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -269,12 +269,28 @@ Status RealtimeContextImpl::AdvanceCommittedProgress(int64_t snapshot_id,
             if (partition_bucket.bucket < 0 || committed_end_offset < 0) {
                 return Status::Invalid("invalid partition-bucket committed offset");
             }
+        }
+        // Only stores created by this context can contain state which cannot be restored in
+        // place. Offsets for other partition-buckets are reference state for lazy store creation
+        // and may be removed or rolled back without rebuilding the context.
+        std::lock_guard<std::mutex> registry_lock(mutex_);
+        for (const auto& store_entry : stores_) {
+            const RealtimePartitionBucket& partition_bucket = store_entry.first;
             auto previous_iter = committed_offsets_.find(partition_bucket);
-            if (previous_iter != committed_offsets_.end()) {
-                if (committed_end_offset < previous_iter->second) {
-                    return Status::Invalid(
-                        "real-time partition-bucket committed offset cannot move backwards");
-                }
+            if (previous_iter == committed_offsets_.end()) {
+                continue;
+            }
+
+            auto current_iter = committed_offsets.find(partition_bucket);
+            if (current_iter == committed_offsets.end()) {
+                return Status::Invalid(
+                    "real-time committed progress removed an active partition-bucket; recreate "
+                    "RealtimeContext");
+            }
+            if (current_iter->second < previous_iter->second) {
+                return Status::Invalid(
+                    "real-time committed offset moved backwards for an active partition-bucket; "
+                    "recreate RealtimeContext");
             }
         }
         committed_offsets_ = committed_offsets;
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index 9fa145e99..f5118c18f 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -88,6 +88,9 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
 
     Status ReleaseReadView(const std::string& opaque_ticket);
 
+    // Returns an error requiring a new context if a newer snapshot removes or moves committed
+    // progress backwards for a store created by this context. Progress for inactive stores is
+    // only reference state and can be replaced in place.
     Status AdvanceCommittedProgress(int64_t snapshot_id,
                                     const RealtimeOffsetMap& committed_offsets);
 
diff --git a/src/paimon/core/realtime/realtime_reader.h b/src/paimon/core/realtime/realtime_reader.h
index a041e0caa..6c25fd853 100644
--- a/src/paimon/core/realtime/realtime_reader.h
+++ b/src/paimon/core/realtime/realtime_reader.h
@@ -44,16 +44,10 @@ class RealtimeReader final : public BatchReader {
     }
 
     Result<ReadBatch> NextBatch() override {
-        if (closed_) {
-            return MakeEofBatch();
-        }
         return reader_->NextBatch();
     }
 
     Result<ReadBatchWithBitmap> NextBatchWithBitmap() override {
-        if (closed_) {
-            return MakeEofBatchWithBitmap();
-        }
         return reader_->NextBatchWithBitmap();
     }
 
@@ -62,10 +56,6 @@ class RealtimeReader final : public BatchReader {
     }
 
     void Close() override {
-        if (closed_) {
-            return;
-        }
-        closed_ = true;
         reader_->Close();
         read_view_.reset();
     }
@@ -78,7 +68,6 @@ class RealtimeReader final : public BatchReader {
     // before releasing the data it references.
     std::shared_ptr<RealtimeReadView> read_view_;
     std::unique_ptr<BatchReader> reader_;
-    bool closed_ = false;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_reader_test.cpp b/src/paimon/core/realtime/realtime_reader_test.cpp
index 10f6ce5be..ded060989 100644
--- a/src/paimon/core/realtime/realtime_reader_test.cpp
+++ b/src/paimon/core/realtime/realtime_reader_test.cpp
@@ -21,6 +21,7 @@
 
 #include <memory>
 #include <optional>
+#include <utility>
 
 #include "paimon/arrow/abi.h"
 #include "paimon/testing/utils/testharness.h"
@@ -66,20 +67,18 @@ TEST(RealtimeReaderTest, TestRejectsIncompleteReader) {
                         "inner reader is null");
 }
 
-TEST(RealtimeReaderTest, TestCloseIsIdempotentAndReturnsEof) {
+TEST(RealtimeReaderTest, TestCloseReleasesResources) {
     int32_t close_count = 0;
+    std::shared_ptr<TestingReadView> read_view = std::make_shared<TestingReadView>();
+    std::weak_ptr<TestingReadView> weak_read_view = read_view;
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<RealtimeReader> reader,
-        RealtimeReader::Create(std::make_shared<TestingReadView>(),
+        RealtimeReader::Create(std::move(read_view),
                                std::make_unique<TestingBatchReader>(&close_count)));
-    reader->Close();
+    ASSERT_FALSE(weak_read_view.expired());
     reader->Close();
     ASSERT_EQ(1, close_count);
-    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
-    ASSERT_TRUE(BatchReader::IsEofBatch(batch));
-    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch_with_bitmap,
-                         reader->NextBatchWithBitmap());
-    ASSERT_TRUE(BatchReader::IsEofBatch(batch_with_bitmap));
+    ASSERT_TRUE(weak_read_view.expired());
 }
 
 }  // namespace
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index e27b36903..0e6f83b70 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -1939,7 +1939,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRecovery) {
                          seed_commit_builder.SetOptions(options_).Finish());
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> seed_commit,
                          FileStoreCommit::Create(std::move(seed_commit_context)));
-    ASSERT_OK(seed_commit->Commit(seed_messages));
+    ASSERT_OK(seed_commit->Commit(seed_messages, /*commit_identifier=*/0));
     ASSERT_OK(seed_writer->Close());
     const std::vector<Row> mutations = {
         {1, "one", "p0"}, {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};

From 352ce215288ee17b8f0a99d84a2e6452471a7066 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Mon, 24 Aug 2026 23:01:26 +0800
Subject: [PATCH 44/93] fix(realtime): harden primary-key prepared batches

---
 include/paimon/realtime/realtime_store.h      |   2 +
 include/paimon/utils/special_field_ids.h      |   2 +
 .../io/merged_key_value_record_reader.cpp     |  10 +-
 .../core/io/merged_key_value_record_reader.h  |   1 +
 .../merged_key_value_record_reader_test.cpp   |  93 ++++++++-
 src/paimon/core/mergetree/merge_tree_writer.h |   3 +
 .../realtime/prepared_key_value_reader.cpp    | 144 ++++++++------
 .../core/realtime/prepared_key_value_reader.h |   2 +
 .../realtime/primary_key_realtime_store.cpp   | 135 ++++++++++---
 .../primary_key_realtime_store_test.cpp       | 128 ++++++++++++-
 src/paimon/core/realtime/realtime_fields.h    |   6 +-
 .../realtime/realtime_primary_key_writer.cpp  |  14 --
 .../table/source/append_only_table_read.cpp   |  37 +++-
 .../table/source/key_value_table_read.cpp     |  12 +-
 .../core/table/source/realtime_table_scan.cpp |  20 +-
 .../core/table/source/realtime_table_scan.h   |   3 +-
 src/paimon/core/table/source/table_scan.cpp   |   2 +-
 test/inte/realtime_write_inte_test.cpp        | 180 ++++++++++++++++--
 18 files changed, 652 insertions(+), 142 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 792bb1c56..90c6ce0a8 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -109,6 +109,8 @@ class PAIMON_EXPORT RealtimeReadView {
 struct PAIMON_EXPORT RealtimeQueryContext {
     /// Append mode receives the requested output fields before the mandatory leading
     /// `_VALUE_KIND` field is added. Primary-key mode receives the complete prepared schema.
+    /// This schema is borrowed and remains valid only during `CreateQueryReaders`; plugins must
+    /// import or copy it synchronously.
     ::ArrowSchema* read_schema;
     /// Predicate using field indexes from `read_schema`.
     std::shared_ptr<Predicate> predicate;
diff --git a/include/paimon/utils/special_field_ids.h b/include/paimon/utils/special_field_ids.h
index 829f29889..5219d72db 100644
--- a/include/paimon/utils/special_field_ids.h
+++ b/include/paimon/utils/special_field_ids.h
@@ -42,6 +42,8 @@ class SpecialFieldIds {
 
     /// Special field ID reserved for index score. Value: CPP_FIELD_ID_END - 1
     inline static constexpr int32_t INDEX_SCORE = CPP_FIELD_ID_END - 1;
+    /// Special field ID reserved for realtime offset. Value: CPP_FIELD_ID_END - 2
+    inline static constexpr int32_t REALTIME_OFFSET = CPP_FIELD_ID_END - 2;
 
     /// Lowest field ID reserved for system fields; IDs at or above it are excluded from the
     /// highest field ID of a schema. Value: INT32_MAX / 2
diff --git a/src/paimon/core/io/merged_key_value_record_reader.cpp b/src/paimon/core/io/merged_key_value_record_reader.cpp
index 70f2bcfb9..8c3952874 100644
--- a/src/paimon/core/io/merged_key_value_record_reader.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader.cpp
@@ -117,13 +117,21 @@ Status MergedKeyValueRecordReader::Iterator::LoadNextRawKeyValue() const {
 }
 
 Result<std::unique_ptr<KeyValueRecordReader::Iterator>> MergedKeyValueRecordReader::NextBatch() {
+    if (initialization_error_.has_value()) {
+        return initialization_error_.value();
+    }
     if (visited_) {
         return std::unique_ptr<KeyValueRecordReader::Iterator>();
     }
     visited_ = true;
 
     auto iterator = std::make_unique<Iterator>(this);
-    PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext());
+    Result<bool> has_next_result = iterator->HasNext();
+    if (!has_next_result.ok()) {
+        initialization_error_ = has_next_result.status();
+        return initialization_error_.value();
+    }
+    bool has_next = std::move(has_next_result).value();
     if (!has_next) {
         return std::unique_ptr<KeyValueRecordReader::Iterator>();
     }
diff --git a/src/paimon/core/io/merged_key_value_record_reader.h b/src/paimon/core/io/merged_key_value_record_reader.h
index a1b7aa5e4..227a1593a 100644
--- a/src/paimon/core/io/merged_key_value_record_reader.h
+++ b/src/paimon/core/io/merged_key_value_record_reader.h
@@ -67,6 +67,7 @@ class MergedKeyValueRecordReader : public KeyValueRecordReader {
 
  private:
     bool visited_ = false;
+    std::optional<Status> initialization_error_;
     std::unique_ptr<KeyValueRecordReader> reader_;
     std::shared_ptr<FieldsComparator> key_comparator_;
     std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper_;
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index a83c9bbd6..a0d65205c 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -19,7 +19,6 @@
 #include "paimon/core/io/merged_key_value_record_reader.h"
 
 #include <cstdint>
-#include <limits>
 #include <memory>
 #include <optional>
 #include <string>
@@ -45,6 +44,7 @@
 #include "paimon/testing/utils/key_value_checker.h"
 #include "paimon/testing/utils/read_result_collector.h"
 #include "paimon/testing/utils/testharness.h"
+#include "paimon/utils/special_field_ids.h"
 
 namespace paimon::test {
 
@@ -107,7 +107,7 @@ class MergedKeyValueRecordReaderTest : public testing::Test {
 
 TEST_F(MergedKeyValueRecordReaderTest, TestRealtimeOffsetField) {
     const DataField& field = RealtimeOffsetField();
-    ASSERT_EQ(std::numeric_limits<int32_t>::max() - 10002, field.Id());
+    ASSERT_EQ(SpecialFieldIds::REALTIME_OFFSET, field.Id());
     ASSERT_EQ("_REALTIME_OFFSET", field.Name());
     ASSERT_EQ(arrow::Type::INT64, field.Type()->id());
     ASSERT_FALSE(field.Nullable());
@@ -296,6 +296,95 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
                         "exact");
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestBadCommitBatch) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> value = MakeField("value", arrow::int32(), 1);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key, value});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, value});
+    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
+    std::shared_ptr<arrow::Array> actual =
+        arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+
+    auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, std::nullopt,
+                                 arrow::schema({key}), value_schema, pool_));
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestMissingCompositeKey) {
+    std::shared_ptr<arrow::Field> key0 = MakeField("key0", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> key1 = MakeField("key1", arrow::int32(), 1);
+    std::shared_ptr<arrow::Field> value = MakeField("value", arrow::int32(), 2);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key0, key1, value});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key0, key1, value});
+    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key0, value});
+    std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
+    std::shared_ptr<arrow::Array> actual =
+        arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1, 20]])")
+            .ValueOrDie();
+
+    auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                 arrow::schema({key0, key1}), value_schema, pool_));
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "cannot find field id 1");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryAddRename) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> old_value = MakeField("old_value", arrow::int32(), 1);
+    std::shared_ptr<arrow::Field> renamed_value = MakeField("renamed_value", arrow::int32(), 1);
+    std::shared_ptr<arrow::Field> added = MakeField("added", arrow::int32(), 2);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key, renamed_value, added});
+    std::shared_ptr<arrow::Schema> prepared_schema =
+        MakePreparedSchema({key, renamed_value, added});
+    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key, old_value});
+    std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
+    std::shared_ptr<arrow::Array> actual =
+        arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1, 20]])")
+            .ValueOrDie();
+
+    auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                 arrow::schema({key}), value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader::Iterator> iterator,
+                         reader->NextBatch());
+    ASSERT_OK_AND_ASSIGN(KeyValue key_value, iterator->Next());
+    ASSERT_EQ(20, key_value.value->GetInt(1));
+    ASSERT_TRUE(key_value.value->IsNullAt(2));
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestMergedReaderErrorRetry) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+    auto failing_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    failing_reader->SetNextBatchStatus(Status::IOError("stable prepared error"));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReader(std::move(failing_reader), prepared_schema, OffsetRange(0, 1),
+                                 value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FieldsComparator> key_comparator,
+                         FieldsComparator::Create({DataField(0, key)}, true));
+    MergedKeyValueRecordReader merged_reader(std::move(reader), key_comparator,
+                                             merge_function_wrapper_);
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> first = merged_reader.NextBatch();
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> retry = merged_reader.NextBatch();
+    ASSERT_NOK(first);
+    ASSERT_NOK(retry);
+    ASSERT_EQ(first.status().ToString(), retry.status().ToString());
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h
index 542affd81..cea07f3e4 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.h
+++ b/src/paimon/core/mergetree/merge_tree_writer.h
@@ -70,6 +70,9 @@ class MergeTreeWriter : public BatchWriter {
 
     Status Write(std::unique_ptr<RecordBatch>&& batch) override;
 
+    /// Consumes readers whose complete streams are individually sorted by primary key and
+    /// sequence number. Readers are closed on success or failure; an error may leave generated
+    /// file state unpublished, so the caller must discard this writer and replay its input.
     Status WriteSortedReaders(std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers);
 
     Status Compact(bool full_compaction) override;
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 5b0375ad1..864456818 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -62,8 +62,18 @@ constexpr int32_t kSequenceNumberIndex = 1;
 constexpr int32_t kRealtimeOffsetIndex = 2;
 constexpr int32_t kPreparedValueStartIndex = 3;
 
+template <typename Reader>
+void CloseReaders(const std::vector<std::unique_ptr<Reader>>& readers) {
+    for (const std::unique_ptr<Reader>& reader : readers) {
+        if (reader) {
+            reader->Close();
+        }
+    }
+}
+
 Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
-    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type);
+    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
+    arrow::MemoryPool* arrow_pool);
 
 class RealtimeOffsetCoverage {
  public:
@@ -237,22 +247,9 @@ Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_
     return Status::OK();
 }
 
-Status ValidatePreparedSchema(const std::shared_ptr<arrow::Schema>& prepared_schema) {
-    if (!prepared_schema || prepared_schema->num_fields() < kPreparedValueStartIndex) {
-        return Status::Invalid("prepared schema must contain realtime transport fields");
-    }
-    PAIMON_RETURN_NOT_OK(
-        CheckPreparedField(prepared_schema, kValueKindIndex, SpecialFields::ValueKind()));
-    PAIMON_RETURN_NOT_OK(
-        CheckPreparedField(prepared_schema, kSequenceNumberIndex, SpecialFields::SequenceNumber()));
-    PAIMON_RETURN_NOT_OK(
-        CheckPreparedField(prepared_schema, kRealtimeOffsetIndex, RealtimeOffsetField()));
-    return Status::OK();
-}
-
 Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
     const std::shared_ptr<arrow::StructArray>& array,
-    const std::shared_ptr<arrow::StructType>& read_type) {
+    const std::shared_ptr<arrow::StructType>& read_type, arrow::MemoryPool* arrow_pool) {
     const std::shared_ptr<arrow::StructType> data_type =
         checked_pointer_cast<arrow::StructType>(array->type());
     std::unordered_map<int32_t, int32_t> data_field_id_to_idx;
@@ -273,12 +270,16 @@ Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
                                NestedProjectionUtils::GetPaimonFieldId(read_field));
         auto data_iter = data_field_id_to_idx.find(read_field_id);
         if (data_iter == data_field_id_to_idx.end()) {
-            return Status::Invalid(
-                fmt::format("cannot find field id {} in prepared value struct", read_field_id));
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> null_child,
+                arrow::MakeArrayOfNull(read_field->type(), array->offset() + array->length(),
+                                       arrow_pool));
+            aligned_arrays.push_back(std::move(null_child));
+            continue;
         }
         std::shared_ptr<arrow::Array> child =
             arrow::MakeArray(array->data()->child_data[data_iter->second]);
-        PAIMON_ASSIGN_OR_RAISE(child, AlignArrayByPaimonIds(child, read_field->type()));
+        PAIMON_ASSIGN_OR_RAISE(child, AlignArrayByPaimonIds(child, read_field->type(), arrow_pool));
         aligned_arrays.push_back(std::move(child));
     }
 
@@ -294,9 +295,10 @@ Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
 
 Result<std::shared_ptr<arrow::Array>> AlignListArrayByPaimonIds(
     const std::shared_ptr<arrow::ListArray>& array,
-    const std::shared_ptr<arrow::ListType>& read_type) {
+    const std::shared_ptr<arrow::ListType>& read_type, arrow::MemoryPool* arrow_pool) {
     std::shared_ptr<arrow::Array> values = array->values();
-    PAIMON_ASSIGN_OR_RAISE(values, AlignArrayByPaimonIds(values, read_type->value_type()));
+    PAIMON_ASSIGN_OR_RAISE(values,
+                           AlignArrayByPaimonIds(values, read_type->value_type(), arrow_pool));
     std::shared_ptr<arrow::ArrayData> new_data = array->data()->Copy();
     new_data->type = read_type;
     new_data->child_data = {values->data()};
@@ -304,12 +306,12 @@ Result<std::shared_ptr<arrow::Array>> AlignListArrayByPaimonIds(
 }
 
 Result<std::shared_ptr<arrow::Array>> AlignMapArrayByPaimonIds(
-    const std::shared_ptr<arrow::MapArray>& array,
-    const std::shared_ptr<arrow::MapType>& read_type) {
+    const std::shared_ptr<arrow::MapArray>& array, const std::shared_ptr<arrow::MapType>& read_type,
+    arrow::MemoryPool* arrow_pool) {
     std::shared_ptr<arrow::Array> keys = array->keys();
-    PAIMON_ASSIGN_OR_RAISE(keys, AlignArrayByPaimonIds(keys, read_type->key_type()));
+    PAIMON_ASSIGN_OR_RAISE(keys, AlignArrayByPaimonIds(keys, read_type->key_type(), arrow_pool));
     std::shared_ptr<arrow::Array> items = array->items();
-    PAIMON_ASSIGN_OR_RAISE(items, AlignArrayByPaimonIds(items, read_type->item_type()));
+    PAIMON_ASSIGN_OR_RAISE(items, AlignArrayByPaimonIds(items, read_type->item_type(), arrow_pool));
 
     const std::shared_ptr<arrow::ArrayData>& entries_data = array->data()->child_data[0];
     std::shared_ptr<arrow::ArrayData> new_entries = entries_data->Copy();
@@ -323,7 +325,8 @@ Result<std::shared_ptr<arrow::Array>> AlignMapArrayByPaimonIds(
 }
 
 Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
-    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type) {
+    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
+    arrow::MemoryPool* arrow_pool) {
     if (array->type()->id() != read_type->id()) {
         return Status::Invalid(fmt::format("prepared value type {} does not match query type {}",
                                            array->type()->ToString(), read_type->ToString()));
@@ -331,13 +334,16 @@ Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
     switch (read_type->id()) {
         case arrow::Type::STRUCT:
             return AlignStructArrayByPaimonIds(checked_pointer_cast<arrow::StructArray>(array),
-                                               checked_pointer_cast<arrow::StructType>(read_type));
+                                               checked_pointer_cast<arrow::StructType>(read_type),
+                                               arrow_pool);
         case arrow::Type::LIST:
             return AlignListArrayByPaimonIds(checked_pointer_cast<arrow::ListArray>(array),
-                                             checked_pointer_cast<arrow::ListType>(read_type));
+                                             checked_pointer_cast<arrow::ListType>(read_type),
+                                             arrow_pool);
         case arrow::Type::MAP:
             return AlignMapArrayByPaimonIds(checked_pointer_cast<arrow::MapArray>(array),
-                                            checked_pointer_cast<arrow::MapType>(read_type));
+                                            checked_pointer_cast<arrow::MapType>(read_type),
+                                            arrow_pool);
         default:
             if (!array->type()->Equals(*read_type)) {
                 return Status::Invalid(
@@ -351,7 +357,7 @@ Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
 Result<arrow::ArrayVector> ProjectFieldsByPaimonIds(
     const std::shared_ptr<arrow::StructArray>& data_batch,
     const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::shared_ptr<arrow::Schema>& query_schema) {
+    const std::shared_ptr<arrow::Schema>& query_schema, arrow::MemoryPool* arrow_pool) {
     std::unordered_map<int32_t, int32_t> prepared_field_id_to_idx;
     prepared_field_id_to_idx.reserve(prepared_schema->num_fields());
     for (int32_t i = kPreparedValueStartIndex; i < prepared_schema->num_fields(); ++i) {
@@ -375,7 +381,7 @@ Result<arrow::ArrayVector> ProjectFieldsByPaimonIds(
         }
         std::shared_ptr<arrow::Array> field_array = data_batch->field(prepared_iter->second);
         PAIMON_ASSIGN_OR_RAISE(field_array,
-                               AlignArrayByPaimonIds(field_array, query_field->type()));
+                               AlignArrayByPaimonIds(field_array, query_field->type(), arrow_pool));
         result.push_back(std::move(field_array));
     }
     return result;
@@ -468,8 +474,12 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     };
 
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
+        if (first_error_.has_value()) {
+            return first_error_.value();
+        }
         Result<std::unique_ptr<KeyValueRecordReader::Iterator>> result = NextBatchImpl();
         if (!result.ok()) {
+            first_error_ = result.status();
             Close();
         }
         return result;
@@ -508,6 +518,23 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             }
             std::shared_ptr<arrow::StructArray> data_batch =
                 checked_pointer_cast<arrow::StructArray>(arrow_array);
+            Status transport_status =
+                ValidatePreparedTransportSchema(arrow::schema(data_batch->type()->fields()));
+            if (!transport_status.ok()) {
+                return Status::Invalid(
+                    "prepared batch field does not match prepared transport "
+                    "schema: ",
+                    transport_status.ToString());
+            }
+            if (visible_offsets_.has_value()) {
+                PAIMON_RETURN_NOT_OK(ValidateProjectionSchema(
+                    arrow::schema(data_batch->type()->fields()), key_schema_));
+                PAIMON_ASSIGN_OR_RAISE(
+                    arrow_array,
+                    AlignArrayByPaimonIds(data_batch, arrow::struct_(prepared_schema_->fields()),
+                                          arrow_pool_.get()));
+                data_batch = checked_pointer_cast<arrow::StructArray>(arrow_array);
+            }
             PAIMON_RETURN_NOT_OK(ValidatePreparedBatch(data_batch));
             PAIMON_RETURN_NOT_OK(ValidateOrdering(data_batch));
 
@@ -528,12 +555,12 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                 data_batch->field(kValueKindIndex));
             sequence_number_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
                 data_batch->field(kSequenceNumberIndex));
-            PAIMON_ASSIGN_OR_RAISE(
-                arrow::ArrayVector key_fields,
-                ProjectFieldsByPaimonIds(data_batch, prepared_schema_, key_schema_));
-            PAIMON_ASSIGN_OR_RAISE(
-                arrow::ArrayVector value_fields,
-                ProjectFieldsByPaimonIds(data_batch, prepared_schema_, value_schema_));
+            PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector key_fields,
+                                   ProjectFieldsByPaimonIds(data_batch, prepared_schema_,
+                                                            key_schema_, arrow_pool_.get()));
+            PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector value_fields,
+                                   ProjectFieldsByPaimonIds(data_batch, prepared_schema_,
+                                                            value_schema_, arrow_pool_.get()));
             key_ctx_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
             value_ctx_ = std::make_shared<ColumnarBatchContext>(value_fields, pool_);
             ArrowUtils::TraverseArray(data_batch);
@@ -578,8 +605,9 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         if (data_batch->length() == 0) {
             return Status::OK();
         }
-        PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector key_fields,
-                               ProjectFieldsByPaimonIds(data_batch, prepared_schema_, key_schema_));
+        PAIMON_ASSIGN_OR_RAISE(
+            arrow::ArrayVector key_fields,
+            ProjectFieldsByPaimonIds(data_batch, prepared_schema_, key_schema_, arrow_pool_.get()));
         std::shared_ptr<ColumnarBatchContext> key_context =
             std::make_shared<ColumnarBatchContext>(key_fields, pool_);
         std::shared_ptr<arrow::Int64Array> sequences =
@@ -613,6 +641,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
  private:
     bool closed_ = false;
+    std::optional<Status> first_error_;
     std::unique_ptr<BatchReader> reader_;
     std::shared_ptr<arrow::Schema> prepared_schema_;
     std::optional<OffsetRange> visible_offsets_;
@@ -634,6 +663,19 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
 }  // namespace
 
+Status ValidatePreparedTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema) {
+    if (!prepared_schema || prepared_schema->num_fields() < kPreparedValueStartIndex) {
+        return Status::Invalid("prepared schema must contain realtime transport fields");
+    }
+    PAIMON_RETURN_NOT_OK(
+        CheckPreparedField(prepared_schema, kValueKindIndex, SpecialFields::ValueKind()));
+    PAIMON_RETURN_NOT_OK(
+        CheckPreparedField(prepared_schema, kSequenceNumberIndex, SpecialFields::SequenceNumber()));
+    PAIMON_RETURN_NOT_OK(
+        CheckPreparedField(prepared_schema, kRealtimeOffsetIndex, RealtimeOffsetField()));
+    return Status::OK();
+}
+
 namespace {
 
 Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
@@ -649,7 +691,7 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
         return Status::Invalid("prepared batch reader cannot be null");
     }
     ScopeGuard close_guard([&owned_reader]() -> void { owned_reader->Close(); });
-    PAIMON_RETURN_NOT_OK(ValidatePreparedSchema(prepared_schema));
+    PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(prepared_schema));
     if (!key_schema) {
         return Status::Invalid("prepared key schema cannot be null");
     }
@@ -695,26 +737,23 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool) {
+    std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
+    ScopeGuard readers_guard([&readers, &adapted_readers]() {
+        CloseReaders(readers);
+        CloseReaders(adapted_readers);
+    });
     if (!memory_pool) {
         return Status::Invalid("prepared reader memory pool cannot be null");
     }
+    for (const std::unique_ptr<BatchReader>& reader : readers) {
+        if (!reader) {
+            return Status::Invalid("PK real-time store returned a null commit reader");
+        }
+    }
     std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
         RealtimeOffsetCoverage::Create(sealed_offsets, readers.size(), arrow_pool));
-    ScopeGuard readers_guard([&readers]() {
-        for (const std::unique_ptr<BatchReader>& reader : readers) {
-            if (reader) {
-                reader->Close();
-            }
-        }
-    });
-    std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
-    ScopeGuard adapted_readers_guard([&adapted_readers]() {
-        for (const std::unique_ptr<KeyValueRecordReader>& reader : adapted_readers) {
-            reader->Close();
-        }
-    });
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> adapted_reader,
@@ -724,7 +763,6 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
         adapted_readers.push_back(std::move(adapted_reader));
     }
     readers_guard.Release();
-    adapted_readers_guard.Release();
     return adapted_readers;
 }
 
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
index 064a62958..22a837a76 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/prepared_key_value_reader.h
@@ -33,6 +33,8 @@ class BatchReader;
 class FieldsComparator;
 class MemoryPool;
 
+Status ValidatePreparedTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema);
+
 Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::optional<OffsetRange>& visible_offsets,
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 2f04aae79..e4c480377 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -21,6 +21,8 @@
 #include <cstddef>
 #include <mutex>
 #include <optional>
+#include <queue>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -37,6 +39,7 @@
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/index/pk/primary_key_index_definitions.h"
+#include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/macros.h"
 
@@ -163,9 +166,12 @@ class RawBatchReader final : public BatchReader {
           key_comparator_(key_comparator),
           memory_pool_(memory_pool),
           arrow_pool_(GetArrowPool(memory_pool)),
+          heap_(SourceGreater{this}),
           metrics_(std::make_shared<MetricsImpl>()) {
         key_contexts_.reserve(batches_.size());
-        for (const StoredBatch& batch : batches_) {
+        sequence_arrays_.reserve(batches_.size());
+        for (size_t i = 0; i < batches_.size(); ++i) {
+            const StoredBatch& batch = batches_[i];
             arrow::ArrayVector key_arrays;
             key_arrays.reserve(key_field_indexes_.size());
             for (int32_t field_index : key_field_indexes_) {
@@ -173,34 +179,89 @@ class RawBatchReader final : public BatchReader {
             }
             key_contexts_.push_back(
                 std::make_shared<ColumnarBatchContext>(key_arrays, memory_pool_));
+            sequence_arrays_.push_back(
+                checked_pointer_cast<arrow::Int64Array>(batch.data->field(1)));
+            if (batch.data->length() > 0) {
+                heap_.push(i);
+            }
         }
     }
 
     Result<ReadBatch> NextBatch() override {
-        std::optional<size_t> selected;
-        for (size_t i = 0; i < batches_.size(); ++i) {
-            if (positions_[i] >= batches_[i].data->length()) {
-                continue;
+        if (heap_.empty()) {
+            return MakeEofBatch();
+        }
+
+        struct SelectedRow {
+            size_t selected_source;
+            int64_t source_ordinal;
+        };
+        struct SelectedSource {
+            size_t source;
+            std::vector<int64_t> rows;
+            int64_t base = -1;
+        };
+        std::vector<SelectedRow> selected_rows;
+        selected_rows.reserve(kOutputBatchSize);
+        std::vector<SelectedSource> selected_sources;
+        std::unordered_map<size_t, size_t> selected_source_indexes;
+        while (!heap_.empty() && selected_rows.size() < kOutputBatchSize) {
+            const size_t source = heap_.top();
+            heap_.pop();
+            auto [source_it, inserted] =
+                selected_source_indexes.emplace(source, selected_sources.size());
+            if (inserted) {
+                selected_sources.push_back(SelectedSource{source, {}});
             }
-            if (!selected.has_value() || Less(i, selected.value())) {
-                selected = i;
+            SelectedSource& selected_source = selected_sources[source_it->second];
+            selected_rows.push_back(
+                SelectedRow{source_it->second, static_cast<int64_t>(selected_source.rows.size())});
+            selected_source.rows.push_back(positions_[source]++);
+            if (positions_[source] < batches_[source].data->length()) {
+                heap_.push(source);
             }
         }
-        if (!selected.has_value()) {
-            return MakeEofBatch();
-        }
-        const size_t batch_index = selected.value();
-        arrow::Int64Builder index_builder(arrow_pool_.get());
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(index_builder.Append(positions_[batch_index]));
-        std::shared_ptr<arrow::Array> index;
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(index_builder.Finish(&index));
+
         arrow::compute::ExecContext context(arrow_pool_.get());
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-            arrow::Datum taken,
-            arrow::compute::Take(arrow::Datum(batches_[batch_index].data), arrow::Datum(index),
-                                 arrow::compute::TakeOptions::NoBoundsCheck(), &context));
-        std::shared_ptr<arrow::Array> batch = taken.make_array();
-        ++positions_[batch_index];
+        arrow::ArrayVector grouped_batches;
+        int64_t grouped_row_count = 0;
+        for (SelectedSource& selected_source : selected_sources) {
+            arrow::Int64Builder source_index_builder(arrow_pool_.get());
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(
+                source_index_builder.AppendValues(selected_source.rows));
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> source_indices,
+                                              source_index_builder.Finish());
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                arrow::Datum source_batch,
+                arrow::compute::Take(arrow::Datum(batches_[selected_source.source].data),
+                                     arrow::Datum(source_indices),
+                                     arrow::compute::TakeOptions::NoBoundsCheck(), &context));
+            selected_source.base = grouped_row_count;
+            grouped_row_count += static_cast<int64_t>(selected_source.rows.size());
+            grouped_batches.push_back(source_batch.make_array());
+        }
+
+        std::shared_ptr<arrow::Array> batch;
+        if (grouped_batches.size() == 1) {
+            batch = std::move(grouped_batches[0]);
+        } else {
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> grouped,
+                arrow::Concatenate(grouped_batches, arrow_pool_.get()));
+            arrow::Int64Builder order_builder(arrow_pool_.get());
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(order_builder.Reserve(selected_rows.size()));
+            for (const SelectedRow& selected : selected_rows) {
+                order_builder.UnsafeAppend(selected_sources[selected.selected_source].base +
+                                           selected.source_ordinal);
+            }
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> order,
+                                              order_builder.Finish());
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                arrow::Datum reordered,
+                arrow::compute::Take(arrow::Datum(grouped), arrow::Datum(order),
+                                     arrow::compute::TakeOptions::NoBoundsCheck(), &context));
+            batch = reordered.make_array();
+        }
         auto array = std::make_unique<ArrowArray>();
         auto schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*batch, array.get(), schema.get()));
@@ -211,12 +272,18 @@ class RawBatchReader final : public BatchReader {
         return metrics_;
     }
     void Close() override {
+        while (!heap_.empty()) {
+            heap_.pop();
+        }
         batches_.clear();
         positions_.clear();
         key_contexts_.clear();
+        sequence_arrays_.clear();
     }
 
  private:
+    static constexpr size_t kOutputBatchSize = 1024;
+
     bool Less(size_t left, size_t right) const {
         ColumnarRowRef left_key(key_contexts_[left], positions_[left]);
         ColumnarRowRef right_key(key_contexts_[right], positions_[right]);
@@ -224,13 +291,22 @@ class RawBatchReader final : public BatchReader {
         if (key_comparison != 0) {
             return key_comparison < 0;
         }
-        const std::shared_ptr<arrow::Int64Array> left_sequences =
-            checked_pointer_cast<arrow::Int64Array>(batches_[left].data->field(1));
-        const std::shared_ptr<arrow::Int64Array> right_sequences =
-            checked_pointer_cast<arrow::Int64Array>(batches_[right].data->field(1));
-        return left_sequences->Value(positions_[left]) < right_sequences->Value(positions_[right]);
+        const int64_t left_sequence = sequence_arrays_[left]->Value(positions_[left]);
+        const int64_t right_sequence = sequence_arrays_[right]->Value(positions_[right]);
+        if (left_sequence != right_sequence) {
+            return left_sequence < right_sequence;
+        }
+        return left < right;
     }
 
+    struct SourceGreater {
+        RawBatchReader* reader;
+
+        bool operator()(size_t left, size_t right) const {
+            return reader->Less(right, left);
+        }
+    };
+
     std::vector<StoredBatch> batches_;
     std::vector<int64_t> positions_;
     std::vector<int32_t> key_field_indexes_;
@@ -238,6 +314,8 @@ class RawBatchReader final : public BatchReader {
     std::shared_ptr<MemoryPool> memory_pool_;
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     std::vector<std::shared_ptr<ColumnarBatchContext>> key_contexts_;
+    std::vector<std::shared_ptr<arrow::Int64Array>> sequence_arrays_;
+    std::priority_queue<size_t, std::vector<size_t>, SourceGreater> heap_;
     std::shared_ptr<Metrics> metrics_;
 };
 
@@ -379,8 +457,9 @@ Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create
     const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::vector<std::string>& trimmed_primary_keys,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    if (!prepared_schema || trimmed_primary_keys.empty() || !memory_pool) {
-        return Status::Invalid("PK prepared schema or memory pool is null");
+    PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(prepared_schema));
+    if (trimmed_primary_keys.empty() || !memory_pool) {
+        return Status::Invalid("PK primary keys are empty or memory pool is null");
     }
     std::vector<int32_t> key_field_indexes;
     std::vector<DataField> key_fields;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 116c6e389..dc2ce86b1 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -18,14 +18,18 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
+#include <algorithm>
 #include <memory>
+#include <numeric>
 #include <optional>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
 #include "arrow/ipc/json_simple.h"
+#include "fmt/format.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
@@ -196,6 +200,34 @@ TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
         RealtimeWriteBatch{MakeBatch(R"([[0, 4, 3, 4, "four"]])"), OffsetRange(3, 4)}));
 }
 
+TEST(PrimaryKeyRealtimeStoreTest, TestBadTransportPrefix) {
+    const std::shared_ptr<arrow::Schema> valid = PreparedSchema();
+    std::vector<arrow::FieldVector> invalid_fields;
+
+    arrow::FieldVector wrong_type = valid->fields();
+    wrong_type[0] = DataField::ConvertDataFieldToArrowField(
+                        DataField(SpecialFields::ValueKind().Id(),
+                                  arrow::field("_VALUE_KIND", arrow::int32(), false)))
+                        ->WithNullable(false);
+    invalid_fields.push_back(std::move(wrong_type));
+
+    arrow::FieldVector nullable_sequence = valid->fields();
+    nullable_sequence[1] = nullable_sequence[1]->WithNullable(true);
+    invalid_fields.push_back(std::move(nullable_sequence));
+
+    arrow::FieldVector wrong_offset_id = valid->fields();
+    wrong_offset_id[2] = DataField::ConvertDataFieldToArrowField(
+                             DataField(99, arrow::field("_REALTIME_OFFSET", arrow::int64(), false)))
+                             ->WithNullable(false);
+    invalid_fields.push_back(std::move(wrong_offset_id));
+
+    for (const arrow::FieldVector& fields : invalid_fields) {
+        ASSERT_NOK_WITH_MSG(
+            PrimaryKeyRealtimeStore::Create(arrow::schema(fields), {"id"}, GetDefaultPool()),
+            "prepared schema field");
+    }
+}
+
 TEST(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
     ASSERT_OK_AND_ASSIGN(
         std::shared_ptr<PrimaryKeyRealtimeStore> store,
@@ -233,12 +265,100 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateCommitReaders(segment.value()));
     ASSERT_EQ(1, readers.size());
-    for (int32_t row = 0; row < 2; ++row) {
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
+    ASSERT_FALSE(BatchReader::IsEofBatch(batch));
+    ASSERT_EQ(2, batch.first->length);
+    AssertOffsetsZero(batch.first.get());
+    ASSERT_TRUE(arrow::ImportArray(batch.first.get(), batch.second.get()).ok());
+    ASSERT_OK_AND_ASSIGN(batch, readers[0]->NextBatch());
+    ASSERT_TRUE(BatchReader::IsEofBatch(batch));
+}
+
+TEST(PrimaryKeyRealtimeStoreTest, TestHeapMergeAcrossBatches) {
+    constexpr int64_t kSourceCount = 2057;
+    constexpr int64_t kKeyCount = 257;
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<PrimaryKeyRealtimeStore> store,
+        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+    for (int64_t source = 0; source < kSourceCount; ++source) {
+        const int64_t id = (source * 149) % kKeyCount;
+        const std::string json =
+            fmt::format(R"([[0, {}, {}, {}, "v{}"]])", source, source, id, source);
+        ASSERT_OK(
+            store->Write(RealtimeWriteBatch{MakeBatch(json), OffsetRange(source, source + 1)}));
+    }
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateCommitReaders(segment.value()));
+    ASSERT_EQ(1, readers.size());
+
+    std::vector<int64_t> expected_sources(kSourceCount);
+    std::iota(expected_sources.begin(), expected_sources.end(), 0);
+    std::sort(expected_sources.begin(), expected_sources.end(), [=](int64_t left, int64_t right) {
+        const int64_t left_id = (left * 149) % kKeyCount;
+        const int64_t right_id = (right * 149) % kKeyCount;
+        return left_id != right_id ? left_id < right_id : left < right;
+    });
+
+    int64_t output_row = 0;
+    int64_t output_batches = 0;
+    while (true) {
         ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
-        ASSERT_FALSE(BatchReader::IsEofBatch(batch));
-        AssertOffsetsZero(batch.first.get());
-        ASSERT_TRUE(arrow::ImportArray(batch.first.get(), batch.second.get()).ok());
+        if (BatchReader::IsEofBatch(batch)) {
+            break;
+        }
+        ASSERT_LE(batch.first->length, 1024);
+        ASSERT_GT(batch.first->length, 0);
+        ++output_batches;
+        arrow::Result<std::shared_ptr<arrow::Array>> imported_result =
+            arrow::ImportArray(batch.first.get(), batch.second.get());
+        ASSERT_TRUE(imported_result.ok()) << imported_result.status().ToString();
+        std::shared_ptr<arrow::Array> imported = std::move(imported_result).ValueOrDie();
+        std::shared_ptr<arrow::StructArray> array =
+            std::dynamic_pointer_cast<arrow::StructArray>(imported);
+        ASSERT_NE(nullptr, array);
+        ASSERT_EQ(PreparedSchema()->ToString(), arrow::schema(array->type()->fields())->ToString());
+        std::shared_ptr<arrow::Int64Array> sequences =
+            std::dynamic_pointer_cast<arrow::Int64Array>(array->field(1));
+        std::shared_ptr<arrow::Int64Array> ids =
+            std::dynamic_pointer_cast<arrow::Int64Array>(array->field(3));
+        std::shared_ptr<arrow::StringArray> values =
+            std::dynamic_pointer_cast<arrow::StringArray>(array->field(4));
+        ASSERT_NE(nullptr, sequences);
+        ASSERT_NE(nullptr, ids);
+        ASSERT_NE(nullptr, values);
+        for (int64_t row = 0; row < array->length(); ++row, ++output_row) {
+            ASSERT_LT(output_row, kSourceCount);
+            const int64_t source = expected_sources[output_row];
+            ASSERT_EQ(source, sequences->Value(row));
+            ASSERT_EQ((source * 149) % kKeyCount, ids->Value(row));
+            ASSERT_EQ(fmt::format("v{}", source), values->GetString(row));
+        }
     }
+    ASSERT_EQ(kSourceCount, output_row);
+    ASSERT_EQ(3, output_batches);
+}
+
+TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadMultiSourceReader) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<PrimaryKeyRealtimeStore> store,
+        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 10, 0, 1, "a"]])"), OffsetRange(0, 1)}));
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 20, 1, 2, "b"]])"), OffsetRange(1, 2)}));
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 30, 2, 3, "c"]])"), OffsetRange(2, 3)}));
+    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+                         store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateCommitReaders(segment.value()));
+    ASSERT_EQ(1, readers.size());
+
+    readers[0]->Close();
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
diff --git a/src/paimon/core/realtime/realtime_fields.h b/src/paimon/core/realtime/realtime_fields.h
index 6ed04b38a..270941238 100644
--- a/src/paimon/core/realtime/realtime_fields.h
+++ b/src/paimon/core/realtime/realtime_fields.h
@@ -19,17 +19,15 @@
 
 #pragma once
 
-#include <cstdint>
-#include <limits>
-
 #include "arrow/type.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/utils/special_field_ids.h"
 
 namespace paimon {
 
 inline const DataField& RealtimeOffsetField() {
     static const DataField data_field =
-        DataField(std::numeric_limits<int32_t>::max() - 10002,
+        DataField(SpecialFieldIds::REALTIME_OFFSET,
                   arrow::field("_REALTIME_OFFSET", arrow::int64(), /*nullable=*/false));
     return data_field;
 }
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 82318eadb..185156eb7 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -32,7 +32,6 @@
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
-#include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/io/merged_key_value_record_reader.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
@@ -285,18 +284,6 @@ Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegm
                                               const OffsetRange& sealed_offsets) {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                            realtime_store_->CreateCommitReaders(segment));
-    ScopeGuard readers_guard([&readers]() {
-        for (const std::unique_ptr<BatchReader>& reader : readers) {
-            if (reader) {
-                reader->Close();
-            }
-        }
-    });
-    for (const std::unique_ptr<BatchReader>& reader : readers) {
-        if (!reader) {
-            return Status::Invalid("PK real-time store returned a null commit reader");
-        }
-    }
     PAIMON_ASSIGN_OR_RAISE(
         std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
         AdaptPreparedCommitBatchReaders(std::move(readers), prepared_schema_, sealed_offsets,
@@ -309,7 +296,6 @@ Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegm
             std::move(prepared_reader), key_comparator_,
             std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function))));
     }
-    readers_guard.Release();
     return merge_tree_writer_->WriteSortedReaders(std::move(sorted_readers));
 }
 
diff --git a/src/paimon/core/table/source/append_only_table_read.cpp b/src/paimon/core/table/source/append_only_table_read.cpp
index 6885dc374..34c6ef850 100644
--- a/src/paimon/core/table/source/append_only_table_read.cpp
+++ b/src/paimon/core/table/source/append_only_table_read.cpp
@@ -77,6 +77,13 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateReader(
     std::vector<std::unique_ptr<BatchReader>> readers;
     readers.reserve(splits.size());
     std::vector<std::shared_ptr<RealtimeSplit>> realtime_splits;
+    ScopeGuard cleanup_guard([&]() {
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            if (reader) {
+                reader->Close();
+            }
+        }
+    });
     for (const std::shared_ptr<Split>& split : splits) {
         std::shared_ptr<RealtimeSplit> realtime_split =
             std::dynamic_pointer_cast<RealtimeSplit>(split);
@@ -92,8 +99,6 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateReader(
         }
     }
 
-    std::unique_ptr<BatchReader> result =
-        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
     if (!realtime_splits.empty()) {
         const std::shared_ptr<RealtimeContext> realtime_context = context_->GetRealtimeContext();
         if (!realtime_context) {
@@ -106,6 +111,9 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateReader(
                 realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
         }
     }
+    std::unique_ptr<BatchReader> result =
+        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
+    cleanup_guard.Release();
     return result;
 }
 
@@ -124,6 +132,13 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateRealtimeReader(
                            realtime_context_impl->ResolveReadView(realtime_split->OpaqueTicket()));
     std::vector<std::unique_ptr<BatchReader>> readers;
     readers.reserve(realtime_split->DiskSplits().size() + 1);
+    ScopeGuard readers_guard([&readers]() {
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            if (reader) {
+                reader->Close();
+            }
+        }
+    });
     const RealtimePartitionBucket expected_partition_bucket(realtime_split->Partition(),
                                                             realtime_split->Bucket());
     if (memory.partition_bucket != expected_partition_bucket) {
@@ -150,8 +165,17 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateRealtimeReader(
         std::vector<std::unique_ptr<BatchReader>> memory_readers,
         memory.store->CreateQueryReaders(memory.read_view, realtime_split->CommittedEndOffset(),
                                          query_context));
-
+    const size_t first_memory_reader = readers.size();
+    readers.reserve(readers.size() + memory_readers.size());
     for (std::unique_ptr<BatchReader>& memory_reader : memory_readers) {
+        readers.push_back(std::move(memory_reader));
+    }
+
+    for (size_t i = first_memory_reader; i < readers.size(); ++i) {
+        std::unique_ptr<BatchReader>& memory_reader = readers[i];
+        if (!memory_reader) {
+            return Status::Invalid("append-only real-time store returned a null query reader");
+        }
         if (context_->EnablePredicateFilter() && context_->GetPredicate()) {
             PAIMON_ASSIGN_OR_RAISE(memory_reader, PredicateBatchReader::Create(
                                                       std::move(memory_reader),
@@ -159,14 +183,15 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateRealtimeReader(
         }
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RealtimeReader> realtime_reader,
                                RealtimeReader::Create(memory.read_view, std::move(memory_reader)));
-        readers.push_back(std::move(realtime_reader));
+        memory_reader = std::move(realtime_reader);
     }
-    std::unique_ptr<BatchReader> result =
-        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
     if (release_ticket) {
         PAIMON_RETURN_NOT_OK(
             realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
     }
+    std::unique_ptr<BatchReader> result =
+        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
+    readers_guard.Release();
     return result;
 }
 
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 64d722097..96f3f00d9 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -209,6 +209,13 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
     std::vector<std::unique_ptr<BatchReader>> readers;
     readers.reserve(splits.size());
     std::vector<std::shared_ptr<RealtimeSplit>> realtime_splits;
+    ScopeGuard cleanup_guard([&]() {
+        for (const std::unique_ptr<BatchReader>& reader : readers) {
+            if (reader) {
+                reader->Close();
+            }
+        }
+    });
     for (const std::shared_ptr<Split>& split : splits) {
         std::shared_ptr<RealtimeSplit> realtime_split =
             std::dynamic_pointer_cast<RealtimeSplit>(split);
@@ -223,8 +230,6 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
         }
     }
 
-    std::unique_ptr<BatchReader> result =
-        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
     if (!realtime_splits.empty()) {
         const std::shared_ptr<RealtimeContext> realtime_context = context_->GetRealtimeContext();
         if (!realtime_context) {
@@ -237,6 +242,9 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
                 realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
         }
     }
+    std::unique_ptr<BatchReader> result =
+        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
+    cleanup_guard.Release();
     return result;
 }
 
diff --git a/src/paimon/core/table/source/realtime_table_scan.cpp b/src/paimon/core/table/source/realtime_table_scan.cpp
index 1b496d8a1..4c3968dc3 100644
--- a/src/paimon/core/table/source/realtime_table_scan.cpp
+++ b/src/paimon/core/table/source/realtime_table_scan.cpp
@@ -152,16 +152,17 @@ Result<std::vector<std::shared_ptr<Split>>> RealtimeTableScan::CreateRealtimeSpl
             continue;
         }
 
-        // Append tables can schedule all but the tail disk split independently. The tail split
-        // carries the immutable memory view so disk and memory are still concatenated by one
-        // RealtimeSplit without collapsing the whole partition-bucket into one scheduling unit.
-        auto tail_disk_split = std::prev(grouped_disk_splits.end());
-        result.insert(result.end(), grouped_disk_splits.begin(), tail_disk_split);
-        std::vector<std::shared_ptr<Split>> realtime_disk_splits;
-        realtime_disk_splits.push_back(std::move(*tail_disk_split));
         RealtimePartitionBucketView& memory = memory_iter->second;
+        if (!pk_table_) {
+            // Append tables can schedule all but the tail disk split independently. The tail split
+            // carries the immutable memory view so disk and memory are still concatenated by one
+            // RealtimeSplit without collapsing the whole partition-bucket into one scheduling unit.
+            auto tail_disk_split = std::prev(grouped_disk_splits.end());
+            result.insert(result.end(), grouped_disk_splits.begin(), tail_disk_split);
+            grouped_disk_splits.erase(grouped_disk_splits.begin(), tail_disk_split);
+        }
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Split> realtime_split,
-                               create_realtime_split(key, std::move(realtime_disk_splits), memory));
+                               create_realtime_split(key, std::move(grouped_disk_splits), memory));
         result.push_back(std::move(realtime_split));
         active_memory.erase(memory_iter);
     }
@@ -176,7 +177,7 @@ Result<std::vector<std::shared_ptr<Split>>> RealtimeTableScan::CreateRealtimeSpl
     return result;
 }
 
-RealtimeTableScan::RealtimeTableScan(std::unique_ptr<TableScan>&& disk_scan,
+RealtimeTableScan::RealtimeTableScan(std::unique_ptr<TableScan>&& disk_scan, bool pk_table,
                                      const std::shared_ptr<RealtimeContextImpl>& realtime_context,
                                      const std::shared_ptr<FileStorePathFactory>& path_factory,
                                      const std::shared_ptr<SnapshotManager>& snapshot_manager,
@@ -184,6 +185,7 @@ RealtimeTableScan::RealtimeTableScan(std::unique_ptr<TableScan>&& disk_scan,
                                      const std::shared_ptr<ScanFilter>& scan_filter,
                                      int64_t read_view_ttl_millis)
     : disk_scan_(std::move(disk_scan)),
+      pk_table_(pk_table),
       realtime_context_(realtime_context),
       path_factory_(path_factory),
       snapshot_manager_(snapshot_manager),
diff --git a/src/paimon/core/table/source/realtime_table_scan.h b/src/paimon/core/table/source/realtime_table_scan.h
index 959203ca4..692b749ef 100644
--- a/src/paimon/core/table/source/realtime_table_scan.h
+++ b/src/paimon/core/table/source/realtime_table_scan.h
@@ -38,7 +38,7 @@ class SnapshotManager;
 /// Adds process-local memory splits to a normal data-table batch scan.
 class RealtimeTableScan : public TableScan {
  public:
-    RealtimeTableScan(std::unique_ptr<TableScan>&& disk_scan,
+    RealtimeTableScan(std::unique_ptr<TableScan>&& disk_scan, bool pk_table,
                       const std::shared_ptr<RealtimeContextImpl>& realtime_context,
                       const std::shared_ptr<FileStorePathFactory>& path_factory,
                       const std::shared_ptr<SnapshotManager>& snapshot_manager,
@@ -67,6 +67,7 @@ class RealtimeTableScan : public TableScan {
         const std::optional<int64_t>& snapshot_id) const;
 
     std::unique_ptr<TableScan> disk_scan_;
+    bool pk_table_;
     std::shared_ptr<RealtimeContextImpl> realtime_context_;
     std::shared_ptr<FileStorePathFactory> path_factory_;
     std::shared_ptr<SnapshotManager> snapshot_manager_;
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 0bcd79f61..f894e1a74 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -344,7 +344,7 @@ Result<std::unique_ptr<TableScan>> NewDataTableScan(const std::shared_ptr<ScanCo
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context,
                                RealtimeContextImpl::Cast(context->GetRealtimeContext()));
         return std::make_unique<RealtimeTableScan>(
-            std::move(batch_scan), realtime_context, path_factory,
+            std::move(batch_scan), pk_table, realtime_context, path_factory,
             snapshot_reader->GetSnapshotManager(), core_options.GetFileSystem(),
             context->GetScanFilters(), core_options.GetRealtimeReadViewTtlMillis());
     }
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 0e6f83b70..a393f838d 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -468,21 +468,27 @@ class SwapFirstTwoBatchReader final : public BatchReader {
     Result<ReadBatch> NextBatch() override {
         if (!initialized_) {
             initialized_ = true;
-            PAIMON_ASSIGN_OR_RAISE(ReadBatch first, delegate_->NextBatch());
-            if (BatchReader::IsEofBatch(first)) {
+            PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
+            if (BatchReader::IsEofBatch(batch)) {
                 return MakeEofBatch();
             }
-            PAIMON_ASSIGN_OR_RAISE(ReadBatch second, delegate_->NextBatch());
-            if (BatchReader::IsEofBatch(second)) {
-                return first;
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> array,
+                arrow::ImportArray(batch.first.get(), batch.second.get()));
+            if (array->length() < 2) {
+                return Status::Invalid("cannot make a one-row reader unsorted");
+            }
+            arrow::ArrayVector pieces = {array->Slice(1, 1), array->Slice(0, 1)};
+            if (array->length() > 2) {
+                pieces.push_back(array->Slice(2));
             }
-            first_ = std::move(first);
-            return second;
-        }
-        if (first_.has_value()) {
-            ReadBatch first = std::move(first_.value());
-            first_.reset();
-            return first;
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> swapped,
+                                              arrow::Concatenate(pieces));
+            auto output = std::make_unique<ArrowArray>();
+            auto schema = std::make_unique<ArrowSchema>();
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(
+                arrow::ExportArray(*swapped, output.get(), schema.get()));
+            return ReadBatch(std::move(output), std::move(schema));
         }
         return delegate_->NextBatch();
     }
@@ -492,14 +498,12 @@ class SwapFirstTwoBatchReader final : public BatchReader {
     }
 
     void Close() override {
-        first_.reset();
         delegate_->Close();
     }
 
  private:
     bool initialized_ = false;
     std::unique_ptr<BatchReader> delegate_;
-    std::optional<ReadBatch> first_;
 };
 
 class SubstituteOffsetBatchReader final : public BatchReader {
@@ -1253,6 +1257,8 @@ class RealtimeWriteInteTest : public ::testing::Test {
         return Status::OK();
     }
 
+    void RunConcurrencyTest(bool primary_key);
+
     Result<RealtimeOffsetMap> ReadCommittedOffsets() const {
         PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_));
         SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_);
@@ -1326,8 +1332,10 @@ class RealtimeWriteInteTest : public ::testing::Test {
         } else {
             CreateTable(/*partition_keys=*/{"pt"});
         }
+        auto close_state = std::make_shared<CloseTrackingReaderState>();
+        auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(close_state);
         ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                             RealtimeContext::Create());
+                             RealtimeContext::Create(factory));
         ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                              CreateRealtimeWriter(realtime_context));
         std::vector<Row> p0_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
@@ -1363,6 +1371,9 @@ class RealtimeWriteInteTest : public ::testing::Test {
                              TableRead::Create(std::move(read_context)));
         ASSERT_NOK_WITH_MSG(table_read->CreateReader(invalid_splits),
                             "unsupported real-time split version");
+        if (!primary_key) {
+            ASSERT_EQ(1, close_state->query_close_count->load(std::memory_order_acquire));
+        }
 
         std::vector<Row> expected_rows = p0_rows;
         expected_rows.insert(expected_rows.end(), p1_rows.begin(), p1_rows.end());
@@ -1631,6 +1642,56 @@ TEST_F(RealtimeWriteInteTest, TestPkMergeDiskSealedAndActive) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkMergeAllDiskSplitsWithMemory) {
+    options_[Options::SOURCE_SPLIT_OPEN_FILE_COST] = "1";
+    options_[Options::SOURCE_SPLIT_TARGET_SIZE] = "1";
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    const std::vector<std::vector<Row>> disk_batches = {
+        {{1, "disk-1", "p0"}, {2, "disk-2", "p0"}},
+        {{10, "disk-10", "p0"}, {11, "disk-11", "p0"}},
+        {{20, "disk-20", "p0"}, {21, "disk-21", "p0"}},
+    };
+    for (int64_t commit_identifier = 0;
+         commit_identifier < static_cast<int64_t>(disk_batches.size()); ++commit_identifier) {
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch(disk_batches[commit_identifier], /*partitioned=*/false));
+        ASSERT_OK(writer->Write(std::move(batch)));
+        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                             writer->PrepareCommitWithProgress(commit_identifier));
+        ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, commit_identifier));
+        ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+    }
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> memory_batch,
+        MakeBatch({Row{1, "memory-1", "p0"}, Row{10, "deleted-10", "p0"}},
+                  /*partitioned=*/false, /*bucket=*/0,
+                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE}));
+    ASSERT_OK(writer->Write(std::move(memory_batch)));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_EQ(1, plan->Splits().size());
+    std::shared_ptr<RealtimeSplit> realtime_split =
+        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
+    ASSERT_NE(nullptr, realtime_split);
+    ASSERT_EQ(3, realtime_split->DiskSplits().size());
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
+    ASSERT_EQ((std::vector<Row>{{1, "memory-1", "p0"},
+                                {2, "disk-2", "p0"},
+                                {11, "disk-11", "p0"},
+                                {20, "disk-20", "p0"},
+                                {21, "disk-21", "p0"}}),
+              actual_rows);
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) {
     const std::shared_ptr<arrow::Field> projected_b = arrow::field("b", arrow::int64());
     fields_ = {
@@ -1712,6 +1773,45 @@ TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkMemoryReadAfterSchemaEvolution) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "old", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+
+    std::shared_ptr<arrow::Field> renamed_payload = arrow::field("renamed_payload", arrow::utf8());
+    std::shared_ptr<arrow::Field> added = arrow::field("added", arrow::int32());
+    ASSERT_OK(TestHelper::WriteNextSchema(dir_->GetFileSystem(), table_path_,
+                                          {DataField(0, fields_[0]), DataField(1, renamed_payload),
+                                           DataField(2, fields_[2]), DataField(3, added)},
+                                          /*highest_field_id=*/3, options_));
+    fields_[1] = renamed_payload;
+    fields_.push_back(added);
+    schema_ = arrow::schema(fields_);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_OK_AND_ASSIGN(CollectedReadResult result,
+                         ReadPlan(plan, realtime_context, {"id", "renamed_payload", "pt", "added"},
+                                  /*predicate=*/nullptr, /*enable_predicate_filter=*/false));
+    ASSERT_EQ(1, result.data->num_chunks());
+    std::shared_ptr<arrow::StructArray> row =
+        std::dynamic_pointer_cast<arrow::StructArray>(result.data->chunk(0));
+    ASSERT_NE(nullptr, row);
+    ASSERT_EQ(1, row->length());
+    std::shared_ptr<arrow::StringArray> renamed_values =
+        std::dynamic_pointer_cast<arrow::StringArray>(row->field(2));
+    ASSERT_NE(nullptr, renamed_values);
+    ASSERT_EQ("old", renamed_values->GetString(0));
+    ASSERT_TRUE(row->field(4)->IsNull(0));
+    result.reader->Close();
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkCompositeMerge) {
     CreatePkTable(/*partition_keys=*/{}, /*primary_keys=*/{"id", "payload"});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
@@ -2277,6 +2377,40 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderCloseFailure) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestAppendQueryReaderCloseFailure) {
+    CreateTable(/*partition_keys=*/{});
+    auto state = std::make_shared<CloseTrackingReaderState>();
+    state->query_null_index = 1;
+    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    std::vector<Row> rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch(rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+
+    ReadContextBuilder read_builder(table_path_);
+    read_builder.SetOptions(options_)
+        .SetReadFieldNames({"id", "payload", "pt"})
+        .WithRealtimeContext(realtime_context)
+        .WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+                         TableRead::Create(std::move(read_context)));
+    ASSERT_NOK_WITH_MSG(table_read->CreateReader(plan->Splits()),
+                        "append-only real-time store returned a null query reader");
+    ASSERT_EQ(1, state->query_close_count->load(std::memory_order_acquire));
+
+    state->query_null_index = -1;
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
+    ASSERT_EQ(rows, actual_rows);
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkCommitReaderCloseFailure) {
     CreatePkTable();
     auto state = std::make_shared<CloseTrackingReaderState>();
@@ -3705,8 +3839,12 @@ TEST_F(RealtimeWriteInteTest, TestReopenRealtimeContextAfterRollback) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestConcurrentWritePrepareCommitReadAndRefresh) {
-    CreateTable(/*partition_keys=*/{});
+void RealtimeWriteInteTest::RunConcurrencyTest(bool primary_key) {
+    if (primary_key) {
+        CreatePkTable();
+    } else {
+        CreateTable(/*partition_keys=*/{});
+    }
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create());
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -3942,6 +4080,14 @@ TEST_F(RealtimeWriteInteTest, TestConcurrentWritePrepareCommitReadAndRefresh) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestConcurrentWritePrepareCommitReadAndRefresh) {
+    RunConcurrencyTest(/*primary_key=*/false);
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkConcurrency) {
+    RunConcurrencyTest(/*primary_key=*/true);
+}
+
 TEST_F(RealtimeWriteInteTest, TestMultiplePartitions) {
     CreateTable(/*partition_keys=*/{"pt"});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,

From d818466ba925ed719694bdb9ed389cefc730f825 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 00:31:55 +0800
Subject: [PATCH 45/93] refactor(realtime): simplify primary-key contracts

---
 .../realtime/arrow_realtime_store_factory.h   |  1 -
 include/paimon/realtime/realtime_store.h      | 39 ++++++++-----------
 src/paimon/core/mergetree/merge_tree_writer.h |  5 +--
 .../realtime/primary_key_realtime_store.h     |  2 +-
 .../realtime/realtime_primary_key_writer.h    |  1 -
 5 files changed, 19 insertions(+), 29 deletions(-)

diff --git a/include/paimon/realtime/arrow_realtime_store_factory.h b/include/paimon/realtime/arrow_realtime_store_factory.h
index da1b8de36..153d524d4 100644
--- a/include/paimon/realtime/arrow_realtime_store_factory.h
+++ b/include/paimon/realtime/arrow_realtime_store_factory.h
@@ -26,7 +26,6 @@ namespace paimon {
 /// Factory for Paimon's default Arrow-backed `RealtimeStore`.
 class PAIMON_EXPORT ArrowRealtimeStoreFactory : public RealtimeStoreFactory {
  public:
-    /// Creates the built-in append or in-memory primary-key store.
     Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override;
 };
 
diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 90c6ce0a8..60d1afc39 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -69,10 +69,10 @@ struct PAIMON_EXPORT RealtimeStoreCreateRequest {
 
 /// A record batch and its framework-assigned contiguous offset range.
 ///
-/// Append-mode batches contain table write fields, and row `i` is associated with
-/// `offset_range.begin + i`. Primary-key batches contain the prepared transport schema supplied
-/// to the factory and are physically sorted by full primary key then sequence number; their
-/// per-row `_REALTIME_OFFSET` field preserves the original write-order offset after sorting.
+/// Append-mode batches contain table write fields, and row `i` has offset
+/// `offset_range.begin + i`. Primary-key batches use the prepared transport schema, are sorted
+/// by full primary key then sequence number, and retain the original offset in
+/// `_REALTIME_OFFSET`.
 struct PAIMON_EXPORT RealtimeWriteBatch {
     /// Input batch whose ownership is transferred to `RealtimeStore::Write`.
     std::unique_ptr<RecordBatch> batch;
@@ -147,14 +147,10 @@ class PAIMON_EXPORT RealtimeStore {
 
     /// Creates readers that expose all rows in a sealed segment for Paimon file writing.
     ///
-    /// Concatenating the returned readers must produce every sealed row exactly once. Append-mode
-    /// readers preserve write order and contain `_VALUE_KIND` followed by table write fields.
-    /// Primary-key readers expose raw prepared rows. Each returned reader's complete stream,
-    /// including across `NextBatch` boundaries, is sorted by full primary key then sequence
-    /// number; all readers collectively cover sealed mutations exactly once. Reader cardinality is
-    /// independent of the number of writes. Paimon adapts and merges those rows before writing
-    /// files. Paimon validates the complete ordering and coverage before publishing generated file
-    /// state; a violation fails the prepare operation.
+    /// The returned readers collectively expose every sealed row exactly once. Append-mode readers
+    /// preserve write order and contain `_VALUE_KIND` followed by table write fields. Primary-key
+    /// readers use the prepared transport schema; each reader's complete stream is sorted by full
+    /// primary key then sequence number.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) = 0;
 
@@ -164,18 +160,15 @@ class PAIMON_EXPORT RealtimeStore {
     /// also provide a consistent snapshot when a write or seal is in progress.
     virtual Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() = 0;
 
-    /// Creates readers over rows in `view`. Append mode returns rows whose offsets are greater
-    /// than or equal to `offset_begin`. Primary-key mode ignores `offset_begin` and returns raw
-    /// prepared rows; Paimon applies offset filtering, projection, and merge-on-read adaptation.
+    /// Creates readers over rows in `view`. Append mode returns rows whose offsets are greater than
+    /// or equal to `offset_begin`; primary-key mode ignores `offset_begin`.
     ///
-    /// Append-mode output batches contain `_VALUE_KIND` first, followed by requested fields except
-    /// a duplicate `_VALUE_KIND`; all returned append readers collectively cover every matching
-    /// row exactly once. Primary-key output batches use the prepared transport schema and may
-    /// contain multiple mutations per key. Each returned primary-key reader's complete stream is
-    /// sorted by full primary key then sequence number, and all readers collectively cover raw
-    /// mutations exactly once. Reader cardinality is independent of the number of writes. Paimon
-    /// validates ordering while adapting each complete reader stream and retains `view` for the
-    /// lifetime of the resulting framework reader.
+    /// Append-mode batches contain `_VALUE_KIND` followed by the requested fields except a duplicate
+    /// `_VALUE_KIND`, and collectively expose every matching row exactly once. Primary-key batches
+    /// use the prepared transport schema and may contain multiple mutations per key; each reader's
+    /// complete stream is sorted by full primary key then sequence number, and the readers
+    /// collectively expose every raw mutation exactly once. Paimon retains `view` for the lifetime
+    /// of the resulting framework reader.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) = 0;
diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h
index cea07f3e4..01efd975c 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.h
+++ b/src/paimon/core/mergetree/merge_tree_writer.h
@@ -70,9 +70,8 @@ class MergeTreeWriter : public BatchWriter {
 
     Status Write(std::unique_ptr<RecordBatch>&& batch) override;
 
-    /// Consumes readers whose complete streams are individually sorted by primary key and
-    /// sequence number. Readers are closed on success or failure; an error may leave generated
-    /// file state unpublished, so the caller must discard this writer and replay its input.
+    /// Consumes readers whose complete streams are individually sorted by primary key and sequence
+    /// number. Readers are closed on success or failure.
     Status WriteSortedReaders(std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers);
 
     Status Compact(bool full_compaction) override;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index d6a23ccf9..f779b4d7d 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -35,7 +35,7 @@ class TableSchema;
 
 Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const TableSchema& schema);
 
-/// In-memory store for prepared primary-key real-time batches.
+/// Internal in-memory implementation of the default primary-key `RealtimeStore`.
 class PrimaryKeyRealtimeStore final : public RealtimeStore {
  public:
     static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index 2eaf7ce24..d65c7e533 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -42,7 +42,6 @@ class FieldsComparator;
 class RealtimeContextImpl;
 struct RealtimeStoreState;
 
-/// Coordinates framework-prepared primary-key real-time writes.
 class RealtimePrimaryKeyWriter final : public BatchWriter {
  public:
     static Result<std::shared_ptr<RealtimePrimaryKeyWriter>> Create(

From ff0c7bdbef7f50eaf79f26d9645202735868d802 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 00:32:24 +0800
Subject: [PATCH 46/93] fix(realtime): strengthen primary-key recovery coverage

---
 .../core/mergetree/merge_tree_writer_test.cpp |  58 +++++++
 .../core/realtime/realtime_context_impl.cpp   |  16 +-
 .../core/realtime/realtime_context_test.cpp   |   6 +-
 .../table/source/key_value_table_read.cpp     |   3 +
 test/inte/realtime_write_inte_test.cpp        | 158 ++++++++++++++++++
 5 files changed, 237 insertions(+), 4 deletions(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 63e896573..9ce5498cb 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -530,6 +530,64 @@ TEST_P(MergeTreeWriterTest, TestSortedReaders) {
     ASSERT_EQ(1, new_file->delete_row_count);
 }
 
+TEST_P(MergeTreeWriterTest, TestMergeSortedReaders) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+    ASSERT_OK_AND_ASSIGN(auto merge_writer,
+                         CreateMergeWriter(/*last_sequence_number=*/4, dir->Str(), path_factory,
+                                           /*schema_id=*/7, options));
+
+    auto first_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
+      [5, 0, "Alice", 10, 0, 15.1],
+      [7, 0, "Carol", 20, 1, 17.1],
+      [10, 0, "Eve", 30, 2, 20.1]
+    ])")
+            .ValueOrDie());
+    auto second_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
+      [6, 0, "Bob", 11, 0, 16.1],
+      [8, 3, "Carol", 21, 1, null],
+      [9, 0, "David", 22, 2, 19.1]
+    ])")
+            .ValueOrDie());
+    bool first_closed = false;
+    bool second_closed = false;
+    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
+    sorted_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
+        CreateSingleReader(first_array), &first_closed));
+    sorted_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
+        CreateSingleReader(second_array), &second_closed));
+
+    ASSERT_OK(merge_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_TRUE(first_closed);
+    ASSERT_TRUE(second_closed);
+    ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, merge_writer->PrepareCommit(false));
+    ASSERT_OK(merge_writer->Close());
+
+    ASSERT_EQ(1, commit_increment.GetNewFilesIncrement().NewFiles().size());
+    const std::shared_ptr<DataFileMeta>& new_file =
+        commit_increment.GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(5, new_file->row_count);
+    ASSERT_EQ(1, new_file->delete_row_count);
+    std::shared_ptr<arrow::ChunkedArray> expected_array;
+    ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(write_type_, {R"([
+      [5, 0, "Alice", 10, 0, 15.1],
+      [6, 0, "Bob", 11, 0, 16.1],
+      [8, 3, "Carol", 21, 1, null],
+      [9, 0, "David", 22, 2, 19.1],
+      [10, 0, "Eve", 30, 2, 20.1]
+    ])"},
+                                                                 &expected_array)
+                    .ok());
+    CheckFileContent(path_factory->ToPath(new_file), expected_array);
+}
+
 TEST_P(MergeTreeWriterTest, TestSortedReaderOwnership) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
                          CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index ba4c8b7a6..736ebb02d 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -59,6 +59,17 @@ bool SameMode(const RealtimeStoreCreateConfig& left, const RealtimeStoreCreateCo
     return true;
 }
 
+std::string PartitionToString(const std::map<std::string, std::string>& partition) {
+    std::string result = "{";
+    for (auto iter = partition.begin(); iter != partition.end(); ++iter) {
+        if (iter != partition.begin()) {
+            result += ", ";
+        }
+        result += iter->first + "=" + iter->second;
+    }
+    return result + "}";
+}
+
 }  // namespace
 
 RealtimeContextImpl::RealtimeContextImpl(const std::shared_ptr<RealtimeStoreFactory>& factory)
@@ -120,8 +131,9 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     if (iter != stores_.end()) {
         if (!SameMode(iter->second.mode_config, request.mode_config) ||
             !iter->second.write_schema->Equals(*requested_schema, /*check_metadata=*/true)) {
-            return Status::Invalid(
-                "real-time store schema or mode does not match the registered store");
+            return Status::Invalid("real-time store schema or mode mismatch for partition " +
+                                   PartitionToString(key.partition) + ", bucket " +
+                                   std::to_string(key.bucket) + "; recreate the RealtimeContext");
         }
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
                                iter->second.store->AcquireReadView());
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 2b47e9dc9..916b46aad 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -171,13 +171,15 @@ TEST(RealtimeContextTest, TestRejectsMismatchedSchemaOnStoreReuse) {
     ASSERT_NOK_WITH_MSG(
         GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(arrow::int32(), metadata), {},
                                GetDefaultPool()),
-        "schema or mode does not match");
+        "schema or mode mismatch for partition {dt=2026-08-02}, bucket 0; recreate the "
+        "RealtimeContext");
     ASSERT_NOK_WITH_MSG(
         GetOrCreateAppendStore(
             context, partition, 0,
             MakeWriteSchema(arrow::int64(), arrow::key_value_metadata({"identity"}, {"v2"})), {},
             GetDefaultPool()),
-        "schema or mode does not match");
+        "schema or mode mismatch for partition {dt=2026-08-02}, bucket 0; recreate the "
+        "RealtimeContext");
     ASSERT_EQ(1, factory->stores.size());
 }
 
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 96f3f00d9..3532b59b4 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -253,6 +253,9 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
     if (realtime_split->Version() != RealtimeSplit::kCurrentVersion) {
         return Status::Invalid("unsupported real-time split version");
     }
+    if (realtime_split->MemoryEndOffset() < realtime_split->CommittedEndOffset()) {
+        return Status::Invalid("real-time split memory end offset precedes committed end offset");
+    }
     const std::shared_ptr<RealtimeContext> realtime_context = context_->GetRealtimeContext();
     if (!realtime_context) {
         return Status::Invalid("reading a real-time split requires a real-time context");
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index a393f838d..53737fc2b 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -1281,6 +1281,30 @@ class RealtimeWriteInteTest : public ::testing::Test {
         ASSERT_EQ(expected_rows, actual_rows);
     }
 
+    void ReplayPkWalAndCommit(const std::vector<Row>& wal,
+                              const std::vector<RecordBatch::RowKind>& row_kinds,
+                              int64_t commit_identifier,
+                              const std::vector<Row>& expected_rows) const {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                             RealtimeContext::Create());
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                             CreateRealtimeWriter(realtime_context));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
+        ASSERT_OK(writer->Write(std::move(batch)));
+        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                             writer->PrepareCommitWithProgress(commit_identifier));
+        ASSERT_EQ(1, progress.size());
+        ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, commit_identifier));
+        ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+        ASSERT_OK(writer->Close());
+        writer.reset();
+        realtime_context.reset();
+
+        ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows());
+        ASSERT_EQ(expected_rows, actual_rows);
+    }
+
     void CheckDropDatePartitionRemovesOffset(bool legacy_partition_name_enabled) {
         fields_ = {arrow::field("id", arrow::int64()), arrow::field("payload", arrow::utf8()),
                    arrow::field("pt", arrow::date32())};
@@ -2861,6 +2885,44 @@ TEST_F(RealtimeWriteInteTest, TestPlanExcludesRowsWrittenAfterMemoryEndOffset) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkRejectsReversedVisibleOffsets) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    std::vector<Row> rows = {{1, "one", "p0"}, {2, "two", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch(rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ASSERT_EQ(1, plan->Splits().size());
+    std::shared_ptr<RealtimeSplit> split =
+        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
+    ASSERT_NE(nullptr, split);
+    std::vector<std::shared_ptr<Split>> disk_splits = split->DiskSplits();
+    std::vector<std::shared_ptr<Split>> invalid_splits = {std::make_shared<RealtimeSplit>(
+        split->Version(), split->SnapshotId(), split->Partition(), split->Bucket(),
+        std::move(disk_splits), split->MemoryEndOffset() + 1, split->MemoryEndOffset(),
+        split->OpaqueTicket())};
+
+    ReadContextBuilder read_builder(table_path_);
+    read_builder.SetOptions(options_)
+        .SetReadFieldNames({"id", "payload", "pt"})
+        .WithRealtimeContext(realtime_context)
+        .WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+                         TableRead::Create(std::move(read_context)));
+    ASSERT_NOK_WITH_MSG(table_read->CreateReader(invalid_splits),
+                        "memory end offset precedes committed end offset");
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
+    ASSERT_EQ(rows, actual_rows);
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestReadFailsAfterRealtimeSplitTicketExpires) {
     options_[Options::REALTIME_READ_VIEW_TTL] = "10 ms";
     CreateTable(/*partition_keys=*/{});
@@ -4392,4 +4454,100 @@ TEST_F(RealtimeWriteInteTest, TestRestoreOffsetFromCommittedSnapshot) {
     ASSERT_EQ(5, second_committed_offsets.at(partition_bucket));
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkExternalCommitRecovery) {
+    CreatePkTable();
+    const std::vector<Row> seed_rows = {{99, "seed", "p0"}};
+    ReplayPkWalAndCommit(seed_rows, /*row_kinds=*/{}, /*commit_identifier=*/0, seed_rows);
+
+    const std::vector<Row> wal = {
+        {1, "one", "p0"}, {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
+    const std::vector<RecordBatch::RowKind> row_kinds = {
+        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
+        RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT};
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
+                         CreateRealtimeWriter(failed_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
+                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
+    ASSERT_OK(failed_writer->Write(std::move(failed_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> failed_progress,
+                         failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, failed_progress.size());
+    CommitContextBuilder commit_builder(table_path_, commit_user_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
+                         commit_builder.SetOptions(options_).Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
+                         FileStoreCommit::Create(std::move(commit_context)));
+    IOHook* io_hook = IOHook::GetInstance();
+    ScopeGuard hook_guard([io_hook]() { io_hook->Clear(); });
+    io_hook->Reset(/*pos=*/0, IOHook::Mode::RETURN_ERROR);
+    Result<int64_t> failed_commit =
+        commit->CommitWithProgress(failed_progress, /*commit_identifier=*/1,
+                                   /*watermark=*/std::nullopt);
+    io_hook->Clear();
+    ASSERT_TRUE(failed_commit.status().IsIOError()) << failed_commit.status().ToString();
+    ASSERT_OK(failed_writer->Close());
+    failed_writer.reset();
+    failed_context.reset();
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failure, ReadRows());
+    ASSERT_EQ(seed_rows, rows_after_failure);
+
+    const std::vector<Row> expected_rows = {
+        {1, "one-new", "p0"}, {3, "three", "p0"}, {99, "seed", "p0"}};
+    ReplayPkWalAndCommit(wal, row_kinds, /*commit_identifier=*/1, expected_rows);
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkRefreshRecovery) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
+                         CreateRealtimeWriter(failed_context));
+
+    const std::vector<Row> base_rows = {{1, "one", "p0"}, {2, "two", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
+                         MakeBatch(base_rows, /*partitioned=*/false));
+    ASSERT_OK(failed_writer->Write(std::move(base_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
+                         failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t base_snapshot_id, Commit(base_progress, /*commit_identifier=*/0));
+    ASSERT_OK(failed_writer->RefreshCommittedSnapshot(base_snapshot_id));
+
+    const std::vector<Row> committed_wal = {
+        {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
+    const std::vector<RecordBatch::RowKind> committed_kinds = {RecordBatch::RowKind::UPDATE_AFTER,
+                                                               RecordBatch::RowKind::DELETE,
+                                                               RecordBatch::RowKind::INSERT};
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> committed_batch,
+        MakeBatch(committed_wal, /*partitioned=*/false, /*bucket=*/0, committed_kinds));
+    ASSERT_OK(failed_writer->Write(std::move(committed_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> committed_progress,
+                         failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(int64_t committed_snapshot_id,
+                         Commit(committed_progress, /*commit_identifier=*/1));
+
+    const std::vector<Row> replay_wal = {{4, "four", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> replay_batch,
+                         MakeBatch(replay_wal, /*partitioned=*/false));
+    ASSERT_OK(failed_writer->Write(std::move(replay_batch)));
+    IOHook* io_hook = IOHook::GetInstance();
+    ScopeGuard hook_guard([io_hook]() { io_hook->Clear(); });
+    io_hook->Reset(/*pos=*/0, IOHook::Mode::RETURN_ERROR);
+    Status failed_refresh = failed_writer->RefreshCommittedSnapshot(committed_snapshot_id);
+    io_hook->Clear();
+    ASSERT_TRUE(failed_refresh.IsIOError()) << failed_refresh.ToString();
+    ASSERT_OK(failed_writer->Close());
+    failed_writer.reset();
+    failed_context.reset();
+    const std::vector<Row> committed_rows = {{1, "one-new", "p0"}, {3, "three", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failure, ReadRows());
+    ASSERT_EQ(committed_rows, rows_after_failure);
+
+    const std::vector<Row> expected_rows = {
+        {1, "one-new", "p0"}, {3, "three", "p0"}, {4, "four", "p0"}};
+    ReplayPkWalAndCommit(replay_wal, /*row_kinds=*/{}, /*commit_identifier=*/2, expected_rows);
+}
+
 }  // namespace paimon::test

From 83ff3cbf98fee672910fed7228648ade0fef5e2b Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 10:15:22 +0800
Subject: [PATCH 47/93] test(realtime): strengthen failure recovery coverage

---
 include/paimon/realtime/realtime_store.h      |  12 +-
 .../core/io/single_file_writer_test.cpp       |   4 +-
 .../realtime/primary_key_realtime_store.cpp   |   8 +-
 .../realtime/primary_key_realtime_store.h     |   2 +-
 .../table/source/key_value_table_read.cpp     |   5 +-
 test/inte/realtime_write_inte_test.cpp        | 124 ++++++++++++++++++
 6 files changed, 141 insertions(+), 14 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 60d1afc39..03ef279a3 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -163,12 +163,12 @@ class PAIMON_EXPORT RealtimeStore {
     /// Creates readers over rows in `view`. Append mode returns rows whose offsets are greater than
     /// or equal to `offset_begin`; primary-key mode ignores `offset_begin`.
     ///
-    /// Append-mode batches contain `_VALUE_KIND` followed by the requested fields except a duplicate
-    /// `_VALUE_KIND`, and collectively expose every matching row exactly once. Primary-key batches
-    /// use the prepared transport schema and may contain multiple mutations per key; each reader's
-    /// complete stream is sorted by full primary key then sequence number, and the readers
-    /// collectively expose every raw mutation exactly once. Paimon retains `view` for the lifetime
-    /// of the resulting framework reader.
+    /// Append-mode batches contain `_VALUE_KIND` followed by the requested fields except a
+    /// duplicate `_VALUE_KIND`, and collectively expose every matching row exactly once.
+    /// Primary-key batches use the prepared transport schema and may contain multiple mutations
+    /// per key; each reader's complete stream is sorted by full primary key then sequence number,
+    /// and the readers collectively expose every raw mutation exactly once. Paimon retains `view`
+    /// for the lifetime of the resulting framework reader.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) = 0;
diff --git a/src/paimon/core/io/single_file_writer_test.cpp b/src/paimon/core/io/single_file_writer_test.cpp
index 78fce54c7..4136702e8 100644
--- a/src/paimon/core/io/single_file_writer_test.cpp
+++ b/src/paimon/core/io/single_file_writer_test.cpp
@@ -18,8 +18,10 @@
 
 #include "paimon/core/io/single_file_writer.h"
 
+#include <functional>
 #include <map>
-#include <utility>
+#include <memory>
+#include <string>
 
 #include "arrow/api.h"
 #include "arrow/c/abi.h"
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index e4c480377..22fab1b08 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -418,9 +418,9 @@ class PrimaryKeyRealtimeStore::Impl {
         return readers;
     }
 
-    Status AdvanceCommittedOffset(int64_t committed_end) {
+    Status AdvanceCommittedOffset(int64_t committed_end_offset) {
         std::lock_guard<std::mutex> lock(mutex_);
-        while (!sealed_.empty() && sealed_.front()->GetOffsetRange().end <= committed_end) {
+        while (!sealed_.empty() && sealed_.front()->GetOffsetRange().end <= committed_end_offset) {
             sealed_.erase(sealed_.begin());
         }
         return Status::OK();
@@ -499,8 +499,8 @@ Result<std::vector<std::unique_ptr<BatchReader>>> PrimaryKeyRealtimeStore::Creat
     const RealtimeQueryContext& context) {
     return impl_->CreateQueryReaders(view, offset, context);
 }
-Status PrimaryKeyRealtimeStore::AdvanceCommittedOffset(int64_t offset) {
-    return impl_->AdvanceCommittedOffset(offset);
+Status PrimaryKeyRealtimeStore::AdvanceCommittedOffset(int64_t committed_end_offset) {
+    return impl_->AdvanceCommittedOffset(committed_end_offset);
 }
 uint64_t PrimaryKeyRealtimeStore::GetMemoryUsage() const {
     return impl_->GetMemoryUsage();
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index f779b4d7d..52f9a6076 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -53,7 +53,7 @@ class PrimaryKeyRealtimeStore final : public RealtimeStore {
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) override;
-    Status AdvanceCommittedOffset(int64_t committed_offset) override;
+    Status AdvanceCommittedOffset(int64_t committed_end_offset) override;
     uint64_t GetMemoryUsage() const override;
 
  private:
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 3532b59b4..af585a1bb 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -150,7 +150,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
     const std::shared_ptr<Split>& split) {
     std::shared_ptr<RealtimeSplit> realtime_split = std::dynamic_pointer_cast<RealtimeSplit>(split);
     if (realtime_split) {
-        return CreateRealtimeReader(realtime_split, true);
+        return CreateRealtimeReader(realtime_split, /*release_ticket=*/true);
     }
 
     std::shared_ptr<Split> dispatch_split = split;
@@ -221,7 +221,8 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
             std::dynamic_pointer_cast<RealtimeSplit>(split);
         if (realtime_split) {
             PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
-                                   CreateRealtimeReader(realtime_split, false));
+                                   CreateRealtimeReader(realtime_split,
+                                                        /*release_ticket=*/false));
             readers.push_back(std::move(reader));
             realtime_splits.push_back(std::move(realtime_split));
         } else {
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 53737fc2b..918d1dd35 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -28,6 +28,7 @@
 #include <map>
 #include <memory>
 #include <mutex>
+#include <new>
 #include <set>
 #include <string>
 #include <thread>
@@ -78,6 +79,61 @@
 namespace paimon::test {
 namespace {
 
+class FailAllocationMemoryPool final : public MemoryPool {
+ public:
+    explicit FailAllocationMemoryPool(const std::shared_ptr<MemoryPool>& delegate)
+        : delegate_(delegate) {}
+
+    void FailAfterAllocations(int64_t successful_allocations) {
+        allocations_before_failure_.store(successful_allocations, std::memory_order_release);
+    }
+
+    void* Malloc(uint64_t size, uint64_t alignment = 0) override {
+        if (ShouldFail()) {
+            throw std::bad_alloc();
+        }
+        return delegate_->Malloc(size, alignment);
+    }
+
+    void* Realloc(void* p, size_t old_size, size_t new_size, uint64_t alignment = 0) override {
+        if (ShouldFail()) {
+            throw std::bad_alloc();
+        }
+        return delegate_->Realloc(p, old_size, new_size, alignment);
+    }
+
+    void Free(void* p, uint64_t size) override {
+        delegate_->Free(p, size);
+    }
+
+    void Free(void* p, uint64_t size, uint64_t alignment) override {
+        delegate_->Free(p, size, alignment);
+    }
+
+    uint64_t CurrentUsage() const override {
+        return delegate_->CurrentUsage();
+    }
+
+    uint64_t MaxMemoryUsage() const override {
+        return delegate_->MaxMemoryUsage();
+    }
+
+ private:
+    bool ShouldFail() {
+        int64_t remaining = allocations_before_failure_.load(std::memory_order_acquire);
+        while (remaining >= 0) {
+            if (allocations_before_failure_.compare_exchange_weak(remaining, remaining - 1,
+                                                                  std::memory_order_acq_rel)) {
+                return remaining == 0;
+            }
+        }
+        return false;
+    }
+
+    std::shared_ptr<MemoryPool> delegate_;
+    std::atomic<int64_t> allocations_before_failure_{-1};
+};
+
 class TrackingRealtimeReadView final : public RealtimeReadView {
  public:
     explicit TrackingRealtimeReadView(std::shared_ptr<RealtimeReadView> delegate)
@@ -2106,6 +2162,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRecovery) {
     ASSERT_EQ(1, NewFiles(progress).size());
     ASSERT_EQ(2, NewFiles(progress)[0]->min_sequence_number);
     ASSERT_EQ(memory_sequences.back(), NewFiles(progress)[0]->max_sequence_number);
+    ASSERT_EQ(1, NewFiles(progress)[0]->delete_row_count);
     ASSERT_OK(Commit(progress, /*commit_identifier=*/1));
     ASSERT_OK(first_writer->Close());
     first_context.reset();
@@ -4454,6 +4511,73 @@ TEST_F(RealtimeWriteInteTest, TestRestoreOffsetFromCommittedSnapshot) {
     ASSERT_EQ(5, second_committed_offsets.at(partition_bucket));
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkWriteFailureRecovery) {
+    CreatePkTable();
+    const std::vector<Row> seed_rows = {{99, "seed", "p0"}};
+    ReplayPkWalAndCommit(seed_rows, /*row_kinds=*/{}, /*commit_identifier=*/0, seed_rows);
+
+    const std::vector<Row> wal = {
+        {1, "one", "p0"}, {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
+    const std::vector<RecordBatch::RowKind> row_kinds = {
+        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
+        RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT};
+    std::shared_ptr<FailAllocationMemoryPool> failing_pool =
+        std::make_shared<FailAllocationMemoryPool>(pool_);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
+                         RealtimeContext::Create());
+    WriteContextBuilder failed_builder(table_path_, commit_user_);
+    failed_builder.SetOptions(options_)
+        .WithStreamingMode(true)
+        .WithRealtimeContext(failed_context)
+        .WithMemoryPool(failing_pool);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> failed_write_context,
+                         failed_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
+                         FileStoreWrite::Create(std::move(failed_write_context)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> empty_batch,
+                         MakeUnpartitionedBatchFromJson("[]"));
+    ASSERT_OK(failed_writer->Write(std::move(empty_batch)));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
+                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
+    failing_pool->FailAfterAllocations(1);
+    Status failed_write = failed_writer->Write(std::move(failed_batch));
+    ASSERT_TRUE(failed_write.IsOutOfMemory()) << failed_write.ToString();
+    ASSERT_OK(failed_writer->Close());
+    failed_writer.reset();
+    failed_context.reset();
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failure, ReadRows());
+    ASSERT_EQ(seed_rows, rows_after_failure);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> replay_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> replay_writer,
+                         CreateRealtimeWriter(replay_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> replay_batch,
+                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
+    ASSERT_OK(replay_writer->Write(std::move(replay_batch)));
+    const std::vector<Row> expected_rows = {
+        {1, "one-new", "p0"}, {3, "three", "p0"}, {99, "seed", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> replayed_rows, ReadRows(replay_context));
+    ASSERT_EQ(expected_rows, replayed_rows);
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         replay_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+    ASSERT_EQ(1, progress.size());
+    ASSERT_EQ(OffsetRange(1, 5), progress[0].offset_range);
+    ASSERT_EQ(1, NewFiles(progress).size());
+    ASSERT_EQ(1, NewFiles(progress)[0]->delete_row_count);
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/1));
+    ASSERT_OK(replay_writer->RefreshCommittedSnapshot(snapshot_id));
+    ASSERT_OK(replay_writer->Close());
+    replay_writer.reset();
+    replay_context.reset();
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> persisted_rows, ReadRows());
+    ASSERT_EQ(expected_rows, persisted_rows);
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
+    ASSERT_EQ(5, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkExternalCommitRecovery) {
     CreatePkTable();
     const std::vector<Row> seed_rows = {{99, "seed", "p0"}};

From 55581822a7d8ff87177711719a5f870431f5c719 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 11:24:04 +0800
Subject: [PATCH 48/93] refactor(realtime): simplify primary key state and
 tests

---
 src/paimon/common/table/special_fields.h      |   7 +
 .../common/table/special_fields_test.cpp      |   8 +
 .../merged_key_value_record_reader_test.cpp   |  12 +-
 .../operation/key_value_file_store_write.cpp  |   7 +-
 .../realtime/prepared_key_value_reader.cpp    |   3 +-
 .../primary_key_realtime_store_test.cpp       |   5 +-
 .../core/realtime/realtime_context_impl.cpp   |  12 +-
 .../core/realtime/realtime_context_impl.h     |  16 +-
 .../core/realtime/realtime_context_test.cpp   |  58 +++
 src/paimon/core/realtime/realtime_fields.h    |  35 --
 .../realtime/realtime_primary_key_writer.cpp  |   3 +-
 .../table/source/append_only_table_read.cpp   |   5 +-
 .../table/source/key_value_table_read.cpp     |   8 +-
 test/inte/realtime_write_inte_test.cpp        | 401 ++++++------------
 14 files changed, 234 insertions(+), 346 deletions(-)
 delete mode 100644 src/paimon/core/realtime/realtime_fields.h

diff --git a/src/paimon/common/table/special_fields.h b/src/paimon/common/table/special_fields.h
index 74b95b19c..3279bfed6 100644
--- a/src/paimon/common/table/special_fields.h
+++ b/src/paimon/common/table/special_fields.h
@@ -66,6 +66,13 @@ struct SpecialFields {
         return data_field;
     }
 
+    static const DataField& RealtimeOffset() {
+        static const DataField data_field =
+            DataField(SpecialFieldIds::REALTIME_OFFSET,
+                      arrow::field("_REALTIME_OFFSET", arrow::int64(), false));
+        return data_field;
+    }
+
     static bool IsSystemField(const std::string& field_name) {
         if (StringUtils::StartsWith(field_name, KEY_FIELD_PREFIX)) {
             return true;
diff --git a/src/paimon/common/table/special_fields_test.cpp b/src/paimon/common/table/special_fields_test.cpp
index 68e805fd6..b61d289b0 100644
--- a/src/paimon/common/table/special_fields_test.cpp
+++ b/src/paimon/common/table/special_fields_test.cpp
@@ -55,6 +55,13 @@ TEST(SpecialFieldsTest, TestIndexScore) {
     ASSERT_EQ(SpecialFields::IndexScore().Type()->id(), arrow::Type::FLOAT);
 }
 
+TEST(SpecialFieldsTest, TestRealtimeOffset) {
+    ASSERT_EQ(SpecialFields::RealtimeOffset().Id(), SpecialFieldIds::REALTIME_OFFSET);
+    ASSERT_EQ(SpecialFields::RealtimeOffset().Name(), "_REALTIME_OFFSET");
+    ASSERT_EQ(SpecialFields::RealtimeOffset().Type()->id(), arrow::Type::INT64);
+    ASSERT_FALSE(SpecialFields::RealtimeOffset().Nullable());
+}
+
 TEST(SpecialFieldsTest, TestKeyValueSpecialFieldCount) {
     ASSERT_EQ(SpecialFields::KEY_VALUE_SPECIAL_FIELD_COUNT, 2);
 }
@@ -66,6 +73,7 @@ TEST(SpecialFieldsTest, TestIsSystemField) {
     ASSERT_TRUE(SpecialFields::IsSystemField("rowkind"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_ROW_ID"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_INDEX_SCORE"));
+    ASSERT_FALSE(SpecialFields::IsSystemField("_REALTIME_OFFSET"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_KEY_0"));
 }
 
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index a0d65205c..79217828c 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -36,7 +36,6 @@
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
-#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/realtime/offset_range.h"
 #include "paimon/testing/mock/mock_file_batch_reader.h"
@@ -44,7 +43,6 @@
 #include "paimon/testing/utils/key_value_checker.h"
 #include "paimon/testing/utils/read_result_collector.h"
 #include "paimon/testing/utils/testharness.h"
-#include "paimon/utils/special_field_ids.h"
 
 namespace paimon::test {
 
@@ -62,7 +60,7 @@ std::shared_ptr<arrow::Schema> MakePreparedSchema(const arrow::FieldVector& valu
         DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
             ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
     prepared_fields.insert(prepared_fields.end(), value_fields.begin(), value_fields.end());
     return arrow::schema(prepared_fields);
 }
@@ -105,14 +103,6 @@ class MergedKeyValueRecordReaderTest : public testing::Test {
     std::shared_ptr<ReducerMergeFunctionWrapper> merge_function_wrapper_;
 };
 
-TEST_F(MergedKeyValueRecordReaderTest, TestRealtimeOffsetField) {
-    const DataField& field = RealtimeOffsetField();
-    ASSERT_EQ(SpecialFieldIds::REALTIME_OFFSET, field.Id());
-    ASSERT_EQ("_REALTIME_OFFSET", field.Name());
-    ASSERT_EQ(arrow::Type::INT64, field.Type()->id());
-    ASSERT_FALSE(field.Nullable());
-}
-
 TEST_F(MergedKeyValueRecordReaderTest, TestMergeAcrossUnderlyingBatches) {
     std::vector<DataField> fields = {DataField(0, arrow::field("k0", arrow::int32())),
                                      DataField(1, arrow::field("k1", arrow::int32())),
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index ee6445057..737ece080 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -36,7 +36,6 @@
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
-#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/realtime/realtime_primary_key_writer.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/utils/file_store_path_factory.h"
@@ -136,16 +135,16 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         partition_map =
             std::map<std::string, std::string>(partition_values.begin(), partition_values.end());
         PAIMON_ASSIGN_OR_RAISE(realtime_context_impl, RealtimeContextImpl::Cast(realtime_context_));
-        if (schema_->GetFieldByName(RealtimeOffsetField().Name())) {
+        if (schema_->GetFieldByName(SpecialFields::RealtimeOffset().Name())) {
             return Status::Invalid("PK real-time write schema contains reserved transport field " +
-                                   RealtimeOffsetField().Name());
+                                   SpecialFields::RealtimeOffset().Name());
         }
         arrow::FieldVector prepared_fields = {
             DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
                 ->WithNullable(false),
             DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
                 ->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+            DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
         prepared_fields.insert(prepared_fields.end(), schema_->fields().begin(),
                                schema_->fields().end());
         auto c_write_schema = std::make_unique<ArrowSchema>();
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 864456818..623e3f3fd 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -47,7 +47,6 @@
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/common/utils/scope_guard.h"
-#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
 #include "paimon/reader/batch_reader.h"
@@ -672,7 +671,7 @@ Status ValidatePreparedTransportSchema(const std::shared_ptr<arrow::Schema>& pre
     PAIMON_RETURN_NOT_OK(
         CheckPreparedField(prepared_schema, kSequenceNumberIndex, SpecialFields::SequenceNumber()));
     PAIMON_RETURN_NOT_OK(
-        CheckPreparedField(prepared_schema, kRealtimeOffsetIndex, RealtimeOffsetField()));
+        CheckPreparedField(prepared_schema, kRealtimeOffsetIndex, SpecialFields::RealtimeOffset()));
     return Status::OK();
 }
 
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index dc2ce86b1..384b937cb 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -34,7 +34,6 @@
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/core/core_options.h"
-#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/macros.h"
 #include "paimon/memory/memory_pool.h"
@@ -48,7 +47,7 @@ std::shared_ptr<arrow::Schema> PreparedSchema() {
         {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
          DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
              ->WithNullable(false),
-         DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false),
+         DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
          DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64()))),
          DataField::ConvertDataFieldToArrowField(
              DataField(1, arrow::field("value", arrow::utf8())))});
@@ -59,7 +58,7 @@ std::shared_ptr<arrow::Schema> NestedPreparedSchema() {
         {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
          DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
              ->WithNullable(false),
-         DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false),
+         DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
          DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64()))),
          DataField::ConvertDataFieldToArrowField(DataField(
              1,
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 736ebb02d..9b12aa5bd 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -161,8 +161,7 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     RealtimeStoreCreateConfig mode_config = request.mode_config;
     Result<std::shared_ptr<RealtimeStore>> store_result = factory_->Create(std::move(request));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
-    stores_.emplace(key,
-                    RealtimeStoreRegistryEntry{store, requested_schema, std::move(mode_config)});
+    stores_.emplace(key, StoreEntry{store, requested_schema, std::move(mode_config)});
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
     }
@@ -172,12 +171,11 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
 int64_t RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
     const RealtimePartitionBucket& partition_bucket, int64_t max_sequence_number) {
     std::lock_guard<std::mutex> lock(mutex_);
-    auto [iter, inserted] =
-        materialized_max_sequence_numbers_.emplace(partition_bucket, max_sequence_number);
-    if (!inserted && max_sequence_number > iter->second) {
-        iter->second = max_sequence_number;
+    StoreEntry& entry = stores_.at(partition_bucket);
+    if (max_sequence_number > entry.materialized_max_sequence_number) {
+        entry.materialized_max_sequence_number = max_sequence_number;
     }
-    return iter->second;
+    return entry.materialized_max_sequence_number;
 }
 
 Result<std::vector<RealtimePartitionBucketView>> RealtimeContextImpl::AcquireReadViews() {
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index f5118c18f..f0014176d 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -59,12 +59,6 @@ struct RealtimePartitionBucketView {
     std::shared_ptr<RealtimeReadView> read_view;
 };
 
-struct RealtimeStoreRegistryEntry {
-    std::shared_ptr<RealtimeStore> store;
-    std::shared_ptr<arrow::Schema> write_schema;
-    RealtimeStoreCreateConfig mode_config;
-};
-
 class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
  public:
     static Result<std::shared_ptr<RealtimeContextImpl>> Create(
@@ -102,6 +96,13 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
         std::chrono::steady_clock::time_point expire_at;
     };
 
+    struct StoreEntry {
+        std::shared_ptr<RealtimeStore> store;
+        std::shared_ptr<arrow::Schema> write_schema;
+        RealtimeStoreCreateConfig mode_config;
+        int64_t materialized_max_sequence_number = -1;
+    };
+
     explicit RealtimeContextImpl(const std::shared_ptr<RealtimeStoreFactory>& factory);
 
     Status Start();
@@ -111,8 +112,7 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     std::shared_ptr<RealtimeStoreFactory> factory_;
     std::mutex mutex_;
     std::mutex progress_mutex_;
-    std::map<RealtimePartitionBucket, RealtimeStoreRegistryEntry> stores_;
-    std::map<RealtimePartitionBucket, int64_t> materialized_max_sequence_numbers_;
+    std::map<RealtimePartitionBucket, StoreEntry> stores_;
     // Full-table progress used as the initial offset when a store is created lazily.
     RealtimeOffsetMap committed_offsets_;
     // Progress already reflected in stores owned by this context.
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 916b46aad..5dc5d8b4f 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -189,6 +189,9 @@ TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
     const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
     const RealtimePartitionBucket partition_bucket(partition, /*bucket=*/0);
 
+    ASSERT_OK(
+        GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, GetDefaultPool()));
+
     ASSERT_EQ(4, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
                                                                /*max_sequence_number=*/4));
     ASSERT_EQ(8, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
@@ -243,6 +246,28 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     ASSERT_EQ(std::vector<int64_t>({8}), factory->stores[1]->committed_offsets);
 }
 
+TEST(RealtimeContextTest, TestRemovedInactivePartitionDoesNotRequireReopen) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    const std::map<std::string, std::string> active_partition = {{"dt", "2026-08-02"}};
+    const std::map<std::string, std::string> inactive_partition = {{"dt", "2026-08-03"}};
+    const RealtimePartitionBucket active_partition_bucket(active_partition, /*bucket=*/0);
+    const RealtimePartitionBucket inactive_partition_bucket(inactive_partition, /*bucket=*/0);
+
+    ASSERT_OK(context->AdvanceCommittedProgress(
+        5, {{active_partition_bucket, /*offset=*/7}, {inactive_partition_bucket, /*offset=*/9}}));
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState active_state,
+                         GetOrCreateAppendStore(context, active_partition, 0, MakeWriteSchema(), {},
+                                                GetDefaultPool()));
+    ASSERT_EQ(7, active_state.initial_offset);
+
+    ASSERT_OK(context->AdvanceCommittedProgress(6, {{active_partition_bucket, /*offset=*/7}}));
+    ASSERT_OK_AND_ASSIGN(RealtimeStoreState inactive_state,
+                         GetOrCreateAppendStore(context, inactive_partition, 0, MakeWriteSchema(),
+                                                {}, GetDefaultPool()));
+    ASSERT_EQ(0, inactive_state.initial_offset);
+}
+
 TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
@@ -279,6 +304,39 @@ TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
     ASSERT_EQ(std::vector<int64_t>({8}), factory->stores[1]->committed_offsets);
 }
 
+TEST(RealtimeContextTest, TestRequiresReopenWhenCommittedProgressMovesBackwards) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    const std::map<std::string, std::string> first_partition = {{"dt", "2026-08-02"}};
+    const std::map<std::string, std::string> second_partition = {{"dt", "2026-08-03"}};
+    const RealtimePartitionBucket first_partition_bucket(first_partition, /*bucket=*/0);
+    const RealtimePartitionBucket second_partition_bucket(second_partition, /*bucket=*/0);
+
+    ASSERT_OK(GetOrCreateAppendStore(context, first_partition, 0, MakeWriteSchema(), {},
+                                     GetDefaultPool()));
+    ASSERT_OK(GetOrCreateAppendStore(context, second_partition, 0, MakeWriteSchema(), {},
+                                     GetDefaultPool()));
+    ASSERT_OK(context->AdvanceCommittedProgress(
+        5, {{first_partition_bucket, /*offset=*/7}, {second_partition_bucket, /*offset=*/9}}));
+    ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
+    ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[1]->committed_offsets);
+
+    ASSERT_NOK_WITH_MSG(
+        context->AdvanceCommittedProgress(
+            6, {{first_partition_bucket, /*offset=*/6}, {second_partition_bucket, /*offset=*/10}}),
+        "recreate RealtimeContext");
+    ASSERT_NOK_WITH_MSG(
+        context->AdvanceCommittedProgress(6, {{first_partition_bucket, /*offset=*/10}}),
+        "recreate RealtimeContext");
+    ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
+    ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[1]->committed_offsets);
+
+    ASSERT_OK(context->AdvanceCommittedProgress(
+        6, {{first_partition_bucket, /*offset=*/10}, {second_partition_bucket, /*offset=*/11}}));
+    ASSERT_EQ(std::vector<int64_t>({7, 10}), factory->stores[0]->committed_offsets);
+    ASSERT_EQ(std::vector<int64_t>({9, 11}), factory->stores[1]->committed_offsets);
+}
+
 TEST(RealtimeContextTest, TestPinsResolvesAndReleasesReadViewTicket) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
diff --git a/src/paimon/core/realtime/realtime_fields.h b/src/paimon/core/realtime/realtime_fields.h
deleted file mode 100644
index 270941238..000000000
--- a/src/paimon/core/realtime/realtime_fields.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include "arrow/type.h"
-#include "paimon/common/types/data_field.h"
-#include "paimon/utils/special_field_ids.h"
-
-namespace paimon {
-
-inline const DataField& RealtimeOffsetField() {
-    static const DataField data_field =
-        DataField(SpecialFieldIds::REALTIME_OFFSET,
-                  arrow::field("_REALTIME_OFFSET", arrow::int64(), /*nullable=*/false));
-    return data_field;
-}
-
-}  // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 185156eb7..7f4d4b5f7 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -38,7 +38,6 @@
 #include "paimon/core/mergetree/merge_tree_writer.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
-#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/utils/commit_increment.h"
 #include "paimon/macros.h"
 
@@ -169,7 +168,7 @@ Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Crea
         DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
             ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
     prepared_fields.insert(prepared_fields.end(), write_schema->fields().begin(),
                            write_schema->fields().end());
     const RealtimePartitionBucket partition_bucket(partition, bucket);
diff --git a/src/paimon/core/table/source/append_only_table_read.cpp b/src/paimon/core/table/source/append_only_table_read.cpp
index 34c6ef850..12b3823a7 100644
--- a/src/paimon/core/table/source/append_only_table_read.cpp
+++ b/src/paimon/core/table/source/append_only_table_read.cpp
@@ -111,10 +111,7 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateReader(
                 realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
         }
     }
-    std::unique_ptr<BatchReader> result =
-        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
-    cleanup_guard.Release();
-    return result;
+    return std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
 }
 
 Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateRealtimeReader(
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index af585a1bb..32231140e 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -38,7 +38,6 @@
 #include "paimon/core/operation/raw_file_split_read.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
-#include "paimon/core/realtime/realtime_fields.h"
 #include "paimon/core/realtime/realtime_reader.h"
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/table/source/pk_count_reader.h"
@@ -68,7 +67,7 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
         DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
             ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(RealtimeOffsetField())->WithNullable(false)};
+        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
     prepared_fields.insert(prepared_fields.end(), full_value_schema->fields().begin(),
                            full_value_schema->fields().end());
     std::shared_ptr<arrow::Schema> prepared_schema = arrow::schema(std::move(prepared_fields));
@@ -243,10 +242,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateReader(
                 realtime_context_impl->ReleaseReadView(realtime_split->OpaqueTicket()));
         }
     }
-    std::unique_ptr<BatchReader> result =
-        std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
-    cleanup_guard.Release();
-    return result;
+    return std::make_unique<ConcatBatchReader>(std::move(readers), GetMemoryPool());
 }
 
 Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 918d1dd35..7c143c995 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -23,6 +23,7 @@
 #include <condition_variable>
 #include <cstdint>
 #include <deque>
+#include <functional>
 #include <iterator>
 #include <limits>
 #include <map>
@@ -177,12 +178,10 @@ class ReadViewCheckingBatchReader final : public BatchReader {
     std::weak_ptr<RealtimeReadView> read_view_;
 };
 
-class QueryTrackingRealtimeStore final : public RealtimeStore {
+class DelegatingRealtimeStore : public RealtimeStore {
  public:
-    QueryTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                               const std::shared_ptr<std::atomic<bool>>& saw_query_predicate,
-                               const std::shared_ptr<std::weak_ptr<RealtimeReadView>>& query_view)
-        : delegate_(delegate), saw_query_predicate_(saw_query_predicate), query_view_(query_view) {}
+    explicit DelegatingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
+        : delegate_(delegate) {}
 
     Status Write(RealtimeWriteBatch&& batch) override {
         return delegate_->Write(std::move(batch));
@@ -197,6 +196,64 @@ class QueryTrackingRealtimeStore final : public RealtimeStore {
         return delegate_->CreateCommitReaders(segment);
     }
 
+    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
+        return delegate_->AcquireReadView();
+    }
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        return delegate_->CreateQueryReaders(view, offset_begin, context);
+    }
+
+    Status AdvanceCommittedOffset(int64_t committed_offset) override {
+        return delegate_->AdvanceCommittedOffset(committed_offset);
+    }
+
+    uint64_t GetMemoryUsage() const override {
+        return delegate_->GetMemoryUsage();
+    }
+
+ protected:
+    std::shared_ptr<RealtimeStore> delegate_;
+};
+
+class DecoratingRealtimeStoreFactory final : public RealtimeStoreFactory {
+ public:
+    using Decorator =
+        std::function<std::shared_ptr<RealtimeStore>(const std::shared_ptr<RealtimeStore>&)>;
+
+    explicit DecoratingRealtimeStoreFactory(Decorator decorator)
+        : decorator_(std::move(decorator)) {}
+
+    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
+                               delegate_.Create(std::move(request)));
+        return decorator_(delegate);
+    }
+
+ private:
+    ArrowRealtimeStoreFactory delegate_;
+    Decorator decorator_;
+};
+
+template <typename Store, typename... Args>
+std::shared_ptr<RealtimeStoreFactory> MakeDecoratingFactory(Args... args) {
+    return std::make_shared<DecoratingRealtimeStoreFactory>(
+        [=](const std::shared_ptr<RealtimeStore>& delegate) -> std::shared_ptr<RealtimeStore> {
+            return std::make_shared<Store>(delegate, args...);
+        });
+}
+
+class QueryTrackingRealtimeStore final : public DelegatingRealtimeStore {
+ public:
+    QueryTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
+                               const std::shared_ptr<std::atomic<bool>>& saw_query_predicate,
+                               const std::shared_ptr<std::weak_ptr<RealtimeReadView>>& query_view)
+        : DelegatingRealtimeStore(delegate),
+          saw_query_predicate_(saw_query_predicate),
+          query_view_(query_view) {}
+
     Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> delegate_view,
                                delegate_->AcquireReadView());
@@ -225,36 +282,7 @@ class QueryTrackingRealtimeStore final : public RealtimeStore {
         return readers;
     }
 
-    Status AdvanceCommittedOffset(int64_t committed_offset) override {
-        return delegate_->AdvanceCommittedOffset(committed_offset);
-    }
-
-    uint64_t GetMemoryUsage() const override {
-        return delegate_->GetMemoryUsage();
-    }
-
  private:
-    std::shared_ptr<RealtimeStore> delegate_;
-    std::shared_ptr<std::atomic<bool>> saw_query_predicate_;
-    std::shared_ptr<std::weak_ptr<RealtimeReadView>> query_view_;
-};
-
-class QueryTrackingRealtimeStoreFactory final : public RealtimeStoreFactory {
- public:
-    explicit QueryTrackingRealtimeStoreFactory(
-        const std::shared_ptr<std::atomic<bool>>& saw_query_predicate,
-        const std::shared_ptr<std::weak_ptr<RealtimeReadView>>& query_view)
-        : saw_query_predicate_(saw_query_predicate), query_view_(query_view) {}
-
-    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
-                               delegate_.Create(std::move(request)));
-        return std::shared_ptr<RealtimeStore>(std::make_shared<QueryTrackingRealtimeStore>(
-            delegate, saw_query_predicate_, query_view_));
-    }
-
- private:
-    ArrowRealtimeStoreFactory delegate_;
     std::shared_ptr<std::atomic<bool>> saw_query_predicate_;
     std::shared_ptr<std::weak_ptr<RealtimeReadView>> query_view_;
 };
@@ -292,19 +320,11 @@ struct CloseTrackingReaderState {
     int32_t commit_null_index = -1;
 };
 
-class CloseTrackingRealtimeStore final : public RealtimeStore {
+class CloseTrackingRealtimeStore final : public DelegatingRealtimeStore {
  public:
     CloseTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
                                const std::shared_ptr<CloseTrackingReaderState>& state)
-        : delegate_(delegate), state_(state) {}
-
-    Status Write(RealtimeWriteBatch&& batch) override {
-        return delegate_->Write(std::move(batch));
-    }
-
-    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
-        return delegate_->SealForCommit();
-    }
+        : DelegatingRealtimeStore(delegate), state_(state) {}
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
@@ -318,10 +338,6 @@ class CloseTrackingRealtimeStore final : public RealtimeStore {
         return readers;
     }
 
-    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
-        return delegate_->AcquireReadView();
-    }
-
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) override {
@@ -335,14 +351,6 @@ class CloseTrackingRealtimeStore final : public RealtimeStore {
         return readers;
     }
 
-    Status AdvanceCommittedOffset(int64_t committed_offset) override {
-        return delegate_->AdvanceCommittedOffset(committed_offset);
-    }
-
-    uint64_t GetMemoryUsage() const override {
-        return delegate_->GetMemoryUsage();
-    }
-
  private:
     static Status InsertNullReader(int32_t index,
                                    std::vector<std::unique_ptr<BatchReader>>* readers) {
@@ -356,25 +364,6 @@ class CloseTrackingRealtimeStore final : public RealtimeStore {
         return Status::OK();
     }
 
-    std::shared_ptr<RealtimeStore> delegate_;
-    std::shared_ptr<CloseTrackingReaderState> state_;
-};
-
-class CloseTrackingRealtimeStoreFactory final : public RealtimeStoreFactory {
- public:
-    explicit CloseTrackingRealtimeStoreFactory(
-        const std::shared_ptr<CloseTrackingReaderState>& state)
-        : state_(state) {}
-
-    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
-                               delegate_.Create(std::move(request)));
-        return std::shared_ptr<RealtimeStore>(
-            std::make_shared<CloseTrackingRealtimeStore>(delegate, state_));
-    }
-
- private:
-    ArrowRealtimeStoreFactory delegate_;
     std::shared_ptr<CloseTrackingReaderState> state_;
 };
 
@@ -421,18 +410,10 @@ class SplitBatchReader final : public BatchReader {
     int64_t next_row_ = 0;
 };
 
-class SplitCommitReaderRealtimeStore final : public RealtimeStore {
+class SplitCommitReaderRealtimeStore final : public DelegatingRealtimeStore {
  public:
     explicit SplitCommitReaderRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
-        : delegate_(delegate) {}
-
-    Status Write(RealtimeWriteBatch&& batch) override {
-        return delegate_->Write(std::move(batch));
-    }
-
-    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
-        return delegate_->SealForCommit();
-    }
+        : DelegatingRealtimeStore(delegate) {}
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
@@ -443,48 +424,39 @@ class SplitCommitReaderRealtimeStore final : public RealtimeStore {
         }
         return readers;
     }
+};
 
-    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
-        return delegate_->AcquireReadView();
-    }
+enum class CommitReaderMalformation { DROP_LAST, UNSORTED, SUBSTITUTE_OFFSET };
 
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
-        const RealtimeQueryContext& context) override {
-        return delegate_->CreateQueryReaders(view, offset_begin, context);
-    }
+class CorruptingBatchReader final : public BatchReader {
+ public:
+    CorruptingBatchReader(std::unique_ptr<BatchReader> delegate,
+                          CommitReaderMalformation malformation)
+        : delegate_(std::move(delegate)), malformation_(malformation) {}
 
-    Status AdvanceCommittedOffset(int64_t committed_offset) override {
-        return delegate_->AdvanceCommittedOffset(committed_offset);
+    Result<ReadBatch> NextBatch() override {
+        switch (malformation_) {
+            case CommitReaderMalformation::DROP_LAST:
+                return DropLast();
+            case CommitReaderMalformation::UNSORTED:
+                return SwapFirstTwo();
+            case CommitReaderMalformation::SUBSTITUTE_OFFSET:
+                return SubstituteOffset();
+        }
+        return Status::Invalid("unknown commit reader malformation");
     }
 
-    uint64_t GetMemoryUsage() const override {
-        return delegate_->GetMemoryUsage();
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
     }
 
- private:
-    std::shared_ptr<RealtimeStore> delegate_;
-};
-
-class SplitCommitReaderRealtimeStoreFactory final : public RealtimeStoreFactory {
- public:
-    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
-                               delegate_.Create(std::move(request)));
-        return std::shared_ptr<RealtimeStore>(
-            std::make_shared<SplitCommitReaderRealtimeStore>(delegate));
+    void Close() override {
+        buffered_.reset();
+        delegate_->Close();
     }
 
  private:
-    ArrowRealtimeStoreFactory delegate_;
-};
-
-class DropLastBatchReader final : public BatchReader {
- public:
-    explicit DropLastBatchReader(std::unique_ptr<BatchReader> delegate)
-        : delegate_(std::move(delegate)) {}
-
-    Result<ReadBatch> NextBatch() override {
+    Result<ReadBatch> DropLast() {
         if (!buffered_.has_value()) {
             PAIMON_ASSIGN_OR_RAISE(ReadBatch first, delegate_->NextBatch());
             if (BatchReader::IsEofBatch(first)) {
@@ -502,72 +474,34 @@ class DropLastBatchReader final : public BatchReader {
         return result;
     }
 
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        buffered_.reset();
-        delegate_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> delegate_;
-    std::optional<ReadBatch> buffered_;
-};
-
-class SwapFirstTwoBatchReader final : public BatchReader {
- public:
-    explicit SwapFirstTwoBatchReader(std::unique_ptr<BatchReader> delegate)
-        : delegate_(std::move(delegate)) {}
-
-    Result<ReadBatch> NextBatch() override {
-        if (!initialized_) {
-            initialized_ = true;
-            PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
-            if (BatchReader::IsEofBatch(batch)) {
-                return MakeEofBatch();
-            }
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-                std::shared_ptr<arrow::Array> array,
-                arrow::ImportArray(batch.first.get(), batch.second.get()));
-            if (array->length() < 2) {
-                return Status::Invalid("cannot make a one-row reader unsorted");
-            }
-            arrow::ArrayVector pieces = {array->Slice(1, 1), array->Slice(0, 1)};
-            if (array->length() > 2) {
-                pieces.push_back(array->Slice(2));
-            }
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> swapped,
-                                              arrow::Concatenate(pieces));
-            auto output = std::make_unique<ArrowArray>();
-            auto schema = std::make_unique<ArrowSchema>();
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(
-                arrow::ExportArray(*swapped, output.get(), schema.get()));
-            return ReadBatch(std::move(output), std::move(schema));
+    Result<ReadBatch> SwapFirstTwo() {
+        if (corrupted_) {
+            return delegate_->NextBatch();
         }
-        return delegate_->NextBatch();
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        delegate_->Close();
+        corrupted_ = true;
+        PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
+        if (BatchReader::IsEofBatch(batch)) {
+            return MakeEofBatch();
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> array,
+            arrow::ImportArray(batch.first.get(), batch.second.get()));
+        if (array->length() < 2) {
+            return Status::Invalid("cannot make a one-row reader unsorted");
+        }
+        arrow::ArrayVector pieces = {array->Slice(1, 1), array->Slice(0, 1)};
+        if (array->length() > 2) {
+            pieces.push_back(array->Slice(2));
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> swapped,
+                                          arrow::Concatenate(pieces));
+        auto output = std::make_unique<ArrowArray>();
+        auto schema = std::make_unique<ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*swapped, output.get(), schema.get()));
+        return ReadBatch(std::move(output), std::move(schema));
     }
 
- private:
-    bool initialized_ = false;
-    std::unique_ptr<BatchReader> delegate_;
-};
-
-class SubstituteOffsetBatchReader final : public BatchReader {
- public:
-    explicit SubstituteOffsetBatchReader(std::unique_ptr<BatchReader> delegate)
-        : delegate_(std::move(delegate)) {}
-
-    Result<ReadBatch> NextBatch() override {
+    Result<ReadBatch> SubstituteOffset() {
         PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
         if (BatchReader::IsEofBatch(batch)) {
             return batch;
@@ -602,86 +536,29 @@ class SubstituteOffsetBatchReader final : public BatchReader {
         return ReadBatch(std::move(output), std::move(schema));
     }
 
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        delegate_->Close();
-    }
-
- private:
     std::unique_ptr<BatchReader> delegate_;
+    CommitReaderMalformation malformation_;
+    bool corrupted_ = false;
+    std::optional<ReadBatch> buffered_;
 };
 
-enum class CommitReaderMalformation { DROP_LAST, UNSORTED, SUBSTITUTE_OFFSET };
-
-class MalformedCoverageRealtimeStore final : public RealtimeStore {
+class MalformedCoverageRealtimeStore final : public DelegatingRealtimeStore {
  public:
     MalformedCoverageRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
                                    CommitReaderMalformation malformation)
-        : delegate_(delegate), malformation_(malformation) {}
+        : DelegatingRealtimeStore(delegate), malformation_(malformation) {}
 
-    Status Write(RealtimeWriteBatch&& batch) override {
-        return delegate_->Write(std::move(batch));
-    }
-    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
-        return delegate_->SealForCommit();
-    }
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
         PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                                delegate_->CreateCommitReaders(segment));
         for (std::unique_ptr<BatchReader>& reader : readers) {
-            switch (malformation_) {
-                case CommitReaderMalformation::DROP_LAST:
-                    reader = std::make_unique<DropLastBatchReader>(std::move(reader));
-                    break;
-                case CommitReaderMalformation::UNSORTED:
-                    reader = std::make_unique<SwapFirstTwoBatchReader>(std::move(reader));
-                    break;
-                case CommitReaderMalformation::SUBSTITUTE_OFFSET:
-                    reader = std::make_unique<SubstituteOffsetBatchReader>(std::move(reader));
-                    break;
-            }
+            reader = std::make_unique<CorruptingBatchReader>(std::move(reader), malformation_);
         }
         return readers;
     }
-    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
-        return delegate_->AcquireReadView();
-    }
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
-        const RealtimeQueryContext& context) override {
-        return delegate_->CreateQueryReaders(view, offset_begin, context);
-    }
-    Status AdvanceCommittedOffset(int64_t committed_offset) override {
-        return delegate_->AdvanceCommittedOffset(committed_offset);
-    }
-    uint64_t GetMemoryUsage() const override {
-        return delegate_->GetMemoryUsage();
-    }
 
  private:
-    std::shared_ptr<RealtimeStore> delegate_;
-    CommitReaderMalformation malformation_;
-};
-
-class MalformedCoverageRealtimeStoreFactory final : public RealtimeStoreFactory {
- public:
-    explicit MalformedCoverageRealtimeStoreFactory(
-        CommitReaderMalformation malformation = CommitReaderMalformation::DROP_LAST)
-        : malformation_(malformation) {}
-
-    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
-                               delegate_.Create(std::move(request)));
-        return std::shared_ptr<RealtimeStore>(
-            std::make_shared<MalformedCoverageRealtimeStore>(delegate, malformation_));
-    }
-
- private:
-    ArrowRealtimeStoreFactory delegate_;
     CommitReaderMalformation malformation_;
 };
 
@@ -1062,16 +939,6 @@ class RealtimeWriteInteTest : public ::testing::Test {
         return commit->Expire();
     }
 
-    Status CommitMessages(const std::vector<std::shared_ptr<CommitMessage>>& messages,
-                          int64_t commit_identifier) const {
-        CommitContextBuilder builder(table_path_, commit_user_);
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
-                               builder.SetOptions(options_).Finish());
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
-                               FileStoreCommit::Create(std::move(context)));
-        return commit->Commit(messages, commit_identifier);
-    }
-
     Result<std::shared_ptr<Plan>> CreatePlan(
         const std::shared_ptr<RealtimeContext>& realtime_context,
         const std::shared_ptr<Predicate>& predicate) const {
@@ -1413,7 +1280,7 @@ class RealtimeWriteInteTest : public ::testing::Test {
             CreateTable(/*partition_keys=*/{"pt"});
         }
         auto close_state = std::make_shared<CloseTrackingReaderState>();
-        auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(close_state);
+        auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(close_state);
         ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                              RealtimeContext::Create(factory));
         ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -1524,7 +1391,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) {
     auto saw_query_predicate = std::make_shared<std::atomic<bool>>(false);
     auto query_view = std::make_shared<std::weak_ptr<RealtimeReadView>>();
     auto factory =
-        std::make_shared<QueryTrackingRealtimeStoreFactory>(saw_query_predicate, query_view);
+        MakeDecoratingFactory<QueryTrackingRealtimeStore>(saw_query_predicate, query_view);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -2257,7 +2124,12 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
             std::max(compacted_live_max_sequence_number, file->max_sequence_number);
     }
     ASSERT_LT(compacted_live_max_sequence_number, kHistoricalMaxSequenceNumber);
-    ASSERT_OK(CommitMessages(compact_messages, /*commit_identifier=*/4));
+    CommitContextBuilder commit_builder(table_path_, commit_user_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
+                         commit_builder.SetOptions(options_).Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
+                         FileStoreCommit::Create(std::move(commit_context)));
+    ASSERT_OK(commit->Commit(compact_messages, /*commit_identifier=*/4));
     ASSERT_OK(compact_writer->Close());
 
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(options_));
@@ -2304,7 +2176,7 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
 
 TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
     CreatePkTable();
-    auto factory = std::make_shared<SplitCommitReaderRealtimeStoreFactory>();
+    auto factory = MakeDecoratingFactory<SplitCommitReaderRealtimeStore>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -2332,7 +2204,8 @@ TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsMalformedCoverage) {
     CreatePkTable();
-    auto factory = std::make_shared<MalformedCoverageRealtimeStoreFactory>();
+    auto factory =
+        MakeDecoratingFactory<MalformedCoverageRealtimeStore>(CommitReaderMalformation::DROP_LAST);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -2348,7 +2221,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRejectsMalformedCoverage) {
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsEqualCardinalityOffsetSubstitution) {
     CreatePkTable();
-    auto factory = std::make_shared<MalformedCoverageRealtimeStoreFactory>(
+    auto factory = MakeDecoratingFactory<MalformedCoverageRealtimeStore>(
         CommitReaderMalformation::SUBSTITUTE_OFFSET);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
@@ -2366,7 +2239,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRejectsEqualCardinalityOffsetSubstitution) {
 TEST_F(RealtimeWriteInteTest, TestPkRejectsUnsortedPluginRows) {
     CreatePkTable();
     auto factory =
-        std::make_shared<MalformedCoverageRealtimeStoreFactory>(CommitReaderMalformation::UNSORTED);
+        MakeDecoratingFactory<MalformedCoverageRealtimeStore>(CommitReaderMalformation::UNSORTED);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -2383,7 +2256,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRejectsUnsortedPluginRows) {
 TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
     CreatePkTable();
     auto state = std::make_shared<CloseTrackingReaderState>();
-    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
+    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -2421,7 +2294,7 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
 TEST_F(RealtimeWriteInteTest, TestPkQueryReaderCloseFailure) {
     CreatePkTable();
     auto state = std::make_shared<CloseTrackingReaderState>();
-    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
+    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -2462,7 +2335,7 @@ TEST_F(RealtimeWriteInteTest, TestAppendQueryReaderCloseFailure) {
     CreateTable(/*partition_keys=*/{});
     auto state = std::make_shared<CloseTrackingReaderState>();
     state->query_null_index = 1;
-    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
+    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
@@ -2496,7 +2369,7 @@ TEST_F(RealtimeWriteInteTest, TestPkCommitReaderCloseFailure) {
     CreatePkTable();
     auto state = std::make_shared<CloseTrackingReaderState>();
     state->commit_null_index = 1;
-    auto factory = std::make_shared<CloseTrackingRealtimeStoreFactory>(state);
+    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                          RealtimeContext::Create(factory));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,

From e68ac16f7fb8ccac9c4e09ad1588dab934d4d681 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 11:43:44 +0800
Subject: [PATCH 49/93] test(realtime): simplify integration test setup

---
 test/inte/realtime_write_inte_test.cpp | 140 ++++++++++---------------
 1 file changed, 57 insertions(+), 83 deletions(-)

diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 7c143c995..86f8a4a54 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -954,6 +954,27 @@ class RealtimeWriteInteTest : public ::testing::Test {
         return scan->CreatePlan();
     }
 
+    Result<std::unique_ptr<BatchReader>> CreateQueryReader(
+        const std::shared_ptr<Plan>& plan,
+        const std::shared_ptr<RealtimeContext>& realtime_context) const {
+        ReadContextBuilder read_builder(table_path_);
+        read_builder.SetOptions(options_)
+            .SetReadFieldNames({"id", "payload", "pt"})
+            .WithRealtimeContext(realtime_context)
+            .WithMemoryPool(pool_);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
+                               TableRead::Create(std::move(read_context)));
+        return table_read->CreateReader(plan->Splits());
+    }
+
+    Result<std::unique_ptr<BatchReader>> CreateQueryReader(
+        const std::shared_ptr<RealtimeContext>& realtime_context) const {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
+                               CreatePlan(realtime_context, /*predicate=*/nullptr));
+        return CreateQueryReader(plan, realtime_context);
+    }
+
     Result<CollectedReadResult> ReadPlan(const std::shared_ptr<Plan>& plan,
                                          const std::shared_ptr<RealtimeContext>& realtime_context,
                                          const std::vector<std::string>& read_fields,
@@ -1329,6 +1350,23 @@ class RealtimeWriteInteTest : public ::testing::Test {
         ASSERT_OK(writer->Close());
     }
 
+    void CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation malformation,
+                                                const std::string& expected_error) {
+        CreatePkTable();
+        auto factory = MakeDecoratingFactory<MalformedCoverageRealtimeStore>(malformation);
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                             RealtimeContext::Create(factory));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                             CreateRealtimeWriter(realtime_context));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
+                                       /*partitioned=*/false));
+        ASSERT_OK(writer->Write(std::move(batch)));
+        ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
+                            expected_error);
+        ASSERT_OK(writer->Close());
+    }
+
     std::unique_ptr<UniqueTestDirectory> dir_;
     std::string table_path_;
     std::string commit_user_ = "realtime_commit_user";
@@ -2203,54 +2241,19 @@ TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsMalformedCoverage) {
-    CreatePkTable();
-    auto factory =
-        MakeDecoratingFactory<MalformedCoverageRealtimeStore>(CommitReaderMalformation::DROP_LAST);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
-                                   /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
-                        "commit readers did not cover the sealed range");
-    ASSERT_OK(writer->Close());
+    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::DROP_LAST,
+                                           "commit readers did not cover the sealed range");
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsEqualCardinalityOffsetSubstitution) {
-    CreatePkTable();
-    auto factory = MakeDecoratingFactory<MalformedCoverageRealtimeStore>(
-        CommitReaderMalformation::SUBSTITUTE_OFFSET);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
-                                   /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
-                        "duplicate REALTIME_OFFSET");
-    ASSERT_OK(writer->Close());
+    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::SUBSTITUTE_OFFSET,
+                                           "duplicate REALTIME_OFFSET");
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsUnsortedPluginRows) {
-    CreatePkTable();
-    auto factory =
-        MakeDecoratingFactory<MalformedCoverageRealtimeStore>(CommitReaderMalformation::UNSORTED);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
-                                   /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
-                        "not globally sorted by primary key and sequence number");
-    ASSERT_OK(writer->Close());
+    CheckPkRejectsCommitReaderMalformation(
+        CommitReaderMalformation::UNSORTED,
+        "not globally sorted by primary key and sequence number");
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
@@ -2265,27 +2268,19 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
                          MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
     ASSERT_OK(writer->Write(std::move(batch)));
 
-    auto create_reader = [&]() -> Result<std::unique_ptr<BatchReader>> {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
-                               CreatePlan(realtime_context, /*predicate=*/nullptr));
-        ReadContextBuilder read_builder(table_path_);
-        read_builder.SetOptions(options_)
-            .SetReadFieldNames({"id", "payload", "pt"})
-            .WithRealtimeContext(realtime_context)
-            .WithMemoryPool(pool_);
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
-                               TableRead::Create(std::move(read_context)));
-        return table_read->CreateReader(plan->Splits());
+    auto release_reader = [&](bool explicit_close) -> Status {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
+                               CreateQueryReader(realtime_context));
+        if (explicit_close) {
+            reader->Close();
+        }
+        return Status::OK();
     };
 
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> explicitly_closed_reader, create_reader());
-    explicitly_closed_reader->Close();
-    explicitly_closed_reader.reset();
+    ASSERT_OK(release_reader(/*explicit_close=*/true));
     ASSERT_EQ(1, state->query_close_count->load(std::memory_order_acquire));
 
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> destroyed_reader, create_reader());
-    destroyed_reader.reset();
+    ASSERT_OK(release_reader(/*explicit_close=*/false));
     ASSERT_EQ(2, state->query_close_count->load(std::memory_order_acquire));
 
     ASSERT_OK(writer->Close());
@@ -2309,23 +2304,10 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderCloseFailure) {
                          MakeBatch({Row{2, "two", "p0"}}, /*partitioned=*/false));
     ASSERT_OK(writer->Write(std::move(second_batch)));
 
-    auto create_reader = [&]() -> Result<std::unique_ptr<BatchReader>> {
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
-                               CreatePlan(realtime_context, /*predicate=*/nullptr));
-        ReadContextBuilder read_builder(table_path_);
-        read_builder.SetOptions(options_)
-            .SetReadFieldNames({"id", "payload", "pt"})
-            .WithRealtimeContext(realtime_context)
-            .WithMemoryPool(pool_);
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
-                               TableRead::Create(std::move(read_context)));
-        return table_read->CreateReader(plan->Splits());
-    };
-
     for (int32_t null_index = 0; null_index <= 1; ++null_index) {
         state->query_null_index = null_index;
-        ASSERT_NOK_WITH_MSG(create_reader(), "PK real-time store returned a null query reader");
+        ASSERT_NOK_WITH_MSG(CreateQueryReader(realtime_context),
+                            "PK real-time store returned a null query reader");
         ASSERT_EQ(null_index + 1, state->query_close_count->load(std::memory_order_acquire));
     }
     ASSERT_OK(writer->Close());
@@ -2347,15 +2329,7 @@ TEST_F(RealtimeWriteInteTest, TestAppendQueryReaderCloseFailure) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                          CreatePlan(realtime_context, /*predicate=*/nullptr));
 
-    ReadContextBuilder read_builder(table_path_);
-    read_builder.SetOptions(options_)
-        .SetReadFieldNames({"id", "payload", "pt"})
-        .WithRealtimeContext(realtime_context)
-        .WithMemoryPool(pool_);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
-                         TableRead::Create(std::move(read_context)));
-    ASSERT_NOK_WITH_MSG(table_read->CreateReader(plan->Splits()),
+    ASSERT_NOK_WITH_MSG(CreateQueryReader(plan, realtime_context),
                         "append-only real-time store returned a null query reader");
     ASSERT_EQ(1, state->query_close_count->load(std::memory_order_acquire));
 

From f92fe6a5cafff2092d0102d83a1def57ffd15840 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 13:18:36 +0800
Subject: [PATCH 50/93] fix(realtime): reject PK read-optimized scans

---
 src/paimon/core/table/source/table_scan.cpp   |   9 +-
 .../system/read_optimized_system_table.cpp    |   4 +
 test/inte/realtime_write_inte_test.cpp        | 258 +++++++++++++++++-
 3 files changed, 258 insertions(+), 13 deletions(-)

diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index f894e1a74..7f9fe568b 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -219,7 +219,7 @@ Result<std::unique_ptr<TableScan>> TableScan::Create(std::unique_ptr<ScanContext
 namespace {
 
 Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options,
-                            const ScanContext& context) {
+                            const ScanContext& context, bool read_optimized) {
     if (!context.GetRealtimeContext()) {
         return Status::OK();
     }
@@ -233,6 +233,10 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
         return Status::Invalid("real-time union read does not support data evolution");
     }
     if (!table_schema.PrimaryKeys().empty()) {
+        if (read_optimized) {
+            return Status::NotImplemented(
+                "PK real-time union read does not support read-optimized scans");
+        }
         PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(core_options, table_schema));
     }
     if (context.IsStreamingMode()) {
@@ -282,7 +286,8 @@ Result<std::unique_ptr<TableScan>> NewDataTableScan(const std::shared_ptr<ScanCo
                            CoreOptions::FromMap(options, context->GetSpecificFileSystem(), {}));
     core_options.WithCache(context->GetCache());
 
-    PAIMON_RETURN_NOT_OK(ValidateRealtimeScan(*table_schema, core_options, *context));
+    PAIMON_RETURN_NOT_OK(
+        ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized));
     // validate options
     if (core_options.GetBucket() == -1) {
         if (!table_schema->PrimaryKeys().empty()) {
diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp
index 6abec946b..d7bfa0912 100644
--- a/src/paimon/core/table/system/read_optimized_system_table.cpp
+++ b/src/paimon/core/table/system/read_optimized_system_table.cpp
@@ -58,6 +58,10 @@ std::map<std::string, std::string> ReadOptimizedSystemTable::ReadOptimizedOption
 
 Result<std::unique_ptr<TableScan>> ReadOptimizedSystemTable::NewScan(
     const std::shared_ptr<ScanContext>& context) const {
+    if (context->GetRealtimeContext() && !table_schema_->PrimaryKeys().empty()) {
+        return Status::NotImplemented(
+            "PK real-time union read does not support read-optimized scans");
+    }
     auto options = ReadOptimizedOptions();
     ScanContextBuilder builder(table_path_);
     builder.SetOptions(options)
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 86f8a4a54..f10e93858 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -58,6 +58,7 @@
 #include "paimon/defs.h"
 #include "paimon/file_store_commit.h"
 #include "paimon/file_store_write.h"
+#include "paimon/fs/file_system.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/orphan_files_cleaner.h"
 #include "paimon/predicate/function.h"
@@ -80,6 +81,33 @@
 namespace paimon::test {
 namespace {
 
+bool HasSuffix(const std::string& value, const std::string& suffix) {
+    return value.size() >= suffix.size() &&
+           value.compare(value.size() - suffix.size(), suffix.size(), suffix) == 0;
+}
+
+Result<std::set<std::string>> ListPhysicalArtifacts(const std::shared_ptr<FileSystem>& file_system,
+                                                    const std::string& root) {
+    std::set<std::string> artifacts;
+    std::vector<std::string> directories = {root};
+    while (!directories.empty()) {
+        std::string directory = std::move(directories.back());
+        directories.pop_back();
+        std::vector<BasicFileStatus> statuses;
+        PAIMON_RETURN_NOT_OK(file_system->ListDir(directory, &statuses));
+        for (const BasicFileStatus& status : statuses) {
+            if (status.IsDir()) {
+                directories.push_back(status.GetPath());
+            } else if (HasSuffix(status.GetPath(), ".orc") ||
+                       HasSuffix(status.GetPath(), ".index") ||
+                       HasSuffix(status.GetPath(), ".channel")) {
+                artifacts.insert(status.GetPath());
+            }
+        }
+    }
+    return artifacts;
+}
+
 class FailAllocationMemoryPool final : public MemoryPool {
  public:
     explicit FailAllocationMemoryPool(const std::shared_ptr<MemoryPool>& delegate)
@@ -426,6 +454,90 @@ class SplitCommitReaderRealtimeStore final : public DelegatingRealtimeStore {
     }
 };
 
+class FailAfterPhysicalFileBatchReader final : public BatchReader {
+ public:
+    FailAfterPhysicalFileBatchReader(std::unique_ptr<BatchReader> delegate,
+                                     const std::shared_ptr<FileSystem>& file_system,
+                                     std::string root, size_t baseline_artifact_count,
+                                     const std::shared_ptr<std::atomic<bool>>& saw_artifacts)
+        : delegate_(std::move(delegate)),
+          file_system_(file_system),
+          root_(std::move(root)),
+          baseline_artifact_count_(baseline_artifact_count),
+          saw_artifacts_(saw_artifacts) {}
+
+    Result<ReadBatch> NextBatch() override {
+        if (returned_batch_count_ < 4) {
+            ++returned_batch_count_;
+            return delegate_->NextBatch();
+        }
+        const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30);
+        while (std::chrono::steady_clock::now() < deadline) {
+            PAIMON_ASSIGN_OR_RAISE(std::set<std::string> artifacts,
+                                   ListPhysicalArtifacts(file_system_, root_));
+            bool has_data = false;
+            for (const std::string& artifact : artifacts) {
+                has_data = has_data || HasSuffix(artifact, ".orc");
+            }
+            if (artifacts.size() > baseline_artifact_count_ && has_data) {
+                saw_artifacts_->store(true, std::memory_order_release);
+                return Status::IOError(
+                    "injected commit reader failure after physical file creation");
+            }
+            std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        }
+        return Status::IOError("timed out waiting for partial physical files");
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    std::shared_ptr<FileSystem> file_system_;
+    std::string root_;
+    size_t baseline_artifact_count_;
+    std::shared_ptr<std::atomic<bool>> saw_artifacts_;
+    int32_t returned_batch_count_ = 0;
+};
+
+class FailAfterPhysicalFileRealtimeStore final : public DelegatingRealtimeStore {
+ public:
+    FailAfterPhysicalFileRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
+                                       const std::shared_ptr<FileSystem>& file_system,
+                                       const std::string& root, size_t baseline_artifact_count,
+                                       const std::shared_ptr<std::atomic<bool>>& saw_artifacts)
+        : DelegatingRealtimeStore(delegate),
+          file_system_(file_system),
+          root_(root),
+          baseline_artifact_count_(baseline_artifact_count),
+          saw_artifacts_(saw_artifacts) {}
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               delegate_->CreateCommitReaders(segment));
+        if (readers.empty()) {
+            return Status::Invalid("commit reader failure test requires a reader");
+        }
+        readers[0] = std::make_unique<FailAfterPhysicalFileBatchReader>(
+            std::make_unique<SplitBatchReader>(std::move(readers[0])), file_system_, root_,
+            baseline_artifact_count_, saw_artifacts_);
+        return readers;
+    }
+
+ private:
+    std::shared_ptr<FileSystem> file_system_;
+    std::string root_;
+    size_t baseline_artifact_count_;
+    std::shared_ptr<std::atomic<bool>> saw_artifacts_;
+};
+
 enum class CommitReaderMalformation { DROP_LAST, UNSORTED, SUBSTITUTE_OFFSET };
 
 class CorruptingBatchReader final : public BatchReader {
@@ -1339,9 +1451,7 @@ class RealtimeWriteInteTest : public ::testing::Test {
                              TableRead::Create(std::move(read_context)));
         ASSERT_NOK_WITH_MSG(table_read->CreateReader(invalid_splits),
                             "unsupported real-time split version");
-        if (!primary_key) {
-            ASSERT_EQ(1, close_state->query_close_count->load(std::memory_order_acquire));
-        }
+        ASSERT_EQ(1, close_state->query_close_count->load(std::memory_order_acquire));
 
         std::vector<Row> expected_rows = p0_rows;
         expected_rows.insert(expected_rows.end(), p1_rows.begin(), p1_rows.end());
@@ -1510,6 +1620,33 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) {
     ASSERT_TRUE(query_view->expired());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkRealtimeReadOptimizedScanUnsupported) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    const std::vector<Row> rows = {{1, "one", "p0"}, {2, "two", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch(rows, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
+    ASSERT_EQ(rows, disk_rows);
+
+    ScanContextBuilder scan_builder(table_path_ + "$ro");
+    scan_builder.SetOptions(options_).WithRealtimeContext(realtime_context).WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ScanContext> scan_context, scan_builder.Finish());
+    Result<std::unique_ptr<TableScan>> scan = TableScan::Create(std::move(scan_context));
+    ASSERT_TRUE(scan.status().IsNotImplemented()) << scan.status().ToString();
+    ASSERT_NE(std::string::npos, scan.status().ToString().find(
+                                     "PK real-time union read does not support read-optimized"));
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkDeleteInsertAndPinnedReadsAcrossRefresh) {
     CreatePkTable();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
@@ -2358,6 +2495,59 @@ TEST_F(RealtimeWriteInteTest, TestPkCommitReaderCloseFailure) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkPrepareFailureCleansPartialPhysicalFiles) {
+    options_[Options::WRITE_BATCH_SIZE] = "1";
+    options_[Options::TARGET_FILE_ROW_NUM] = "1";
+    options_["file-index.bitmap.columns"] = "payload";
+    options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
+    CreatePkTable();
+    std::shared_ptr<FileSystem> file_system = dir_->GetFileSystem();
+    ASSERT_OK_AND_ASSIGN(std::set<std::string> baseline_artifacts,
+                         ListPhysicalArtifacts(file_system, dir_->Str()));
+    auto saw_artifacts = std::make_shared<std::atomic<bool>>(false);
+    auto factory = MakeDecoratingFactory<FailAfterPhysicalFileRealtimeStore>(
+        file_system, dir_->Str(), baseline_artifacts.size(), saw_artifacts);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
+                         RealtimeContext::Create(factory));
+    WriteContextBuilder failed_builder(table_path_, commit_user_);
+    failed_builder.SetOptions(options_)
+        .WithStreamingMode(true)
+        .WithRealtimeContext(failed_context)
+        .WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp"));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> failed_write_context,
+                         failed_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
+                         FileStoreWrite::Create(std::move(failed_write_context)));
+
+    const std::vector<Row> wal = {
+        {1, "old", "p0"}, {1, "new", "p0"}, {2, "two", "p0"}, {2, "gone", "p0"}};
+    const std::vector<RecordBatch::RowKind> row_kinds = {
+        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
+        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
+                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
+    ASSERT_OK(failed_writer->Write(std::move(failed_batch)));
+    Result<std::vector<RealtimeCommitProgress>> failed_prepare =
+        failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/0);
+    ASSERT_TRUE(failed_prepare.status().IsIOError()) << failed_prepare.status().ToString();
+    ASSERT_NE(std::string::npos,
+              failed_prepare.status().ToString().find(
+                  "injected commit reader failure after physical file creation"));
+    ASSERT_TRUE(saw_artifacts->load(std::memory_order_acquire));
+    ASSERT_OK_AND_ASSIGN(std::set<std::string> artifacts_after_abort,
+                         ListPhysicalArtifacts(file_system, dir_->Str()));
+    ASSERT_EQ(baseline_artifacts, artifacts_after_abort);
+
+    ASSERT_OK(failed_writer->Close());
+    failed_writer.reset();
+    failed_context.reset();
+
+    const std::vector<Row> expected_rows = {{1, "new", "p0"}};
+    ReplayPkWalAndCommit(wal, row_kinds, /*commit_identifier=*/0, expected_rows);
+    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
+    ASSERT_EQ(4, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+}
+
 TEST_F(RealtimeWriteInteTest, TestRollingFilesPreserveProgress) {
     options_[Options::TARGET_FILE_ROW_NUM] = "10";
     CreateTable(/*partition_keys=*/{});
@@ -3819,7 +4009,45 @@ void RealtimeWriteInteTest::RunConcurrencyTest(bool primary_key) {
     constexpr int32_t kReadThreadCount = 4;
     constexpr int64_t kBatchCount = 12;
     constexpr int64_t kRowsPerBatch = 2;
-    constexpr int64_t kTotalRows = kBatchCount * kRowsPerBatch;
+    const int64_t total_rows = kBatchCount * (primary_key ? 3 : kRowsPerBatch);
+
+    std::vector<std::vector<Row>> pk_batches;
+    std::vector<std::vector<RecordBatch::RowKind>> pk_row_kinds;
+    std::vector<std::vector<Row>> pk_expected_states(1);
+    if (primary_key) {
+        std::map<int64_t, Row> current_rows;
+        for (int64_t batch_index = 0; batch_index < kBatchCount; ++batch_index) {
+            const int64_t key = batch_index % 4;
+            const int64_t deleted_key = (key + 2) % 4;
+            std::vector<Row> rows = {{key, "update-" + std::to_string(batch_index), "p0"},
+                                     {key, "latest-" + std::to_string(batch_index), "p0"},
+                                     {deleted_key, "deleted-" + std::to_string(batch_index), "p0"}};
+            pk_batches.push_back(rows);
+            pk_row_kinds.push_back({batch_index < 4 ? RecordBatch::RowKind::INSERT
+                                                    : RecordBatch::RowKind::UPDATE_AFTER,
+                                    RecordBatch::RowKind::UPDATE_AFTER,
+                                    RecordBatch::RowKind::DELETE});
+            current_rows[key] = rows[1];
+            current_rows.erase(deleted_key);
+            std::vector<Row> expected;
+            for (const auto& [id, row] : current_rows) {
+                static_cast<void>(id);
+                expected.push_back(row);
+            }
+            pk_expected_states.push_back(std::move(expected));
+        }
+    }
+
+    auto validate_read = [&](const std::vector<Row>& rows) {
+        if (!primary_key) {
+            return ValidateReadPrefix(rows, total_rows);
+        }
+        if (std::find(pk_expected_states.begin(), pk_expected_states.end(), rows) ==
+            pk_expected_states.end()) {
+            return Status::Invalid("PK real-time read does not match any completed write");
+        }
+        return Status::OK();
+    };
 
     std::atomic<bool> writer_done{false};
     std::atomic<bool> prepare_done{false};
@@ -3856,10 +4084,14 @@ void RealtimeWriteInteTest::RunConcurrencyTest(bool primary_key) {
         state.WaitForStart();
         for (int64_t batch_index = 0; batch_index < kBatchCount && !state.ShouldStop();
              ++batch_index) {
-            std::vector<Row> rows =
-                MakeRows(batch_index * kRowsPerBatch, kRowsPerBatch, /*partition=*/"p0");
+            std::vector<Row> rows = primary_key
+                                        ? pk_batches[static_cast<size_t>(batch_index)]
+                                        : MakeRows(batch_index * kRowsPerBatch, kRowsPerBatch,
+                                                   /*partition=*/"p0");
             Result<std::unique_ptr<RecordBatch>> batch_result =
-                MakeBatch(rows, /*partitioned=*/false);
+                primary_key ? MakeBatch(rows, /*partitioned=*/false, /*bucket=*/0,
+                                        pk_row_kinds[static_cast<size_t>(batch_index)])
+                            : MakeBatch(rows, /*partitioned=*/false);
             if (state.RecordErrorIfNotOk(batch_result)) {
                 break;
             }
@@ -3994,7 +4226,7 @@ void RealtimeWriteInteTest::RunConcurrencyTest(bool primary_key) {
                 if (state.RecordErrorIfNotOk(result)) {
                     break;
                 }
-                Status status = ValidateReadPrefix(result.value(), kTotalRows);
+                Status status = validate_read(result.value());
                 if (state.RecordErrorIfNotOk(status)) {
                     break;
                 }
@@ -4036,10 +4268,14 @@ void RealtimeWriteInteTest::RunConcurrencyTest(bool primary_key) {
     ASSERT_GE(commit_count.load(), 2);
     ASSERT_GE(refresh_count.load(), 2);
     ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows(realtime_context));
-    ASSERT_EQ(kTotalRows, static_cast<int64_t>(final_rows.size()));
-    ASSERT_OK(ValidateReadPrefix(final_rows, kTotalRows));
+    if (primary_key) {
+        ASSERT_EQ(pk_expected_states.back(), final_rows);
+    } else {
+        ASSERT_EQ(total_rows, static_cast<int64_t>(final_rows.size()));
+        ASSERT_OK(ValidateReadPrefix(final_rows, total_rows));
+    }
     ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
-    ASSERT_EQ(kTotalRows,
+    ASSERT_EQ(total_rows,
               committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
     ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
     ASSERT_EQ(0, memory_usage);

From b187e07dc3ba056d4f46620a38f6e489d65c00a2 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 14:17:59 +0800
Subject: [PATCH 51/93] fix(realtime): harden prepared store handling

---
 include/paimon/realtime/realtime_store.h      |  6 ++
 .../merged_key_value_record_reader_test.cpp   | 97 +++++++++++++------
 .../realtime/prepared_key_value_reader.cpp    | 20 +---
 .../core/realtime/prepared_key_value_reader.h | 10 +-
 .../core/realtime/realtime_context_impl.cpp   | 10 +-
 .../core/realtime/realtime_context_impl.h     |  4 +-
 .../core/realtime/realtime_context_test.cpp   | 31 ++++--
 .../realtime/realtime_primary_key_writer.cpp  | 12 ++-
 8 files changed, 119 insertions(+), 71 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 03ef279a3..9ed1e4361 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -55,15 +55,21 @@ struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {
 using RealtimeStoreCreateConfig =
     std::variant<AppendRealtimeStoreCreateConfig, PrimaryKeyRealtimeStoreCreateConfig>;
 
+/// Parameters used by a `RealtimeStoreFactory` to create a store.
 struct PAIMON_EXPORT RealtimeStoreCreateRequest {
     /// Schema whose ownership is transferred to the factory. Append mode receives the complete
     /// table write schema. Primary-key mode receives the prepared transport schema:
     /// [_VALUE_KIND, _SEQUENCE_NUMBER, _REALTIME_OFFSET, table write fields].
     std::unique_ptr<::ArrowSchema> write_schema;
+    /// Table options available to the store implementation.
     std::map<std::string, std::string> options;
+    /// Memory pool for allocations retained by the store.
     std::shared_ptr<MemoryPool> memory_pool;
+    /// Partition values identifying the store.
     std::map<std::string, std::string> partition;
+    /// Bucket identifying the store within its partition.
     int32_t bucket = -1;
+    /// Mode-specific store configuration.
     RealtimeStoreCreateConfig mode_config;
 };
 
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 79217828c..81a1f133e 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -65,6 +65,23 @@ std::shared_ptr<arrow::Schema> MakePreparedSchema(const arrow::FieldVector& valu
     return arrow::schema(prepared_fields);
 }
 
+Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderForTest(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::optional<OffsetRange>& visible_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    if (!key_schema) {
+        return Status::Invalid("prepared key schema cannot be null");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> key_fields,
+                           DataField::ConvertArrowSchemaToDataFields(key_schema));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
+                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
+    return AdaptPreparedBatchReader(std::move(reader), prepared_schema, visible_offsets, key_schema,
+                                    value_schema, key_comparator, memory_pool);
+}
+
 class TrackingBatchReader : public BatchReader {
  public:
     TrackingBatchReader(std::unique_ptr<BatchReader>&& delegate, int32_t* close_count)
@@ -217,8 +234,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderOffsetFilter) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(2, 4),
-                                 key_schema, value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(2, 4),
+                                        key_schema, value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> results,
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
@@ -232,6 +249,23 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderOffsetFilter) {
     KeyValueChecker::CheckResult(expected, results, 1, 2);
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsReversedVisibleOffsets) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+    auto batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1);
+
+    Result<std::unique_ptr<KeyValueRecordReader>> result =
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(2, 1),
+                                        value_schema, value_schema, pool_);
+    ASSERT_TRUE(result.status().IsInvalid());
+    ASSERT_NOK_WITH_MSG(result, "prepared visible offset range begin exceeds end");
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestRejectsUnsortedPluginRowsAcrossBatches) {
     std::vector<DataField> value_fields = {DataField(0, arrow::field("id", arrow::int32()))};
     std::shared_ptr<arrow::Schema> value_schema =
@@ -247,9 +281,10 @@ TEST_F(MergedKeyValueRecordReaderTest, TestRejectsUnsortedPluginRowsAcrossBatche
             .ValueOrDie();
     auto batch_reader =
         std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
-                         AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema,
-                                                  std::nullopt, key_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, std::nullopt,
+                                        key_schema, value_schema, pool_));
     Result<std::vector<KeyValue>> result =
         ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
                                                    KeyValueRecordReader::Iterator>(reader.get());
@@ -270,8 +305,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
         std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> query_reader,
-        AdaptPreparedBatchReader(std::move(query_batch_reader), prepared_schema, OffsetRange(0, 1),
-                                 value_schema, value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(query_batch_reader), prepared_schema,
+                                        OffsetRange(0, 1), value_schema, value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> query_results,
         (ReadResultCollector::CollectKeyValueResult<
@@ -281,9 +316,10 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
     ASSERT_EQ(query_results[0].value->GetInt(0), 1);
 
     auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-    ASSERT_NOK_WITH_MSG(AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema,
-                                                 std::nullopt, value_schema, value_schema, pool_),
-                        "exact");
+    ASSERT_NOK_WITH_MSG(
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, std::nullopt,
+                                        value_schema, value_schema, pool_),
+        "exact");
 }
 
 TEST_F(MergedKeyValueRecordReaderTest, TestBadCommitBatch) {
@@ -299,8 +335,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestBadCommitBatch) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, std::nullopt,
-                                 arrow::schema({key}), value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, std::nullopt,
+                                        arrow::schema({key}), value_schema, pool_));
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
@@ -319,8 +355,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestMissingCompositeKey) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                 arrow::schema({key0, key1}), value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                        arrow::schema({key0, key1}), value_schema, pool_));
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "cannot find field id 1");
 }
 
@@ -341,8 +377,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestQueryAddRename) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                 arrow::schema({key}), value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                        arrow::schema({key}), value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader::Iterator> iterator,
                          reader->NextBatch());
     ASSERT_OK_AND_ASSIGN(KeyValue key_value, iterator->Next());
@@ -361,8 +397,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestMergedReaderErrorRetry) {
     failing_reader->SetNextBatchStatus(Status::IOError("stable prepared error"));
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(failing_reader), prepared_schema, OffsetRange(0, 1),
-                                 value_schema, value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(failing_reader), prepared_schema,
+                                        OffsetRange(0, 1), value_schema, value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<FieldsComparator> key_comparator,
                          FieldsComparator::Create({DataField(0, key)}, true));
     MergedKeyValueRecordReader merged_reader(std::move(reader), key_comparator,
@@ -390,8 +426,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                 value_schema, value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                        value_schema, value_schema, pool_));
     ASSERT_NOK_WITH_MSG(
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
                                                     KeyValueRecordReader::Iterator>(reader.get())),
@@ -448,8 +484,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedProjection) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReader(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                 key_schema, query_value_schema, pool_));
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                        key_schema, query_value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> results,
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
@@ -516,8 +552,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
             &explicit_close_count);
         ASSERT_OK_AND_ASSIGN(
             std::unique_ptr<KeyValueRecordReader> reader,
-            AdaptPreparedBatchReader(std::move(tracking_reader), prepared_schema, OffsetRange(0, 1),
-                                     key_schema, value_schema, pool_));
+            AdaptPreparedBatchReaderForTest(std::move(tracking_reader), prepared_schema,
+                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
         reader->Close();
     }
     ASSERT_EQ(explicit_close_count, 1);
@@ -529,8 +565,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
             &destructor_close_count);
         ASSERT_OK_AND_ASSIGN(
             std::unique_ptr<KeyValueRecordReader> reader,
-            AdaptPreparedBatchReader(std::move(tracking_reader), prepared_schema, OffsetRange(0, 1),
-                                     key_schema, value_schema, pool_));
+            AdaptPreparedBatchReaderForTest(std::move(tracking_reader), prepared_schema,
+                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
     }
     ASSERT_EQ(destructor_close_count, 1);
 
@@ -540,8 +576,9 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
             std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
             &factory_failure_close_count);
         std::shared_ptr<arrow::Schema> invalid_schema = arrow::schema(value_schema->fields());
-        ASSERT_NOK(AdaptPreparedBatchReader(std::move(tracking_reader), invalid_schema,
-                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
+        ASSERT_NOK(AdaptPreparedBatchReaderForTest(std::move(tracking_reader), invalid_schema,
+                                                   OffsetRange(0, 1), key_schema, value_schema,
+                                                   pool_));
         ASSERT_EQ(nullptr, tracking_reader);
     }
     ASSERT_EQ(factory_failure_close_count, 1);
@@ -555,8 +592,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
                                                                      &read_failure_close_count);
         ASSERT_OK_AND_ASSIGN(
             std::unique_ptr<KeyValueRecordReader> reader,
-            AdaptPreparedBatchReader(std::move(tracking_reader), prepared_schema, OffsetRange(0, 1),
-                                     key_schema, value_schema, pool_));
+            AdaptPreparedBatchReaderForTest(std::move(tracking_reader), prepared_schema,
+                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
         ASSERT_NOK_WITH_MSG(reader->NextBatch(), "prepared reader failure");
         ASSERT_EQ(read_failure_close_count, 1);
     }
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 623e3f3fd..fa1dba7ef 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -690,6 +690,9 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
         return Status::Invalid("prepared batch reader cannot be null");
     }
     ScopeGuard close_guard([&owned_reader]() -> void { owned_reader->Close(); });
+    if (visible_offsets.has_value() && visible_offsets->begin > visible_offsets->end) {
+        return Status::Invalid("prepared visible offset range begin exceeds end");
+    }
     PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(prepared_schema));
     if (!key_schema) {
         return Status::Invalid("prepared key schema cannot be null");
@@ -765,21 +768,4 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
     return adapted_readers;
 }
 
-Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::optional<OffsetRange>& visible_offsets,
-    const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
-    if (!key_schema) {
-        return Status::Invalid("prepared key schema cannot be null");
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> key_fields,
-                           DataField::ConvertArrowSchemaToDataFields(key_schema));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
-                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
-    return AdaptPreparedBatchReader(std::move(reader), prepared_schema, visible_offsets, key_schema,
-                                    value_schema, key_comparator, memory_pool);
-}
-
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
index 22a837a76..81ae0abc7 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/prepared_key_value_reader.h
@@ -33,8 +33,10 @@ class BatchReader;
 class FieldsComparator;
 class MemoryPool;
 
+/// Validates the required leading fields of a prepared real-time transport schema.
 Status ValidatePreparedTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema);
 
+/// Adapts a plugin query reader and limits its rows to `visible_offsets` when present.
 Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::optional<OffsetRange>& visible_offsets,
@@ -43,6 +45,7 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool);
 
+/// Adapts commit readers and validates that they collectively cover `sealed_offsets` exactly.
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBatchReaders(
     std::vector<std::unique_ptr<BatchReader>>&& readers,
     const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
@@ -51,11 +54,4 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
     const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool);
 
-Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::optional<OffsetRange>& visible_offsets,
-    const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool);
-
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 9b12aa5bd..c9f719b7e 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -168,10 +168,16 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     return RealtimeStoreState{std::move(store), initial_offset};
 }
 
-int64_t RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
+Result<int64_t> RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
     const RealtimePartitionBucket& partition_bucket, int64_t max_sequence_number) {
     std::lock_guard<std::mutex> lock(mutex_);
-    StoreEntry& entry = stores_.at(partition_bucket);
+    auto iter = stores_.find(partition_bucket);
+    if (iter == stores_.end()) {
+        return Status::KeyError("real-time store not found for partition " +
+                                PartitionToString(partition_bucket.partition) + ", bucket " +
+                                std::to_string(partition_bucket.bucket));
+    }
+    StoreEntry& entry = iter->second;
     if (max_sequence_number > entry.materialized_max_sequence_number) {
         entry.materialized_max_sequence_number = max_sequence_number;
     }
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index f0014176d..fd65fc246 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -71,8 +71,8 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
 
     Result<RealtimeStoreState> GetOrCreateRealtimeStore(RealtimeStoreCreateRequest&& request);
 
-    int64_t AdvanceMaterializedMaxSequenceNumber(const RealtimePartitionBucket& partition_bucket,
-                                                 int64_t max_sequence_number);
+    Result<int64_t> AdvanceMaterializedMaxSequenceNumber(
+        const RealtimePartitionBucket& partition_bucket, int64_t max_sequence_number);
 
     Result<std::vector<RealtimePartitionBucketView>> AcquireReadViews();
 
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 5dc5d8b4f..538e4c56c 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -192,14 +192,29 @@ TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
     ASSERT_OK(
         GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, GetDefaultPool()));
 
-    ASSERT_EQ(4, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
-                                                               /*max_sequence_number=*/4));
-    ASSERT_EQ(8, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
-                                                               /*max_sequence_number=*/8));
-    ASSERT_EQ(8, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
-                                                               /*max_sequence_number=*/6));
-    ASSERT_EQ(10, context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
-                                                                /*max_sequence_number=*/10));
+    ASSERT_OK_AND_ASSIGN(int64_t first, context->AdvanceMaterializedMaxSequenceNumber(
+                                            partition_bucket, /*max_sequence_number=*/4));
+    ASSERT_EQ(4, first);
+    ASSERT_OK_AND_ASSIGN(int64_t second, context->AdvanceMaterializedMaxSequenceNumber(
+                                             partition_bucket, /*max_sequence_number=*/8));
+    ASSERT_EQ(8, second);
+    ASSERT_OK_AND_ASSIGN(int64_t third, context->AdvanceMaterializedMaxSequenceNumber(
+                                            partition_bucket, /*max_sequence_number=*/6));
+    ASSERT_EQ(8, third);
+    ASSERT_OK_AND_ASSIGN(int64_t fourth, context->AdvanceMaterializedMaxSequenceNumber(
+                                             partition_bucket, /*max_sequence_number=*/10));
+    ASSERT_EQ(10, fourth);
+}
+
+TEST(RealtimeContextTest, TestMaterializedSequenceRejectsMissingStore) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+
+    Result<int64_t> result = context->AdvanceMaterializedMaxSequenceNumber(
+        RealtimePartitionBucket({{"dt", "missing"}}, /*bucket=*/3),
+        /*max_sequence_number=*/4);
+    ASSERT_TRUE(result.status().IsKeyError());
+    ASSERT_NOK_WITH_MSG(result, "real-time store not found for partition {dt=missing}, bucket 3");
 }
 
 TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 7f4d4b5f7..b04cc8e25 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -172,9 +172,9 @@ Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Crea
     prepared_fields.insert(prepared_fields.end(), write_schema->fields().begin(),
                            write_schema->fields().end());
     const RealtimePartitionBucket partition_bucket(partition, bucket);
-    const int64_t initial_max_sequence_number =
-        realtime_context->AdvanceMaterializedMaxSequenceNumber(partition_bucket,
-                                                               restored_max_sequence_number);
+    PAIMON_ASSIGN_OR_RAISE(int64_t initial_max_sequence_number,
+                           realtime_context->AdvanceMaterializedMaxSequenceNumber(
+                               partition_bucket, restored_max_sequence_number));
     return std::shared_ptr<RealtimePrimaryKeyWriter>(new RealtimePrimaryKeyWriter(
         store_state.store, merge_tree_writer, realtime_context, partition_bucket, write_schema,
         arrow::schema(std::move(prepared_fields)), arrow::schema(std::move(key_fields)),
@@ -246,8 +246,10 @@ Status RealtimePrimaryKeyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
         std::move(prepared_batch), OffsetRange(next_offset_, next_offset_ + count)}));
     next_offset_ += count;
     last_sequence_number_ += count;
-    realtime_context_->AdvanceMaterializedMaxSequenceNumber(partition_bucket_,
-                                                            last_sequence_number_);
+    PAIMON_RETURN_NOT_OK(
+        realtime_context_
+            ->AdvanceMaterializedMaxSequenceNumber(partition_bucket_, last_sequence_number_)
+            .status());
     return Status::OK();
 }
 

From 5a6dcb657515ea35570b012d24bcd279faeb3ea3 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 15:56:16 +0800
Subject: [PATCH 52/93] refactor(realtime): simplify PK store boundary

---
 include/paimon/realtime/realtime_store.h      |   5 +-
 .../operation/key_value_file_store_write.cpp  |   2 +-
 .../realtime/arrow_realtime_store_factory.cpp |   5 +-
 .../realtime/prepared_key_value_reader.cpp    | 102 ++++-----
 .../realtime/primary_key_realtime_store.cpp   | 215 ++----------------
 .../realtime/primary_key_realtime_store.h     |   5 +-
 .../primary_key_realtime_store_test.cpp       | 131 +++--------
 .../core/realtime/realtime_context_impl.cpp   |   9 +-
 test/inte/realtime_write_inte_test.cpp        |  25 +-
 9 files changed, 111 insertions(+), 388 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 9ed1e4361..e61051e29 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -47,10 +47,7 @@ struct PAIMON_EXPORT AppendRealtimeStoreCreateConfig {
     StatisticsMode statistics_mode;
 };
 
-struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {
-    /// Primary-key fields after removing partition fields, in comparison order.
-    std::vector<std::string> trimmed_primary_keys;
-};
+struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {};
 
 using RealtimeStoreCreateConfig =
     std::variant<AppendRealtimeStoreCreateConfig, PrimaryKeyRealtimeStoreCreateConfig>;
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 737ece080..31889ee84 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -154,7 +154,7 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
             RealtimeStoreState store_state,
             realtime_context_impl->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
                 std::move(c_write_schema), options_.ToMap(), pool_, partition_map, bucket,
-                PrimaryKeyRealtimeStoreCreateConfig{trimmed_primary_keys}}));
+                PrimaryKeyRealtimeStoreCreateConfig{}}));
         realtime_store_state = std::move(store_state);
         compact_manager = std::make_shared<NoopCompactManager>();
     } else {
diff --git a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
index babc55a3d..d336394ad 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
@@ -50,11 +50,8 @@ Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
                                                     request.memory_pool, arrow_pool);
     }
 
-    const PrimaryKeyRealtimeStoreCreateConfig& config =
-        std::get<PrimaryKeyRealtimeStoreCreateConfig>(request.mode_config);
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                           PrimaryKeyRealtimeStore::Create(
-                               imported_schema, config.trimmed_primary_keys, request.memory_pool));
+                           PrimaryKeyRealtimeStore::Create(imported_schema));
     return std::shared_ptr<RealtimeStore>(std::move(store));
 }
 
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index fa1dba7ef..c17513aef 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -32,7 +32,6 @@
 #include "arrow/array/builder_primitive.h"
 #include "arrow/buffer.h"
 #include "arrow/c/bridge.h"
-#include "arrow/compute/api.h"
 #include "arrow/type.h"
 #include "arrow/util/bit_util.h"
 #include "fmt/format.h"
@@ -386,39 +385,6 @@ Result<arrow::ArrayVector> ProjectFieldsByPaimonIds(
     return result;
 }
 
-Result<std::shared_ptr<arrow::StructArray>> ApplyOffsetFilter(
-    const std::shared_ptr<arrow::StructArray>& data_batch,
-    const std::shared_ptr<arrow::NumericArray<arrow::Int64Type>>& offset_array,
-    const std::optional<OffsetRange>& visible_offsets, arrow::MemoryPool* arrow_pool) {
-    if (!visible_offsets.has_value()) {
-        return data_batch;
-    }
-
-    arrow::BooleanBuilder filter_builder(arrow_pool);
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(filter_builder.Reserve(offset_array->length()));
-    int64_t visible_row_count = 0;
-    for (int64_t i = 0; i < offset_array->length(); ++i) {
-        int64_t offset = offset_array->Value(i);
-        bool visible = offset >= visible_offsets->begin && offset < visible_offsets->end;
-        filter_builder.UnsafeAppend(visible);
-        visible_row_count += visible;
-    }
-    if (visible_row_count == 0) {
-        return std::shared_ptr<arrow::StructArray>();
-    }
-    if (visible_row_count == data_batch->length()) {
-        return data_batch;
-    }
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> filter,
-                                      filter_builder.Finish());
-    arrow::compute::ExecContext exec_context(arrow_pool);
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-        arrow::Datum filtered,
-        arrow::compute::Filter(data_batch, filter, arrow::compute::FilterOptions::Defaults(),
-                               &exec_context));
-    return checked_pointer_cast<arrow::StructArray>(filtered.make_array());
-}
-
 class PreparedKeyValueReader final : public KeyValueRecordReader {
  public:
     PreparedKeyValueReader(std::unique_ptr<BatchReader>&& reader,
@@ -448,20 +414,20 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         explicit Iterator(PreparedKeyValueReader* reader) : reader_(reader) {}
 
         Result<bool> HasNext() const override {
-            return cursor_ < reader_->row_kind_array_->length();
+            return cursor_ < reader_->RowCount();
         }
 
         Result<KeyValue> Next() override {
-            if (cursor_ >= reader_->row_kind_array_->length()) {
+            if (cursor_ >= reader_->RowCount()) {
                 return Status::Invalid("No more prepared key values in current iterator");
             }
+            const int64_t row = reader_->RowAt(cursor_);
             std::shared_ptr<InternalRow> key =
-                std::make_shared<ColumnarRowRef>(reader_->key_ctx_, cursor_);
-            auto value = std::make_unique<ColumnarRowRef>(reader_->value_ctx_, cursor_);
-            PAIMON_ASSIGN_OR_RAISE(
-                const RowKind* row_kind,
-                RowKind::FromByteValue(reader_->row_kind_array_->Value(cursor_)));
-            int64_t sequence_number = reader_->sequence_number_array_->Value(cursor_);
+                std::make_shared<ColumnarRowRef>(reader_->key_ctx_, row);
+            auto value = std::make_unique<ColumnarRowRef>(reader_->value_ctx_, row);
+            PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind,
+                                   RowKind::FromByteValue(reader_->row_kind_array_->Value(row)));
+            int64_t sequence_number = reader_->sequence_number_array_->Value(row);
             ++cursor_;
             return KeyValue(row_kind, sequence_number, KeyValue::UNKNOWN_LEVEL, std::move(key),
                             std::move(value));
@@ -535,7 +501,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                 data_batch = checked_pointer_cast<arrow::StructArray>(arrow_array);
             }
             PAIMON_RETURN_NOT_OK(ValidatePreparedBatch(data_batch));
-            PAIMON_RETURN_NOT_OK(ValidateOrdering(data_batch));
 
             std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> offset_array =
                 checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
@@ -543,12 +508,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             if (offset_coverage_) {
                 PAIMON_RETURN_NOT_OK(offset_coverage_->Add(*offset_array));
             }
-            PAIMON_ASSIGN_OR_RAISE(
-                data_batch,
-                ApplyOffsetFilter(data_batch, offset_array, visible_offsets_, arrow_pool_.get()));
-            if (!data_batch) {
-                continue;
-            }
 
             row_kind_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int8Type>>(
                 data_batch->field(kValueKindIndex));
@@ -562,6 +521,10 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                                                             value_schema_, arrow_pool_.get()));
             key_ctx_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
             value_ctx_ = std::make_shared<ColumnarBatchContext>(value_fields, pool_);
+            PAIMON_RETURN_NOT_OK(ValidateOrdering(key_ctx_, sequence_number_array_));
+            if (!SelectVisibleRows(*offset_array)) {
+                continue;
+            }
             ArrowUtils::TraverseArray(data_batch);
             return std::make_unique<Iterator>(this);
         }
@@ -600,18 +563,13 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         return Status::OK();
     }
 
-    Status ValidateOrdering(const std::shared_ptr<arrow::StructArray>& data_batch) {
-        if (data_batch->length() == 0) {
+    Status ValidateOrdering(
+        const std::shared_ptr<ColumnarBatchContext>& key_context,
+        const std::shared_ptr<arrow::NumericArray<arrow::Int64Type>>& sequences) {
+        if (sequences->length() == 0) {
             return Status::OK();
         }
-        PAIMON_ASSIGN_OR_RAISE(
-            arrow::ArrayVector key_fields,
-            ProjectFieldsByPaimonIds(data_batch, prepared_schema_, key_schema_, arrow_pool_.get()));
-        std::shared_ptr<ColumnarBatchContext> key_context =
-            std::make_shared<ColumnarBatchContext>(key_fields, pool_);
-        std::shared_ptr<arrow::Int64Array> sequences =
-            checked_pointer_cast<arrow::Int64Array>(data_batch->field(kSequenceNumberIndex));
-        for (int64_t row = 0; row < data_batch->length(); ++row) {
+        for (int64_t row = 0; row < sequences->length(); ++row) {
             ColumnarRowRef current_key(key_context, row);
             if (previous_key_context_) {
                 ColumnarRowRef previous_key(previous_key_context_, previous_key_row_);
@@ -631,11 +589,36 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         return Status::OK();
     }
 
+    bool SelectVisibleRows(const arrow::Int64Array& offsets) {
+        if (!visible_offsets_.has_value()) {
+            return true;
+        }
+        visible_rows_.emplace();
+        visible_rows_->reserve(offsets.length());
+        for (int64_t row = 0; row < offsets.length(); ++row) {
+            const int64_t offset = offsets.Value(row);
+            if (offset >= visible_offsets_->begin && offset < visible_offsets_->end) {
+                visible_rows_->push_back(row);
+            }
+        }
+        return !visible_rows_->empty();
+    }
+
+    int64_t RowCount() const {
+        return visible_rows_.has_value() ? static_cast<int64_t>(visible_rows_->size())
+                                         : row_kind_array_->length();
+    }
+
+    int64_t RowAt(int64_t ordinal) const {
+        return visible_rows_.has_value() ? (*visible_rows_)[ordinal] : ordinal;
+    }
+
     void ResetBatchState() {
         key_ctx_.reset();
         value_ctx_.reset();
         row_kind_array_.reset();
         sequence_number_array_.reset();
+        visible_rows_.reset();
     }
 
  private:
@@ -655,6 +638,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     std::shared_ptr<ColumnarBatchContext> value_ctx_;
     std::shared_ptr<arrow::NumericArray<arrow::Int8Type>> row_kind_array_;
     std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> sequence_number_array_;
+    std::optional<std::vector<int64_t>> visible_rows_;
     std::shared_ptr<ColumnarBatchContext> previous_key_context_;
     int64_t previous_key_row_ = 0;
     int64_t previous_sequence_ = 0;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 22fab1b08..68f4bf500 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -18,25 +18,17 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
-#include <cstddef>
 #include <mutex>
 #include <optional>
-#include <queue>
-#include <unordered_map>
 #include <utility>
 #include <vector>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
-#include "arrow/compute/api.h"
-#include "paimon/common/data/columnar/columnar_batch_context.h"
-#include "paimon/common/data/columnar/columnar_row_ref.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/types/data_field.h"
-#include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
-#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/index/pk/primary_key_index_definitions.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
@@ -155,116 +147,19 @@ class ReadView final : public RealtimeReadView {
     std::optional<OffsetRange> range_;
 };
 
-class RawBatchReader final : public BatchReader {
+class StoredBatchReader final : public BatchReader {
  public:
-    RawBatchReader(std::vector<StoredBatch> batches, std::vector<int32_t> key_field_indexes,
-                   const std::shared_ptr<FieldsComparator>& key_comparator,
-                   const std::shared_ptr<MemoryPool>& memory_pool)
-        : batches_(std::move(batches)),
-          positions_(batches_.size(), 0),
-          key_field_indexes_(std::move(key_field_indexes)),
-          key_comparator_(key_comparator),
-          memory_pool_(memory_pool),
-          arrow_pool_(GetArrowPool(memory_pool)),
-          heap_(SourceGreater{this}),
-          metrics_(std::make_shared<MetricsImpl>()) {
-        key_contexts_.reserve(batches_.size());
-        sequence_arrays_.reserve(batches_.size());
-        for (size_t i = 0; i < batches_.size(); ++i) {
-            const StoredBatch& batch = batches_[i];
-            arrow::ArrayVector key_arrays;
-            key_arrays.reserve(key_field_indexes_.size());
-            for (int32_t field_index : key_field_indexes_) {
-                key_arrays.push_back(batch.data->field(field_index));
-            }
-            key_contexts_.push_back(
-                std::make_shared<ColumnarBatchContext>(key_arrays, memory_pool_));
-            sequence_arrays_.push_back(
-                checked_pointer_cast<arrow::Int64Array>(batch.data->field(1)));
-            if (batch.data->length() > 0) {
-                heap_.push(i);
-            }
-        }
-    }
+    explicit StoredBatchReader(const StoredBatch& batch)
+        : data_(batch.data), metrics_(std::make_shared<MetricsImpl>()) {}
 
     Result<ReadBatch> NextBatch() override {
-        if (heap_.empty()) {
+        if (!data_) {
             return MakeEofBatch();
         }
-
-        struct SelectedRow {
-            size_t selected_source;
-            int64_t source_ordinal;
-        };
-        struct SelectedSource {
-            size_t source;
-            std::vector<int64_t> rows;
-            int64_t base = -1;
-        };
-        std::vector<SelectedRow> selected_rows;
-        selected_rows.reserve(kOutputBatchSize);
-        std::vector<SelectedSource> selected_sources;
-        std::unordered_map<size_t, size_t> selected_source_indexes;
-        while (!heap_.empty() && selected_rows.size() < kOutputBatchSize) {
-            const size_t source = heap_.top();
-            heap_.pop();
-            auto [source_it, inserted] =
-                selected_source_indexes.emplace(source, selected_sources.size());
-            if (inserted) {
-                selected_sources.push_back(SelectedSource{source, {}});
-            }
-            SelectedSource& selected_source = selected_sources[source_it->second];
-            selected_rows.push_back(
-                SelectedRow{source_it->second, static_cast<int64_t>(selected_source.rows.size())});
-            selected_source.rows.push_back(positions_[source]++);
-            if (positions_[source] < batches_[source].data->length()) {
-                heap_.push(source);
-            }
-        }
-
-        arrow::compute::ExecContext context(arrow_pool_.get());
-        arrow::ArrayVector grouped_batches;
-        int64_t grouped_row_count = 0;
-        for (SelectedSource& selected_source : selected_sources) {
-            arrow::Int64Builder source_index_builder(arrow_pool_.get());
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(
-                source_index_builder.AppendValues(selected_source.rows));
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> source_indices,
-                                              source_index_builder.Finish());
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-                arrow::Datum source_batch,
-                arrow::compute::Take(arrow::Datum(batches_[selected_source.source].data),
-                                     arrow::Datum(source_indices),
-                                     arrow::compute::TakeOptions::NoBoundsCheck(), &context));
-            selected_source.base = grouped_row_count;
-            grouped_row_count += static_cast<int64_t>(selected_source.rows.size());
-            grouped_batches.push_back(source_batch.make_array());
-        }
-
-        std::shared_ptr<arrow::Array> batch;
-        if (grouped_batches.size() == 1) {
-            batch = std::move(grouped_batches[0]);
-        } else {
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-                std::shared_ptr<arrow::Array> grouped,
-                arrow::Concatenate(grouped_batches, arrow_pool_.get()));
-            arrow::Int64Builder order_builder(arrow_pool_.get());
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(order_builder.Reserve(selected_rows.size()));
-            for (const SelectedRow& selected : selected_rows) {
-                order_builder.UnsafeAppend(selected_sources[selected.selected_source].base +
-                                           selected.source_ordinal);
-            }
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> order,
-                                              order_builder.Finish());
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-                arrow::Datum reordered,
-                arrow::compute::Take(arrow::Datum(grouped), arrow::Datum(order),
-                                     arrow::compute::TakeOptions::NoBoundsCheck(), &context));
-            batch = reordered.make_array();
-        }
         auto array = std::make_unique<ArrowArray>();
         auto schema = std::make_unique<ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*batch, array.get(), schema.get()));
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*data_, array.get(), schema.get()));
+        data_.reset();
         return ReadBatch(std::move(array), std::move(schema));
     }
 
@@ -272,50 +167,11 @@ class RawBatchReader final : public BatchReader {
         return metrics_;
     }
     void Close() override {
-        while (!heap_.empty()) {
-            heap_.pop();
-        }
-        batches_.clear();
-        positions_.clear();
-        key_contexts_.clear();
-        sequence_arrays_.clear();
+        data_.reset();
     }
 
  private:
-    static constexpr size_t kOutputBatchSize = 1024;
-
-    bool Less(size_t left, size_t right) const {
-        ColumnarRowRef left_key(key_contexts_[left], positions_[left]);
-        ColumnarRowRef right_key(key_contexts_[right], positions_[right]);
-        const int32_t key_comparison = key_comparator_->CompareTo(left_key, right_key);
-        if (key_comparison != 0) {
-            return key_comparison < 0;
-        }
-        const int64_t left_sequence = sequence_arrays_[left]->Value(positions_[left]);
-        const int64_t right_sequence = sequence_arrays_[right]->Value(positions_[right]);
-        if (left_sequence != right_sequence) {
-            return left_sequence < right_sequence;
-        }
-        return left < right;
-    }
-
-    struct SourceGreater {
-        RawBatchReader* reader;
-
-        bool operator()(size_t left, size_t right) const {
-            return reader->Less(right, left);
-        }
-    };
-
-    std::vector<StoredBatch> batches_;
-    std::vector<int64_t> positions_;
-    std::vector<int32_t> key_field_indexes_;
-    std::shared_ptr<FieldsComparator> key_comparator_;
-    std::shared_ptr<MemoryPool> memory_pool_;
-    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
-    std::vector<std::shared_ptr<ColumnarBatchContext>> key_contexts_;
-    std::vector<std::shared_ptr<arrow::Int64Array>> sequence_arrays_;
-    std::priority_queue<size_t, std::vector<size_t>, SourceGreater> heap_;
+    std::shared_ptr<arrow::StructArray> data_;
     std::shared_ptr<Metrics> metrics_;
 };
 
@@ -323,13 +179,8 @@ class RawBatchReader final : public BatchReader {
 
 class PrimaryKeyRealtimeStore::Impl {
  public:
-    Impl(std::shared_ptr<arrow::Schema> prepared_schema, std::vector<int32_t> key_field_indexes,
-         const std::shared_ptr<FieldsComparator>& key_comparator,
-         const std::shared_ptr<MemoryPool>& memory_pool)
-        : prepared_schema_(std::move(prepared_schema)),
-          key_field_indexes_(std::move(key_field_indexes)),
-          key_comparator_(key_comparator),
-          memory_pool_(memory_pool) {}
+    explicit Impl(std::shared_ptr<arrow::Schema> prepared_schema)
+        : prepared_schema_(std::move(prepared_schema)) {}
 
     Status Write(RealtimeWriteBatch&& write_batch) {
         if (!write_batch.batch || !write_batch.batch->GetData()) {
@@ -349,7 +200,6 @@ class PrimaryKeyRealtimeStore::Impl {
         }
         std::shared_ptr<arrow::StructArray> prepared =
             checked_pointer_cast<arrow::StructArray>(array);
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(prepared->ValidateFull());
         std::lock_guard<std::mutex> lock(mutex_);
         if (last_offset_ && write_batch.offset_range.begin != last_offset_.value()) {
             return Status::Invalid("PK real-time offset ranges must be contiguous");
@@ -381,9 +231,9 @@ class PrimaryKeyRealtimeStore::Impl {
             return Status::Invalid("segment was not created by the PK real-time store");
         }
         std::vector<std::unique_ptr<BatchReader>> readers;
-        if (!segment->Batches().empty()) {
-            readers.push_back(std::make_unique<RawBatchReader>(
-                segment->Batches(), key_field_indexes_, key_comparator_, memory_pool_));
+        readers.reserve(segment->Batches().size());
+        for (const StoredBatch& batch : segment->Batches()) {
+            readers.push_back(std::make_unique<StoredBatchReader>(batch));
         }
         return readers;
     }
@@ -407,13 +257,10 @@ class PrimaryKeyRealtimeStore::Impl {
             return Status::Invalid("read view was not created by the PK real-time store");
         }
         std::vector<std::unique_ptr<BatchReader>> readers;
-        std::vector<StoredBatch> batches;
         for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
-            batches.insert(batches.end(), segment->Batches().begin(), segment->Batches().end());
-        }
-        if (!batches.empty()) {
-            readers.push_back(std::make_unique<RawBatchReader>(
-                std::move(batches), key_field_indexes_, key_comparator_, memory_pool_));
+            for (const StoredBatch& batch : segment->Batches()) {
+                readers.push_back(std::make_unique<StoredBatchReader>(batch));
+            }
         }
         return readers;
     }
@@ -439,9 +286,6 @@ class PrimaryKeyRealtimeStore::Impl {
 
  private:
     std::shared_ptr<arrow::Schema> prepared_schema_;
-    std::vector<int32_t> key_field_indexes_;
-    std::shared_ptr<FieldsComparator> key_comparator_;
-    std::shared_ptr<MemoryPool> memory_pool_;
     mutable std::mutex mutex_;
     std::vector<StoredBatch> building_;
     std::vector<std::shared_ptr<Segment>> sealed_;
@@ -454,31 +298,10 @@ PrimaryKeyRealtimeStore::PrimaryKeyRealtimeStore(std::unique_ptr<Impl>&& impl)
 PrimaryKeyRealtimeStore::~PrimaryKeyRealtimeStore() = default;
 
 Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::vector<std::string>& trimmed_primary_keys,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
+    const std::shared_ptr<arrow::Schema>& prepared_schema) {
     PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(prepared_schema));
-    if (trimmed_primary_keys.empty() || !memory_pool) {
-        return Status::Invalid("PK primary keys are empty or memory pool is null");
-    }
-    std::vector<int32_t> key_field_indexes;
-    std::vector<DataField> key_fields;
-    key_field_indexes.reserve(trimmed_primary_keys.size());
-    key_fields.reserve(trimmed_primary_keys.size());
-    for (const std::string& key : trimmed_primary_keys) {
-        const int32_t field_index = prepared_schema->GetFieldIndex(key);
-        if (field_index < 3) {
-            return Status::Invalid("PK field is missing from prepared schema: ", key);
-        }
-        key_field_indexes.push_back(field_index);
-        PAIMON_ASSIGN_OR_RAISE(DataField field, DataField::ConvertArrowFieldToDataField(
-                                                    prepared_schema->field(field_index)));
-        key_fields.push_back(std::move(field));
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
-                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
-    return std::shared_ptr<PrimaryKeyRealtimeStore>(new PrimaryKeyRealtimeStore(
-        std::make_unique<Impl>(prepared_schema, key_field_indexes, key_comparator, memory_pool)));
+    return std::shared_ptr<PrimaryKeyRealtimeStore>(
+        new PrimaryKeyRealtimeStore(std::make_unique<Impl>(prepared_schema)));
 }
 Status PrimaryKeyRealtimeStore::Write(RealtimeWriteBatch&& batch) {
     return impl_->Write(std::move(batch));
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index 52f9a6076..35f04485b 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -30,7 +30,6 @@ class Schema;
 namespace paimon {
 
 class CoreOptions;
-class MemoryPool;
 class TableSchema;
 
 Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const TableSchema& schema);
@@ -39,9 +38,7 @@ Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const Table
 class PrimaryKeyRealtimeStore final : public RealtimeStore {
  public:
     static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
-        const std::shared_ptr<arrow::Schema>& prepared_schema,
-        const std::vector<std::string>& trimmed_primary_keys,
-        const std::shared_ptr<MemoryPool>& memory_pool);
+        const std::shared_ptr<arrow::Schema>& prepared_schema);
 
     ~PrimaryKeyRealtimeStore() override;
 
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 384b937cb..49da66fc2 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -18,9 +18,7 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
-#include <algorithm>
 #include <memory>
-#include <numeric>
 #include <optional>
 #include <string>
 #include <utility>
@@ -29,7 +27,6 @@
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
 #include "arrow/ipc/json_simple.h"
-#include "fmt/format.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
@@ -168,9 +165,8 @@ TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsEnabledGlobalIndex) {
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<PrimaryKeyRealtimeStore> store,
-        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
     ASSERT_FALSE(segment.has_value());
@@ -221,16 +217,14 @@ TEST(PrimaryKeyRealtimeStoreTest, TestBadTransportPrefix) {
     invalid_fields.push_back(std::move(wrong_offset_id));
 
     for (const arrow::FieldVector& fields : invalid_fields) {
-        ASSERT_NOK_WITH_MSG(
-            PrimaryKeyRealtimeStore::Create(arrow::schema(fields), {"id"}, GetDefaultPool()),
-            "prepared schema field");
+        ASSERT_NOK_WITH_MSG(PrimaryKeyRealtimeStore::Create(arrow::schema(fields)),
+                            "prepared schema field");
     }
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<PrimaryKeyRealtimeStore> store,
-        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderPerStoredBatch) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeBatch(R"([[1, 6, 1, 1, "before"], [0, 5, 0, 3, "three"]])"), OffsetRange(0, 2)}));
     ASSERT_OK(store->Write(
@@ -240,21 +234,23 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitBatches) {
     ASSERT_TRUE(segment.has_value());
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateCommitReaders(segment.value()));
-    ASSERT_EQ(1, readers.size());
+    ASSERT_EQ(2, readers.size());
     ASSERT_OK_AND_ASSIGN(std::string actual, ReadJson(readers));
     ASSERT_EQ(
-        "-- is_valid: all not null\n-- child 0 type: int8\n  [\n    1,\n    2,\n    0\n  ]\n-- "
-        "child 1 type: int64\n  [\n    6,\n    7,\n    5\n  ]\n-- child 2 type: int64\n  [\n    "
-        "1,\n    2,\n    0\n  ]\n-- child 3 type: int64\n  [\n    1,\n    2,\n    3\n  ]\n-- child "
-        "4 type: string\n  [\n    \"before\",\n    \"after\",\n    \"three\"\n  ]",
+        "-- is_valid: all not null\n-- child 0 type: int8\n  [\n    1,\n    0,\n    2\n  ]\n-- "
+        "child 1 type: int64\n  [\n    6,\n    5,\n    7\n  ]\n-- child 2 type: int64\n  [\n    "
+        "1,\n    0,\n    2\n  ]\n-- child 3 type: int64\n  [\n    1,\n    3,\n    2\n  ]\n-- child "
+        "4 type: string\n  [\n    \"before\",\n    \"three\",\n    \"after\"\n  ]",
         actual);
-    readers[0]->Close();
+    for (const std::unique_ptr<BatchReader>& reader : readers) {
+        reader->Close();
+    }
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
     std::shared_ptr<arrow::Schema> schema = NestedPreparedSchema();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(schema, {"id"}, GetDefaultPool()));
+                         PrimaryKeyRealtimeStore::Create(schema));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeBatch(schema, R"([[0, 1, 0, 1, ["one", [1, 2]]], [0, 2, 1, 2, ["two", [3, 4]]]])"),
         OffsetRange(0, 2)}));
@@ -273,77 +269,9 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
     ASSERT_TRUE(BatchReader::IsEofBatch(batch));
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestHeapMergeAcrossBatches) {
-    constexpr int64_t kSourceCount = 2057;
-    constexpr int64_t kKeyCount = 257;
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<PrimaryKeyRealtimeStore> store,
-        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
-    for (int64_t source = 0; source < kSourceCount; ++source) {
-        const int64_t id = (source * 149) % kKeyCount;
-        const std::string json =
-            fmt::format(R"([[0, {}, {}, {}, "v{}"]])", source, source, id, source);
-        ASSERT_OK(
-            store->Write(RealtimeWriteBatch{MakeBatch(json), OffsetRange(source, source + 1)}));
-    }
-    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
-                         store->SealForCommit());
-    ASSERT_TRUE(segment.has_value());
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateCommitReaders(segment.value()));
-    ASSERT_EQ(1, readers.size());
-
-    std::vector<int64_t> expected_sources(kSourceCount);
-    std::iota(expected_sources.begin(), expected_sources.end(), 0);
-    std::sort(expected_sources.begin(), expected_sources.end(), [=](int64_t left, int64_t right) {
-        const int64_t left_id = (left * 149) % kKeyCount;
-        const int64_t right_id = (right * 149) % kKeyCount;
-        return left_id != right_id ? left_id < right_id : left < right;
-    });
-
-    int64_t output_row = 0;
-    int64_t output_batches = 0;
-    while (true) {
-        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
-        if (BatchReader::IsEofBatch(batch)) {
-            break;
-        }
-        ASSERT_LE(batch.first->length, 1024);
-        ASSERT_GT(batch.first->length, 0);
-        ++output_batches;
-        arrow::Result<std::shared_ptr<arrow::Array>> imported_result =
-            arrow::ImportArray(batch.first.get(), batch.second.get());
-        ASSERT_TRUE(imported_result.ok()) << imported_result.status().ToString();
-        std::shared_ptr<arrow::Array> imported = std::move(imported_result).ValueOrDie();
-        std::shared_ptr<arrow::StructArray> array =
-            std::dynamic_pointer_cast<arrow::StructArray>(imported);
-        ASSERT_NE(nullptr, array);
-        ASSERT_EQ(PreparedSchema()->ToString(), arrow::schema(array->type()->fields())->ToString());
-        std::shared_ptr<arrow::Int64Array> sequences =
-            std::dynamic_pointer_cast<arrow::Int64Array>(array->field(1));
-        std::shared_ptr<arrow::Int64Array> ids =
-            std::dynamic_pointer_cast<arrow::Int64Array>(array->field(3));
-        std::shared_ptr<arrow::StringArray> values =
-            std::dynamic_pointer_cast<arrow::StringArray>(array->field(4));
-        ASSERT_NE(nullptr, sequences);
-        ASSERT_NE(nullptr, ids);
-        ASSERT_NE(nullptr, values);
-        for (int64_t row = 0; row < array->length(); ++row, ++output_row) {
-            ASSERT_LT(output_row, kSourceCount);
-            const int64_t source = expected_sources[output_row];
-            ASSERT_EQ(source, sequences->Value(row));
-            ASSERT_EQ((source * 149) % kKeyCount, ids->Value(row));
-            ASSERT_EQ(fmt::format("v{}", source), values->GetString(row));
-        }
-    }
-    ASSERT_EQ(kSourceCount, output_row);
-    ASSERT_EQ(3, output_batches);
-}
-
-TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadMultiSourceReader) {
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<PrimaryKeyRealtimeStore> store,
-        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadBatchReaders) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 10, 0, 1, "a"]])"), OffsetRange(0, 1)}));
     ASSERT_OK(
@@ -355,15 +283,15 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadMultiSourceReader) {
     ASSERT_TRUE(segment.has_value());
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateCommitReaders(segment.value()));
-    ASSERT_EQ(1, readers.size());
-
-    readers[0]->Close();
+    ASSERT_EQ(3, readers.size());
+    for (const std::unique_ptr<BatchReader>& reader : readers) {
+        reader->Close();
+    }
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<PrimaryKeyRealtimeStore> store,
-        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 0, 4, 1, "one"]])"), OffsetRange(4, 5)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -373,10 +301,9 @@ TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(4, 5)), view->GetOffsetRange());
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderCardinalityIsConstant) {
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<PrimaryKeyRealtimeStore> store,
-        PrimaryKeyRealtimeStore::Create(PreparedSchema(), {"id"}, GetDefaultPool()));
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 2, "two"]])"), OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -389,7 +316,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderCardinalityIsConstant) {
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    ASSERT_EQ(1, readers.size());
+    ASSERT_EQ(2, readers.size());
     ASSERT_OK_AND_ASSIGN(std::string actual, ReadJson(readers));
     ASSERT_NE(std::string::npos, actual.find("\"one\""));
     ASSERT_NE(std::string::npos, actual.find("\"two\""));
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index c9f719b7e..0ea4c61d6 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -49,14 +49,7 @@ namespace paimon {
 namespace {
 
 bool SameMode(const RealtimeStoreCreateConfig& left, const RealtimeStoreCreateConfig& right) {
-    if (left.index() != right.index()) {
-        return false;
-    }
-    if (const auto* left_pk = std::get_if<PrimaryKeyRealtimeStoreCreateConfig>(&left)) {
-        const auto& right_pk = std::get<PrimaryKeyRealtimeStoreCreateConfig>(right);
-        return left_pk->trimmed_primary_keys == right_pk.trimmed_primary_keys;
-    }
-    return true;
+    return left.index() == right.index();
 }
 
 std::string PartitionToString(const std::map<std::string, std::string>& partition) {
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index f10e93858..173981527 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -2349,7 +2349,7 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
               final_rows);
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
+TEST_F(RealtimeWriteInteTest, TestPkMultipleStoredBatchesMergeForQueryAndCommit) {
     CreatePkTable();
     auto factory = MakeDecoratingFactory<SplitCommitReaderRealtimeStore>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
@@ -2357,23 +2357,28 @@ TEST_F(RealtimeWriteInteTest, TestPkPluginContract) {
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                          CreateRealtimeWriter(realtime_context));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
-                         MakeBatch({Row{4, "four", "p0"}, Row{3, "three", "p0"}},
+                         MakeBatch({Row{4, "four", "p0"}, Row{2, "two", "p0"}, Row{1, "one", "p0"}},
                                    /*partitioned=*/false));
     ASSERT_OK(writer->Write(std::move(first_batch)));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
-                         MakeBatch({Row{2, "two", "p0"}, Row{1, "one", "p0"}},
-                                   /*partitioned=*/false));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> second_batch,
+        MakeBatch({Row{3, "three", "p0"}, Row{2, "deleted", "p0"}, Row{1, "one-new", "p0"}},
+                  /*partitioned=*/false, /*bucket=*/0,
+                  {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE,
+                   RecordBatch::RowKind::UPDATE_AFTER}));
     ASSERT_OK(writer->Write(std::move(second_batch)));
 
+    const std::vector<Row> expected = {{1, "one-new", "p0"}, {3, "three", "p0"}, {4, "four", "p0"}};
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> query_rows, ReadRows(realtime_context));
+    ASSERT_EQ(expected, query_rows);
+
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                          writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
     ASSERT_EQ(1, progress.size());
-    ASSERT_EQ(OffsetRange(0, 4), progress[0].offset_range);
+    ASSERT_EQ(OffsetRange(0, 6), progress[0].offset_range);
     ASSERT_OK(Commit(progress, /*commit_identifier=*/0));
     ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, ReadRows());
-    ASSERT_EQ((std::vector<Row>{
-                  {1, "one", "p0"}, {2, "two", "p0"}, {3, "three", "p0"}, {4, "four", "p0"}}),
-              rows);
+    ASSERT_EQ(expected, rows);
     ASSERT_OK(writer->Close());
 }
 
@@ -2445,7 +2450,7 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderCloseFailure) {
         state->query_null_index = null_index;
         ASSERT_NOK_WITH_MSG(CreateQueryReader(realtime_context),
                             "PK real-time store returned a null query reader");
-        ASSERT_EQ(null_index + 1, state->query_close_count->load(std::memory_order_acquire));
+        ASSERT_EQ((null_index + 1) * 2, state->query_close_count->load(std::memory_order_acquire));
     }
     ASSERT_OK(writer->Close());
 }

From 808cc593d2fab91caccd715ab7f6122c30a4fe49 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 15:56:43 +0800
Subject: [PATCH 53/93] refactor(realtime): simplify PK offset coverage

---
 .../realtime/prepared_key_value_reader.cpp    | 53 +++++++------------
 test/inte/realtime_write_inte_test.cpp        | 23 +++++---
 2 files changed, 33 insertions(+), 43 deletions(-)

diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index c17513aef..ae575ca58 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -18,9 +18,10 @@
 
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 
+#include <algorithm>
 #include <cstdint>
+#include <limits>
 #include <memory>
-#include <mutex>
 #include <optional>
 #include <unordered_map>
 #include <utility>
@@ -29,11 +30,8 @@
 #include "arrow/array/array_base.h"
 #include "arrow/array/array_nested.h"
 #include "arrow/array/array_primitive.h"
-#include "arrow/array/builder_primitive.h"
-#include "arrow/buffer.h"
 #include "arrow/c/bridge.h"
 #include "arrow/type.h"
-#include "arrow/util/bit_util.h"
 #include "fmt/format.h"
 #include "paimon/common/data/columnar/columnar_batch_context.h"
 #include "paimon/common/data/columnar/columnar_row_ref.h"
@@ -75,42 +73,35 @@ Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
 
 class RealtimeOffsetCoverage {
  public:
-    static Result<std::shared_ptr<RealtimeOffsetCoverage>> Create(
-        const OffsetRange& sealed_offsets, size_t reader_count,
-        const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+    static Result<std::shared_ptr<RealtimeOffsetCoverage>> Create(const OffsetRange& sealed_offsets,
+                                                                  size_t reader_count) {
         if (sealed_offsets.begin < 0 || sealed_offsets.end < sealed_offsets.begin) {
             return Status::Invalid("PK real-time store returned an invalid sealed offset range");
         }
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-            std::shared_ptr<arrow::Buffer> seen_offsets,
-            arrow::AllocateEmptyBitmap(sealed_offsets.Count(), arrow_pool.get()));
-        return std::shared_ptr<RealtimeOffsetCoverage>(new RealtimeOffsetCoverage(
-            sealed_offsets, reader_count, std::move(seen_offsets), arrow_pool));
+        return std::shared_ptr<RealtimeOffsetCoverage>(
+            new RealtimeOffsetCoverage(sealed_offsets, reader_count));
     }
 
     Status Add(const arrow::Int64Array& offsets) {
-        std::lock_guard<std::mutex> lock(mutex_);
         for (int64_t row = 0; row < offsets.length(); ++row) {
             const int64_t offset = offsets.Value(row);
             if (offset < sealed_offsets_.begin || offset >= sealed_offsets_.end) {
                 return Status::Invalid(
                     "PK real-time store commit reader offset is outside the sealed range");
             }
-            const int64_t index = offset - sealed_offsets_.begin;
-            if (arrow::bit_util::GetBit(seen_offsets_->data(), index)) {
-                return Status::Invalid(
-                    "PK real-time store commit readers contain duplicate REALTIME_OFFSET");
-            }
-            arrow::bit_util::SetBit(seen_offsets_->mutable_data(), index);
+            min_seen_offset_ = std::min(min_seen_offset_, offset);
+            max_seen_offset_ = std::max(max_seen_offset_, offset);
             ++seen_count_;
         }
         return Status::OK();
     }
 
     Status FinishReader() {
-        std::lock_guard<std::mutex> lock(mutex_);
         ++finished_reader_count_;
-        if (finished_reader_count_ == reader_count_ && seen_count_ != sealed_offsets_.Count()) {
+        if (finished_reader_count_ == reader_count_ &&
+            (seen_count_ != sealed_offsets_.Count() ||
+             (seen_count_ > 0 && (min_seen_offset_ != sealed_offsets_.begin ||
+                                  max_seen_offset_ != sealed_offsets_.end - 1)))) {
             return Status::Invalid(
                 "PK real-time store commit readers did not cover the sealed range");
         }
@@ -118,21 +109,15 @@ class RealtimeOffsetCoverage {
     }
 
  private:
-    RealtimeOffsetCoverage(const OffsetRange& sealed_offsets, size_t reader_count,
-                           std::shared_ptr<arrow::Buffer> seen_offsets,
-                           const std::shared_ptr<arrow::MemoryPool>& arrow_pool)
-        : sealed_offsets_(sealed_offsets),
-          reader_count_(reader_count),
-          arrow_pool_(arrow_pool),
-          seen_offsets_(std::move(seen_offsets)) {}
+    RealtimeOffsetCoverage(const OffsetRange& sealed_offsets, size_t reader_count)
+        : sealed_offsets_(sealed_offsets), reader_count_(reader_count) {}
 
     OffsetRange sealed_offsets_;
     size_t reader_count_;
-    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
-    std::shared_ptr<arrow::Buffer> seen_offsets_;
+    int64_t min_seen_offset_ = std::numeric_limits<int64_t>::max();
+    int64_t max_seen_offset_ = std::numeric_limits<int64_t>::min();
     int64_t seen_count_ = 0;
     size_t finished_reader_count_ = 0;
-    std::mutex mutex_;
 };
 
 Status CheckPreparedField(const std::shared_ptr<arrow::Schema>& schema, int32_t field_idx,
@@ -736,10 +721,8 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
             return Status::Invalid("PK real-time store returned a null commit reader");
         }
     }
-    std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
-        RealtimeOffsetCoverage::Create(sealed_offsets, readers.size(), arrow_pool));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
+                           RealtimeOffsetCoverage::Create(sealed_offsets, readers.size()));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> adapted_reader,
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 173981527..ed75b554e 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -538,7 +538,7 @@ class FailAfterPhysicalFileRealtimeStore final : public DelegatingRealtimeStore
     std::shared_ptr<std::atomic<bool>> saw_artifacts_;
 };
 
-enum class CommitReaderMalformation { DROP_LAST, UNSORTED, SUBSTITUTE_OFFSET };
+enum class CommitReaderMalformation { DROP_LAST, UNSORTED, DUPLICATE_OFFSET, OUT_OF_RANGE_OFFSET };
 
 class CorruptingBatchReader final : public BatchReader {
  public:
@@ -552,8 +552,10 @@ class CorruptingBatchReader final : public BatchReader {
                 return DropLast();
             case CommitReaderMalformation::UNSORTED:
                 return SwapFirstTwo();
-            case CommitReaderMalformation::SUBSTITUTE_OFFSET:
-                return SubstituteOffset();
+            case CommitReaderMalformation::DUPLICATE_OFFSET:
+                return SubstituteOffset(/*offset=*/0);
+            case CommitReaderMalformation::OUT_OF_RANGE_OFFSET:
+                return SubstituteOffset(/*offset=*/-1);
         }
         return Status::Invalid("unknown commit reader malformation");
     }
@@ -613,7 +615,7 @@ class CorruptingBatchReader final : public BatchReader {
         return ReadBatch(std::move(output), std::move(schema));
     }
 
-    Result<ReadBatch> SubstituteOffset() {
+    Result<ReadBatch> SubstituteOffset(int64_t offset) {
         PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
         if (BatchReader::IsEofBatch(batch)) {
             return batch;
@@ -634,7 +636,7 @@ class CorruptingBatchReader final : public BatchReader {
         arrow::Int64Builder builder;
         PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(offsets->length()));
         for (int64_t row = 0; row < offsets->length(); ++row) {
-            builder.UnsafeAppend(0);
+            builder.UnsafeAppend(offset);
         }
         std::shared_ptr<arrow::Array> substituted_offsets;
         PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&substituted_offsets));
@@ -2387,9 +2389,14 @@ TEST_F(RealtimeWriteInteTest, TestPkRejectsMalformedCoverage) {
                                            "commit readers did not cover the sealed range");
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkRejectsEqualCardinalityOffsetSubstitution) {
-    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::SUBSTITUTE_OFFSET,
-                                           "duplicate REALTIME_OFFSET");
+TEST_F(RealtimeWriteInteTest, TestPkRejectsDuplicateOffset) {
+    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::DUPLICATE_OFFSET,
+                                           "commit readers did not cover the sealed range");
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkRejectsOutOfRangeOffset) {
+    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::OUT_OF_RANGE_OFFSET,
+                                           "offset is outside the sealed range");
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsUnsortedPluginRows) {

From 7e592537123f6547ad7eaaecdd456660e1e5a5cc Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 17:58:31 +0800
Subject: [PATCH 54/93] refactor(realtime): simplify stores around
 framework-owned PK offsets

---
 include/paimon/realtime/realtime_store.h      | 17 +++----
 .../merged_key_value_record_reader_test.cpp   | 34 +------------
 .../operation/key_value_file_store_write.cpp  |  6 +--
 .../realtime/arrow_realtime_store_factory.cpp | 22 ++++----
 .../realtime/arrow_realtime_store_test.cpp    | 15 +++++-
 .../realtime/prepared_key_value_reader.cpp    | 50 ++-----------------
 .../core/realtime/prepared_key_value_reader.h |  5 +-
 .../realtime/primary_key_realtime_store.cpp   |  5 --
 .../primary_key_realtime_store_test.cpp       |  6 ---
 .../realtime/realtime_append_only_writer.cpp  |  4 +-
 .../core/realtime/realtime_context_impl.cpp   | 10 ++--
 .../core/realtime/realtime_context_impl.h     |  2 +-
 .../core/realtime/realtime_context_test.cpp   | 27 ++++++++--
 .../realtime/realtime_primary_key_writer.cpp  |  2 +-
 .../table/source/key_value_table_read.cpp     | 11 ++--
 test/inte/realtime_write_inte_test.cpp        | 38 +-------------
 16 files changed, 78 insertions(+), 176 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index e61051e29..241413b31 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -25,7 +25,6 @@
 #include <optional>
 #include <string>
 #include <utility>
-#include <variant>
 #include <vector>
 
 #include "arrow/c/abi.h"
@@ -43,15 +42,11 @@ namespace paimon {
 class MemoryPool;
 class Predicate;
 
-struct PAIMON_EXPORT AppendRealtimeStoreCreateConfig {
-    StatisticsMode statistics_mode;
+enum class PAIMON_EXPORT RealtimeStoreMode {
+    APPEND_ONLY,
+    PRIMARY_KEY,
 };
 
-struct PAIMON_EXPORT PrimaryKeyRealtimeStoreCreateConfig {};
-
-using RealtimeStoreCreateConfig =
-    std::variant<AppendRealtimeStoreCreateConfig, PrimaryKeyRealtimeStoreCreateConfig>;
-
 /// Parameters used by a `RealtimeStoreFactory` to create a store.
 struct PAIMON_EXPORT RealtimeStoreCreateRequest {
     /// Schema whose ownership is transferred to the factory. Append mode receives the complete
@@ -66,8 +61,10 @@ struct PAIMON_EXPORT RealtimeStoreCreateRequest {
     std::map<std::string, std::string> partition;
     /// Bucket identifying the store within its partition.
     int32_t bucket = -1;
-    /// Mode-specific store configuration.
-    RealtimeStoreCreateConfig mode_config;
+    /// Table mode implemented by the store.
+    RealtimeStoreMode mode = RealtimeStoreMode::APPEND_ONLY;
+    /// Statistics collected by append-only stores.
+    StatisticsMode statistics_mode = StatisticsMode::NONE;
 };
 
 /// A record batch and its framework-assigned contiguous offset range.
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 81a1f133e..3ba81f03f 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -71,15 +71,8 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderForTest(
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    if (!key_schema) {
-        return Status::Invalid("prepared key schema cannot be null");
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> key_fields,
-                           DataField::ConvertArrowSchemaToDataFields(key_schema));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FieldsComparator> key_comparator,
-                           FieldsComparator::Create(key_fields, /*is_ascending_order=*/true));
     return AdaptPreparedBatchReader(std::move(reader), prepared_schema, visible_offsets, key_schema,
-                                    value_schema, key_comparator, memory_pool);
+                                    value_schema, memory_pool);
 }
 
 class TrackingBatchReader : public BatchReader {
@@ -266,31 +259,6 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsReversedVisibleO
     ASSERT_NOK_WITH_MSG(result, "prepared visible offset range begin exceeds end");
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestRejectsUnsortedPluginRowsAcrossBatches) {
-    std::vector<DataField> value_fields = {DataField(0, arrow::field("id", arrow::int32()))};
-    std::shared_ptr<arrow::Schema> value_schema =
-        DataField::ConvertDataFieldsToArrowSchema(value_fields);
-    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
-            [0, 10, 0, 2],
-            [0, 11, 1, 1]
-        ])")
-            .ValueOrDie();
-    auto batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, std::nullopt,
-                                        key_schema, value_schema, pool_));
-    Result<std::vector<KeyValue>> result =
-        ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                   KeyValueRecordReader::Iterator>(reader.get());
-    ASSERT_NOK_WITH_MSG(result, "not globally sorted by primary key and sequence number");
-}
-
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 31889ee84..d8e7f5d15 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -152,9 +152,9 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
             arrow::ExportSchema(*arrow::schema(std::move(prepared_fields)), c_write_schema.get()));
         PAIMON_ASSIGN_OR_RAISE(
             RealtimeStoreState store_state,
-            realtime_context_impl->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
-                std::move(c_write_schema), options_.ToMap(), pool_, partition_map, bucket,
-                PrimaryKeyRealtimeStoreCreateConfig{}}));
+            realtime_context_impl->GetOrCreateRealtimeStore(
+                RealtimeStoreCreateRequest{std::move(c_write_schema), options_.ToMap(), pool_,
+                                           partition_map, bucket, RealtimeStoreMode::PRIMARY_KEY}));
         realtime_store_state = std::move(store_state);
         compact_manager = std::make_shared<NoopCompactManager>();
     } else {
diff --git a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
index d336394ad..d0d4ae704 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
@@ -42,17 +42,19 @@ Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
     }
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> imported_schema,
                                       arrow::ImportSchema(request.write_schema.get()));
-    if (std::holds_alternative<AppendRealtimeStoreCreateConfig>(request.mode_config)) {
-        const AppendRealtimeStoreCreateConfig& append_config =
-            std::get<AppendRealtimeStoreCreateConfig>(request.mode_config);
-        std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(request.memory_pool);
-        return std::make_shared<ArrowRealtimeStore>(imported_schema, append_config.statistics_mode,
-                                                    request.memory_pool, arrow_pool);
+    switch (request.mode) {
+        case RealtimeStoreMode::APPEND_ONLY: {
+            std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(request.memory_pool);
+            return std::make_shared<ArrowRealtimeStore>(imported_schema, request.statistics_mode,
+                                                        request.memory_pool, arrow_pool);
+        }
+        case RealtimeStoreMode::PRIMARY_KEY: {
+            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                                   PrimaryKeyRealtimeStore::Create(imported_schema));
+            return std::shared_ptr<RealtimeStore>(std::move(store));
+        }
     }
-
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                           PrimaryKeyRealtimeStore::Create(imported_schema));
-    return std::shared_ptr<RealtimeStore>(std::move(store));
+    return Status::Invalid("invalid real-time store mode: ", static_cast<int32_t>(request.mode));
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
index f186a8161..0a3e52353 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
@@ -237,7 +237,8 @@ TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) {
                                        pool_,
                                        /*partition=*/{},
                                        /*bucket=*/0,
-                                       AppendRealtimeStoreCreateConfig{StatisticsMode::FULL}};
+                                       RealtimeStoreMode::APPEND_ONLY,
+                                       StatisticsMode::FULL};
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> realtime_store,
                          factory.Create(std::move(request)));
     std::shared_ptr<ArrowRealtimeStore> store =
@@ -274,6 +275,18 @@ TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) {
     ASSERT_EQ(std::vector<int64_t>({0, 1}), ReadIds(unfiltered_batch));
 }
 
+TEST_F(ArrowRealtimeStoreTest, TestFactoryRejectsInvalidMode) {
+    ArrowRealtimeStoreFactory factory;
+    std::unique_ptr<ArrowSchema> write_schema = MakeReadSchema(schema_);
+    RealtimeStoreCreateRequest request{std::move(write_schema),
+                                       /*options=*/{},
+                                       pool_,
+                                       /*partition=*/{},
+                                       /*bucket=*/0,
+                                       static_cast<RealtimeStoreMode>(-1)};
+    ASSERT_NOK_WITH_MSG(factory.Create(std::move(request)), "invalid real-time store mode: -1");
+}
+
 TEST_F(ArrowRealtimeStoreTest, TestMissingStatisticsRetainsNonMatchingBatch) {
     ASSERT_OK(
         store_->Write(RealtimeWriteBatch{MakeBatch(R"([[0, "a"], [1, "b"]])"), OffsetRange(0, 2)}));
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index ae575ca58..a34ccea43 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -42,7 +42,6 @@
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
-#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
@@ -377,7 +376,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                            const std::optional<OffsetRange>& visible_offsets,
                            const std::shared_ptr<arrow::Schema>& key_schema,
                            const std::shared_ptr<arrow::Schema>& value_schema,
-                           const std::shared_ptr<FieldsComparator>& key_comparator,
                            const std::shared_ptr<MemoryPool>& pool,
                            const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage)
         : reader_(std::move(reader)),
@@ -385,7 +383,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
           visible_offsets_(visible_offsets),
           key_schema_(key_schema),
           value_schema_(value_schema),
-          key_comparator_(key_comparator),
           pool_(pool),
           arrow_pool_(GetArrowPool(pool)),
           offset_coverage_(offset_coverage) {}
@@ -506,7 +503,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                                                             value_schema_, arrow_pool_.get()));
             key_ctx_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
             value_ctx_ = std::make_shared<ColumnarBatchContext>(value_fields, pool_);
-            PAIMON_RETURN_NOT_OK(ValidateOrdering(key_ctx_, sequence_number_array_));
             if (!SelectVisibleRows(*offset_array)) {
                 continue;
             }
@@ -548,32 +544,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         return Status::OK();
     }
 
-    Status ValidateOrdering(
-        const std::shared_ptr<ColumnarBatchContext>& key_context,
-        const std::shared_ptr<arrow::NumericArray<arrow::Int64Type>>& sequences) {
-        if (sequences->length() == 0) {
-            return Status::OK();
-        }
-        for (int64_t row = 0; row < sequences->length(); ++row) {
-            ColumnarRowRef current_key(key_context, row);
-            if (previous_key_context_) {
-                ColumnarRowRef previous_key(previous_key_context_, previous_key_row_);
-                const int32_t key_comparison =
-                    key_comparator_->CompareTo(previous_key, current_key);
-                if (key_comparison > 0 ||
-                    (key_comparison == 0 && previous_sequence_ > sequences->Value(row))) {
-                    return Status::Invalid(
-                        "PK real-time plugin reader is not globally sorted by primary key and "
-                        "sequence number");
-                }
-            }
-            previous_key_context_ = key_context;
-            previous_key_row_ = row;
-            previous_sequence_ = sequences->Value(row);
-        }
-        return Status::OK();
-    }
-
     bool SelectVisibleRows(const arrow::Int64Array& offsets) {
         if (!visible_offsets_.has_value()) {
             return true;
@@ -614,7 +584,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     std::optional<OffsetRange> visible_offsets_;
     std::shared_ptr<arrow::Schema> key_schema_;
     std::shared_ptr<arrow::Schema> value_schema_;
-    std::shared_ptr<FieldsComparator> key_comparator_;
     std::shared_ptr<MemoryPool> pool_;
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     std::shared_ptr<RealtimeOffsetCoverage> offset_coverage_;
@@ -624,9 +593,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     std::shared_ptr<arrow::NumericArray<arrow::Int8Type>> row_kind_array_;
     std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> sequence_number_array_;
     std::optional<std::vector<int64_t>> visible_rows_;
-    std::shared_ptr<ColumnarBatchContext> previous_key_context_;
-    int64_t previous_key_row_ = 0;
-    int64_t previous_sequence_ = 0;
 };
 
 }  // namespace
@@ -651,7 +617,6 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
     const std::optional<OffsetRange>& visible_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool,
     const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage) {
     std::unique_ptr<BatchReader> owned_reader = std::move(reader);
@@ -669,9 +634,6 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
     if (!value_schema) {
         return Status::Invalid("prepared value schema cannot be null");
     }
-    if (!key_comparator) {
-        return Status::Invalid("prepared key comparator cannot be null");
-    }
     if (!memory_pool) {
         return Status::Invalid("prepared reader memory pool cannot be null");
     }
@@ -680,9 +642,9 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
     if (!visible_offsets.has_value()) {
         PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
     }
-    std::unique_ptr<KeyValueRecordReader> result(new PreparedKeyValueReader(
-        std::move(owned_reader), prepared_schema, visible_offsets, key_schema, value_schema,
-        key_comparator, memory_pool, offset_coverage));
+    std::unique_ptr<KeyValueRecordReader> result(
+        new PreparedKeyValueReader(std::move(owned_reader), prepared_schema, visible_offsets,
+                                   key_schema, value_schema, memory_pool, offset_coverage));
     close_guard.Release();
     return result;
 }
@@ -694,10 +656,9 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     const std::optional<OffsetRange>& visible_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     return AdaptPreparedBatchReaderImpl(std::move(reader), prepared_schema, visible_offsets,
-                                        key_schema, value_schema, key_comparator, memory_pool,
+                                        key_schema, value_schema, memory_pool,
                                         /*offset_coverage=*/nullptr);
 }
 
@@ -706,7 +667,6 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
     const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
     ScopeGuard readers_guard([&readers, &adapted_readers]() {
@@ -728,7 +688,7 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> adapted_reader,
                                AdaptPreparedBatchReaderImpl(
                                    std::move(reader), prepared_schema, std::nullopt, key_schema,
-                                   value_schema, key_comparator, memory_pool, offset_coverage));
+                                   value_schema, memory_pool, offset_coverage));
         adapted_readers.push_back(std::move(adapted_reader));
     }
     readers_guard.Release();
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
index 81ae0abc7..4ef4887e6 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/prepared_key_value_reader.h
@@ -30,7 +30,6 @@
 
 namespace paimon {
 class BatchReader;
-class FieldsComparator;
 class MemoryPool;
 
 /// Validates the required leading fields of a prepared real-time transport schema.
@@ -42,16 +41,14 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
     const std::optional<OffsetRange>& visible_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool);
 
-/// Adapts commit readers and validates that they collectively cover `sealed_offsets` exactly.
+/// Adapts commit readers and validates their offsets against `sealed_offsets`.
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBatchReaders(
     std::vector<std::unique_ptr<BatchReader>>&& readers,
     const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<MemoryPool>& memory_pool);
 
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 68f4bf500..7d188609d 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -201,13 +201,9 @@ class PrimaryKeyRealtimeStore::Impl {
         std::shared_ptr<arrow::StructArray> prepared =
             checked_pointer_cast<arrow::StructArray>(array);
         std::lock_guard<std::mutex> lock(mutex_);
-        if (last_offset_ && write_batch.offset_range.begin != last_offset_.value()) {
-            return Status::Invalid("PK real-time offset ranges must be contiguous");
-        }
         building_.push_back(
             StoredBatch{prepared, write_batch.offset_range, GetArrayMemoryUsage(prepared->data())});
         building_memory_usage_ += building_.back().memory_usage;
-        last_offset_ = write_batch.offset_range.end;
         return Status::OK();
     }
 
@@ -290,7 +286,6 @@ class PrimaryKeyRealtimeStore::Impl {
     std::vector<StoredBatch> building_;
     std::vector<std::shared_ptr<Segment>> sealed_;
     uint64_t building_memory_usage_ = 0;
-    std::optional<int64_t> last_offset_;
 };
 
 PrimaryKeyRealtimeStore::PrimaryKeyRealtimeStore(std::unique_ptr<Impl>&& impl)
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 49da66fc2..6ccceba81 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -178,9 +178,6 @@ TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
 
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeBatch(R"([[0, 1, 0, 1, "one"], [0, 2, 1, 2, "two"]])"), OffsetRange(0, 2)}));
-    ASSERT_NOK_WITH_MSG(store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 3, 3, 3, "three"]])"),
-                                                        OffsetRange(3, 4)}),
-                        "offset ranges must be contiguous");
     ASSERT_OK(store->Write(
         RealtimeWriteBatch{MakeBatch(R"([[0, 3, 2, 3, "three"]])"), OffsetRange(2, 3)}));
 
@@ -188,9 +185,6 @@ TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
     ASSERT_TRUE(segment.has_value());
     ASSERT_EQ(OffsetRange(0, 3), segment.value()->GetOffsetRange());
     ASSERT_GT(store->GetMemoryUsage(), 0);
-    ASSERT_NOK_WITH_MSG(
-        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 4, 4, 4, "four"]])"), OffsetRange(4, 5)}),
-        "offset ranges must be contiguous");
     ASSERT_OK(store->Write(
         RealtimeWriteBatch{MakeBatch(R"([[0, 4, 3, 4, "four"]])"), OffsetRange(3, 4)}));
 }
diff --git a/src/paimon/core/realtime/realtime_append_only_writer.cpp b/src/paimon/core/realtime/realtime_append_only_writer.cpp
index ea5feecce..0fb4e58e7 100644
--- a/src/paimon/core/realtime/realtime_append_only_writer.cpp
+++ b/src/paimon/core/realtime/realtime_append_only_writer.cpp
@@ -56,8 +56,8 @@ Result<std::shared_ptr<RealtimeAppendOnlyWriter>> RealtimeAppendOnlyWriter::Crea
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
                            RealtimeContextImpl::Cast(realtime_context));
     RealtimeStoreCreateRequest request{
-        std::move(write_schema),          options, memory_pool, partition, bucket,
-        AppendRealtimeStoreCreateConfig{statistics_mode}};
+        std::move(write_schema),        options,        memory_pool, partition, bucket,
+        RealtimeStoreMode::APPEND_ONLY, statistics_mode};
     PAIMON_ASSIGN_OR_RAISE(RealtimeStoreState store_state,
                            realtime_context_impl->GetOrCreateRealtimeStore(std::move(request)));
     return std::shared_ptr<RealtimeAppendOnlyWriter>(new RealtimeAppendOnlyWriter(
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 0ea4c61d6..404c0faac 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -48,10 +48,6 @@
 namespace paimon {
 namespace {
 
-bool SameMode(const RealtimeStoreCreateConfig& left, const RealtimeStoreCreateConfig& right) {
-    return left.index() == right.index();
-}
-
 std::string PartitionToString(const std::map<std::string, std::string>& partition) {
     std::string result = "{";
     for (auto iter = partition.begin(); iter != partition.end(); ++iter) {
@@ -122,7 +118,7 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
         initial_offset = offset_iter->second;
     }
     if (iter != stores_.end()) {
-        if (!SameMode(iter->second.mode_config, request.mode_config) ||
+        if (iter->second.mode != request.mode ||
             !iter->second.write_schema->Equals(*requested_schema, /*check_metadata=*/true)) {
             return Status::Invalid("real-time store schema or mode mismatch for partition " +
                                    PartitionToString(key.partition) + ", bucket " +
@@ -151,10 +147,10 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     }
     PAIMON_RETURN_NOT_OK_FROM_ARROW(
         arrow::ExportSchema(*requested_schema, request.write_schema.get()));
-    RealtimeStoreCreateConfig mode_config = request.mode_config;
+    RealtimeStoreMode mode = request.mode;
     Result<std::shared_ptr<RealtimeStore>> store_result = factory_->Create(std::move(request));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
-    stores_.emplace(key, StoreEntry{store, requested_schema, std::move(mode_config)});
+    stores_.emplace(key, StoreEntry{store, requested_schema, mode});
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
     }
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index fd65fc246..29ac7c05d 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -99,7 +99,7 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     struct StoreEntry {
         std::shared_ptr<RealtimeStore> store;
         std::shared_ptr<arrow::Schema> write_schema;
-        RealtimeStoreCreateConfig mode_config;
+        RealtimeStoreMode mode;
         int64_t materialized_max_sequence_number = -1;
     };
 
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 538e4c56c..fbdaa86b7 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -117,10 +117,11 @@ Result<RealtimeStoreState> GetOrCreateAppendStore(
     const std::shared_ptr<RealtimeContextImpl>& context,
     const std::map<std::string, std::string>& partition, int32_t bucket,
     std::unique_ptr<ArrowSchema> write_schema, const std::map<std::string, std::string>& options,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
+    const std::shared_ptr<MemoryPool>& memory_pool,
+    StatisticsMode statistics_mode = StatisticsMode::NONE) {
     return context->GetOrCreateRealtimeStore(
         RealtimeStoreCreateRequest{std::move(write_schema), options, memory_pool, partition, bucket,
-                                   AppendRealtimeStoreCreateConfig{StatisticsMode::NONE}});
+                                   RealtimeStoreMode::APPEND_ONLY, statistics_mode});
 }
 
 TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
@@ -130,9 +131,10 @@ TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
                          GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0,
                                                 MakeWriteSchema(), {{"k", "v"}}, GetDefaultPool()));
     ASSERT_EQ(0, first.initial_offset);
-    ASSERT_OK_AND_ASSIGN(RealtimeStoreState second,
-                         GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0,
-                                                MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(
+        RealtimeStoreState second,
+        GetOrCreateAppendStore(context, {{"dt", "2026-08-02"}}, 0, MakeWriteSchema(), {},
+                               GetDefaultPool(), StatisticsMode::FULL));
     ASSERT_EQ(first.store, second.store);
     ASSERT_EQ(0, second.initial_offset);
     ASSERT_EQ(1, factory->stores.size());
@@ -160,6 +162,21 @@ TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
     ASSERT_EQ(1, factory->stores[2]->acquire_count);
 }
 
+TEST(RealtimeContextTest, TestRejectsMismatchedModeOnStoreReuse) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
+    ASSERT_OK(
+        GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, GetDefaultPool()));
+
+    ASSERT_NOK_WITH_MSG(
+        context->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
+            MakeWriteSchema(), {}, GetDefaultPool(), partition, 0, RealtimeStoreMode::PRIMARY_KEY}),
+        "schema or mode mismatch for partition {dt=2026-08-02}, bucket 0; recreate the "
+        "RealtimeContext");
+    ASSERT_EQ(1, factory->stores.size());
+}
+
 TEST(RealtimeContextTest, TestRejectsMismatchedSchemaOnStoreReuse) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index b04cc8e25..ef85e8b6e 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -288,7 +288,7 @@ Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegm
     PAIMON_ASSIGN_OR_RAISE(
         std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
         AdaptPreparedCommitBatchReaders(std::move(readers), prepared_schema_, sealed_offsets,
-                                        key_schema_, write_schema_, key_comparator_, memory_pool_));
+                                        key_schema_, write_schema_, memory_pool_));
     std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
     sorted_readers.reserve(prepared_readers.size());
     for (std::unique_ptr<KeyValueRecordReader>& prepared_reader : prepared_readers) {
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 32231140e..0f6b2189b 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -90,12 +90,11 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null query reader");
         }
-        PAIMON_ASSIGN_OR_RAISE(
-            std::unique_ptr<KeyValueRecordReader> prepared_reader,
-            AdaptPreparedBatchReader(
-                std::move(reader), prepared_schema,
-                OffsetRange(split->CommittedEndOffset(), split->MemoryEndOffset()), key_schema,
-                value_schema, key_comparator, memory_pool));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> prepared_reader,
+                               AdaptPreparedBatchReader(std::move(reader), prepared_schema,
+                                                        OffsetRange(split->CommittedEndOffset(),
+                                                                    split->MemoryEndOffset()),
+                                                        key_schema, value_schema, memory_pool));
         auto merge = std::make_unique<DeduplicateMergeFunction>(false);
         result.push_back(std::make_unique<MergedKeyValueRecordReader>(
             std::move(prepared_reader), key_comparator,
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index ed75b554e..78e8badee 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -538,7 +538,7 @@ class FailAfterPhysicalFileRealtimeStore final : public DelegatingRealtimeStore
     std::shared_ptr<std::atomic<bool>> saw_artifacts_;
 };
 
-enum class CommitReaderMalformation { DROP_LAST, UNSORTED, DUPLICATE_OFFSET, OUT_OF_RANGE_OFFSET };
+enum class CommitReaderMalformation { DROP_LAST, DUPLICATE_OFFSET, OUT_OF_RANGE_OFFSET };
 
 class CorruptingBatchReader final : public BatchReader {
  public:
@@ -550,8 +550,6 @@ class CorruptingBatchReader final : public BatchReader {
         switch (malformation_) {
             case CommitReaderMalformation::DROP_LAST:
                 return DropLast();
-            case CommitReaderMalformation::UNSORTED:
-                return SwapFirstTwo();
             case CommitReaderMalformation::DUPLICATE_OFFSET:
                 return SubstituteOffset(/*offset=*/0);
             case CommitReaderMalformation::OUT_OF_RANGE_OFFSET:
@@ -588,33 +586,6 @@ class CorruptingBatchReader final : public BatchReader {
         return result;
     }
 
-    Result<ReadBatch> SwapFirstTwo() {
-        if (corrupted_) {
-            return delegate_->NextBatch();
-        }
-        corrupted_ = true;
-        PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
-        if (BatchReader::IsEofBatch(batch)) {
-            return MakeEofBatch();
-        }
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-            std::shared_ptr<arrow::Array> array,
-            arrow::ImportArray(batch.first.get(), batch.second.get()));
-        if (array->length() < 2) {
-            return Status::Invalid("cannot make a one-row reader unsorted");
-        }
-        arrow::ArrayVector pieces = {array->Slice(1, 1), array->Slice(0, 1)};
-        if (array->length() > 2) {
-            pieces.push_back(array->Slice(2));
-        }
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> swapped,
-                                          arrow::Concatenate(pieces));
-        auto output = std::make_unique<ArrowArray>();
-        auto schema = std::make_unique<ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*swapped, output.get(), schema.get()));
-        return ReadBatch(std::move(output), std::move(schema));
-    }
-
     Result<ReadBatch> SubstituteOffset(int64_t offset) {
         PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
         if (BatchReader::IsEofBatch(batch)) {
@@ -652,7 +623,6 @@ class CorruptingBatchReader final : public BatchReader {
 
     std::unique_ptr<BatchReader> delegate_;
     CommitReaderMalformation malformation_;
-    bool corrupted_ = false;
     std::optional<ReadBatch> buffered_;
 };
 
@@ -2399,12 +2369,6 @@ TEST_F(RealtimeWriteInteTest, TestPkRejectsOutOfRangeOffset) {
                                            "offset is outside the sealed range");
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkRejectsUnsortedPluginRows) {
-    CheckPkRejectsCommitReaderMalformation(
-        CommitReaderMalformation::UNSORTED,
-        "not globally sorted by primary key and sequence number");
-}
-
 TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
     CreatePkTable();
     auto state = std::make_shared<CloseTrackingReaderState>();

From 7a59f9e4329e8539fab7d87a8d3b3f34339ab57a Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 20:46:12 +0800
Subject: [PATCH 55/93] refactor(realtime): align PK query projection with
 store

---
 include/paimon/realtime/realtime_store.h      |  11 +-
 src/paimon/common/utils/arrow/arrow_utils.cpp |  16 +
 src/paimon/common/utils/arrow/arrow_utils.h   |   3 +
 .../merged_key_value_record_reader_test.cpp   |  46 +--
 .../core/operation/file_store_write.cpp       |   2 +-
 .../key_value_file_store_write_test.cpp       |  36 +-
 .../core/realtime/arrow_realtime_store.cpp    |  25 +-
 .../realtime/prepared_key_value_reader.cpp    | 313 ++++--------------
 .../realtime/primary_key_realtime_store.cpp   | 147 +++++++-
 .../realtime/primary_key_realtime_store.h     |   3 +-
 .../primary_key_realtime_store_test.cpp       | 154 ++++++++-
 .../core/realtime/realtime_context_impl.cpp   |   4 +-
 .../realtime/realtime_primary_key_writer.cpp  |   6 +-
 .../table/source/key_value_table_read.cpp     |   9 +-
 src/paimon/core/table/source/table_scan.cpp   |   2 +-
 15 files changed, 424 insertions(+), 353 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 241413b31..fe2b95e1b 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -108,7 +108,7 @@ class PAIMON_EXPORT RealtimeReadView {
 /// Parameters used by a `RealtimeStore` to create readers for a query.
 struct PAIMON_EXPORT RealtimeQueryContext {
     /// Append mode receives the requested output fields before the mandatory leading
-    /// `_VALUE_KIND` field is added. Primary-key mode receives the complete prepared schema.
+    /// `_VALUE_KIND` field is added. Primary-key mode receives the requested prepared schema.
     /// This schema is borrowed and remains valid only during `CreateQueryReaders`; plugins must
     /// import or copy it synchronously.
     ::ArrowSchema* read_schema;
@@ -165,10 +165,11 @@ class PAIMON_EXPORT RealtimeStore {
     ///
     /// Append-mode batches contain `_VALUE_KIND` followed by the requested fields except a
     /// duplicate `_VALUE_KIND`, and collectively expose every matching row exactly once.
-    /// Primary-key batches use the prepared transport schema and may contain multiple mutations
-    /// per key; each reader's complete stream is sorted by full primary key then sequence number,
-    /// and the readers collectively expose every raw mutation exactly once. Paimon retains `view`
-    /// for the lifetime of the resulting framework reader.
+    /// Primary-key batches use the requested prepared transport schema, including nested field-ID
+    /// alignment, and may contain multiple mutations per key; each reader's complete stream is
+    /// sorted by full primary key then sequence number, and the readers collectively expose every
+    /// raw mutation exactly once. Paimon retains `view` for the lifetime of the resulting framework
+    /// reader.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) = 0;
diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp b/src/paimon/common/utils/arrow/arrow_utils.cpp
index f29e1d11e..34721e125 100644
--- a/src/paimon/common/utils/arrow/arrow_utils.cpp
+++ b/src/paimon/common/utils/arrow/arrow_utils.cpp
@@ -355,6 +355,22 @@ void ArrowUtils::TraverseArray(const std::shared_ptr<arrow::Array>& array) {
     }
 }
 
+uint64_t ArrowUtils::GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
+    uint64_t result = 0;
+    for (const std::shared_ptr<arrow::Buffer>& buffer : data->buffers) {
+        if (buffer) {
+            result += static_cast<uint64_t>(buffer->size());
+        }
+    }
+    for (const std::shared_ptr<arrow::ArrayData>& child : data->child_data) {
+        result += GetArrayMemoryUsage(child);
+    }
+    if (data->dictionary) {
+        result += GetArrayMemoryUsage(data->dictionary);
+    }
+    return result;
+}
+
 bool ArrowUtils::EqualsIgnoreNullable(const std::shared_ptr<arrow::DataType>& type,
                                       const std::shared_ptr<arrow::DataType>& other_type) {
     if (type->id() != other_type->id() || type->num_fields() != other_type->num_fields()) {
diff --git a/src/paimon/common/utils/arrow/arrow_utils.h b/src/paimon/common/utils/arrow/arrow_utils.h
index 326b3889e..e8395c4cb 100644
--- a/src/paimon/common/utils/arrow/arrow_utils.h
+++ b/src/paimon/common/utils/arrow/arrow_utils.h
@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include <cstdint>
 #include <vector>
 
 #include "arrow/api.h"
@@ -48,6 +49,8 @@ class PAIMON_EXPORT ArrowUtils {
     // avoid subsequent multi-threading problems.
     static void TraverseArray(const std::shared_ptr<arrow::Array>& array);
 
+    static uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data);
+
     static Result<std::shared_ptr<arrow::StructArray>> RemoveFieldFromStructArray(
         const std::shared_ptr<arrow::StructArray>& struct_array, const std::string& field_name);
 
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 3ba81f03f..c8d80906a 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -325,10 +325,10 @@ TEST_F(MergedKeyValueRecordReaderTest, TestMissingCompositeKey) {
         std::unique_ptr<KeyValueRecordReader> reader,
         AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
                                         arrow::schema({key0, key1}), value_schema, pool_));
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "cannot find field id 1");
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryAddRename) {
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryReaderRequiresStoreAlignedSchema) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Field> old_value = MakeField("old_value", arrow::int32(), 1);
     std::shared_ptr<arrow::Field> renamed_value = MakeField("renamed_value", arrow::int32(), 1);
@@ -347,11 +347,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestQueryAddRename) {
         std::unique_ptr<KeyValueRecordReader> reader,
         AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
                                         arrow::schema({key}), value_schema, pool_));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader::Iterator> iterator,
-                         reader->NextBatch());
-    ASSERT_OK_AND_ASSIGN(KeyValue key_value, iterator->Next());
-    ASSERT_EQ(20, key_value.value->GetInt(1));
-    ASSERT_TRUE(key_value.value->IsNullAt(2));
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
 TEST_F(MergedKeyValueRecordReaderTest, TestMergedReaderErrorRetry) {
@@ -402,35 +398,9 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
         "prepared batch field");
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedProjection) {
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedValues) {
     std::shared_ptr<arrow::Field> id = MakeField("id", arrow::int32(), 0);
-    std::shared_ptr<arrow::Field> item_a = MakeField("a", arrow::int32(), 10);
-    std::shared_ptr<arrow::Field> item_b = MakeField("b", arrow::int32(), 11);
-    std::shared_ptr<arrow::Field> items =
-        MakeField("items", arrow::list(arrow::field("item", arrow::struct_({item_a, item_b}))), 2);
-    std::shared_ptr<arrow::Field> attr_x = MakeField("x", arrow::int32(), 20);
-    std::shared_ptr<arrow::Field> attr_y = MakeField("y", arrow::int32(), 21);
-    std::shared_ptr<arrow::Field> attrs =
-        MakeField("attrs", arrow::map(arrow::utf8(), arrow::struct_({attr_x, attr_y})), 3);
-    std::shared_ptr<arrow::Field> key_left = MakeField("left", arrow::int32(), 30);
-    std::shared_ptr<arrow::Field> key_right = MakeField("right", arrow::int32(), 31);
-    std::shared_ptr<arrow::Field> keyed_values = MakeField(
-        "keyed_values", arrow::map(arrow::struct_({key_left, key_right}), arrow::int32()), 4);
-    std::shared_ptr<arrow::Schema> full_value_schema =
-        arrow::schema({id, items, attrs, keyed_values});
     std::shared_ptr<arrow::Schema> key_schema = arrow::schema({id});
-    std::shared_ptr<arrow::Schema> prepared_schema =
-        MakePreparedSchema(full_value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
-        [0, 9, 9, 0, [[1, 2]], [["prefix", [3, 4]]], [[[5, 6], 7]]],
-        [0, 10, 0, 1, [[100, 200], [300, 400]], [["k1", [7, 8]], ["k2", [9, 10]]], [[[11, 12], 13], [[21, 22], 23]]],
-        [0, 11, 11, 2, [[8, 9]], [["suffix", [10, 11]]], [[[12, 13], 14]]]
-    ])")
-            .ValueOrDie());
-    prepared_array = checked_pointer_cast<arrow::StructArray>(prepared_array->Slice(1, 1));
-
     std::shared_ptr<arrow::Field> query_item_b = MakeField("renamed_b", arrow::int32(), 11);
     std::shared_ptr<arrow::Field> query_item_a = MakeField("renamed_a", arrow::int32(), 10);
     std::shared_ptr<arrow::Field> query_items = MakeField(
@@ -448,6 +418,14 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedProjection) {
                   arrow::map(arrow::struct_({query_key_right, query_key_left}), arrow::int32()), 4);
     std::shared_ptr<arrow::Schema> query_value_schema =
         arrow::schema({id, query_items, query_attrs, query_keyed_values});
+    std::shared_ptr<arrow::Schema> prepared_schema =
+        MakePreparedSchema(query_value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(
+            prepared_type,
+            R"([[0, 10, 0, 1, [[200, 100], [400, 300]], [["k1", [8, 7]], ["k2", [10, 9]]], [[[12, 11], 13], [[22, 21], 23]]]])")
+            .ValueOrDie();
 
     auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
     ASSERT_OK_AND_ASSIGN(
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 84a324762..6c3b4d033 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -198,7 +198,7 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     } else {
         // pk table
         if (ctx->GetRealtimeContext()) {
-            PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(options, *schema));
+            PAIMON_RETURN_NOT_OK(PrimaryKeyRealtimeStore::ValidateOptions(options, *schema));
             if (ignore_previous_files) {
                 return Status::NotImplemented(
                     "PK realtime v1 requires restore from the latest snapshot");
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index a2344e803..0f70235a7 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -44,6 +44,7 @@
 #include "paimon/common/data/shredding/map_shared_shredding_utils.h"
 #include "paimon/common/data/shredding/map_shredding_defs.h"
 #include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/path_util.h"
 #include "paimon/core/io/data_file_meta.h"
@@ -110,7 +111,7 @@ class TestingMemoryPool final : public MemoryPool {
     std::unique_ptr<MemoryPool> delegate_ = GetMemoryPool();
 };
 
-}
+}  // namespace
 
 class KeyValueFileStoreWriteTest : public ::testing::Test {
  protected:
@@ -255,10 +256,23 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
         if (views.size() != 1) {
             return Status::Invalid("expected exactly one real-time store");
         }
-        RealtimeQueryContext query_context{nullptr, nullptr, false};
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               views[0].store->CreateQueryReaders(
-                                   views[0].read_view, 0, query_context));
+        std::shared_ptr<arrow::Schema> prepared_schema = arrow::schema({
+            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
+                ->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+                ->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
+            DataField::ConvertDataFieldToArrowField(
+                DataField(0, arrow::field("id", arrow::int64(), false))),
+            DataField::ConvertDataFieldToArrowField(
+                DataField(1, arrow::field("value", arrow::utf8()))),
+        });
+        auto c_schema = std::make_unique<ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
+        RealtimeQueryContext query_context{c_schema.get(), nullptr, false};
+        PAIMON_ASSIGN_OR_RAISE(
+            std::vector<std::unique_ptr<BatchReader>> readers,
+            views[0].store->CreateQueryReaders(views[0].read_view, 0, query_context));
         std::vector<std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>> rows;
         for (const std::unique_ptr<BatchReader>& reader : readers) {
             while (true) {
@@ -466,8 +480,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
 }
 
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
-    const std::map<std::string, std::string> options = {
-        {Options::BUCKET, "1"}, {Options::REALTIME_ENABLED, "true"}};
+    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"},
+                                                        {Options::REALTIME_ENABLED, "true"}};
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
         arrow::field("id", arrow::int64(), false),
         arrow::field("_REALTIME_OFFSET", arrow::int64()),
@@ -490,8 +504,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
 }
 
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
-    const std::map<std::string, std::string> options = {
-        {Options::BUCKET, "1"}, {Options::REALTIME_ENABLED, "true"}};
+    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"},
+                                                        {Options::REALTIME_ENABLED, "true"}};
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
         arrow::field("id", arrow::int64(), false),
         arrow::field("value", arrow::utf8()),
@@ -549,8 +563,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
 
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) {
     const int64_t max = std::numeric_limits<int64_t>::max();
-    const std::map<std::string, std::string> options = {
-        {Options::BUCKET, "1"}, {Options::REALTIME_ENABLED, "true"}};
+    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"},
+                                                        {Options::REALTIME_ENABLED, "true"}};
     const std::shared_ptr<arrow::Schema> schema = arrow::schema({
         arrow::field("id", arrow::int64(), false),
         arrow::field("value", arrow::utf8()),
diff --git a/src/paimon/core/realtime/arrow_realtime_store.cpp b/src/paimon/core/realtime/arrow_realtime_store.cpp
index 18087a29f..1136243e8 100644
--- a/src/paimon/core/realtime/arrow_realtime_store.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store.cpp
@@ -33,6 +33,7 @@
 #include "paimon/common/reader/complete_row_kind_batch_reader.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/row_kind.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/projected_array.h"
@@ -43,22 +44,6 @@
 namespace paimon {
 namespace {
 
-uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
-    uint64_t result = 0;
-    for (const std::shared_ptr<arrow::Buffer>& buffer : data->buffers) {
-        if (buffer) {
-            result += static_cast<uint64_t>(buffer->size());
-        }
-    }
-    for (const std::shared_ptr<arrow::ArrayData>& child : data->child_data) {
-        result += GetArrayMemoryUsage(child);
-    }
-    if (data->dictionary) {
-        result += GetArrayMemoryUsage(data->dictionary);
-    }
-    return result;
-}
-
 bool SupportsMinMax(const std::shared_ptr<arrow::DataType>& type) {
     switch (type->id()) {
         case arrow::Type::BOOL:
@@ -393,11 +378,11 @@ Status ArrowRealtimeStore::Write(RealtimeWriteBatch&& write_batch) {
     if (building_range_ && write_batch.offset_range.begin != building_range_->end) {
         return Status::Invalid("real-time offset ranges must be contiguous");
     }
-    uint64_t memory_usage = GetArrayMemoryUsage(struct_array->data());
+    uint64_t memory_usage = ArrowUtils::GetArrayMemoryUsage(struct_array->data());
     if (statistics) {
-        memory_usage += GetArrayMemoryUsage(statistics->min_values->data()) +
-                        GetArrayMemoryUsage(statistics->max_values->data()) +
-                        GetArrayMemoryUsage(statistics->null_counts->data());
+        memory_usage += ArrowUtils::GetArrayMemoryUsage(statistics->min_values->data()) +
+                        ArrowUtils::GetArrayMemoryUsage(statistics->max_values->data()) +
+                        ArrowUtils::GetArrayMemoryUsage(statistics->null_counts->data());
     }
     building_memory_usage_ += memory_usage;
     building_batches_.push_back(
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index a34ccea43..53652cc81 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -23,12 +23,10 @@
 #include <limits>
 #include <memory>
 #include <optional>
-#include <unordered_map>
 #include <utility>
 #include <vector>
 
 #include "arrow/array/array_base.h"
-#include "arrow/array/array_nested.h"
 #include "arrow/array/array_primitive.h"
 #include "arrow/c/bridge.h"
 #include "arrow/type.h"
@@ -39,7 +37,6 @@
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/types/row_kind.h"
 #include "paimon/common/utils/arrow/arrow_utils.h"
-#include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/scope_guard.h"
@@ -66,10 +63,6 @@ void CloseReaders(const std::vector<std::unique_ptr<Reader>>& readers) {
     }
 }
 
-Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
-    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
-    arrow::MemoryPool* arrow_pool);
-
 class RealtimeOffsetCoverage {
  public:
     static Result<std::shared_ptr<RealtimeOffsetCoverage>> Create(const OffsetRange& sealed_offsets,
@@ -158,61 +151,29 @@ Result<int32_t> FindFieldIndexByPaimonId(const arrow::FieldVector& fields, int32
     return Status::Invalid(fmt::format("cannot find field id {} in prepared schema", field_id));
 }
 
-Status ValidateProjectionType(const std::shared_ptr<arrow::DataType>& prepared_type,
-                              const std::shared_ptr<arrow::DataType>& query_type) {
-    if (prepared_type->id() != query_type->id()) {
-        return Status::Invalid(fmt::format("prepared value type {} does not match query type {}",
-                                           prepared_type->ToString(), query_type->ToString()));
-    }
-    switch (query_type->id()) {
-        case arrow::Type::STRUCT: {
-            const arrow::FieldVector& prepared_fields = prepared_type->fields();
-            for (const std::shared_ptr<arrow::Field>& query_field : query_type->fields()) {
-                PAIMON_ASSIGN_OR_RAISE(int32_t query_id,
-                                       NestedProjectionUtils::GetPaimonFieldId(query_field));
-                PAIMON_ASSIGN_OR_RAISE(int32_t prepared_idx,
-                                       FindFieldIndexByPaimonId(prepared_fields, query_id));
-                PAIMON_RETURN_NOT_OK(ValidateProjectionType(prepared_fields[prepared_idx]->type(),
-                                                            query_field->type()));
-            }
-            return Status::OK();
-        }
-        case arrow::Type::LIST:
-            return ValidateProjectionType(prepared_type->field(0)->type(),
-                                          query_type->field(0)->type());
-        case arrow::Type::MAP: {
-            const std::shared_ptr<arrow::MapType> prepared_map =
-                checked_pointer_cast<arrow::MapType>(prepared_type);
-            const std::shared_ptr<arrow::MapType> query_map =
-                checked_pointer_cast<arrow::MapType>(query_type);
-            PAIMON_RETURN_NOT_OK(
-                ValidateProjectionType(prepared_map->key_type(), query_map->key_type()));
-            return ValidateProjectionType(prepared_map->item_type(), query_map->item_type());
-        }
-        default:
-            if (!prepared_type->Equals(*query_type)) {
-                return Status::Invalid(
-                    fmt::format("prepared leaf type {} does not match query type {}",
-                                prepared_type->ToString(), query_type->ToString()));
-            }
-            return Status::OK();
-    }
-}
-
-Status ValidateProjectionSchema(const std::shared_ptr<arrow::Schema>& prepared_schema,
-                                const std::shared_ptr<arrow::Schema>& query_schema) {
+Result<std::vector<int32_t>> ResolveFieldIndexes(
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& row_schema) {
     arrow::FieldVector prepared_value_fields(
         prepared_schema->fields().begin() + kPreparedValueStartIndex,
         prepared_schema->fields().end());
-    for (const std::shared_ptr<arrow::Field>& query_field : query_schema->fields()) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t query_id,
-                               NestedProjectionUtils::GetPaimonFieldId(query_field));
-        PAIMON_ASSIGN_OR_RAISE(int32_t prepared_idx,
-                               FindFieldIndexByPaimonId(prepared_value_fields, query_id));
-        PAIMON_RETURN_NOT_OK(ValidateProjectionType(prepared_value_fields[prepared_idx]->type(),
-                                                    query_field->type()));
+    std::vector<int32_t> result;
+    result.reserve(row_schema->num_fields());
+    for (const std::shared_ptr<arrow::Field>& row_field : row_schema->fields()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(row_field));
+        PAIMON_ASSIGN_OR_RAISE(int32_t value_index,
+                               FindFieldIndexByPaimonId(prepared_value_fields, field_id));
+        const std::shared_ptr<arrow::Field>& prepared_field = prepared_value_fields[value_index];
+        if (!prepared_field->type()->Equals(row_field->type())) {
+            return Status::Invalid(fmt::format(
+                "prepared field id {} type {} does not match row "
+                "type {}",
+                field_id, prepared_field->type()->ToString(), row_field->type()->ToString()));
+        }
+        result.push_back(value_index + kPreparedValueStartIndex);
     }
-    return Status::OK();
+    return result;
 }
 
 Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_schema,
@@ -229,162 +190,21 @@ Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_
     return Status::OK();
 }
 
-Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
-    const std::shared_ptr<arrow::StructArray>& array,
-    const std::shared_ptr<arrow::StructType>& read_type, arrow::MemoryPool* arrow_pool) {
-    const std::shared_ptr<arrow::StructType> data_type =
-        checked_pointer_cast<arrow::StructType>(array->type());
-    std::unordered_map<int32_t, int32_t> data_field_id_to_idx;
-    data_field_id_to_idx.reserve(data_type->num_fields());
-    for (int32_t i = 0; i < data_type->num_fields(); ++i) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
-                               NestedProjectionUtils::GetPaimonFieldId(data_type->field(i)));
-        if (!data_field_id_to_idx.emplace(field_id, i).second) {
-            return Status::Invalid(
-                fmt::format("duplicate field id {} in prepared value struct", field_id));
-        }
-    }
-
-    arrow::ArrayVector aligned_arrays;
-    aligned_arrays.reserve(read_type->num_fields());
-    for (const std::shared_ptr<arrow::Field>& read_field : read_type->fields()) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t read_field_id,
-                               NestedProjectionUtils::GetPaimonFieldId(read_field));
-        auto data_iter = data_field_id_to_idx.find(read_field_id);
-        if (data_iter == data_field_id_to_idx.end()) {
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-                std::shared_ptr<arrow::Array> null_child,
-                arrow::MakeArrayOfNull(read_field->type(), array->offset() + array->length(),
-                                       arrow_pool));
-            aligned_arrays.push_back(std::move(null_child));
-            continue;
-        }
-        std::shared_ptr<arrow::Array> child =
-            arrow::MakeArray(array->data()->child_data[data_iter->second]);
-        PAIMON_ASSIGN_OR_RAISE(child, AlignArrayByPaimonIds(child, read_field->type(), arrow_pool));
-        aligned_arrays.push_back(std::move(child));
-    }
-
-    std::shared_ptr<arrow::ArrayData> aligned_data = array->data()->Copy();
-    aligned_data->type = read_type;
-    aligned_data->child_data.clear();
-    aligned_data->child_data.reserve(aligned_arrays.size());
-    for (const std::shared_ptr<arrow::Array>& aligned_array : aligned_arrays) {
-        aligned_data->child_data.push_back(aligned_array->data());
-    }
-    return arrow::MakeArray(std::move(aligned_data));
-}
-
-Result<std::shared_ptr<arrow::Array>> AlignListArrayByPaimonIds(
-    const std::shared_ptr<arrow::ListArray>& array,
-    const std::shared_ptr<arrow::ListType>& read_type, arrow::MemoryPool* arrow_pool) {
-    std::shared_ptr<arrow::Array> values = array->values();
-    PAIMON_ASSIGN_OR_RAISE(values,
-                           AlignArrayByPaimonIds(values, read_type->value_type(), arrow_pool));
-    std::shared_ptr<arrow::ArrayData> new_data = array->data()->Copy();
-    new_data->type = read_type;
-    new_data->child_data = {values->data()};
-    return arrow::MakeArray(new_data);
-}
-
-Result<std::shared_ptr<arrow::Array>> AlignMapArrayByPaimonIds(
-    const std::shared_ptr<arrow::MapArray>& array, const std::shared_ptr<arrow::MapType>& read_type,
-    arrow::MemoryPool* arrow_pool) {
-    std::shared_ptr<arrow::Array> keys = array->keys();
-    PAIMON_ASSIGN_OR_RAISE(keys, AlignArrayByPaimonIds(keys, read_type->key_type(), arrow_pool));
-    std::shared_ptr<arrow::Array> items = array->items();
-    PAIMON_ASSIGN_OR_RAISE(items, AlignArrayByPaimonIds(items, read_type->item_type(), arrow_pool));
-
-    const std::shared_ptr<arrow::ArrayData>& entries_data = array->data()->child_data[0];
-    std::shared_ptr<arrow::ArrayData> new_entries = entries_data->Copy();
-    new_entries->type = arrow::struct_({read_type->key_field(), read_type->item_field()});
-    new_entries->child_data = {keys->data(), items->data()};
-
-    std::shared_ptr<arrow::ArrayData> new_data = array->data()->Copy();
-    new_data->type = read_type;
-    new_data->child_data = {std::move(new_entries)};
-    return arrow::MakeArray(new_data);
-}
-
-Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
-    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
-    arrow::MemoryPool* arrow_pool) {
-    if (array->type()->id() != read_type->id()) {
-        return Status::Invalid(fmt::format("prepared value type {} does not match query type {}",
-                                           array->type()->ToString(), read_type->ToString()));
-    }
-    switch (read_type->id()) {
-        case arrow::Type::STRUCT:
-            return AlignStructArrayByPaimonIds(checked_pointer_cast<arrow::StructArray>(array),
-                                               checked_pointer_cast<arrow::StructType>(read_type),
-                                               arrow_pool);
-        case arrow::Type::LIST:
-            return AlignListArrayByPaimonIds(checked_pointer_cast<arrow::ListArray>(array),
-                                             checked_pointer_cast<arrow::ListType>(read_type),
-                                             arrow_pool);
-        case arrow::Type::MAP:
-            return AlignMapArrayByPaimonIds(checked_pointer_cast<arrow::MapArray>(array),
-                                            checked_pointer_cast<arrow::MapType>(read_type),
-                                            arrow_pool);
-        default:
-            if (!array->type()->Equals(*read_type)) {
-                return Status::Invalid(
-                    fmt::format("prepared leaf type {} does not match query type {}",
-                                array->type()->ToString(), read_type->ToString()));
-            }
-            return array;
-    }
-}
-
-Result<arrow::ArrayVector> ProjectFieldsByPaimonIds(
-    const std::shared_ptr<arrow::StructArray>& data_batch,
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::shared_ptr<arrow::Schema>& query_schema, arrow::MemoryPool* arrow_pool) {
-    std::unordered_map<int32_t, int32_t> prepared_field_id_to_idx;
-    prepared_field_id_to_idx.reserve(prepared_schema->num_fields());
-    for (int32_t i = kPreparedValueStartIndex; i < prepared_schema->num_fields(); ++i) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
-                               NestedProjectionUtils::GetPaimonFieldId(prepared_schema->field(i)));
-        if (!prepared_field_id_to_idx.emplace(field_id, i).second) {
-            return Status::Invalid(
-                fmt::format("duplicate field id {} in prepared schema", field_id));
-        }
-    }
-
-    arrow::ArrayVector result;
-    result.reserve(query_schema->num_fields());
-    for (const std::shared_ptr<arrow::Field>& query_field : query_schema->fields()) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t query_field_id,
-                               NestedProjectionUtils::GetPaimonFieldId(query_field));
-        auto prepared_iter = prepared_field_id_to_idx.find(query_field_id);
-        if (prepared_iter == prepared_field_id_to_idx.end()) {
-            return Status::Invalid(
-                fmt::format("cannot find field id {} in prepared schema", query_field_id));
-        }
-        std::shared_ptr<arrow::Array> field_array = data_batch->field(prepared_iter->second);
-        PAIMON_ASSIGN_OR_RAISE(field_array,
-                               AlignArrayByPaimonIds(field_array, query_field->type(), arrow_pool));
-        result.push_back(std::move(field_array));
-    }
-    return result;
-}
-
 class PreparedKeyValueReader final : public KeyValueRecordReader {
  public:
     PreparedKeyValueReader(std::unique_ptr<BatchReader>&& reader,
                            const std::shared_ptr<arrow::Schema>& prepared_schema,
                            const std::optional<OffsetRange>& visible_offsets,
-                           const std::shared_ptr<arrow::Schema>& key_schema,
-                           const std::shared_ptr<arrow::Schema>& value_schema,
+                           std::vector<int32_t>&& key_field_indexes,
+                           std::vector<int32_t>&& value_field_indexes,
                            const std::shared_ptr<MemoryPool>& pool,
                            const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage)
         : reader_(std::move(reader)),
           prepared_schema_(prepared_schema),
           visible_offsets_(visible_offsets),
-          key_schema_(key_schema),
-          value_schema_(value_schema),
+          key_field_indexes_(std::move(key_field_indexes)),
+          value_field_indexes_(std::move(value_field_indexes)),
           pool_(pool),
-          arrow_pool_(GetArrowPool(pool)),
           offset_coverage_(offset_coverage) {}
 
     ~PreparedKeyValueReader() override {
@@ -449,14 +269,21 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatchImpl() {
         while (true) {
             ResetBatchState();
-            PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
-            if (BatchReader::IsEofBatch(batch)) {
+            BatchReader::ReadBatchWithBitmap batch_with_bitmap;
+            if (visible_offsets_.has_value()) {
+                PAIMON_ASSIGN_OR_RAISE(batch_with_bitmap, reader_->NextBatchWithBitmap());
+            } else {
+                PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
+                batch_with_bitmap.first = std::move(batch);
+            }
+            if (BatchReader::IsEofBatch(batch_with_bitmap)) {
                 if (offset_coverage_ && !offset_coverage_finished_) {
                     offset_coverage_finished_ = true;
                     PAIMON_RETURN_NOT_OK(offset_coverage_->FinishReader());
                 }
                 return std::unique_ptr<KeyValueRecordReader::Iterator>();
             }
+            auto& [batch, selection] = batch_with_bitmap;
             auto& [c_array, c_schema] = batch;
             PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> arrow_array,
                                               arrow::ImportArray(c_array.get(), c_schema.get()));
@@ -473,15 +300,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                     "schema: ",
                     transport_status.ToString());
             }
-            if (visible_offsets_.has_value()) {
-                PAIMON_RETURN_NOT_OK(ValidateProjectionSchema(
-                    arrow::schema(data_batch->type()->fields()), key_schema_));
-                PAIMON_ASSIGN_OR_RAISE(
-                    arrow_array,
-                    AlignArrayByPaimonIds(data_batch, arrow::struct_(prepared_schema_->fields()),
-                                          arrow_pool_.get()));
-                data_batch = checked_pointer_cast<arrow::StructArray>(arrow_array);
-            }
             PAIMON_RETURN_NOT_OK(ValidatePreparedBatch(data_batch));
 
             std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> offset_array =
@@ -495,15 +313,19 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                 data_batch->field(kValueKindIndex));
             sequence_number_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
                 data_batch->field(kSequenceNumberIndex));
-            PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector key_fields,
-                                   ProjectFieldsByPaimonIds(data_batch, prepared_schema_,
-                                                            key_schema_, arrow_pool_.get()));
-            PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector value_fields,
-                                   ProjectFieldsByPaimonIds(data_batch, prepared_schema_,
-                                                            value_schema_, arrow_pool_.get()));
+            arrow::ArrayVector key_fields;
+            key_fields.reserve(key_field_indexes_.size());
+            for (int32_t index : key_field_indexes_) {
+                key_fields.push_back(data_batch->field(index));
+            }
+            arrow::ArrayVector value_fields;
+            value_fields.reserve(value_field_indexes_.size());
+            for (int32_t index : value_field_indexes_) {
+                value_fields.push_back(data_batch->field(index));
+            }
             key_ctx_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
             value_ctx_ = std::make_shared<ColumnarBatchContext>(value_fields, pool_);
-            if (!SelectVisibleRows(*offset_array)) {
+            if (!SelectRows(*offset_array, std::move(selection))) {
                 continue;
             }
             ArrowUtils::TraverseArray(data_batch);
@@ -544,28 +366,30 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         return Status::OK();
     }
 
-    bool SelectVisibleRows(const arrow::Int64Array& offsets) {
+    bool SelectRows(const arrow::Int64Array& offsets, RoaringBitmap32&& selection) {
         if (!visible_offsets_.has_value()) {
+            selected_rows_.reserve(offsets.length());
+            for (int64_t row = 0; row < offsets.length(); ++row) {
+                selected_rows_.push_back(row);
+            }
             return true;
         }
-        visible_rows_.emplace();
-        visible_rows_->reserve(offsets.length());
-        for (int64_t row = 0; row < offsets.length(); ++row) {
+        for (auto iter = selection.Begin(); iter != selection.End(); ++iter) {
+            const int32_t row = *iter;
             const int64_t offset = offsets.Value(row);
             if (offset >= visible_offsets_->begin && offset < visible_offsets_->end) {
-                visible_rows_->push_back(row);
+                selected_rows_.push_back(row);
             }
         }
-        return !visible_rows_->empty();
+        return !selected_rows_.empty();
     }
 
     int64_t RowCount() const {
-        return visible_rows_.has_value() ? static_cast<int64_t>(visible_rows_->size())
-                                         : row_kind_array_->length();
+        return static_cast<int64_t>(selected_rows_.size());
     }
 
     int64_t RowAt(int64_t ordinal) const {
-        return visible_rows_.has_value() ? (*visible_rows_)[ordinal] : ordinal;
+        return selected_rows_[ordinal];
     }
 
     void ResetBatchState() {
@@ -573,7 +397,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         value_ctx_.reset();
         row_kind_array_.reset();
         sequence_number_array_.reset();
-        visible_rows_.reset();
+        selected_rows_.clear();
     }
 
  private:
@@ -582,17 +406,16 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     std::unique_ptr<BatchReader> reader_;
     std::shared_ptr<arrow::Schema> prepared_schema_;
     std::optional<OffsetRange> visible_offsets_;
-    std::shared_ptr<arrow::Schema> key_schema_;
-    std::shared_ptr<arrow::Schema> value_schema_;
+    std::vector<int32_t> key_field_indexes_;
+    std::vector<int32_t> value_field_indexes_;
     std::shared_ptr<MemoryPool> pool_;
-    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     std::shared_ptr<RealtimeOffsetCoverage> offset_coverage_;
     bool offset_coverage_finished_ = false;
     std::shared_ptr<ColumnarBatchContext> key_ctx_;
     std::shared_ptr<ColumnarBatchContext> value_ctx_;
     std::shared_ptr<arrow::NumericArray<arrow::Int8Type>> row_kind_array_;
     std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> sequence_number_array_;
-    std::optional<std::vector<int64_t>> visible_rows_;
+    std::vector<int64_t> selected_rows_;
 };
 
 }  // namespace
@@ -637,14 +460,16 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
     if (!memory_pool) {
         return Status::Invalid("prepared reader memory pool cannot be null");
     }
-    PAIMON_RETURN_NOT_OK(ValidateProjectionSchema(prepared_schema, key_schema));
-    PAIMON_RETURN_NOT_OK(ValidateProjectionSchema(prepared_schema, value_schema));
     if (!visible_offsets.has_value()) {
         PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
     }
-    std::unique_ptr<KeyValueRecordReader> result(
-        new PreparedKeyValueReader(std::move(owned_reader), prepared_schema, visible_offsets,
-                                   key_schema, value_schema, memory_pool, offset_coverage));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> key_field_indexes,
+                           ResolveFieldIndexes(prepared_schema, key_schema));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> value_field_indexes,
+                           ResolveFieldIndexes(prepared_schema, value_schema));
+    std::unique_ptr<KeyValueRecordReader> result(new PreparedKeyValueReader(
+        std::move(owned_reader), prepared_schema, visible_offsets, std::move(key_field_indexes),
+        std::move(value_field_indexes), memory_pool, offset_coverage));
     close_guard.Release();
     return result;
 }
@@ -685,10 +510,10 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBa
                            RealtimeOffsetCoverage::Create(sealed_offsets, readers.size()));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> adapted_reader,
-                               AdaptPreparedBatchReaderImpl(
-                                   std::move(reader), prepared_schema, std::nullopt, key_schema,
-                                   value_schema, memory_pool, offset_coverage));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<KeyValueRecordReader> adapted_reader,
+            AdaptPreparedBatchReaderImpl(std::move(reader), prepared_schema, std::nullopt,
+                                         key_schema, value_schema, memory_pool, offset_coverage));
         adapted_readers.push_back(std::move(adapted_reader));
     }
     readers_guard.Release();
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 7d188609d..cfdf88f49 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -20,24 +20,29 @@
 
 #include <mutex>
 #include <optional>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
+#include "fmt/format.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/index/pk/primary_key_index_definitions.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/schema/table_schema.h"
+#include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
 
 namespace paimon {
 
-Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const TableSchema& schema) {
+Status PrimaryKeyRealtimeStore::ValidateOptions(const CoreOptions& options,
+                                                const TableSchema& schema) {
     if (options.GetBucket() <= 0) {
         return Status::NotImplemented("PK realtime v1 requires fixed buckets");
     }
@@ -86,20 +91,119 @@ Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const Table
 
 namespace {
 
-uint64_t GetArrayMemoryUsage(const std::shared_ptr<arrow::ArrayData>& data) {
-    uint64_t total = 0;
-    for (const std::shared_ptr<arrow::Buffer>& buffer : data->buffers) {
-        if (buffer) {
-            total += static_cast<uint64_t>(buffer->size());
+Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
+    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
+    arrow::MemoryPool* pool);
+
+bool TypesExactlyEqual(const std::shared_ptr<arrow::DataType>& data_type,
+                       const std::shared_ptr<arrow::DataType>& read_type) {
+    if (!data_type->Equals(read_type) || data_type->num_fields() != read_type->num_fields()) {
+        return false;
+    }
+    for (int32_t i = 0; i < data_type->num_fields(); ++i) {
+        if (!data_type->field(i)->Equals(read_type->field(i), /*check_metadata=*/true) ||
+            !TypesExactlyEqual(data_type->field(i)->type(), read_type->field(i)->type())) {
+            return false;
         }
     }
-    for (const std::shared_ptr<arrow::ArrayData>& child : data->child_data) {
-        total += GetArrayMemoryUsage(child);
+    return true;
+}
+
+Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
+    const std::shared_ptr<arrow::StructArray>& array,
+    const std::shared_ptr<arrow::StructType>& read_type, arrow::MemoryPool* pool) {
+    const std::shared_ptr<arrow::StructType> data_type =
+        checked_pointer_cast<arrow::StructType>(array->type());
+    std::unordered_map<int32_t, int32_t> data_field_indexes;
+    data_field_indexes.reserve(data_type->num_fields());
+    for (int32_t i = 0; i < data_type->num_fields(); ++i) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(data_type->field(i)));
+        if (!data_field_indexes.emplace(field_id, i).second) {
+            return Status::Invalid(fmt::format("duplicate field id {} in stored schema", field_id));
+        }
     }
-    if (data->dictionary) {
-        total += GetArrayMemoryUsage(data->dictionary);
+
+    std::unordered_map<int32_t, bool> requested_field_ids;
+    requested_field_ids.reserve(read_type->num_fields());
+    std::vector<std::shared_ptr<arrow::ArrayData>> children;
+    children.reserve(read_type->num_fields());
+    for (const std::shared_ptr<arrow::Field>& read_field : read_type->fields()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(read_field));
+        if (!requested_field_ids.emplace(field_id, true).second) {
+            return Status::Invalid(
+                fmt::format("duplicate field id {} in requested schema", field_id));
+        }
+        const auto data_iter = data_field_indexes.find(field_id);
+        if (data_iter == data_field_indexes.end()) {
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> null_child,
+                arrow::MakeArrayOfNull(read_field->type(), array->offset() + array->length(),
+                                       pool));
+            children.push_back(null_child->data());
+            continue;
+        }
+        std::shared_ptr<arrow::Array> child =
+            arrow::MakeArray(array->data()->child_data[data_iter->second]);
+        PAIMON_ASSIGN_OR_RAISE(child, AlignArrayByPaimonIds(child, read_field->type(), pool));
+        children.push_back(child->data());
+    }
+
+    std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
+    aligned->type = read_type;
+    aligned->child_data = std::move(children);
+    return arrow::MakeArray(std::move(aligned));
+}
+
+Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
+    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
+    arrow::MemoryPool* pool) {
+    if (TypesExactlyEqual(array->type(), read_type)) {
+        return array;
+    }
+    if (array->type_id() != read_type->id()) {
+        return Status::Invalid(fmt::format("stored value type {} does not match requested type {}",
+                                           array->type()->ToString(), read_type->ToString()));
+    }
+    switch (read_type->id()) {
+        case arrow::Type::STRUCT:
+            return AlignStructArrayByPaimonIds(checked_pointer_cast<arrow::StructArray>(array),
+                                               checked_pointer_cast<arrow::StructType>(read_type),
+                                               pool);
+        case arrow::Type::LIST: {
+            std::shared_ptr<arrow::Array> values =
+                checked_pointer_cast<arrow::ListArray>(array)->values();
+            PAIMON_ASSIGN_OR_RAISE(
+                values, AlignArrayByPaimonIds(values, read_type->field(0)->type(), pool));
+            std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
+            aligned->type = read_type;
+            aligned->child_data = {values->data()};
+            return arrow::MakeArray(std::move(aligned));
+        }
+        case arrow::Type::MAP: {
+            const std::shared_ptr<arrow::MapArray> map =
+                checked_pointer_cast<arrow::MapArray>(array);
+            const std::shared_ptr<arrow::MapType> map_type =
+                checked_pointer_cast<arrow::MapType>(read_type);
+            std::shared_ptr<arrow::Array> keys = map->keys();
+            PAIMON_ASSIGN_OR_RAISE(keys, AlignArrayByPaimonIds(keys, map_type->key_type(), pool));
+            std::shared_ptr<arrow::Array> items = map->items();
+            PAIMON_ASSIGN_OR_RAISE(items,
+                                   AlignArrayByPaimonIds(items, map_type->item_type(), pool));
+            std::shared_ptr<arrow::ArrayData> entries = array->data()->child_data[0]->Copy();
+            entries->type = arrow::struct_({map_type->key_field(), map_type->item_field()});
+            entries->child_data = {keys->data(), items->data()};
+            std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
+            aligned->type = read_type;
+            aligned->child_data = {std::move(entries)};
+            return arrow::MakeArray(std::move(aligned));
+        }
+        default:
+            return Status::Invalid(
+                fmt::format("stored leaf type {} does not match requested type {}",
+                            array->type()->ToString(), read_type->ToString()));
     }
-    return total;
 }
 
 struct StoredBatch {
@@ -201,8 +305,8 @@ class PrimaryKeyRealtimeStore::Impl {
         std::shared_ptr<arrow::StructArray> prepared =
             checked_pointer_cast<arrow::StructArray>(array);
         std::lock_guard<std::mutex> lock(mutex_);
-        building_.push_back(
-            StoredBatch{prepared, write_batch.offset_range, GetArrayMemoryUsage(prepared->data())});
+        building_.push_back(StoredBatch{prepared, write_batch.offset_range,
+                                        ArrowUtils::GetArrayMemoryUsage(prepared->data())});
         building_memory_usage_ += building_.back().memory_usage;
         return Status::OK();
     }
@@ -247,15 +351,28 @@ class PrimaryKeyRealtimeStore::Impl {
     }
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t, const RealtimeQueryContext&) {
+        const std::shared_ptr<RealtimeReadView>& view, int64_t,
+        const RealtimeQueryContext& context) {
         std::shared_ptr<ReadView> typed = std::dynamic_pointer_cast<ReadView>(view);
         if (!typed) {
             return Status::Invalid("read view was not created by the PK real-time store");
         }
+        if (context.read_schema == nullptr || context.read_schema->release == nullptr) {
+            return Status::Invalid("PK real-time query read schema is null");
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> read_schema,
+                                          arrow::ImportSchema(context.read_schema));
+        PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(read_schema));
         std::vector<std::unique_ptr<BatchReader>> readers;
         for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
             for (const StoredBatch& batch : segment->Batches()) {
-                readers.push_back(std::make_unique<StoredBatchReader>(batch));
+                PAIMON_ASSIGN_OR_RAISE(
+                    std::shared_ptr<arrow::Array> projected,
+                    AlignArrayByPaimonIds(batch.data, arrow::struct_(read_schema->fields()),
+                                          arrow::default_memory_pool()));
+                StoredBatch query_batch{checked_pointer_cast<arrow::StructArray>(projected),
+                                        batch.offset_range, /*memory_usage=*/0};
+                readers.push_back(std::make_unique<StoredBatchReader>(query_batch));
             }
         }
         return readers;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index 35f04485b..df9a5f0c5 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -32,13 +32,12 @@ namespace paimon {
 class CoreOptions;
 class TableSchema;
 
-Status ValidatePrimaryKeyRealtimeOptions(const CoreOptions& options, const TableSchema& schema);
-
 /// Internal in-memory implementation of the default primary-key `RealtimeStore`.
 class PrimaryKeyRealtimeStore final : public RealtimeStore {
  public:
     static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
         const std::shared_ptr<arrow::Schema>& prepared_schema);
+    static Status ValidateOptions(const CoreOptions& options, const TableSchema& schema);
 
     ~PrimaryKeyRealtimeStore() override;
 
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 6ccceba81..405c0d32a 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -30,6 +30,7 @@
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/macros.h"
@@ -39,6 +40,12 @@
 namespace paimon::test {
 namespace {
 
+std::shared_ptr<arrow::Field> FieldWithId(const std::string& name,
+                                          const std::shared_ptr<arrow::DataType>& type,
+                                          int32_t field_id) {
+    return DataField::ConvertDataFieldToArrowField(DataField(field_id, arrow::field(name, type)));
+}
+
 std::shared_ptr<arrow::Schema> PreparedSchema() {
     return arrow::schema(
         {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
@@ -93,6 +100,18 @@ std::unique_ptr<RecordBatch> MakeBatch(const std::shared_ptr<arrow::Schema>& sch
     return RecordBatchBuilder(c_array.get()).Finish().value();
 }
 
+std::unique_ptr<RecordBatch> MakeSlicedBatch(const std::shared_ptr<arrow::Schema>& schema,
+                                             const std::string& json, int64_t offset,
+                                             int64_t length) {
+    std::shared_ptr<arrow::Array> array =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), json)
+            .ValueOrDie()
+            ->Slice(offset, length);
+    auto c_array = std::make_unique<ArrowArray>();
+    EXPECT_TRUE(arrow::ExportArray(*array, c_array.get()).ok());
+    return RecordBatchBuilder(c_array.get()).Finish().value();
+}
+
 void AssertOffsetsZero(const ArrowArray* array) {
     ASSERT_NE(nullptr, array);
     ASSERT_EQ(0, array->offset);
@@ -125,7 +144,7 @@ Result<std::string> ReadJson(const std::vector<std::unique_ptr<BatchReader>>& re
 
 TEST(PrimaryKeyRealtimeStoreOptionsTest, TestSupportedOptions) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_OK(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema()));
+    ASSERT_OK(PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema()));
 }
 
 TEST(PrimaryKeyRealtimeStoreOptionsTest, TestUnsupportedOptions) {
@@ -143,16 +162,18 @@ TEST(PrimaryKeyRealtimeStoreOptionsTest, TestUnsupportedOptions) {
     };
     for (const std::map<std::string, std::string>& option_map : unsupported_options) {
         ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-        ASSERT_NOK(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema()));
+        ASSERT_NOK(PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema()));
     }
 }
 
 TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsFloatingPrimaryKeys) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_NOK_WITH_MSG(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema(arrow::float32())),
-                        "FLOAT or DOUBLE primary keys");
-    ASSERT_NOK_WITH_MSG(ValidatePrimaryKeyRealtimeOptions(options, *PkSchema(arrow::float64())),
-                        "FLOAT or DOUBLE primary keys");
+    ASSERT_NOK_WITH_MSG(
+        PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema(arrow::float32())),
+        "FLOAT or DOUBLE primary keys");
+    ASSERT_NOK_WITH_MSG(
+        PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema(arrow::float64())),
+        "FLOAT or DOUBLE primary keys");
 }
 
 TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsEnabledGlobalIndex) {
@@ -160,7 +181,7 @@ TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsEnabledGlobalIndex) {
                                                            {Options::PK_BTREE_INDEX_COLUMNS, "id"}};
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
     ASSERT_NOK_WITH_MSG(
-        ValidatePrimaryKeyRealtimeOptions(options, *PkSchema(arrow::int64(), option_map)),
+        PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema(arrow::int64(), option_map)),
         "does not support global indexes");
 }
 
@@ -306,7 +327,9 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 2, 1, 1, "one"]])"), OffsetRange(1, 2)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-    RealtimeQueryContext context{/*read_schema=*/nullptr, /*predicate=*/nullptr,
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*PreparedSchema(), c_schema.get()).ok());
+    RealtimeQueryContext context{/*read_schema=*/c_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateQueryReaders(view, /*offset_begin=*/0, context));
@@ -316,5 +339,120 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
     ASSERT_NE(std::string::npos, actual.find("\"two\""));
 }
 
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsTopLevelFieldsById) {
+    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(stored_schema));
+    ASSERT_OK(store->Write(RealtimeWriteBatch{
+        MakeSlicedBatch(stored_schema,
+                        R"([[0, 1, 0, 6, "six"], [0, 2, 1, 7, "seven"], [0, 3, 2, 8, "eight"]])", 1,
+                        1),
+        OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    arrow::FieldVector requested_fields(stored_schema->fields().begin(),
+                                        stored_schema->fields().begin() + 3);
+    requested_fields.push_back(FieldWithId("renamed_value", arrow::utf8(), 1));
+    requested_fields.push_back(FieldWithId("added", arrow::int32(), 2));
+    std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
+    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(1, readers.size());
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
+    arrow::Result<std::shared_ptr<arrow::Array>> import_result =
+        arrow::ImportArray(batch.first.get(), batch.second.get());
+    ASSERT_TRUE(import_result.ok()) << import_result.status().ToString();
+    std::shared_ptr<arrow::Array> array = std::move(import_result).ValueOrDie();
+    ASSERT_TRUE(array->type()->Equals(arrow::struct_(requested_schema->fields())));
+    std::shared_ptr<arrow::StructArray> projected = checked_pointer_cast<arrow::StructArray>(array);
+    ASSERT_EQ(5, projected->num_fields());
+    ASSERT_EQ("seven", checked_pointer_cast<arrow::StringArray>(projected->field(3))->GetString(0));
+    ASSERT_TRUE(projected->field(4)->IsNull(0));
+}
+
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
+    const std::shared_ptr<arrow::Field> stored_a = FieldWithId("a", arrow::int32(), 10);
+    const std::shared_ptr<arrow::Field> stored_b = FieldWithId("b", arrow::int32(), 11);
+    const std::shared_ptr<arrow::Field> stored_x = FieldWithId("x", arrow::int32(), 20);
+    const std::shared_ptr<arrow::Field> stored_y = FieldWithId("y", arrow::int32(), 21);
+    arrow::FieldVector stored_fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+            ->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
+        FieldWithId("id", arrow::int64(), 0),
+        FieldWithId("items", arrow::list(arrow::struct_({stored_a, stored_b})), 1),
+        FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_x, stored_y})), 2)};
+    std::shared_ptr<arrow::Schema> stored_schema = arrow::schema(std::move(stored_fields));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(stored_schema));
+    ASSERT_OK(store->Write(RealtimeWriteBatch{
+        MakeSlicedBatch(
+            stored_schema,
+            R"([[0, 1, 0, 6, [[1, 2]], [["before", [3, 4]]]], [0, 2, 1, 7, [[100, 200], null], [["k1", [7, 8]], ["k2", null]]], [0, 3, 2, 8, [[9, 10]], [["after", [11, 12]]]]])",
+            1, 1),
+        OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    const std::shared_ptr<arrow::Field> requested_b = FieldWithId("renamed_b", arrow::int32(), 11);
+    const std::shared_ptr<arrow::Field> requested_a = FieldWithId("renamed_a", arrow::int32(), 10);
+    const std::shared_ptr<arrow::Field> requested_item_missing =
+        FieldWithId("added_item", arrow::int32(), 12);
+    const std::shared_ptr<arrow::Field> requested_y = FieldWithId("renamed_y", arrow::int32(), 21);
+    const std::shared_ptr<arrow::Field> requested_x = FieldWithId("renamed_x", arrow::int32(), 20);
+    const std::shared_ptr<arrow::Field> requested_attr_missing =
+        FieldWithId("added_attr", arrow::int32(), 22);
+    arrow::FieldVector requested_fields(stored_schema->fields().begin(),
+                                        stored_schema->fields().begin() + 3);
+    requested_fields.push_back(FieldWithId(
+        "renamed_items",
+        arrow::list(arrow::struct_({requested_b, requested_item_missing, requested_a})), 1));
+    requested_fields.push_back(
+        FieldWithId("renamed_attrs",
+                    arrow::map(arrow::utf8(),
+                               arrow::struct_({requested_y, requested_attr_missing, requested_x})),
+                    2));
+    std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
+    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
+    arrow::Result<std::shared_ptr<arrow::Array>> import_result =
+        arrow::ImportArray(batch.first.get(), batch.second.get());
+    ASSERT_TRUE(import_result.ok()) << import_result.status().ToString();
+    std::shared_ptr<arrow::Array> array = std::move(import_result).ValueOrDie();
+    ASSERT_TRUE(array->type()->Equals(arrow::struct_(requested_schema->fields())));
+    std::shared_ptr<arrow::StructArray> projected = checked_pointer_cast<arrow::StructArray>(array);
+    const std::shared_ptr<arrow::ListArray> items =
+        checked_pointer_cast<arrow::ListArray>(projected->field(3));
+    const std::shared_ptr<arrow::StructArray> item_values =
+        checked_pointer_cast<arrow::StructArray>(items->value_slice(0));
+    ASSERT_EQ(200, checked_pointer_cast<arrow::Int32Array>(item_values->field(0))->Value(0));
+    ASSERT_TRUE(item_values->field(1)->IsNull(0));
+    ASSERT_EQ(100, checked_pointer_cast<arrow::Int32Array>(item_values->field(2))->Value(0));
+    ASSERT_TRUE(item_values->IsNull(1));
+
+    const std::shared_ptr<arrow::MapArray> attrs =
+        checked_pointer_cast<arrow::MapArray>(projected->field(4));
+    const int64_t attr_offset = attrs->value_offset(0);
+    const int64_t attr_length = attrs->value_length(0);
+    const std::shared_ptr<arrow::StringArray> attr_keys =
+        checked_pointer_cast<arrow::StringArray>(attrs->keys()->Slice(attr_offset, attr_length));
+    ASSERT_EQ("k1", attr_keys->GetString(0));
+    const std::shared_ptr<arrow::StructArray> attr_values =
+        checked_pointer_cast<arrow::StructArray>(attrs->items()->Slice(attr_offset, attr_length));
+    ASSERT_EQ(8, checked_pointer_cast<arrow::Int32Array>(attr_values->field(0))->Value(0));
+    ASSERT_TRUE(attr_values->field(1)->IsNull(0));
+    ASSERT_EQ(7, checked_pointer_cast<arrow::Int32Array>(attr_values->field(2))->Value(0));
+    ASSERT_TRUE(attr_values->IsNull(1));
+}
+
 }  // namespace
 }  // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 404c0faac..052b616f6 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -148,8 +148,8 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     PAIMON_RETURN_NOT_OK_FROM_ARROW(
         arrow::ExportSchema(*requested_schema, request.write_schema.get()));
     RealtimeStoreMode mode = request.mode;
-    Result<std::shared_ptr<RealtimeStore>> store_result = factory_->Create(std::move(request));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store, std::move(store_result));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store,
+                           factory_->Create(std::move(request)));
     stores_.emplace(key, StoreEntry{store, requested_schema, mode});
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(key, offset_iter->second);
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index ef85e8b6e..bcc8f6705 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -246,10 +246,8 @@ Status RealtimePrimaryKeyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
         std::move(prepared_batch), OffsetRange(next_offset_, next_offset_ + count)}));
     next_offset_ += count;
     last_sequence_number_ += count;
-    PAIMON_RETURN_NOT_OK(
-        realtime_context_
-            ->AdvanceMaterializedMaxSequenceNumber(partition_bucket_, last_sequence_number_)
-            .status());
+    PAIMON_RETURN_NOT_OK(realtime_context_->AdvanceMaterializedMaxSequenceNumber(
+        partition_bucket_, last_sequence_number_));
     return Status::OK();
 }
 
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 0f6b2189b..64b866439 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -59,17 +59,14 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<FieldsComparator>& key_comparator,
-    const std::shared_ptr<InternalReadContext>& context,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    std::shared_ptr<arrow::Schema> full_value_schema =
-        DataField::ConvertDataFieldsToArrowSchema(context->GetTableSchema()->Fields());
     arrow::FieldVector prepared_fields = {
         DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
             ->WithNullable(false),
         DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
-    prepared_fields.insert(prepared_fields.end(), full_value_schema->fields().begin(),
-                           full_value_schema->fields().end());
+    prepared_fields.insert(prepared_fields.end(), value_schema->fields().begin(),
+                           value_schema->fields().end());
     std::shared_ptr<arrow::Schema> prepared_schema = arrow::schema(std::move(prepared_fields));
     auto c_schema = std::make_unique<ArrowSchema>();
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
@@ -276,7 +273,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
                 std::vector<std::unique_ptr<KeyValueRecordReader>> memory_readers,
                 CreateMemoryReaders(realtime_split, memory, merge_read->GetKeySchema(),
                                     merge_read->GetValueSchema(), merge_read->GetKeyComparator(),
-                                    context_, GetMemoryPool()));
+                                    GetMemoryPool()));
             PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
                                    merge_read->CreateRealtimeReader(realtime_split->DiskSplits(),
                                                                     std::move(memory_readers)));
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 7f9fe568b..70ab0cb8e 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -237,7 +237,7 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
             return Status::NotImplemented(
                 "PK real-time union read does not support read-optimized scans");
         }
-        PAIMON_RETURN_NOT_OK(ValidatePrimaryKeyRealtimeOptions(core_options, table_schema));
+        PAIMON_RETURN_NOT_OK(PrimaryKeyRealtimeStore::ValidateOptions(core_options, table_schema));
     }
     if (context.IsStreamingMode()) {
         return Status::Invalid("real-time union read currently supports batch scans only");

From 884270fed6321944cd1c2dd0407482615369b7a7 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 22:24:26 +0800
Subject: [PATCH 56/93] refactor(realtime): address review feedback

---
 include/paimon/realtime/realtime_store.h      |   6 +-
 src/paimon/common/table/special_fields.h      |   2 +-
 .../common/table/special_fields_test.cpp      |   2 +-
 .../merged_key_value_record_reader_test.cpp   |   4 +-
 .../core/mergetree/merge_tree_writer.cpp      |   4 +-
 src/paimon/core/mergetree/merge_tree_writer.h |   3 +-
 .../core/mergetree/merge_tree_writer_test.cpp |  16 +-
 .../core/operation/file_store_write.cpp       |  47 +++---
 .../operation/key_value_file_store_write.cpp  |   3 +-
 .../core/operation/merge_file_split_read.cpp  | 148 ++++++++++--------
 .../core/operation/merge_file_split_read.h    |  14 ++
 .../realtime/arrow_realtime_store_test.cpp    |  12 +-
 .../realtime/prepared_key_value_reader.cpp    |  14 +-
 .../core/realtime/prepared_key_value_reader.h |  40 ++---
 .../realtime/primary_key_realtime_store.cpp   |   4 +-
 .../realtime/realtime_append_only_writer.cpp  |   8 +-
 .../core/realtime/realtime_context_impl.cpp   |  16 +-
 .../core/realtime/realtime_context_impl.h     |   3 +-
 .../core/realtime/realtime_context_test.cpp   |  11 +-
 .../realtime/realtime_primary_key_writer.cpp  |  10 +-
 .../core/schema/schema_validation_test.cpp    |   5 +-
 .../table/source/key_value_table_read.cpp     |  22 ++-
 test/inte/realtime_write_inte_test.cpp        |  21 ++-
 23 files changed, 233 insertions(+), 182 deletions(-)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index fe2b95e1b..cbfe96595 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -57,10 +57,6 @@ struct PAIMON_EXPORT RealtimeStoreCreateRequest {
     std::map<std::string, std::string> options;
     /// Memory pool for allocations retained by the store.
     std::shared_ptr<MemoryPool> memory_pool;
-    /// Partition values identifying the store.
-    std::map<std::string, std::string> partition;
-    /// Bucket identifying the store within its partition.
-    int32_t bucket = -1;
     /// Table mode implemented by the store.
     RealtimeStoreMode mode = RealtimeStoreMode::APPEND_ONLY;
     /// Statistics collected by append-only stores.
@@ -192,7 +188,7 @@ class PAIMON_EXPORT RealtimeStoreFactory {
  public:
     virtual ~RealtimeStoreFactory() = default;
 
-    /// Creates a store for the requested table mode and partition-bucket.
+    /// Creates a store for the requested table mode.
     /// The factory consumes `request`, including ownership of `request.write_schema`.
     virtual Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) = 0;
 };
diff --git a/src/paimon/common/table/special_fields.h b/src/paimon/common/table/special_fields.h
index 3279bfed6..9771ac232 100644
--- a/src/paimon/common/table/special_fields.h
+++ b/src/paimon/common/table/special_fields.h
@@ -79,7 +79,7 @@ struct SpecialFields {
         }
         return field_name == SequenceNumber().Name() || field_name == ValueKind().Name() ||
                field_name == RowKind().Name() || field_name == RowId().Name() ||
-               field_name == IndexScore().Name();
+               field_name == IndexScore().Name() || field_name == RealtimeOffset().Name();
     }
 
     // TODO(xinyu.lxy): add a func to complete row-tracking fields
diff --git a/src/paimon/common/table/special_fields_test.cpp b/src/paimon/common/table/special_fields_test.cpp
index b61d289b0..58a025ba2 100644
--- a/src/paimon/common/table/special_fields_test.cpp
+++ b/src/paimon/common/table/special_fields_test.cpp
@@ -73,7 +73,7 @@ TEST(SpecialFieldsTest, TestIsSystemField) {
     ASSERT_TRUE(SpecialFields::IsSystemField("rowkind"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_ROW_ID"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_INDEX_SCORE"));
-    ASSERT_FALSE(SpecialFields::IsSystemField("_REALTIME_OFFSET"));
+    ASSERT_TRUE(SpecialFields::IsSystemField("_REALTIME_OFFSET"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_KEY_0"));
 }
 
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index c8d80906a..d86e3fc9a 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -71,8 +71,8 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderForTest(
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    return AdaptPreparedBatchReader(std::move(reader), prepared_schema, visible_offsets, key_schema,
-                                    value_schema, memory_pool);
+    return PreparedKeyValueReaderFactory::Create(
+        std::move(reader), prepared_schema, visible_offsets, key_schema, value_schema, memory_pool);
 }
 
 class TrackingBatchReader : public BatchReader {
diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp
index 49961536a..bcf98e9f7 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer.cpp
@@ -154,7 +154,7 @@ Status MergeTreeWriter::Write(std::unique_ptr<RecordBatch>&& moved_batch) {
     return Status::OK();
 }
 
-Status MergeTreeWriter::WriteSortedReaders(
+Status MergeTreeWriter::WriteSortedReadersToFiles(
     std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers) {
     auto raw_readers_guard = ScopeGuard([&]() -> void {
         for (std::unique_ptr<KeyValueRecordReader>& reader : readers) {
@@ -311,7 +311,7 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction,
         auto cleanup_guard = ScopeGuard([&]() { write_buffer_->Clear(); });
         PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
                                write_buffer_->CreateReaders());
-        PAIMON_RETURN_NOT_OK(WriteSortedReaders(std::move(readers)));
+        PAIMON_RETURN_NOT_OK(WriteSortedReadersToFiles(std::move(readers)));
     }
     PAIMON_RETURN_NOT_OK(TrySyncLatestCompaction(wait_for_latest_compaction));
     PAIMON_RETURN_NOT_OK(compact_manager_->TriggerCompaction(forced_full_compaction));
diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h
index 01efd975c..17a9dc51c 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.h
+++ b/src/paimon/core/mergetree/merge_tree_writer.h
@@ -72,7 +72,8 @@ class MergeTreeWriter : public BatchWriter {
 
     /// Consumes readers whose complete streams are individually sorted by primary key and sequence
     /// number. Readers are closed on success or failure.
-    Status WriteSortedReaders(std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers);
+    Status WriteSortedReadersToFiles(
+        std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers);
 
     Status Compact(bool full_compaction) override;
 
diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 9ce5498cb..e93935a41 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -359,7 +359,7 @@ TEST_P(MergeTreeWriterTest, TestSimple) {
                          CreateMergeWriter(-1, dir->Str(), sorted_reader_path_factory, 1, options));
     std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
     sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
-    ASSERT_OK(sorted_reader_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK(sorted_reader_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
     ASSERT_OK_AND_ASSIGN(CommitIncrement sorted_reader_commit_increment,
                          sorted_reader_writer->PrepareCommit(false));
     ASSERT_OK(sorted_reader_writer->Close());
@@ -464,7 +464,7 @@ TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) {
                          CreateMergeWriter(9, dir->Str(), sorted_reader_path_factory, 0, options));
     std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
     sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
-    ASSERT_OK(sorted_reader_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK(sorted_reader_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
     ASSERT_OK_AND_ASSIGN(CommitIncrement sorted_reader_commit_increment,
                          sorted_reader_writer->PrepareCommit(false));
     ASSERT_OK(sorted_reader_writer->Close());
@@ -498,7 +498,7 @@ TEST_P(MergeTreeWriterTest, TestSortedReaders) {
     std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
     sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
 
-    ASSERT_OK(merge_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK(merge_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
     ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, merge_writer->PrepareCommit(false));
     ASSERT_OK(merge_writer->Close());
 
@@ -564,7 +564,7 @@ TEST_P(MergeTreeWriterTest, TestMergeSortedReaders) {
     sorted_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
         CreateSingleReader(second_array), &second_closed));
 
-    ASSERT_OK(merge_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK(merge_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
     ASSERT_TRUE(first_closed);
     ASSERT_TRUE(second_closed);
     ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, merge_writer->PrepareCommit(false));
@@ -611,7 +611,7 @@ TEST_P(MergeTreeWriterTest, TestSortedReaderOwnership) {
     sorted_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
         CreateSingleReader(sorted_reader_array), &closed));
 
-    ASSERT_OK(merge_writer->WriteSortedReaders(std::move(sorted_readers)));
+    ASSERT_OK(merge_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
     ASSERT_TRUE(closed);
     ASSERT_OK(merge_writer->Close());
 }
@@ -629,12 +629,12 @@ TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
                          CreateMergeWriter(-1, dir->Str(), path_factory, 0, options));
 
     std::vector<std::unique_ptr<KeyValueRecordReader>> empty_readers;
-    Status empty_status = merge_writer->WriteSortedReaders(std::move(empty_readers));
+    Status empty_status = merge_writer->WriteSortedReadersToFiles(std::move(empty_readers));
     ASSERT_TRUE(empty_status.IsInvalid());
 
     std::vector<std::unique_ptr<KeyValueRecordReader>> null_readers;
     null_readers.push_back(nullptr);
-    Status null_status = merge_writer->WriteSortedReaders(std::move(null_readers));
+    Status null_status = merge_writer->WriteSortedReadersToFiles(std::move(null_readers));
     ASSERT_TRUE(null_status.IsInvalid());
 
     auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
@@ -648,7 +648,7 @@ TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
     failing_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
         CreateSingleReader(sorted_reader_array, /*batch_size=*/16, expected_status),
         &failing_reader_closed));
-    Status failing_status = merge_writer->WriteSortedReaders(std::move(failing_readers));
+    Status failing_status = merge_writer->WriteSortedReadersToFiles(std::move(failing_readers));
     ASSERT_EQ(expected_status, failing_status);
     ASSERT_TRUE(failing_reader_closed);
     ASSERT_OK(merge_writer->Close());
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 6c3b4d033..26befb7ee 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -58,6 +58,27 @@ struct KeyValue;
 template <typename T>
 class MergeFunctionWrapper;
 
+namespace {
+
+Status RestoreRealtimeCommittedProgress(const std::shared_ptr<RealtimeContext>& realtime_context,
+                                        const std::shared_ptr<SnapshotManager>& snapshot_manager,
+                                        const CoreOptions& options) {
+    PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
+                           snapshot_manager->LatestSnapshot());
+    if (latest_snapshot) {
+        PAIMON_ASSIGN_OR_RAISE(
+            RealtimeOffsetMap realtime_committed_offsets,
+            RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem()));
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+                               RealtimeContextImpl::Cast(realtime_context));
+        PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress(
+            latest_snapshot->Id(), realtime_committed_offsets));
+    }
+    return Status::OK();
+}
+
+}  // namespace
+
 Result<std::vector<RealtimeCommitProgress>> FileStoreWrite::PrepareCommitWithProgress(int64_t) {
     return Status::Invalid("prepare commit with progress requires a real-time writer");
 }
@@ -144,17 +165,8 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
             if (options.DeletionVectorsEnabled()) {
                 return Status::Invalid("real-time append write does not support deletion vectors");
             }
-            PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
-                                   snapshot_manager->LatestSnapshot());
-            if (latest_snapshot) {
-                PAIMON_ASSIGN_OR_RAISE(RealtimeOffsetMap realtime_committed_offsets,
-                                       RealtimeCommitProperties::ReadOffsets(
-                                           latest_snapshot, options.GetFileSystem()));
-                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
-                                       RealtimeContextImpl::Cast(ctx->GetRealtimeContext()));
-                PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress(
-                    latest_snapshot->Id(), realtime_committed_offsets));
-            }
+            PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(),
+                                                                  snapshot_manager, options));
         }
         std::shared_ptr<arrow::Schema> write_schema = arrow_schema;
         const auto& write_field_names = ctx->GetWriteSchema();
@@ -207,17 +219,8 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
                 return Status::NotImplemented(
                     "PK realtime v1 does not support a custom write schema");
             }
-            PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
-                                   snapshot_manager->LatestSnapshot());
-            if (latest_snapshot) {
-                PAIMON_ASSIGN_OR_RAISE(RealtimeOffsetMap realtime_committed_offsets,
-                                       RealtimeCommitProperties::ReadOffsets(
-                                           latest_snapshot, options.GetFileSystem()));
-                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
-                                       RealtimeContextImpl::Cast(ctx->GetRealtimeContext()));
-                PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress(
-                    latest_snapshot->Id(), realtime_committed_offsets));
-            }
+            PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(),
+                                                                  snapshot_manager, options));
         }
         if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) {
             return PostponeBucketFileStoreWrite::Create(
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index d8e7f5d15..c6a62c107 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -154,7 +154,8 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
             RealtimeStoreState store_state,
             realtime_context_impl->GetOrCreateRealtimeStore(
                 RealtimeStoreCreateRequest{std::move(c_write_schema), options_.ToMap(), pool_,
-                                           partition_map, bucket, RealtimeStoreMode::PRIMARY_KEY}));
+                                           RealtimeStoreMode::PRIMARY_KEY},
+                RealtimePartitionBucket(partition_map, bucket)));
         realtime_store_state = std::move(store_state);
         compact_manager = std::make_shared<NoopCompactManager>();
     } else {
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index c85e75ee0..02a7fac20 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -81,8 +81,9 @@ class MergeFunctionWrapper;
 class MergeFileSplitRead::RealtimeReaderBuilder {
  public:
     static Result<std::unique_ptr<BatchReader>> Create(
-        MergeFileSplitRead* owner, const std::vector<std::shared_ptr<Split>>& disk_splits,
-        std::vector<std::unique_ptr<KeyValueRecordReader>>&& additional_readers) {
+        const std::vector<std::shared_ptr<Split>>& disk_splits,
+        std::vector<std::unique_ptr<KeyValueRecordReader>>&& additional_readers,
+        MergeFileSplitRead* owner) {
         RealtimeReaderBuilder builder(owner);
         std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
         if (!disk_splits.empty()) {
@@ -140,18 +141,18 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
             }
         }
 
-        DeletionVector::Factory dv_factory = DeletionVector::CreateFactory(
-            owner_->options_.GetFileSystem(),
-            DeletionVector::CreateDeletionFileMap(data_files, deletion_files), owner_->pool_);
-        std::vector<std::vector<SortedRun>> disk_sections =
-            IntervalPartition(data_files, owner_->key_comparator_).Partition();
+        DeletionVector::Factory dv_factory;
+        std::vector<std::vector<SortedRun>> disk_sections;
+        PAIMON_RETURN_NOT_OK(
+            owner_->CreateDiskSections(data_files, deletion_files, &dv_factory, &disk_sections));
         for (const std::vector<SortedRun>& section : disk_sections) {
-            for (const SortedRun& run : section) {
-                PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> disk_reader,
-                                       owner_->CreateReaderForRun(partition, run, dv_factory,
-                                                                  owner_->predicate_for_keys_,
-                                                                  data_file_path_factory));
-                readers->push_back(std::move(disk_reader));
+            PAIMON_ASSIGN_OR_RAISE(
+                std::vector<std::unique_ptr<KeyValueRecordReader>> section_readers,
+                owner_->CreateRecordReadersForSection(section, partition, dv_factory,
+                                                      owner_->predicate_for_keys_,
+                                                      data_file_path_factory));
+            for (std::unique_ptr<KeyValueRecordReader>& reader : section_readers) {
+                readers->push_back(std::move(reader));
             }
         }
         return Status::OK();
@@ -165,34 +166,9 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
         }
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> sort_merge_reader,
                                owner_->CreateSortMergeReader(std::move(record_readers)));
-        return CreateProjectedReader(std::move(sort_merge_reader));
-    }
-
-    Result<std::unique_ptr<BatchReader>> CreateProjectedReader(
-        std::unique_ptr<SortMergeReader>&& sort_merge_reader) {
-        if (!owner_->force_keep_delete_) {
-            sort_merge_reader = std::make_unique<DropDeleteReader>(std::move(sort_merge_reader));
-        }
-
-        std::unique_ptr<BatchReader> projection_reader;
-        if (!owner_->context_->EnableMultiThreadRowToBatch()) {
-            PAIMON_ASSIGN_OR_RAISE(
-                projection_reader,
-                KeyValueProjectionReader::Create(
-                    std::move(sort_merge_reader), owner_->raw_read_schema_, owner_->projection_,
-                    owner_->options_.GetReadBatchSize(), owner_->pool_));
-        } else {
-            const int32_t thread_number = owner_->context_->GetRowToBatchThreadNumber();
-            assert(thread_number > 0);
-            projection_reader = std::make_unique<AsyncKeyValueProjectionReader>(
-                std::move(sort_merge_reader), owner_->raw_read_schema_, owner_->projection_,
-                owner_->options_.GetReadBatchSize(), thread_number, owner_->pool_);
-        }
-        PAIMON_ASSIGN_OR_RAISE(projection_reader,
-                               owner_->ApplyPredicateFilterIfNeeded(
-                                   std::move(projection_reader), owner_->context_->GetPredicate()));
-        return std::make_unique<CompleteRowKindBatchReader>(std::move(projection_reader),
-                                                            owner_->pool_);
+        return owner_->CreateProjectedReader(std::move(sort_merge_reader),
+                                             owner_->context_->GetPredicate(),
+                                             /*complete_row_kind=*/true);
     }
 
     MergeFileSplitRead* owner_;
@@ -281,7 +257,7 @@ Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateReader(
 Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateRealtimeReader(
     const std::vector<std::shared_ptr<Split>>& disk_splits,
     std::vector<std::unique_ptr<KeyValueRecordReader>>&& additional_readers) {
-    return RealtimeReaderBuilder::Create(this, disk_splits, std::move(additional_readers));
+    return RealtimeReaderBuilder::Create(disk_splits, std::move(additional_readers), this);
 }
 
 void MergeFileSplitRead::SetMergeFunctionWrapper(
@@ -362,13 +338,10 @@ Result<std::unique_ptr<FileBatchReader>> MergeFileSplitRead::ApplyIndexAndDvRead
 Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateMergeReader(
     const std::shared_ptr<DataSplitImpl>& data_split,
     const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) {
-    auto dv_factory = DeletionVector::CreateFactory(
-        options_.GetFileSystem(),
-        DeletionVector::CreateDeletionFileMap(data_split->DataFiles(), data_split->DeletionFiles()),
-        pool_);
-
-    std::vector<std::vector<SortedRun>> sections =
-        IntervalPartition(data_split->DataFiles(), key_comparator_).Partition();
+    DeletionVector::Factory dv_factory;
+    std::vector<std::vector<SortedRun>> sections;
+    PAIMON_RETURN_NOT_OK(CreateDiskSections(data_split->DataFiles(), data_split->DeletionFiles(),
+                                            &dv_factory, &sections));
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
     batch_readers.reserve(sections.size());
     // no overlap through multiple sections
@@ -579,36 +552,75 @@ Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateReaderForSection(
     } else {
         predicate = context_->GetPredicate();
     }
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> sort_merge_reader,
-                           CreateSortMergeReaderForSection(section, partition, dv_factory,
-                                                           predicate, data_file_path_factory,
-                                                           /*drop_delete=*/!force_keep_delete_));
-    // KeyValueProjectionReader converts KeyValue objects to arrow array according to projection
-    if (!context_->EnableMultiThreadRowToBatch()) {
-        return KeyValueProjectionReader::Create(std::move(sort_merge_reader), raw_read_schema_,
-                                                projection_, options_.GetReadBatchSize(), pool_);
-    }
-    int32_t thread_number = context_->GetRowToBatchThreadNumber();
-    assert(thread_number > 0);
-    return std::make_unique<AsyncKeyValueProjectionReader>(
-        std::move(sort_merge_reader), raw_read_schema_, projection_, options_.GetReadBatchSize(),
-        thread_number, pool_);
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<SortMergeReader> sort_merge_reader,
+        CreateSortMergeReaderForSection(section, partition, dv_factory, predicate,
+                                        data_file_path_factory, /*drop_delete=*/false));
+    return CreateProjectedReader(std::move(sort_merge_reader), /*predicate=*/nullptr,
+                                 /*complete_row_kind=*/false);
 }
 
-Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeReaderForSection(
+Status MergeFileSplitRead::CreateDiskSections(
+    const std::vector<std::shared_ptr<DataFileMeta>>& data_files,
+    const std::vector<std::optional<DeletionFile>>& deletion_files,
+    DeletionVector::Factory* dv_factory, std::vector<std::vector<SortedRun>>* sections) const {
+    *dv_factory = DeletionVector::CreateFactory(
+        options_.GetFileSystem(), DeletionVector::CreateDeletionFileMap(data_files, deletion_files),
+        pool_);
+    *sections = IntervalPartition(data_files, key_comparator_).Partition();
+    return Status::OK();
+}
+
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
+MergeFileSplitRead::CreateRecordReadersForSection(
     const std::vector<SortedRun>& section, const BinaryRow& partition,
     DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
-    const std::shared_ptr<DataFilePathFactory>& data_file_path_factory, bool drop_delete) {
-    // with overlap in one section
+    const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) const {
     std::vector<std::unique_ptr<KeyValueRecordReader>> record_readers;
     record_readers.reserve(section.size());
-    for (const auto& run : section) {
-        // no overlap in a run
+    for (const SortedRun& run : section) {
         PAIMON_ASSIGN_OR_RAISE(
             std::unique_ptr<KeyValueRecordReader> run_reader,
             CreateReaderForRun(partition, run, dv_factory, predicate, data_file_path_factory));
         record_readers.emplace_back(std::move(run_reader));
     }
+    return record_readers;
+}
+
+Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateProjectedReader(
+    std::unique_ptr<SortMergeReader>&& sort_merge_reader,
+    const std::shared_ptr<Predicate>& predicate, bool complete_row_kind) {
+    if (!force_keep_delete_) {
+        sort_merge_reader = std::make_unique<DropDeleteReader>(std::move(sort_merge_reader));
+    }
+    std::unique_ptr<BatchReader> projection_reader;
+    if (!context_->EnableMultiThreadRowToBatch()) {
+        PAIMON_ASSIGN_OR_RAISE(
+            projection_reader,
+            KeyValueProjectionReader::Create(std::move(sort_merge_reader), raw_read_schema_,
+                                             projection_, options_.GetReadBatchSize(), pool_));
+    } else {
+        const int32_t thread_number = context_->GetRowToBatchThreadNumber();
+        assert(thread_number > 0);
+        projection_reader = std::make_unique<AsyncKeyValueProjectionReader>(
+            std::move(sort_merge_reader), raw_read_schema_, projection_,
+            options_.GetReadBatchSize(), thread_number, pool_);
+    }
+    PAIMON_ASSIGN_OR_RAISE(projection_reader,
+                           ApplyPredicateFilterIfNeeded(std::move(projection_reader), predicate));
+    if (complete_row_kind) {
+        return std::make_unique<CompleteRowKindBatchReader>(std::move(projection_reader), pool_);
+    }
+    return projection_reader;
+}
+
+Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeReaderForSection(
+    const std::vector<SortedRun>& section, const BinaryRow& partition,
+    DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
+    const std::shared_ptr<DataFilePathFactory>& data_file_path_factory, bool drop_delete) {
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> record_readers,
+                           CreateRecordReadersForSection(section, partition, dv_factory, predicate,
+                                                         data_file_path_factory));
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> sort_merge_reader,
                            CreateSortMergeReader(std::move(record_readers)));
     if (drop_delete) {
diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h
index f01b252be..ad45e4cf5 100644
--- a/src/paimon/core/operation/merge_file_split_read.h
+++ b/src/paimon/core/operation/merge_file_split_read.h
@@ -143,6 +143,20 @@ class MergeFileSplitRead : public AbstractSplitRead {
         DeletionVector::Factory dv_factory,
         const std::shared_ptr<DataFilePathFactory>& data_file_path_factory);
 
+    Status CreateDiskSections(const std::vector<std::shared_ptr<DataFileMeta>>& data_files,
+                              const std::vector<std::optional<DeletionFile>>& deletion_files,
+                              DeletionVector::Factory* dv_factory,
+                              std::vector<std::vector<SortedRun>>* sections) const;
+
+    Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateRecordReadersForSection(
+        const std::vector<SortedRun>& section, const BinaryRow& partition,
+        DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
+        const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) const;
+
+    Result<std::unique_ptr<BatchReader>> CreateProjectedReader(
+        std::unique_ptr<SortMergeReader>&& sort_merge_reader,
+        const std::shared_ptr<Predicate>& predicate, bool complete_row_kind);
+
     Result<std::unique_ptr<KeyValueRecordReader>> CreateReaderForRun(
         const BinaryRow& partition, const SortedRun& sorted_run, DeletionVector::Factory dv_factory,
         const std::shared_ptr<Predicate>& predicate,
diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
index 0a3e52353..d4bda2000 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
@@ -233,11 +233,7 @@ TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) {
     ArrowRealtimeStoreFactory factory;
     std::unique_ptr<ArrowSchema> write_schema = MakeReadSchema(schema_);
     RealtimeStoreCreateRequest request{std::move(write_schema),
-                                       /*options=*/{},
-                                       pool_,
-                                       /*partition=*/{},
-                                       /*bucket=*/0,
-                                       RealtimeStoreMode::APPEND_ONLY,
+                                       /*options=*/{}, pool_, RealtimeStoreMode::APPEND_ONLY,
                                        StatisticsMode::FULL};
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> realtime_store,
                          factory.Create(std::move(request)));
@@ -279,11 +275,7 @@ TEST_F(ArrowRealtimeStoreTest, TestFactoryRejectsInvalidMode) {
     ArrowRealtimeStoreFactory factory;
     std::unique_ptr<ArrowSchema> write_schema = MakeReadSchema(schema_);
     RealtimeStoreCreateRequest request{std::move(write_schema),
-                                       /*options=*/{},
-                                       pool_,
-                                       /*partition=*/{},
-                                       /*bucket=*/0,
-                                       static_cast<RealtimeStoreMode>(-1)};
+                                       /*options=*/{}, pool_, static_cast<RealtimeStoreMode>(-1)};
     ASSERT_NOK_WITH_MSG(factory.Create(std::move(request)), "invalid real-time store mode: -1");
 }
 
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 53652cc81..d8a4cfdc0 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -292,8 +292,8 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             }
             std::shared_ptr<arrow::StructArray> data_batch =
                 checked_pointer_cast<arrow::StructArray>(arrow_array);
-            Status transport_status =
-                ValidatePreparedTransportSchema(arrow::schema(data_batch->type()->fields()));
+            Status transport_status = PreparedKeyValueReaderFactory::ValidateTransportSchema(
+                arrow::schema(data_batch->type()->fields()));
             if (!transport_status.ok()) {
                 return Status::Invalid(
                     "prepared batch field does not match prepared transport "
@@ -420,7 +420,8 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
 }  // namespace
 
-Status ValidatePreparedTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema) {
+Status PreparedKeyValueReaderFactory::ValidateTransportSchema(
+    const std::shared_ptr<arrow::Schema>& prepared_schema) {
     if (!prepared_schema || prepared_schema->num_fields() < kPreparedValueStartIndex) {
         return Status::Invalid("prepared schema must contain realtime transport fields");
     }
@@ -450,7 +451,7 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
     if (visible_offsets.has_value() && visible_offsets->begin > visible_offsets->end) {
         return Status::Invalid("prepared visible offset range begin exceeds end");
     }
-    PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(prepared_schema));
+    PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
     if (!key_schema) {
         return Status::Invalid("prepared key schema cannot be null");
     }
@@ -476,7 +477,7 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
 
 }  // namespace
 
-Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
+Result<std::unique_ptr<KeyValueRecordReader>> PreparedKeyValueReaderFactory::Create(
     std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::optional<OffsetRange>& visible_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
@@ -487,7 +488,8 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
                                         /*offset_coverage=*/nullptr);
 }
 
-Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBatchReaders(
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
+PreparedKeyValueReaderFactory::CreateForCommit(
     std::vector<std::unique_ptr<BatchReader>>&& readers,
     const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
index 4ef4887e6..389c75f9a 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/prepared_key_value_reader.h
@@ -32,23 +32,27 @@ namespace paimon {
 class BatchReader;
 class MemoryPool;
 
-/// Validates the required leading fields of a prepared real-time transport schema.
-Status ValidatePreparedTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema);
-
-/// Adapts a plugin query reader and limits its rows to `visible_offsets` when present.
-Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReader(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::optional<OffsetRange>& visible_offsets,
-    const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool);
-
-/// Adapts commit readers and validates their offsets against `sealed_offsets`.
-Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> AdaptPreparedCommitBatchReaders(
-    std::vector<std::unique_ptr<BatchReader>>&& readers,
-    const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
-    const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool);
+class PreparedKeyValueReaderFactory {
+ public:
+    PreparedKeyValueReaderFactory() = delete;
+    ~PreparedKeyValueReaderFactory() = delete;
+
+    static Status ValidateTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema);
+
+    static Result<std::unique_ptr<KeyValueRecordReader>> Create(
+        std::unique_ptr<BatchReader>&& reader,
+        const std::shared_ptr<arrow::Schema>& prepared_schema,
+        const std::optional<OffsetRange>& visible_offsets,
+        const std::shared_ptr<arrow::Schema>& key_schema,
+        const std::shared_ptr<arrow::Schema>& value_schema,
+        const std::shared_ptr<MemoryPool>& memory_pool);
+
+    static Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateForCommit(
+        std::vector<std::unique_ptr<BatchReader>>&& readers,
+        const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
+        const std::shared_ptr<arrow::Schema>& key_schema,
+        const std::shared_ptr<arrow::Schema>& value_schema,
+        const std::shared_ptr<MemoryPool>& memory_pool);
+};
 
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index cfdf88f49..75aa15271 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -362,7 +362,7 @@ class PrimaryKeyRealtimeStore::Impl {
         }
         PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> read_schema,
                                           arrow::ImportSchema(context.read_schema));
-        PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(read_schema));
+        PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(read_schema));
         std::vector<std::unique_ptr<BatchReader>> readers;
         for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
             for (const StoredBatch& batch : segment->Batches()) {
@@ -411,7 +411,7 @@ PrimaryKeyRealtimeStore::~PrimaryKeyRealtimeStore() = default;
 
 Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
     const std::shared_ptr<arrow::Schema>& prepared_schema) {
-    PAIMON_RETURN_NOT_OK(ValidatePreparedTransportSchema(prepared_schema));
+    PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
     return std::shared_ptr<PrimaryKeyRealtimeStore>(
         new PrimaryKeyRealtimeStore(std::make_unique<Impl>(prepared_schema)));
 }
diff --git a/src/paimon/core/realtime/realtime_append_only_writer.cpp b/src/paimon/core/realtime/realtime_append_only_writer.cpp
index 0fb4e58e7..632d64e16 100644
--- a/src/paimon/core/realtime/realtime_append_only_writer.cpp
+++ b/src/paimon/core/realtime/realtime_append_only_writer.cpp
@@ -55,11 +55,11 @@ Result<std::shared_ptr<RealtimeAppendOnlyWriter>> RealtimeAppendOnlyWriter::Crea
     }
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
                            RealtimeContextImpl::Cast(realtime_context));
-    RealtimeStoreCreateRequest request{
-        std::move(write_schema),        options,        memory_pool, partition, bucket,
-        RealtimeStoreMode::APPEND_ONLY, statistics_mode};
+    RealtimeStoreCreateRequest request{std::move(write_schema), options, memory_pool,
+                                       RealtimeStoreMode::APPEND_ONLY, statistics_mode};
     PAIMON_ASSIGN_OR_RAISE(RealtimeStoreState store_state,
-                           realtime_context_impl->GetOrCreateRealtimeStore(std::move(request)));
+                           realtime_context_impl->GetOrCreateRealtimeStore(
+                               std::move(request), RealtimePartitionBucket(partition, bucket)));
     return std::shared_ptr<RealtimeAppendOnlyWriter>(new RealtimeAppendOnlyWriter(
         store_state.store, file_writer, input_schema, store_state.initial_offset, memory_pool));
 }
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 052b616f6..4b5da718e 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -96,7 +96,7 @@ Status RealtimeContextImpl::Start() {
 }
 
 Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
-    RealtimeStoreCreateRequest&& request) {
+    RealtimeStoreCreateRequest&& request, const RealtimePartitionBucket& partition_bucket) {
     if (!request.write_schema || !request.write_schema->release) {
         return Status::Invalid("real-time store write schema is null");
     }
@@ -107,10 +107,9 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     schema_guard.Release();
     std::lock_guard<std::mutex> progress_lock(progress_mutex_);
     std::lock_guard<std::mutex> registry_lock(mutex_);
-    const RealtimePartitionBucket key(request.partition, request.bucket);
-    auto iter = stores_.find(key);
+    auto iter = stores_.find(partition_bucket);
     int64_t initial_offset = 0;
-    auto offset_iter = committed_offsets_.find(key);
+    auto offset_iter = committed_offsets_.find(partition_bucket);
     if (offset_iter != committed_offsets_.end()) {
         if (offset_iter->second == std::numeric_limits<int64_t>::max()) {
             return Status::Invalid("real-time offset has reached INT64_MAX");
@@ -121,8 +120,9 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
         if (iter->second.mode != request.mode ||
             !iter->second.write_schema->Equals(*requested_schema, /*check_metadata=*/true)) {
             return Status::Invalid("real-time store schema or mode mismatch for partition " +
-                                   PartitionToString(key.partition) + ", bucket " +
-                                   std::to_string(key.bucket) + "; recreate the RealtimeContext");
+                                   PartitionToString(partition_bucket.partition) + ", bucket " +
+                                   std::to_string(partition_bucket.bucket) +
+                                   "; recreate the RealtimeContext");
         }
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
                                iter->second.store->AcquireReadView());
@@ -150,9 +150,9 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     RealtimeStoreMode mode = request.mode;
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store,
                            factory_->Create(std::move(request)));
-    stores_.emplace(key, StoreEntry{store, requested_schema, mode});
+    stores_.emplace(partition_bucket, StoreEntry{store, requested_schema, mode});
     if (offset_iter != committed_offsets_.end()) {
-        reclaimed_offsets_.emplace(key, offset_iter->second);
+        reclaimed_offsets_.emplace(partition_bucket, offset_iter->second);
     }
     return RealtimeStoreState{std::move(store), initial_offset};
 }
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index 29ac7c05d..ea069a5cd 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -69,7 +69,8 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
     static Result<std::shared_ptr<RealtimeContextImpl>> Cast(
         const std::shared_ptr<RealtimeContext>& context);
 
-    Result<RealtimeStoreState> GetOrCreateRealtimeStore(RealtimeStoreCreateRequest&& request);
+    Result<RealtimeStoreState> GetOrCreateRealtimeStore(
+        RealtimeStoreCreateRequest&& request, const RealtimePartitionBucket& partition_bucket);
 
     Result<int64_t> AdvanceMaterializedMaxSequenceNumber(
         const RealtimePartitionBucket& partition_bucket, int64_t max_sequence_number);
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index fbdaa86b7..a418c17a8 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -120,8 +120,9 @@ Result<RealtimeStoreState> GetOrCreateAppendStore(
     const std::shared_ptr<MemoryPool>& memory_pool,
     StatisticsMode statistics_mode = StatisticsMode::NONE) {
     return context->GetOrCreateRealtimeStore(
-        RealtimeStoreCreateRequest{std::move(write_schema), options, memory_pool, partition, bucket,
-                                   RealtimeStoreMode::APPEND_ONLY, statistics_mode});
+        RealtimeStoreCreateRequest{std::move(write_schema), options, memory_pool,
+                                   RealtimeStoreMode::APPEND_ONLY, statistics_mode},
+        RealtimePartitionBucket(partition, bucket));
 }
 
 TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
@@ -170,8 +171,10 @@ TEST(RealtimeContextTest, TestRejectsMismatchedModeOnStoreReuse) {
         GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, GetDefaultPool()));
 
     ASSERT_NOK_WITH_MSG(
-        context->GetOrCreateRealtimeStore(RealtimeStoreCreateRequest{
-            MakeWriteSchema(), {}, GetDefaultPool(), partition, 0, RealtimeStoreMode::PRIMARY_KEY}),
+        context->GetOrCreateRealtimeStore(
+            RealtimeStoreCreateRequest{
+                MakeWriteSchema(), {}, GetDefaultPool(), RealtimeStoreMode::PRIMARY_KEY},
+            RealtimePartitionBucket(partition, 0)),
         "schema or mode mismatch for partition {dt=2026-08-02}, bucket 0; recreate the "
         "RealtimeContext");
     ASSERT_EQ(1, factory->stores.size());
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index bcc8f6705..692eff1bd 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -283,10 +283,10 @@ Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegm
                                               const OffsetRange& sealed_offsets) {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                            realtime_store_->CreateCommitReaders(segment));
-    PAIMON_ASSIGN_OR_RAISE(
-        std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
-        AdaptPreparedCommitBatchReaders(std::move(readers), prepared_schema_, sealed_offsets,
-                                        key_schema_, write_schema_, memory_pool_));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
+                           PreparedKeyValueReaderFactory::CreateForCommit(
+                               std::move(readers), prepared_schema_, sealed_offsets, key_schema_,
+                               write_schema_, memory_pool_));
     std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
     sorted_readers.reserve(prepared_readers.size());
     for (std::unique_ptr<KeyValueRecordReader>& prepared_reader : prepared_readers) {
@@ -295,7 +295,7 @@ Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegm
             std::move(prepared_reader), key_comparator_,
             std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function))));
     }
-    return merge_tree_writer_->WriteSortedReaders(std::move(sorted_readers));
+    return merge_tree_writer_->WriteSortedReadersToFiles(std::move(sorted_readers));
 }
 
 Status RealtimePrimaryKeyWriter::Compact(bool) {
diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp
index 050f09701..cc1e6a076 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -46,11 +46,12 @@ TEST(SchemaValidationTest, TestSimple) {
     ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
 }
 
-TEST(SchemaValidationTest, TestRealtimeOffsetIsNotGloballyReserved) {
+TEST(SchemaValidationTest, TestRealtimeOffsetIsGloballyReserved) {
     auto schema = arrow::schema({arrow::field("_REALTIME_OFFSET", arrow::int64())});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
                          TableSchema::Create(0, schema, {}, {}, {}));
-    ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+    ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                        "field name '_REALTIME_OFFSET' in schema cannot be special field");
 }
 
 TEST(SchemaValidationTest, TestVectorType) {
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 64b866439..e13140f54 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -32,7 +32,7 @@
 #include "paimon/core/global_index/indexed_split_impl.h"
 #include "paimon/core/io/merged_key_value_record_reader.h"
 #include "paimon/core/key_value.h"
-#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
+#include "paimon/core/mergetree/compact/merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/operation/merge_file_split_read.h"
 #include "paimon/core/operation/raw_file_split_read.h"
@@ -42,6 +42,7 @@
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/table/source/pk_count_reader.h"
 #include "paimon/core/table/source/realtime_split.h"
+#include "paimon/core/utils/primary_key_table_utils.h"
 #include "paimon/status.h"
 
 namespace paimon {
@@ -59,6 +60,7 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<InternalReadContext>& context,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     arrow::FieldVector prepared_fields = {
         DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
@@ -87,12 +89,16 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null query reader");
         }
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> prepared_reader,
-                               AdaptPreparedBatchReader(std::move(reader), prepared_schema,
-                                                        OffsetRange(split->CommittedEndOffset(),
-                                                                    split->MemoryEndOffset()),
-                                                        key_schema, value_schema, memory_pool));
-        auto merge = std::make_unique<DeduplicateMergeFunction>(false);
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<KeyValueRecordReader> prepared_reader,
+            PreparedKeyValueReaderFactory::Create(
+                std::move(reader), prepared_schema,
+                OffsetRange(split->CommittedEndOffset(), split->MemoryEndOffset()), key_schema,
+                value_schema, memory_pool));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<MergeFunction> merge,
+                               PrimaryKeyTableUtils::CreateMergeFunction(
+                                   value_schema, context->GetTableSchema()->PrimaryKeys(),
+                                   context->GetCoreOptions(), memory_pool));
         result.push_back(std::make_unique<MergedKeyValueRecordReader>(
             std::move(prepared_reader), key_comparator,
             std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge))));
@@ -273,7 +279,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
                 std::vector<std::unique_ptr<KeyValueRecordReader>> memory_readers,
                 CreateMemoryReaders(realtime_split, memory, merge_read->GetKeySchema(),
                                     merge_read->GetValueSchema(), merge_read->GetKeyComparator(),
-                                    GetMemoryPool()));
+                                    context_, GetMemoryPool()));
             PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
                                    merge_read->CreateRealtimeReader(realtime_split->DiskSplits(),
                                                                     std::move(memory_readers)));
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 78e8badee..94bf81edc 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -52,6 +52,8 @@
 #include "paimon/core/core_options.h"
 #include "paimon/core/operation/commit/realtime_commit_properties.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/table_schema.h"
 #include "paimon/core/table/sink/commit_message_impl.h"
 #include "paimon/core/table/source/realtime_split.h"
 #include "paimon/core/utils/snapshot_manager.h"
@@ -1200,11 +1202,24 @@ class RealtimeWriteInteTest : public ::testing::Test {
         if (views.size() != 1) {
             return Status::Invalid("expected one PK real-time read view");
         }
+        PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options_));
+        SchemaManager schema_manager(core_options.GetFileSystem(), table_path_);
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> table_schema,
+                               schema_manager.Latest());
+        if (!table_schema) {
+            return Status::Invalid("expected a table schema");
+        }
         auto read_schema = std::make_unique<ArrowSchema>();
         arrow::FieldVector requested_fields = {
-            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())};
-        requested_fields.insert(requested_fields.end(), schema_->fields().begin(),
-                                schema_->fields().end());
+            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
+                ->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+                ->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
+        std::shared_ptr<arrow::Schema> value_schema =
+            DataField::ConvertDataFieldsToArrowSchema(table_schema.value()->Fields());
+        requested_fields.insert(requested_fields.end(), value_schema->fields().begin(),
+                                value_schema->fields().end());
         PAIMON_RETURN_NOT_OK_FROM_ARROW(
             arrow::ExportSchema(*arrow::schema(requested_fields), read_schema.get()));
         ScopeGuard schema_guard([schema = read_schema.get()]() { ArrowSchemaRelease(schema); });

From bf00d530445fcd52c82427ec25b0354887879b89 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 22:55:52 +0800
Subject: [PATCH 57/93] fix(realtime): tighten primary key framework boundaries

---
 src/paimon/CMakeLists.txt                     |   2 +
 .../core/operation/file_store_write.cpp       |   4 +-
 .../key_value_file_store_write_test.cpp       |  24 ++-
 .../realtime/arrow_realtime_store_factory.cpp |   5 +-
 .../primary_key_realtime_validator.cpp        |  80 ++++++++++
 .../primary_key_realtime_validator.h          |  36 +++++
 .../primary_key_realtime_validator_test.cpp   |  91 ++++++++++++
 .../realtime/primary_key_realtime_store.cpp   |  76 +++-------
 .../realtime/primary_key_realtime_store.h     |   7 +-
 .../primary_key_realtime_store_test.cpp       | 139 ++++++++++--------
 .../realtime/realtime_primary_key_writer.cpp  |  23 ++-
 src/paimon/core/table/source/table_scan.cpp   |   5 +-
 12 files changed, 346 insertions(+), 146 deletions(-)
 create mode 100644 src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp
 create mode 100644 src/paimon/core/realtime/framework/primary_key_realtime_validator.h
 create mode 100644 src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 0a78b0902..7ff15100b 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -382,6 +382,7 @@ set(PAIMON_CORE_SRCS
     core/operation/write_restore.cpp
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
+    core/realtime/framework/primary_key_realtime_validator.cpp
     core/realtime/prepared_key_value_reader.cpp
     core/realtime/primary_key_realtime_store.cpp
     core/realtime/realtime_append_only_writer.cpp
@@ -790,6 +791,7 @@ if(PAIMON_BUILD_TESTS)
                     core/manifest/index_manifest_file_handler_test.cpp
                     core/memory/writer_memory_manager_test.cpp
                     core/realtime/arrow_realtime_store_test.cpp
+                    core/realtime/framework/primary_key_realtime_validator_test.cpp
                     core/realtime/primary_key_realtime_store_test.cpp
                     core/realtime/realtime_context_test.cpp
                     core/realtime/realtime_reader_test.cpp
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 26befb7ee..f77dadda2 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -36,7 +36,7 @@
 #include "paimon/core/operation/key_value_file_store_write.h"
 #include "paimon/core/options/merge_engine.h"
 #include "paimon/core/postpone/postpone_bucket_file_store_write.h"
-#include "paimon/core/realtime/primary_key_realtime_store.h"
+#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
@@ -210,7 +210,7 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     } else {
         // pk table
         if (ctx->GetRealtimeContext()) {
-            PAIMON_RETURN_NOT_OK(PrimaryKeyRealtimeStore::ValidateOptions(options, *schema));
+            PAIMON_RETURN_NOT_OK(PrimaryKeyRealtimeValidator::ValidateOptions(options, *schema));
             if (ignore_previous_files) {
                 return Status::NotImplemented(
                     "PK realtime v1 requires restore from the latest snapshot");
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index 0f70235a7..d52841f95 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -488,19 +488,17 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
     });
     std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
     ASSERT_TRUE(dir);
-    CreateTable(dir->Str(), schema, options);
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    WriteContextBuilder builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test");
-    builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         FileStoreWrite::Create(std::move(write_context)));
-
-    ASSERT_NOK_WITH_MSG(writer->Write(MakeBatch(schema, R"([[1, 10]])")),
-                        "PK real-time write schema contains reserved transport field");
-    ASSERT_OK(writer->Close());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<Catalog> catalog, Catalog::Create(dir->Str(), options));
+    ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false));
+    ArrowSchema c_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
+    Status create_status =
+        catalog->CreateTable(Identifier("foo", "bar"), &c_schema,
+                             /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options,
+                             /*ignore_if_exists=*/false);
+    ArrowSchemaRelease(&c_schema);
+    ASSERT_NOK_WITH_MSG(create_status,
+                        "field name '_REALTIME_OFFSET' in schema cannot be special field");
 }
 
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
diff --git a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
index d0d4ae704..dff12b589 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
@@ -49,8 +49,9 @@ Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
                                                         request.memory_pool, arrow_pool);
         }
         case RealtimeStoreMode::PRIMARY_KEY: {
-            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                                   PrimaryKeyRealtimeStore::Create(imported_schema));
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                PrimaryKeyRealtimeStore::Create(imported_schema, request.memory_pool));
             return std::shared_ptr<RealtimeStore>(std::move(store));
         }
     }
diff --git a/src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp b/src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp
new file mode 100644
index 000000000..61c640ee6
--- /dev/null
+++ b/src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
+
+#include <vector>
+
+#include "arrow/type.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/index/pk/primary_key_index_definitions.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/macros.h"
+
+namespace paimon {
+
+Status PrimaryKeyRealtimeValidator::ValidateOptions(const CoreOptions& options,
+                                                    const TableSchema& schema) {
+    if (options.GetBucket() <= 0) {
+        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
+    }
+    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
+        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
+    }
+    if (options.DataEvolutionEnabled()) {
+        return Status::NotImplemented("PK realtime v1 does not support data evolution");
+    }
+    if (!options.GetFieldsSequenceGroups().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
+    }
+    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
+        options.AggregationRemoveRecordOnDelete() ||
+        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
+        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
+    }
+    if (!options.GetSequenceField().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
+    }
+    if (!options.SequenceFieldSortOrderIsAscending()) {
+        return Status::NotImplemented(
+            "PK realtime v1 supports only ascending sequence.field.sort-order");
+    }
+    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
+        options.GetChangelogProducer() != ChangelogProducer::NONE) {
+        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> primary_key_fields,
+                           schema.TrimmedPrimaryKeyFields());
+    for (const DataField& field : primary_key_fields) {
+        if (field.Type()->id() == arrow::Type::FLOAT || field.Type()->id() == arrow::Type::DOUBLE) {
+            return Status::NotImplemented(
+                "PK realtime v1 does not support FLOAT or DOUBLE primary keys");
+        }
+    }
+    if (options.GlobalIndexEnabled()) {
+        PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions,
+                               PrimaryKeyIndexDefinitions::Create(schema));
+        if (!definitions.Definitions().empty()) {
+            return Status::NotImplemented("PK realtime v1 does not support global indexes");
+        }
+    }
+    return Status::OK();
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/framework/primary_key_realtime_validator.h b/src/paimon/core/realtime/framework/primary_key_realtime_validator.h
new file mode 100644
index 000000000..fa9432d3b
--- /dev/null
+++ b/src/paimon/core/realtime/framework/primary_key_realtime_validator.h
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "paimon/status.h"
+
+namespace paimon {
+
+class CoreOptions;
+class TableSchema;
+
+class PrimaryKeyRealtimeValidator {
+ public:
+    PrimaryKeyRealtimeValidator() = delete;
+    ~PrimaryKeyRealtimeValidator() = delete;
+
+    static Status ValidateOptions(const CoreOptions& options, const TableSchema& schema);
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp b/src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp
new file mode 100644
index 000000000..3b7793b14
--- /dev/null
+++ b/src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/type.h"
+#include "gtest/gtest.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/defs.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+namespace {
+
+std::shared_ptr<TableSchema> PkSchema(
+    const std::shared_ptr<arrow::DataType>& key_type = arrow::int64(),
+    const std::map<std::string, std::string>& options = {}) {
+    return TableSchema::Create(
+               /*schema_id=*/0,
+               arrow::schema({arrow::field("id", key_type), arrow::field("value", arrow::utf8())}),
+               /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options)
+        .value();
+}
+
+}  // namespace
+
+TEST(PrimaryKeyRealtimeValidatorTest, TestSupportedOptions) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_OK(PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema()));
+}
+
+TEST(PrimaryKeyRealtimeValidatorTest, TestUnsupportedOptions) {
+    const std::string sequence_group =
+        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
+    const std::vector<std::map<std::string, std::string>> unsupported_options = {
+        {{Options::BUCKET, "0"}},
+        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
+        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
+        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
+    };
+    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
+        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+        ASSERT_NOK(PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema()));
+    }
+}
+
+TEST(PrimaryKeyRealtimeValidatorTest, TestRejectsFloatingPrimaryKeys) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_NOK_WITH_MSG(
+        PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema(arrow::float32())),
+        "FLOAT or DOUBLE primary keys");
+    ASSERT_NOK_WITH_MSG(
+        PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema(arrow::float64())),
+        "FLOAT or DOUBLE primary keys");
+}
+
+TEST(PrimaryKeyRealtimeValidatorTest, TestRejectsEnabledGlobalIndex) {
+    const std::map<std::string, std::string> option_map = {{Options::BUCKET, "1"},
+                                                           {Options::PK_BTREE_INDEX_COLUMNS, "id"}};
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+    ASSERT_NOK_WITH_MSG(PrimaryKeyRealtimeValidator::ValidateOptions(
+                            options, *PkSchema(arrow::int64(), option_map)),
+                        "does not support global indexes");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 75aa15271..88e78680a 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -28,67 +28,17 @@
 #include "arrow/c/bridge.h"
 #include "fmt/format.h"
 #include "paimon/common/metrics/metrics_impl.h"
-#include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
-#include "paimon/core/core_options.h"
-#include "paimon/core/index/pk/primary_key_index_definitions.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
-#include "paimon/core/schema/table_schema.h"
 #include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
+#include "paimon/memory/memory_pool.h"
 
 namespace paimon {
 
-Status PrimaryKeyRealtimeStore::ValidateOptions(const CoreOptions& options,
-                                                const TableSchema& schema) {
-    if (options.GetBucket() <= 0) {
-        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
-    }
-    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
-        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
-    }
-    if (options.DataEvolutionEnabled()) {
-        return Status::NotImplemented("PK realtime v1 does not support data evolution");
-    }
-    if (!options.GetFieldsSequenceGroups().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
-    }
-    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
-        options.AggregationRemoveRecordOnDelete() ||
-        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
-        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
-    }
-    if (!options.GetSequenceField().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
-    }
-    if (!options.SequenceFieldSortOrderIsAscending()) {
-        return Status::NotImplemented(
-            "PK realtime v1 supports only ascending sequence.field.sort-order");
-    }
-    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
-        options.GetChangelogProducer() != ChangelogProducer::NONE) {
-        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> primary_key_fields,
-                           schema.TrimmedPrimaryKeyFields());
-    for (const DataField& field : primary_key_fields) {
-        if (field.Type()->id() == arrow::Type::FLOAT || field.Type()->id() == arrow::Type::DOUBLE) {
-            return Status::NotImplemented(
-                "PK realtime v1 does not support FLOAT or DOUBLE primary keys");
-        }
-    }
-    if (options.GlobalIndexEnabled()) {
-        PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions,
-                               PrimaryKeyIndexDefinitions::Create(schema));
-        if (!definitions.Definitions().empty()) {
-            return Status::NotImplemented("PK realtime v1 does not support global indexes");
-        }
-    }
-    return Status::OK();
-}
-
 namespace {
 
 Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
@@ -283,8 +233,11 @@ class StoredBatchReader final : public BatchReader {
 
 class PrimaryKeyRealtimeStore::Impl {
  public:
-    explicit Impl(std::shared_ptr<arrow::Schema> prepared_schema)
-        : prepared_schema_(std::move(prepared_schema)) {}
+    Impl(std::shared_ptr<arrow::Schema> prepared_schema, std::shared_ptr<MemoryPool> memory_pool,
+         std::shared_ptr<arrow::MemoryPool> arrow_pool)
+        : prepared_schema_(std::move(prepared_schema)),
+          memory_pool_(std::move(memory_pool)),
+          arrow_pool_(std::move(arrow_pool)) {}
 
     Status Write(RealtimeWriteBatch&& write_batch) {
         if (!write_batch.batch || !write_batch.batch->GetData()) {
@@ -369,7 +322,7 @@ class PrimaryKeyRealtimeStore::Impl {
                 PAIMON_ASSIGN_OR_RAISE(
                     std::shared_ptr<arrow::Array> projected,
                     AlignArrayByPaimonIds(batch.data, arrow::struct_(read_schema->fields()),
-                                          arrow::default_memory_pool()));
+                                          arrow_pool_.get()));
                 StoredBatch query_batch{checked_pointer_cast<arrow::StructArray>(projected),
                                         batch.offset_range, /*memory_usage=*/0};
                 readers.push_back(std::make_unique<StoredBatchReader>(query_batch));
@@ -399,6 +352,8 @@ class PrimaryKeyRealtimeStore::Impl {
 
  private:
     std::shared_ptr<arrow::Schema> prepared_schema_;
+    std::shared_ptr<MemoryPool> memory_pool_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     mutable std::mutex mutex_;
     std::vector<StoredBatch> building_;
     std::vector<std::shared_ptr<Segment>> sealed_;
@@ -410,10 +365,15 @@ PrimaryKeyRealtimeStore::PrimaryKeyRealtimeStore(std::unique_ptr<Impl>&& impl)
 PrimaryKeyRealtimeStore::~PrimaryKeyRealtimeStore() = default;
 
 Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
-    const std::shared_ptr<arrow::Schema>& prepared_schema) {
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
     PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
-    return std::shared_ptr<PrimaryKeyRealtimeStore>(
-        new PrimaryKeyRealtimeStore(std::make_unique<Impl>(prepared_schema)));
+    if (!memory_pool) {
+        return Status::Invalid("PK real-time store memory pool is null");
+    }
+    std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
+    return std::shared_ptr<PrimaryKeyRealtimeStore>(new PrimaryKeyRealtimeStore(
+        std::make_unique<Impl>(prepared_schema, memory_pool, std::move(arrow_pool))));
 }
 Status PrimaryKeyRealtimeStore::Write(RealtimeWriteBatch&& batch) {
     return impl_->Write(std::move(batch));
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index df9a5f0c5..01e1926ab 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -29,15 +29,14 @@ class Schema;
 
 namespace paimon {
 
-class CoreOptions;
-class TableSchema;
+class MemoryPool;
 
 /// Internal in-memory implementation of the default primary-key `RealtimeStore`.
 class PrimaryKeyRealtimeStore final : public RealtimeStore {
  public:
     static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
-        const std::shared_ptr<arrow::Schema>& prepared_schema);
-    static Status ValidateOptions(const CoreOptions& options, const TableSchema& schema);
+        const std::shared_ptr<arrow::Schema>& prepared_schema,
+        const std::shared_ptr<MemoryPool>& memory_pool);
 
     ~PrimaryKeyRealtimeStore() override;
 
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 405c0d32a..7a1a3e27c 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -18,7 +18,10 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
+#include <cstddef>
+#include <cstdint>
 #include <memory>
+#include <new>
 #include <optional>
 #include <string>
 #include <utility>
@@ -31,10 +34,9 @@
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
-#include "paimon/core/core_options.h"
-#include "paimon/core/schema/table_schema.h"
 #include "paimon/macros.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/realtime/arrow_realtime_store_factory.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
@@ -71,16 +73,6 @@ std::shared_ptr<arrow::Schema> NestedPreparedSchema() {
                                           arrow::field("items", arrow::list(arrow::int32()))}))))});
 }
 
-std::shared_ptr<TableSchema> PkSchema(
-    const std::shared_ptr<arrow::DataType>& key_type = arrow::int64(),
-    const std::map<std::string, std::string>& options = {}) {
-    return TableSchema::Create(
-               /*schema_id=*/0,
-               arrow::schema({arrow::field("id", key_type), arrow::field("value", arrow::utf8())}),
-               /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options)
-        .value();
-}
-
 std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) {
     std::shared_ptr<arrow::Array> array =
         arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(PreparedSchema()->fields()), json)
@@ -142,52 +134,50 @@ Result<std::string> ReadJson(const std::vector<std::unique_ptr<BatchReader>>& re
     return result->ToString();
 }
 
-TEST(PrimaryKeyRealtimeStoreOptionsTest, TestSupportedOptions) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_OK(PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema()));
-}
+class TestingMemoryPool final : public MemoryPool {
+ public:
+    void* Malloc(uint64_t size, uint64_t alignment) override {
+        ++allocation_count;
+        if (reject_allocations) {
+            throw std::bad_alloc();
+        }
+        return delegate_->Malloc(size, alignment);
+    }
 
-TEST(PrimaryKeyRealtimeStoreOptionsTest, TestUnsupportedOptions) {
-    const std::string sequence_group =
-        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
-    const std::vector<std::map<std::string, std::string>> unsupported_options = {
-        {{Options::BUCKET, "0"}},
-        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
-        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
-        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
-    };
-    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
-        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-        ASSERT_NOK(PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema()));
+    void* Realloc(void* pointer, size_t old_size, size_t new_size, uint64_t alignment) override {
+        ++allocation_count;
+        if (reject_allocations) {
+            throw std::bad_alloc();
+        }
+        return delegate_->Realloc(pointer, old_size, new_size, alignment);
     }
-}
 
-TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsFloatingPrimaryKeys) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_NOK_WITH_MSG(
-        PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema(arrow::float32())),
-        "FLOAT or DOUBLE primary keys");
-    ASSERT_NOK_WITH_MSG(
-        PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema(arrow::float64())),
-        "FLOAT or DOUBLE primary keys");
-}
+    void Free(void* pointer, uint64_t size) override {
+        delegate_->Free(pointer, size);
+    }
 
-TEST(PrimaryKeyRealtimeStoreOptionsTest, TestRejectsEnabledGlobalIndex) {
-    const std::map<std::string, std::string> option_map = {{Options::BUCKET, "1"},
-                                                           {Options::PK_BTREE_INDEX_COLUMNS, "id"}};
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-    ASSERT_NOK_WITH_MSG(
-        PrimaryKeyRealtimeStore::ValidateOptions(options, *PkSchema(arrow::int64(), option_map)),
-        "does not support global indexes");
-}
+    void Free(void* pointer, uint64_t size, uint64_t alignment) override {
+        delegate_->Free(pointer, size, alignment);
+    }
+
+    uint64_t CurrentUsage() const override {
+        return delegate_->CurrentUsage();
+    }
+
+    uint64_t MaxMemoryUsage() const override {
+        return delegate_->MaxMemoryUsage();
+    }
+
+    bool reject_allocations = false;
+    int64_t allocation_count = 0;
+
+ private:
+    std::unique_ptr<MemoryPool> delegate_ = GetMemoryPool();
+};
 
 TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
     ASSERT_FALSE(segment.has_value());
@@ -232,14 +222,15 @@ TEST(PrimaryKeyRealtimeStoreTest, TestBadTransportPrefix) {
     invalid_fields.push_back(std::move(wrong_offset_id));
 
     for (const arrow::FieldVector& fields : invalid_fields) {
-        ASSERT_NOK_WITH_MSG(PrimaryKeyRealtimeStore::Create(arrow::schema(fields)),
-                            "prepared schema field");
+        ASSERT_NOK_WITH_MSG(
+            PrimaryKeyRealtimeStore::Create(arrow::schema(fields), GetDefaultPool()),
+            "prepared schema field");
     }
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderPerStoredBatch) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeBatch(R"([[1, 6, 1, 1, "before"], [0, 5, 0, 3, "three"]])"), OffsetRange(0, 2)}));
     ASSERT_OK(store->Write(
@@ -265,7 +256,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderPerStoredBatch) {
 TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
     std::shared_ptr<arrow::Schema> schema = NestedPreparedSchema();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(schema));
+                         PrimaryKeyRealtimeStore::Create(schema, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeBatch(schema, R"([[0, 1, 0, 1, ["one", [1, 2]]], [0, 2, 1, 2, ["two", [3, 4]]]])"),
         OffsetRange(0, 2)}));
@@ -286,7 +277,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadBatchReaders) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 10, 0, 1, "a"]])"), OffsetRange(0, 1)}));
     ASSERT_OK(
@@ -306,7 +297,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadBatchReaders) {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 0, 4, 1, "one"]])"), OffsetRange(4, 5)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -318,7 +309,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema()));
+                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 2, "two"]])"), OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -342,7 +333,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsTopLevelFieldsById) {
     const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(stored_schema));
+                         PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeSlicedBatch(stored_schema,
                         R"([[0, 1, 0, 6, "six"], [0, 2, 1, 7, "seven"], [0, 3, 2, 8, "eight"]])", 1,
@@ -374,6 +365,34 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsTopLevelFieldsById) {
     ASSERT_TRUE(projected->field(4)->IsNull(0));
 }
 
+TEST(PrimaryKeyRealtimeStoreTest, TestQuerySchemaAlignmentUsesCallerPool) {
+    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
+    std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
+    auto write_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*stored_schema, write_schema.get()).ok());
+    RealtimeStoreCreateRequest request{std::move(write_schema),
+                                       /*options=*/{}, pool, RealtimeStoreMode::PRIMARY_KEY};
+    ArrowRealtimeStoreFactory factory;
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> store, factory.Create(std::move(request)));
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 7, "seven"]])"), OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    arrow::FieldVector requested_fields = stored_schema->fields();
+    requested_fields.push_back(FieldWithId("added", arrow::int32(), 2));
+    std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
+    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+
+    const int64_t allocations_before_query = pool->allocation_count;
+    pool->reject_allocations = true;
+    ASSERT_NOK_WITH_MSG(store->CreateQueryReaders(view, /*offset_begin=*/0, context),
+                        "Out of memory");
+    ASSERT_GT(pool->allocation_count, allocations_before_query);
+}
+
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
     const std::shared_ptr<arrow::Field> stored_a = FieldWithId("a", arrow::int32(), 10);
     const std::shared_ptr<arrow::Field> stored_b = FieldWithId("b", arrow::int32(), 11);
@@ -389,7 +408,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
         FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_x, stored_y})), 2)};
     std::shared_ptr<arrow::Schema> stored_schema = arrow::schema(std::move(stored_fields));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(stored_schema));
+                         PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeSlicedBatch(
             stored_schema,
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 692eff1bd..bacbfa5e2 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -19,12 +19,14 @@
 #include "paimon/core/realtime/realtime_primary_key_writer.h"
 
 #include <limits>
+#include <new>
 #include <optional>
 #include <utility>
 #include <vector>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
 #include "arrow/compute/api.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
@@ -52,20 +54,31 @@ struct PreparedArrayPrivateData {
 };
 
 void ReleasePreparedArray(ArrowArray* array) {
-    auto* data = static_cast<PreparedArrayPrivateData*>(array->private_data);
+    std::unique_ptr<PreparedArrayPrivateData> data(
+        static_cast<PreparedArrayPrivateData*>(array->private_data));
     array->release = data->release;
     array->private_data = data->private_data;
     array->release(array);
-    delete data;
 }
 
 Status RetainPreparedArrayPool(ArrowArray* array,
                                const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
-    if (!array || !array->release || !arrow_pool) {
+    if (!array || !array->release) {
         return Status::Invalid("cannot retain prepared batch memory pool");
     }
-    array->private_data =
-        new PreparedArrayPrivateData{array->release, array->private_data, arrow_pool};
+    if (!arrow_pool) {
+        ArrowArrayRelease(array);
+        return Status::Invalid("cannot retain prepared batch memory pool");
+    }
+    std::unique_ptr<PreparedArrayPrivateData> data;
+    try {
+        data = std::make_unique<PreparedArrayPrivateData>(
+            PreparedArrayPrivateData{array->release, array->private_data, arrow_pool});
+    } catch (const std::bad_alloc&) {
+        ArrowArrayRelease(array);
+        return Status::OutOfMemory("failed to retain prepared batch memory pool");
+    }
+    array->private_data = data.release();
     array->release = ReleasePreparedArray;
     return Status::OK();
 }
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 70ab0cb8e..85a9e83b3 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -41,7 +41,7 @@
 #include "paimon/core/operation/data_evolution_file_store_scan.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
-#include "paimon/core/realtime/primary_key_realtime_store.h"
+#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/schema_validation.h"
@@ -237,7 +237,8 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
             return Status::NotImplemented(
                 "PK real-time union read does not support read-optimized scans");
         }
-        PAIMON_RETURN_NOT_OK(PrimaryKeyRealtimeStore::ValidateOptions(core_options, table_schema));
+        PAIMON_RETURN_NOT_OK(
+            PrimaryKeyRealtimeValidator::ValidateOptions(core_options, table_schema));
     }
     if (context.IsStreamingMode()) {
         return Status::Invalid("real-time union read currently supports batch scans only");

From 52ee6bd5f95987c1b1ebad8ccf2bc8c0ddee28d0 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 23:40:14 +0800
Subject: [PATCH 58/93] fix(realtime): harden query schema alignment

---
 src/paimon/CMakeLists.txt                     |   1 +
 .../core/realtime/arrow_array_pool_holder.cpp |  69 ++++++++
 .../core/realtime/arrow_array_pool_holder.h   |  36 ++++
 .../realtime/primary_key_realtime_store.cpp   |  75 +++++---
 .../primary_key_realtime_store_test.cpp       | 166 ++++++++++++++++--
 .../realtime/realtime_primary_key_writer.cpp  |  41 +----
 6 files changed, 316 insertions(+), 72 deletions(-)
 create mode 100644 src/paimon/core/realtime/arrow_array_pool_holder.cpp
 create mode 100644 src/paimon/core/realtime/arrow_array_pool_holder.h

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 7ff15100b..fb702bfea 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -380,6 +380,7 @@ set(PAIMON_CORE_SRCS
     core/manifest/snapshot_live_manifest_entries.cpp
     core/operation/write_context.cpp
     core/operation/write_restore.cpp
+    core/realtime/arrow_array_pool_holder.cpp
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
     core/realtime/framework/primary_key_realtime_validator.cpp
diff --git a/src/paimon/core/realtime/arrow_array_pool_holder.cpp b/src/paimon/core/realtime/arrow_array_pool_holder.cpp
new file mode 100644
index 000000000..97a01dd19
--- /dev/null
+++ b/src/paimon/core/realtime/arrow_array_pool_holder.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/realtime/arrow_array_pool_holder.h"
+
+#include <memory>
+#include <new>
+
+#include "arrow/c/abi.h"
+#include "arrow/c/helpers.h"
+#include "arrow/memory_pool.h"
+
+namespace paimon {
+namespace {
+
+struct ArrowArrayPrivateData {
+    void (*release)(ArrowArray*);
+    void* private_data;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool;
+};
+
+void ReleaseArrowArray(ArrowArray* array) {
+    std::unique_ptr<ArrowArrayPrivateData> data(
+        static_cast<ArrowArrayPrivateData*>(array->private_data));
+    array->release = data->release;
+    array->private_data = data->private_data;
+    array->release(array);
+}
+
+}  // namespace
+
+Status RetainArrowArrayMemoryPool(ArrowArray* array,
+                                  const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+    if (!array || !array->release) {
+        return Status::Invalid("cannot retain Arrow array memory pool");
+    }
+    if (!arrow_pool) {
+        ArrowArrayRelease(array);
+        return Status::Invalid("cannot retain Arrow array memory pool");
+    }
+    std::unique_ptr<ArrowArrayPrivateData> data;
+    try {
+        data = std::make_unique<ArrowArrayPrivateData>(
+            ArrowArrayPrivateData{array->release, array->private_data, arrow_pool});
+    } catch (const std::bad_alloc&) {
+        ArrowArrayRelease(array);
+        return Status::OutOfMemory("failed to retain Arrow array memory pool");
+    }
+    array->private_data = data.release();
+    array->release = ReleaseArrowArray;
+    return Status::OK();
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/arrow_array_pool_holder.h b/src/paimon/core/realtime/arrow_array_pool_holder.h
new file mode 100644
index 000000000..05f9ed9dc
--- /dev/null
+++ b/src/paimon/core/realtime/arrow_array_pool_holder.h
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "paimon/status.h"
+
+struct ArrowArray;
+
+namespace arrow {
+class MemoryPool;
+}  // namespace arrow
+
+namespace paimon {
+
+Status RetainArrowArrayMemoryPool(ArrowArray* array,
+                                  const std::shared_ptr<arrow::MemoryPool>& arrow_pool);
+
+}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 88e78680a..cbaef65e2 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -32,6 +32,7 @@
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/realtime/arrow_array_pool_holder.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
@@ -41,9 +42,14 @@ namespace paimon {
 
 namespace {
 
-Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
-    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
-    arrow::MemoryPool* pool);
+struct AlignedArray {
+    std::shared_ptr<arrow::Array> array;
+    bool uses_arrow_pool;
+};
+
+Result<AlignedArray> AlignArrayByPaimonIds(const std::shared_ptr<arrow::Array>& array,
+                                           const std::shared_ptr<arrow::DataType>& read_type,
+                                           arrow::MemoryPool* pool);
 
 bool TypesExactlyEqual(const std::shared_ptr<arrow::DataType>& data_type,
                        const std::shared_ptr<arrow::DataType>& read_type) {
@@ -59,7 +65,7 @@ bool TypesExactlyEqual(const std::shared_ptr<arrow::DataType>& data_type,
     return true;
 }
 
-Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
+Result<AlignedArray> AlignStructArrayByPaimonIds(
     const std::shared_ptr<arrow::StructArray>& array,
     const std::shared_ptr<arrow::StructType>& read_type, arrow::MemoryPool* pool) {
     const std::shared_ptr<arrow::StructType> data_type =
@@ -78,6 +84,7 @@ Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
     requested_field_ids.reserve(read_type->num_fields());
     std::vector<std::shared_ptr<arrow::ArrayData>> children;
     children.reserve(read_type->num_fields());
+    bool uses_arrow_pool = false;
     for (const std::shared_ptr<arrow::Field>& read_field : read_type->fields()) {
         PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
                                NestedProjectionUtils::GetPaimonFieldId(read_field));
@@ -87,30 +94,38 @@ Result<std::shared_ptr<arrow::Array>> AlignStructArrayByPaimonIds(
         }
         const auto data_iter = data_field_indexes.find(field_id);
         if (data_iter == data_field_indexes.end()) {
+            if (!read_field->nullable()) {
+                return Status::Invalid(fmt::format(
+                    "requested non-nullable field '{}' with id {} is absent from stored schema",
+                    read_field->name(), field_id));
+            }
             PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
                 std::shared_ptr<arrow::Array> null_child,
                 arrow::MakeArrayOfNull(read_field->type(), array->offset() + array->length(),
                                        pool));
             children.push_back(null_child->data());
+            uses_arrow_pool = true;
             continue;
         }
         std::shared_ptr<arrow::Array> child =
             arrow::MakeArray(array->data()->child_data[data_iter->second]);
-        PAIMON_ASSIGN_OR_RAISE(child, AlignArrayByPaimonIds(child, read_field->type(), pool));
-        children.push_back(child->data());
+        PAIMON_ASSIGN_OR_RAISE(AlignedArray aligned_child,
+                               AlignArrayByPaimonIds(child, read_field->type(), pool));
+        children.push_back(aligned_child.array->data());
+        uses_arrow_pool = uses_arrow_pool || aligned_child.uses_arrow_pool;
     }
 
     std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
     aligned->type = read_type;
     aligned->child_data = std::move(children);
-    return arrow::MakeArray(std::move(aligned));
+    return AlignedArray{arrow::MakeArray(std::move(aligned)), uses_arrow_pool};
 }
 
-Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
-    const std::shared_ptr<arrow::Array>& array, const std::shared_ptr<arrow::DataType>& read_type,
-    arrow::MemoryPool* pool) {
+Result<AlignedArray> AlignArrayByPaimonIds(const std::shared_ptr<arrow::Array>& array,
+                                           const std::shared_ptr<arrow::DataType>& read_type,
+                                           arrow::MemoryPool* pool) {
     if (TypesExactlyEqual(array->type(), read_type)) {
-        return array;
+        return AlignedArray{array, false};
     }
     if (array->type_id() != read_type->id()) {
         return Status::Invalid(fmt::format("stored value type {} does not match requested type {}",
@@ -125,11 +140,13 @@ Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
             std::shared_ptr<arrow::Array> values =
                 checked_pointer_cast<arrow::ListArray>(array)->values();
             PAIMON_ASSIGN_OR_RAISE(
-                values, AlignArrayByPaimonIds(values, read_type->field(0)->type(), pool));
+                AlignedArray aligned_values,
+                AlignArrayByPaimonIds(values, read_type->field(0)->type(), pool));
             std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
             aligned->type = read_type;
-            aligned->child_data = {values->data()};
-            return arrow::MakeArray(std::move(aligned));
+            aligned->child_data = {aligned_values.array->data()};
+            return AlignedArray{arrow::MakeArray(std::move(aligned)),
+                                aligned_values.uses_arrow_pool};
         }
         case arrow::Type::MAP: {
             const std::shared_ptr<arrow::MapArray> map =
@@ -137,17 +154,19 @@ Result<std::shared_ptr<arrow::Array>> AlignArrayByPaimonIds(
             const std::shared_ptr<arrow::MapType> map_type =
                 checked_pointer_cast<arrow::MapType>(read_type);
             std::shared_ptr<arrow::Array> keys = map->keys();
-            PAIMON_ASSIGN_OR_RAISE(keys, AlignArrayByPaimonIds(keys, map_type->key_type(), pool));
+            PAIMON_ASSIGN_OR_RAISE(AlignedArray aligned_keys,
+                                   AlignArrayByPaimonIds(keys, map_type->key_type(), pool));
             std::shared_ptr<arrow::Array> items = map->items();
-            PAIMON_ASSIGN_OR_RAISE(items,
+            PAIMON_ASSIGN_OR_RAISE(AlignedArray aligned_items,
                                    AlignArrayByPaimonIds(items, map_type->item_type(), pool));
             std::shared_ptr<arrow::ArrayData> entries = array->data()->child_data[0]->Copy();
             entries->type = arrow::struct_({map_type->key_field(), map_type->item_field()});
-            entries->child_data = {keys->data(), items->data()};
+            entries->child_data = {aligned_keys.array->data(), aligned_items.array->data()};
             std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
             aligned->type = read_type;
             aligned->child_data = {std::move(entries)};
-            return arrow::MakeArray(std::move(aligned));
+            return AlignedArray{arrow::MakeArray(std::move(aligned)),
+                                aligned_keys.uses_arrow_pool || aligned_items.uses_arrow_pool};
         }
         default:
             return Status::Invalid(
@@ -203,8 +222,11 @@ class ReadView final : public RealtimeReadView {
 
 class StoredBatchReader final : public BatchReader {
  public:
-    explicit StoredBatchReader(const StoredBatch& batch)
-        : data_(batch.data), metrics_(std::make_shared<MetricsImpl>()) {}
+    explicit StoredBatchReader(const StoredBatch& batch,
+                               std::shared_ptr<arrow::MemoryPool> arrow_pool = nullptr)
+        : arrow_pool_(std::move(arrow_pool)),
+          data_(batch.data),
+          metrics_(std::make_shared<MetricsImpl>()) {}
 
     Result<ReadBatch> NextBatch() override {
         if (!data_) {
@@ -213,7 +235,11 @@ class StoredBatchReader final : public BatchReader {
         auto array = std::make_unique<ArrowArray>();
         auto schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*data_, array.get(), schema.get()));
+        if (arrow_pool_) {
+            PAIMON_RETURN_NOT_OK(RetainArrowArrayMemoryPool(array.get(), arrow_pool_));
+        }
         data_.reset();
+        arrow_pool_.reset();
         return ReadBatch(std::move(array), std::move(schema));
     }
 
@@ -222,9 +248,11 @@ class StoredBatchReader final : public BatchReader {
     }
     void Close() override {
         data_.reset();
+        arrow_pool_.reset();
     }
 
  private:
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     std::shared_ptr<arrow::StructArray> data_;
     std::shared_ptr<Metrics> metrics_;
 };
@@ -320,12 +348,13 @@ class PrimaryKeyRealtimeStore::Impl {
         for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
             for (const StoredBatch& batch : segment->Batches()) {
                 PAIMON_ASSIGN_OR_RAISE(
-                    std::shared_ptr<arrow::Array> projected,
+                    AlignedArray projected,
                     AlignArrayByPaimonIds(batch.data, arrow::struct_(read_schema->fields()),
                                           arrow_pool_.get()));
-                StoredBatch query_batch{checked_pointer_cast<arrow::StructArray>(projected),
+                StoredBatch query_batch{checked_pointer_cast<arrow::StructArray>(projected.array),
                                         batch.offset_range, /*memory_usage=*/0};
-                readers.push_back(std::make_unique<StoredBatchReader>(query_batch));
+                readers.push_back(std::make_unique<StoredBatchReader>(
+                    query_batch, projected.uses_arrow_pool ? arrow_pool_ : nullptr));
             }
         }
         return readers;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 7a1a3e27c..a63d79eec 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -44,8 +44,10 @@ namespace {
 
 std::shared_ptr<arrow::Field> FieldWithId(const std::string& name,
                                           const std::shared_ptr<arrow::DataType>& type,
-                                          int32_t field_id) {
-    return DataField::ConvertDataFieldToArrowField(DataField(field_id, arrow::field(name, type)));
+                                          int32_t field_id, bool nullable = true) {
+    return DataField::ConvertDataFieldToArrowField(
+               DataField(field_id, arrow::field(name, type, nullable)))
+        ->WithNullable(nullable);
 }
 
 std::shared_ptr<arrow::Schema> PreparedSchema() {
@@ -365,6 +367,26 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsTopLevelFieldsById) {
     ASSERT_TRUE(projected->field(4)->IsNull(0));
 }
 
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderRejectsMissingNonNullableTopLevelField) {
+    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 7, "seven"]])"), OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    arrow::FieldVector requested_fields = stored_schema->fields();
+    requested_fields.push_back(
+        FieldWithId("required_added", arrow::int32(), 2, /*nullable=*/false));
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(
+        arrow::ExportSchema(*arrow::schema(std::move(requested_fields)), c_schema.get()).ok());
+    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_NOK_WITH_MSG(store->CreateQueryReaders(view, /*offset_begin=*/0, context),
+                        "requested non-nullable field 'required_added' with id 2 is absent");
+}
+
 TEST(PrimaryKeyRealtimeStoreTest, TestQuerySchemaAlignmentUsesCallerPool) {
     const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
     std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
@@ -386,14 +408,81 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQuerySchemaAlignmentUsesCallerPool) {
     RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
 
-    const int64_t allocations_before_query = pool->allocation_count;
+    std::vector<arrow::FieldVector> zero_copy_schemas;
+    zero_copy_schemas.push_back(stored_schema->fields());
+    arrow::FieldVector reordered_fields(stored_schema->fields().begin(),
+                                        stored_schema->fields().begin() + 3);
+    reordered_fields.push_back(FieldWithId("renamed_value", arrow::utf8(), 1));
+    reordered_fields.push_back(FieldWithId("renamed_id", arrow::int64(), 0));
+    zero_copy_schemas.push_back(std::move(reordered_fields));
     pool->reject_allocations = true;
+    for (const arrow::FieldVector& fields : zero_copy_schemas) {
+        auto zero_copy_schema = std::make_unique<ArrowSchema>();
+        ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(fields), zero_copy_schema.get()).ok());
+        RealtimeQueryContext zero_copy_context{zero_copy_schema.get(), /*predicate=*/nullptr,
+                                               /*enable_predicate_pushdown=*/false};
+        const int64_t allocations_before_query = pool->allocation_count;
+        ASSERT_OK_AND_ASSIGN(
+            std::vector<std::unique_ptr<BatchReader>> readers,
+            store->CreateQueryReaders(view, /*offset_begin=*/0, zero_copy_context));
+        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
+        ASSERT_TRUE(arrow::ImportArray(batch.first.get(), batch.second.get()).ok());
+        ASSERT_EQ(allocations_before_query, pool->allocation_count);
+    }
+
+    const int64_t allocations_before_query = pool->allocation_count;
     ASSERT_NOK_WITH_MSG(store->CreateQueryReaders(view, /*offset_begin=*/0, context),
                         "Out of memory");
     ASSERT_GT(pool->allocation_count, allocations_before_query);
 }
 
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryAlignmentPoolOutlivesStoreReaderAndExport) {
+    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
+    std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
+    std::weak_ptr<TestingMemoryPool> pool_lifetime = pool;
+    auto write_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*stored_schema, write_schema.get()).ok());
+    RealtimeStoreCreateRequest request{std::move(write_schema),
+                                       /*options=*/{}, pool, RealtimeStoreMode::PRIMARY_KEY};
+    ArrowRealtimeStoreFactory factory;
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> store, factory.Create(std::move(request)));
+    request.memory_pool.reset();
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 7, "seven"]])"), OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    arrow::FieldVector requested_fields = stored_schema->fields();
+    requested_fields.push_back(FieldWithId("added", arrow::int32(), 2));
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(
+        arrow::ExportSchema(*arrow::schema(std::move(requested_fields)), c_schema.get()).ok());
+    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(1, readers.size());
+    ASSERT_GT(pool->allocation_count, 0);
+
+    view.reset();
+    store.reset();
+    pool.reset();
+    ASSERT_FALSE(pool_lifetime.expired());
+
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
+    ASSERT_FALSE(BatchReader::IsEofBatch(batch));
+    arrow::Result<std::shared_ptr<arrow::Array>> import_result =
+        arrow::ImportArray(batch.first.get(), batch.second.get());
+    ASSERT_TRUE(import_result.ok()) << import_result.status().ToString();
+    std::shared_ptr<arrow::Array> imported = std::move(import_result).ValueOrDie();
+    readers.clear();
+    ASSERT_FALSE(pool_lifetime.expired());
+    imported.reset();
+    ASSERT_TRUE(pool_lifetime.expired());
+}
+
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
+    const std::shared_ptr<arrow::Field> stored_profile_a =
+        FieldWithId("profile_a", arrow::int32(), 30);
     const std::shared_ptr<arrow::Field> stored_a = FieldWithId("a", arrow::int32(), 10);
     const std::shared_ptr<arrow::Field> stored_b = FieldWithId("b", arrow::int32(), 11);
     const std::shared_ptr<arrow::Field> stored_x = FieldWithId("x", arrow::int32(), 20);
@@ -404,19 +493,24 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
             ->WithNullable(false),
         DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
         FieldWithId("id", arrow::int64(), 0),
-        FieldWithId("items", arrow::list(arrow::struct_({stored_a, stored_b})), 1),
-        FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_x, stored_y})), 2)};
+        FieldWithId("profile", arrow::struct_({stored_profile_a}), 1),
+        FieldWithId("items", arrow::list(arrow::struct_({stored_a, stored_b})), 2),
+        FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_x, stored_y})), 3)};
     std::shared_ptr<arrow::Schema> stored_schema = arrow::schema(std::move(stored_fields));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
                          PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeSlicedBatch(
             stored_schema,
-            R"([[0, 1, 0, 6, [[1, 2]], [["before", [3, 4]]]], [0, 2, 1, 7, [[100, 200], null], [["k1", [7, 8]], ["k2", null]]], [0, 3, 2, 8, [[9, 10]], [["after", [11, 12]]]]])",
+            R"([[0, 1, 0, 6, [5], [[1, 2]], [["before", [3, 4]]]], [0, 2, 1, 7, [50], [[100, 200], null], [["k1", [7, 8]], ["k2", null]]], [0, 3, 2, 8, [500], [[9, 10]], [["after", [11, 12]]]]])",
             1, 1),
         OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
 
+    const std::shared_ptr<arrow::Field> requested_profile_missing =
+        FieldWithId("added_profile", arrow::int32(), 31);
+    const std::shared_ptr<arrow::Field> requested_profile_a =
+        FieldWithId("renamed_profile_a", arrow::int32(), 30);
     const std::shared_ptr<arrow::Field> requested_b = FieldWithId("renamed_b", arrow::int32(), 11);
     const std::shared_ptr<arrow::Field> requested_a = FieldWithId("renamed_a", arrow::int32(), 10);
     const std::shared_ptr<arrow::Field> requested_item_missing =
@@ -427,14 +521,16 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
         FieldWithId("added_attr", arrow::int32(), 22);
     arrow::FieldVector requested_fields(stored_schema->fields().begin(),
                                         stored_schema->fields().begin() + 3);
+    requested_fields.push_back(FieldWithId(
+        "renamed_profile", arrow::struct_({requested_profile_missing, requested_profile_a}), 1));
     requested_fields.push_back(FieldWithId(
         "renamed_items",
-        arrow::list(arrow::struct_({requested_b, requested_item_missing, requested_a})), 1));
+        arrow::list(arrow::struct_({requested_b, requested_item_missing, requested_a})), 2));
     requested_fields.push_back(
         FieldWithId("renamed_attrs",
                     arrow::map(arrow::utf8(),
                                arrow::struct_({requested_y, requested_attr_missing, requested_x})),
-                    2));
+                    3));
     std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
     auto c_schema = std::make_unique<ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
@@ -449,8 +545,12 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
     std::shared_ptr<arrow::Array> array = std::move(import_result).ValueOrDie();
     ASSERT_TRUE(array->type()->Equals(arrow::struct_(requested_schema->fields())));
     std::shared_ptr<arrow::StructArray> projected = checked_pointer_cast<arrow::StructArray>(array);
+    const std::shared_ptr<arrow::StructArray> profile =
+        checked_pointer_cast<arrow::StructArray>(projected->field(3));
+    ASSERT_TRUE(profile->field(0)->IsNull(0));
+    ASSERT_EQ(50, checked_pointer_cast<arrow::Int32Array>(profile->field(1))->Value(0));
     const std::shared_ptr<arrow::ListArray> items =
-        checked_pointer_cast<arrow::ListArray>(projected->field(3));
+        checked_pointer_cast<arrow::ListArray>(projected->field(4));
     const std::shared_ptr<arrow::StructArray> item_values =
         checked_pointer_cast<arrow::StructArray>(items->value_slice(0));
     ASSERT_EQ(200, checked_pointer_cast<arrow::Int32Array>(item_values->field(0))->Value(0));
@@ -459,7 +559,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
     ASSERT_TRUE(item_values->IsNull(1));
 
     const std::shared_ptr<arrow::MapArray> attrs =
-        checked_pointer_cast<arrow::MapArray>(projected->field(4));
+        checked_pointer_cast<arrow::MapArray>(projected->field(5));
     const int64_t attr_offset = attrs->value_offset(0);
     const int64_t attr_length = attrs->value_length(0);
     const std::shared_ptr<arrow::StringArray> attr_keys =
@@ -473,5 +573,51 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
     ASSERT_TRUE(attr_values->IsNull(1));
 }
 
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderRejectsMissingNonNullableNestedFields) {
+    const std::shared_ptr<arrow::Field> stored_profile_a =
+        FieldWithId("profile_a", arrow::int32(), 30);
+    const std::shared_ptr<arrow::Field> stored_item_a = FieldWithId("item_a", arrow::int32(), 10);
+    const std::shared_ptr<arrow::Field> stored_attr_a = FieldWithId("attr_a", arrow::int32(), 20);
+    arrow::FieldVector stored_fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+            ->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
+        FieldWithId("profile", arrow::struct_({stored_profile_a}), 1),
+        FieldWithId("items", arrow::list(arrow::struct_({stored_item_a})), 2),
+        FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_attr_a})), 3)};
+    const std::shared_ptr<arrow::Schema> stored_schema = arrow::schema(std::move(stored_fields));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
+                         PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(stored_schema, R"([[0, 1, 0, [5], [[10]], [["key", [20]]]]])"),
+                           OffsetRange(0, 1)}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+
+    const std::shared_ptr<arrow::Field> required =
+        FieldWithId("required_nested", arrow::int32(), 99, /*nullable=*/false);
+    std::vector<arrow::FieldVector> requested_schemas;
+    arrow::FieldVector struct_fields = stored_schema->fields();
+    struct_fields[3] = FieldWithId("profile", arrow::struct_({stored_profile_a, required}), 1);
+    requested_schemas.push_back(std::move(struct_fields));
+    arrow::FieldVector list_fields = stored_schema->fields();
+    list_fields[4] =
+        FieldWithId("items", arrow::list(arrow::struct_({stored_item_a, required})), 2);
+    requested_schemas.push_back(std::move(list_fields));
+    arrow::FieldVector map_fields = stored_schema->fields();
+    map_fields[5] = FieldWithId(
+        "attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_attr_a, required})), 3);
+    requested_schemas.push_back(std::move(map_fields));
+
+    for (const arrow::FieldVector& fields : requested_schemas) {
+        auto c_schema = std::make_unique<ArrowSchema>();
+        ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(fields), c_schema.get()).ok());
+        RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                     /*enable_predicate_pushdown=*/false};
+        ASSERT_NOK_WITH_MSG(store->CreateQueryReaders(view, /*offset_begin=*/0, context),
+                            "requested non-nullable field 'required_nested' with id 99 is absent");
+    }
+}
+
 }  // namespace
 }  // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index bacbfa5e2..d2b2b86ce 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -19,14 +19,12 @@
 #include "paimon/core/realtime/realtime_primary_key_writer.h"
 
 #include <limits>
-#include <new>
 #include <optional>
 #include <utility>
 #include <vector>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
-#include "arrow/c/helpers.h"
 #include "arrow/compute/api.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
@@ -38,6 +36,7 @@
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/mergetree/merge_tree_writer.h"
+#include "paimon/core/realtime/arrow_array_pool_holder.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/utils/commit_increment.h"
@@ -47,42 +46,6 @@ namespace paimon {
 
 namespace {
 
-struct PreparedArrayPrivateData {
-    void (*release)(ArrowArray*);
-    void* private_data;
-    std::shared_ptr<arrow::MemoryPool> arrow_pool;
-};
-
-void ReleasePreparedArray(ArrowArray* array) {
-    std::unique_ptr<PreparedArrayPrivateData> data(
-        static_cast<PreparedArrayPrivateData*>(array->private_data));
-    array->release = data->release;
-    array->private_data = data->private_data;
-    array->release(array);
-}
-
-Status RetainPreparedArrayPool(ArrowArray* array,
-                               const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
-    if (!array || !array->release) {
-        return Status::Invalid("cannot retain prepared batch memory pool");
-    }
-    if (!arrow_pool) {
-        ArrowArrayRelease(array);
-        return Status::Invalid("cannot retain prepared batch memory pool");
-    }
-    std::unique_ptr<PreparedArrayPrivateData> data;
-    try {
-        data = std::make_unique<PreparedArrayPrivateData>(
-            PreparedArrayPrivateData{array->release, array->private_data, arrow_pool});
-    } catch (const std::bad_alloc&) {
-        ArrowArrayRelease(array);
-        return Status::OutOfMemory("failed to retain prepared batch memory pool");
-    }
-    array->private_data = data.release();
-    array->release = ReleasePreparedArray;
-    return Status::OK();
-}
-
 Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
     std::unique_ptr<RecordBatch>&& batch, const std::shared_ptr<arrow::Schema>& write_schema,
     const std::shared_ptr<arrow::Schema>& prepared_schema,
@@ -252,7 +215,7 @@ Status RealtimePrimaryKeyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
                      first_sequence, next_offset_, arrow_pool_.get()));
     auto output = std::make_unique<ArrowArray>();
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*prepared, output.get()));
-    PAIMON_RETURN_NOT_OK(RetainPreparedArrayPool(output.get(), arrow_pool_));
+    PAIMON_RETURN_NOT_OK(RetainArrowArrayMemoryPool(output.get(), arrow_pool_));
     RecordBatchBuilder builder(output.get());
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> prepared_batch, builder.Finish());
     PAIMON_RETURN_NOT_OK(realtime_store_->Write(RealtimeWriteBatch{

From ace7e1d9bbaaac1c51c2251e1c22b3dcf0773183 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Tue, 25 Aug 2026 23:57:27 +0800
Subject: [PATCH 59/93] fix(realtime): validate exact commit reader coverage

---
 .../merged_key_value_record_reader_test.cpp   | 57 +++++++++++++++++++
 .../realtime/prepared_key_value_reader.cpp    | 18 +++---
 .../realtime/primary_key_realtime_store.cpp   |  7 +++
 3 files changed, 71 insertions(+), 11 deletions(-)

diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index d86e3fc9a..bf1aaec9e 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -290,6 +290,63 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
         "exact");
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestCommitOffsetCoverage) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> first_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 2, 1], [0, 11, 0, 3]])")
+            .ValueOrDie();
+    std::shared_ptr<arrow::Array> second_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 12, 1, 2], [0, 13, 3, 4]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForCommit(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
+                             value_schema, value_schema, pool_));
+    int64_t row_count = 0;
+    for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
+        ASSERT_OK_AND_ASSIGN(
+            std::vector<KeyValue> rows,
+            (ReadResultCollector::CollectKeyValueResult<
+                KeyValueRecordReader, KeyValueRecordReader::Iterator>(reader.get())));
+        row_count += static_cast<int64_t>(rows.size());
+    }
+    ASSERT_EQ(4, row_count);
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestRejectsDuplicateCommitOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(
+            prepared_type, R"([[0, 10, 0, 1], [0, 11, 0, 2], [0, 12, 2, 3]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                                  /*read_batch_size=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForCommit(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 3),
+                             value_schema, value_schema, pool_));
+    ASSERT_NOK_WITH_MSG((ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                                    KeyValueRecordReader::Iterator>(
+                            readers[0].get())),
+                        "did not cover the sealed range");
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestBadCommitBatch) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Field> value = MakeField("value", arrow::int32(), 1);
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index d8a4cfdc0..ed1f9e158 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -18,9 +18,7 @@
 
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 
-#include <algorithm>
 #include <cstdint>
-#include <limits>
 #include <memory>
 #include <optional>
 #include <utility>
@@ -44,6 +42,7 @@
 #include "paimon/macros.h"
 #include "paimon/reader/batch_reader.h"
 #include "paimon/status.h"
+#include "paimon/utils/roaring_bitmap64.h"
 
 namespace paimon {
 
@@ -81,9 +80,10 @@ class RealtimeOffsetCoverage {
                 return Status::Invalid(
                     "PK real-time store commit reader offset is outside the sealed range");
             }
-            min_seen_offset_ = std::min(min_seen_offset_, offset);
-            max_seen_offset_ = std::max(max_seen_offset_, offset);
-            ++seen_count_;
+            if (!seen_offsets_.CheckedAdd(offset)) {
+                return Status::Invalid(
+                    "PK real-time store commit readers did not cover the sealed range");
+            }
         }
         return Status::OK();
     }
@@ -91,9 +91,7 @@ class RealtimeOffsetCoverage {
     Status FinishReader() {
         ++finished_reader_count_;
         if (finished_reader_count_ == reader_count_ &&
-            (seen_count_ != sealed_offsets_.Count() ||
-             (seen_count_ > 0 && (min_seen_offset_ != sealed_offsets_.begin ||
-                                  max_seen_offset_ != sealed_offsets_.end - 1)))) {
+            seen_offsets_.Cardinality() != sealed_offsets_.Count()) {
             return Status::Invalid(
                 "PK real-time store commit readers did not cover the sealed range");
         }
@@ -106,9 +104,7 @@ class RealtimeOffsetCoverage {
 
     OffsetRange sealed_offsets_;
     size_t reader_count_;
-    int64_t min_seen_offset_ = std::numeric_limits<int64_t>::max();
-    int64_t max_seen_offset_ = std::numeric_limits<int64_t>::min();
-    int64_t seen_count_ = 0;
+    RoaringBitmap64 seen_offsets_;
     size_t finished_reader_count_ = 0;
 };
 
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index cbaef65e2..cffe1e6d0 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -26,12 +26,14 @@
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
 #include "fmt/format.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/realtime/arrow_array_pool_holder.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/utils/nested_projection_utils.h"
@@ -234,12 +236,17 @@ class StoredBatchReader final : public BatchReader {
         }
         auto array = std::make_unique<ArrowArray>();
         auto schema = std::make_unique<ArrowSchema>();
+        ScopeGuard export_guard([array_ptr = array.get(), schema_ptr = schema.get()]() {
+            ArrowArrayRelease(array_ptr);
+            ArrowSchemaRelease(schema_ptr);
+        });
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*data_, array.get(), schema.get()));
         if (arrow_pool_) {
             PAIMON_RETURN_NOT_OK(RetainArrowArrayMemoryPool(array.get(), arrow_pool_));
         }
         data_.reset();
         arrow_pool_.reset();
+        export_guard.Release();
         return ReadBatch(std::move(array), std::move(schema));
     }
 

From 607cac7dcc30d6150a8bea61bf6ff0e8ccc317c6 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 00:18:02 +0800
Subject: [PATCH 60/93] fix(realtime): enforce store reader boundaries

---
 .../realtime/primary_key_realtime_store.cpp   | 20 ++++---
 .../primary_key_realtime_store_test.cpp       | 54 +++++++++++++++----
 .../core/realtime/realtime_context_impl.cpp   |  6 +++
 .../core/realtime/realtime_context_test.cpp   | 23 ++++++++
 4 files changed, 86 insertions(+), 17 deletions(-)

diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index cffe1e6d0..4f6249985 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -225,7 +225,7 @@ class ReadView final : public RealtimeReadView {
 class StoredBatchReader final : public BatchReader {
  public:
     explicit StoredBatchReader(const StoredBatch& batch,
-                               std::shared_ptr<arrow::MemoryPool> arrow_pool = nullptr)
+                               std::shared_ptr<arrow::MemoryPool> arrow_pool)
         : arrow_pool_(std::move(arrow_pool)),
           data_(batch.data),
           metrics_(std::make_shared<MetricsImpl>()) {}
@@ -240,10 +240,15 @@ class StoredBatchReader final : public BatchReader {
             ArrowArrayRelease(array_ptr);
             ArrowSchemaRelease(schema_ptr);
         });
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*data_, array.get(), schema.get()));
-        if (arrow_pool_) {
-            PAIMON_RETURN_NOT_OK(RetainArrowArrayMemoryPool(array.get(), arrow_pool_));
-        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::RecordBatch> record_batch,
+            arrow::RecordBatch::FromStructArray(data_, arrow_pool_.get()));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr<arrow::RecordBatch> normalized_batch,
+            ArrowUtils::NormalizeRecordBatchOffsets(record_batch, arrow_pool_.get()));
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(
+            arrow::ExportRecordBatch(*normalized_batch, array.get(), schema.get()));
+        PAIMON_RETURN_NOT_OK(RetainArrowArrayMemoryPool(array.get(), arrow_pool_));
         data_.reset();
         arrow_pool_.reset();
         export_guard.Release();
@@ -321,7 +326,7 @@ class PrimaryKeyRealtimeStore::Impl {
         std::vector<std::unique_ptr<BatchReader>> readers;
         readers.reserve(segment->Batches().size());
         for (const StoredBatch& batch : segment->Batches()) {
-            readers.push_back(std::make_unique<StoredBatchReader>(batch));
+            readers.push_back(std::make_unique<StoredBatchReader>(batch, arrow_pool_));
         }
         return readers;
     }
@@ -360,8 +365,7 @@ class PrimaryKeyRealtimeStore::Impl {
                                           arrow_pool_.get()));
                 StoredBatch query_batch{checked_pointer_cast<arrow::StructArray>(projected.array),
                                         batch.offset_range, /*memory_usage=*/0};
-                readers.push_back(std::make_unique<StoredBatchReader>(
-                    query_batch, projected.uses_arrow_pool ? arrow_pool_ : nullptr));
+                readers.push_back(std::make_unique<StoredBatchReader>(query_batch, arrow_pool_));
             }
         }
         return readers;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index a63d79eec..7b2e09245 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -255,12 +255,45 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderPerStoredBatch) {
     }
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
+void AssertSlicedBatch(BatchReader* reader) {
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
+    ASSERT_FALSE(BatchReader::IsEofBatch(batch));
+    ASSERT_EQ(2, batch.first->length);
+    AssertOffsetsZero(batch.first.get());
+    arrow::Result<std::shared_ptr<arrow::Array>> import_result =
+        arrow::ImportArray(batch.first.get(), batch.second.get());
+    ASSERT_TRUE(import_result.ok()) << import_result.status().ToString();
+    std::shared_ptr<arrow::Array> array = std::move(import_result).ValueOrDie();
+    std::shared_ptr<arrow::StructArray> values = checked_pointer_cast<arrow::StructArray>(array);
+    ASSERT_EQ(2, checked_pointer_cast<arrow::Int64Array>(values->field(3))->Value(0));
+    ASSERT_EQ(3, checked_pointer_cast<arrow::Int64Array>(values->field(3))->Value(1));
+    std::shared_ptr<arrow::StructArray> nested =
+        checked_pointer_cast<arrow::StructArray>(values->field(4));
+    ASSERT_EQ("two", checked_pointer_cast<arrow::StringArray>(nested->field(0))->GetString(0));
+    ASSERT_EQ("three", checked_pointer_cast<arrow::StringArray>(nested->field(0))->GetString(1));
+    std::shared_ptr<arrow::ListArray> items =
+        checked_pointer_cast<arrow::ListArray>(nested->field(1));
+    std::shared_ptr<arrow::Int32Array> first_items =
+        checked_pointer_cast<arrow::Int32Array>(items->value_slice(0));
+    ASSERT_EQ(3, first_items->Value(0));
+    ASSERT_EQ(4, first_items->Value(1));
+    std::shared_ptr<arrow::Int32Array> second_items =
+        checked_pointer_cast<arrow::Int32Array>(items->value_slice(1));
+    ASSERT_EQ(5, second_items->Value(0));
+    ASSERT_EQ(6, second_items->Value(1));
+    ASSERT_OK_AND_ASSIGN(batch, reader->NextBatch());
+    ASSERT_TRUE(BatchReader::IsEofBatch(batch));
+}
+
+TEST(PrimaryKeyRealtimeStoreTest, TestSlicedReadersExportZeroOffsets) {
     std::shared_ptr<arrow::Schema> schema = NestedPreparedSchema();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
                          PrimaryKeyRealtimeStore::Create(schema, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
-        MakeBatch(schema, R"([[0, 1, 0, 1, ["one", [1, 2]]], [0, 2, 1, 2, ["two", [3, 4]]]])"),
+        MakeSlicedBatch(
+            schema,
+            R"([[0, 1, 0, 1, ["one", [1, 2]]], [0, 2, 1, 2, ["two", [3, 4]]], [0, 3, 2, 3, ["three", [5, 6]]], [0, 4, 3, 4, ["four", [7, 8]]]])",
+            1, 2),
         OffsetRange(0, 2)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
@@ -268,13 +301,16 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderExportsZeroOffsets) {
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateCommitReaders(segment.value()));
     ASSERT_EQ(1, readers.size());
-    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
-    ASSERT_FALSE(BatchReader::IsEofBatch(batch));
-    ASSERT_EQ(2, batch.first->length);
-    AssertOffsetsZero(batch.first.get());
-    ASSERT_TRUE(arrow::ImportArray(batch.first.get(), batch.second.get()).ok());
-    ASSERT_OK_AND_ASSIGN(batch, readers[0]->NextBatch());
-    ASSERT_TRUE(BatchReader::IsEofBatch(batch));
+    AssertSlicedBatch(readers[0].get());
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok());
+    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(readers, store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_EQ(1, readers.size());
+    AssertSlicedBatch(readers[0].get());
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadBatchReaders) {
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 4b5da718e..ad17ee695 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -150,6 +150,9 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     RealtimeStoreMode mode = request.mode;
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store,
                            factory_->Create(std::move(request)));
+    if (!store) {
+        return Status::Invalid("real-time store factory returned a null store");
+    }
     stores_.emplace(partition_bucket, StoreEntry{store, requested_schema, mode});
     if (offset_iter != committed_offsets_.end()) {
         reclaimed_offsets_.emplace(partition_bucket, offset_iter->second);
@@ -180,6 +183,9 @@ Result<std::vector<RealtimePartitionBucketView>> RealtimeContextImpl::AcquireRea
     for (const auto& [partition_bucket, store] : stores_) {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
                                store.store->AcquireReadView());
+        if (!read_view) {
+            return Status::Invalid("real-time store returned a null read view");
+        }
         result.push_back(
             RealtimePartitionBucketView{partition_bucket, store.store, std::move(read_view)});
     }
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index a418c17a8..6dd1d7577 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -56,6 +56,9 @@ class TestingRealtimeStore : public RealtimeStore {
     }
     Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
         ++acquire_count;
+        if (return_null_read_view) {
+            return std::shared_ptr<RealtimeReadView>();
+        }
         return std::make_shared<TestingReadView>();
     }
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
@@ -78,6 +81,7 @@ class TestingRealtimeStore : public RealtimeStore {
     int32_t acquire_count = 0;
     int32_t advance_count = 0;
     bool fail_next_advance = false;
+    bool return_null_read_view = false;
     std::vector<int64_t> committed_offsets;
 };
 
@@ -88,11 +92,15 @@ class TestingRealtimeStoreFactory : public RealtimeStoreFactory {
             return Status::Invalid("testing write schema is null");
         }
         ArrowSchemaRelease(request.write_schema.get());
+        if (return_null_store) {
+            return std::shared_ptr<RealtimeStore>();
+        }
         auto store = std::make_shared<TestingRealtimeStore>();
         stores.push_back(store);
         return store;
     }
 
+    bool return_null_store = false;
     std::vector<std::shared_ptr<TestingRealtimeStore>> stores;
 };
 
@@ -421,5 +429,20 @@ TEST(RealtimeContextTest, TestRejectsNullFactory) {
                         "real-time store factory is null");
 }
 
+TEST(RealtimeContextTest, TestRejectsNullPluginResults) {
+    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+    factory->return_null_store = true;
+    ASSERT_NOK_WITH_MSG(GetOrCreateAppendStore(context, /*partition=*/{}, /*bucket=*/0,
+                                               MakeWriteSchema(), {}, GetDefaultPool()),
+                        "real-time store factory returned a null store");
+
+    factory->return_null_store = false;
+    ASSERT_OK(GetOrCreateAppendStore(context, /*partition=*/{}, /*bucket=*/0, MakeWriteSchema(), {},
+                                     GetDefaultPool()));
+    factory->stores[0]->return_null_read_view = true;
+    ASSERT_NOK_WITH_MSG(context->AcquireReadViews(), "real-time store returned a null read view");
+}
+
 }  // namespace
 }  // namespace paimon::test

From 86123e0b54d57d92c50ca6e7bf5990f79be436b7 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 11:22:23 +0800
Subject: [PATCH 61/93] refactor(realtime): align primary key query projection

---
 src/paimon/CMakeLists.txt                     |   2 -
 .../core/operation/file_store_write.cpp       |   3 +-
 .../primary_key_realtime_validator.cpp        |  80 -------
 .../primary_key_realtime_validator.h          |  36 ---
 .../primary_key_realtime_validator_test.cpp   |  91 --------
 .../realtime/primary_key_realtime_store.cpp   | 147 +-----------
 .../primary_key_realtime_store_test.cpp       | 217 +-----------------
 src/paimon/core/table/source/table_scan.cpp   |   4 +-
 .../core/utils/primary_key_table_utils.cpp    |  50 ++++
 .../core/utils/primary_key_table_utils.h      |   3 +
 .../utils/primary_key_table_utils_test.cpp    |  58 +++++
 test/inte/realtime_write_inte_test.cpp        |  39 ----
 12 files changed, 132 insertions(+), 598 deletions(-)
 delete mode 100644 src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp
 delete mode 100644 src/paimon/core/realtime/framework/primary_key_realtime_validator.h
 delete mode 100644 src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index fb702bfea..55d033570 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -383,7 +383,6 @@ set(PAIMON_CORE_SRCS
     core/realtime/arrow_array_pool_holder.cpp
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
-    core/realtime/framework/primary_key_realtime_validator.cpp
     core/realtime/prepared_key_value_reader.cpp
     core/realtime/primary_key_realtime_store.cpp
     core/realtime/realtime_append_only_writer.cpp
@@ -792,7 +791,6 @@ if(PAIMON_BUILD_TESTS)
                     core/manifest/index_manifest_file_handler_test.cpp
                     core/memory/writer_memory_manager_test.cpp
                     core/realtime/arrow_realtime_store_test.cpp
-                    core/realtime/framework/primary_key_realtime_validator_test.cpp
                     core/realtime/primary_key_realtime_store_test.cpp
                     core/realtime/realtime_context_test.cpp
                     core/realtime/realtime_reader_test.cpp
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index f77dadda2..9710b57f0 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -36,7 +36,6 @@
 #include "paimon/core/operation/key_value_file_store_write.h"
 #include "paimon/core/options/merge_engine.h"
 #include "paimon/core/postpone/postpone_bucket_file_store_write.h"
-#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
@@ -210,7 +209,7 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
     } else {
         // pk table
         if (ctx->GetRealtimeContext()) {
-            PAIMON_RETURN_NOT_OK(PrimaryKeyRealtimeValidator::ValidateOptions(options, *schema));
+            PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema));
             if (ignore_previous_files) {
                 return Status::NotImplemented(
                     "PK realtime v1 requires restore from the latest snapshot");
diff --git a/src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp b/src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp
deleted file mode 100644
index 61c640ee6..000000000
--- a/src/paimon/core/realtime/framework/primary_key_realtime_validator.cpp
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
-
-#include <vector>
-
-#include "arrow/type.h"
-#include "paimon/common/types/data_field.h"
-#include "paimon/core/core_options.h"
-#include "paimon/core/index/pk/primary_key_index_definitions.h"
-#include "paimon/core/schema/table_schema.h"
-#include "paimon/macros.h"
-
-namespace paimon {
-
-Status PrimaryKeyRealtimeValidator::ValidateOptions(const CoreOptions& options,
-                                                    const TableSchema& schema) {
-    if (options.GetBucket() <= 0) {
-        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
-    }
-    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
-        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
-    }
-    if (options.DataEvolutionEnabled()) {
-        return Status::NotImplemented("PK realtime v1 does not support data evolution");
-    }
-    if (!options.GetFieldsSequenceGroups().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
-    }
-    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
-        options.AggregationRemoveRecordOnDelete() ||
-        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
-        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
-    }
-    if (!options.GetSequenceField().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
-    }
-    if (!options.SequenceFieldSortOrderIsAscending()) {
-        return Status::NotImplemented(
-            "PK realtime v1 supports only ascending sequence.field.sort-order");
-    }
-    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
-        options.GetChangelogProducer() != ChangelogProducer::NONE) {
-        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> primary_key_fields,
-                           schema.TrimmedPrimaryKeyFields());
-    for (const DataField& field : primary_key_fields) {
-        if (field.Type()->id() == arrow::Type::FLOAT || field.Type()->id() == arrow::Type::DOUBLE) {
-            return Status::NotImplemented(
-                "PK realtime v1 does not support FLOAT or DOUBLE primary keys");
-        }
-    }
-    if (options.GlobalIndexEnabled()) {
-        PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions,
-                               PrimaryKeyIndexDefinitions::Create(schema));
-        if (!definitions.Definitions().empty()) {
-            return Status::NotImplemented("PK realtime v1 does not support global indexes");
-        }
-    }
-    return Status::OK();
-}
-
-}  // namespace paimon
diff --git a/src/paimon/core/realtime/framework/primary_key_realtime_validator.h b/src/paimon/core/realtime/framework/primary_key_realtime_validator.h
deleted file mode 100644
index fa9432d3b..000000000
--- a/src/paimon/core/realtime/framework/primary_key_realtime_validator.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include "paimon/status.h"
-
-namespace paimon {
-
-class CoreOptions;
-class TableSchema;
-
-class PrimaryKeyRealtimeValidator {
- public:
-    PrimaryKeyRealtimeValidator() = delete;
-    ~PrimaryKeyRealtimeValidator() = delete;
-
-    static Status ValidateOptions(const CoreOptions& options, const TableSchema& schema);
-};
-
-}  // namespace paimon
diff --git a/src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp b/src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp
deleted file mode 100644
index 3b7793b14..000000000
--- a/src/paimon/core/realtime/framework/primary_key_realtime_validator_test.cpp
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
-
-#include <map>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "arrow/type.h"
-#include "gtest/gtest.h"
-#include "paimon/core/core_options.h"
-#include "paimon/core/schema/table_schema.h"
-#include "paimon/defs.h"
-#include "paimon/testing/utils/testharness.h"
-
-namespace paimon::test {
-namespace {
-
-std::shared_ptr<TableSchema> PkSchema(
-    const std::shared_ptr<arrow::DataType>& key_type = arrow::int64(),
-    const std::map<std::string, std::string>& options = {}) {
-    return TableSchema::Create(
-               /*schema_id=*/0,
-               arrow::schema({arrow::field("id", key_type), arrow::field("value", arrow::utf8())}),
-               /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options)
-        .value();
-}
-
-}  // namespace
-
-TEST(PrimaryKeyRealtimeValidatorTest, TestSupportedOptions) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_OK(PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema()));
-}
-
-TEST(PrimaryKeyRealtimeValidatorTest, TestUnsupportedOptions) {
-    const std::string sequence_group =
-        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
-    const std::vector<std::map<std::string, std::string>> unsupported_options = {
-        {{Options::BUCKET, "0"}},
-        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
-        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
-        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
-    };
-    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
-        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-        ASSERT_NOK(PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema()));
-    }
-}
-
-TEST(PrimaryKeyRealtimeValidatorTest, TestRejectsFloatingPrimaryKeys) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
-    ASSERT_NOK_WITH_MSG(
-        PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema(arrow::float32())),
-        "FLOAT or DOUBLE primary keys");
-    ASSERT_NOK_WITH_MSG(
-        PrimaryKeyRealtimeValidator::ValidateOptions(options, *PkSchema(arrow::float64())),
-        "FLOAT or DOUBLE primary keys");
-}
-
-TEST(PrimaryKeyRealtimeValidatorTest, TestRejectsEnabledGlobalIndex) {
-    const std::map<std::string, std::string> option_map = {{Options::BUCKET, "1"},
-                                                           {Options::PK_BTREE_INDEX_COLUMNS, "id"}};
-    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
-    ASSERT_NOK_WITH_MSG(PrimaryKeyRealtimeValidator::ValidateOptions(
-                            options, *PkSchema(arrow::int64(), option_map)),
-                        "does not support global indexes");
-}
-
-}  // namespace paimon::test
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 4f6249985..4e1defdf5 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -20,14 +20,12 @@
 
 #include <mutex>
 #include <optional>
-#include <unordered_map>
 #include <utility>
 #include <vector>
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
-#include "fmt/format.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
@@ -44,139 +42,6 @@ namespace paimon {
 
 namespace {
 
-struct AlignedArray {
-    std::shared_ptr<arrow::Array> array;
-    bool uses_arrow_pool;
-};
-
-Result<AlignedArray> AlignArrayByPaimonIds(const std::shared_ptr<arrow::Array>& array,
-                                           const std::shared_ptr<arrow::DataType>& read_type,
-                                           arrow::MemoryPool* pool);
-
-bool TypesExactlyEqual(const std::shared_ptr<arrow::DataType>& data_type,
-                       const std::shared_ptr<arrow::DataType>& read_type) {
-    if (!data_type->Equals(read_type) || data_type->num_fields() != read_type->num_fields()) {
-        return false;
-    }
-    for (int32_t i = 0; i < data_type->num_fields(); ++i) {
-        if (!data_type->field(i)->Equals(read_type->field(i), /*check_metadata=*/true) ||
-            !TypesExactlyEqual(data_type->field(i)->type(), read_type->field(i)->type())) {
-            return false;
-        }
-    }
-    return true;
-}
-
-Result<AlignedArray> AlignStructArrayByPaimonIds(
-    const std::shared_ptr<arrow::StructArray>& array,
-    const std::shared_ptr<arrow::StructType>& read_type, arrow::MemoryPool* pool) {
-    const std::shared_ptr<arrow::StructType> data_type =
-        checked_pointer_cast<arrow::StructType>(array->type());
-    std::unordered_map<int32_t, int32_t> data_field_indexes;
-    data_field_indexes.reserve(data_type->num_fields());
-    for (int32_t i = 0; i < data_type->num_fields(); ++i) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
-                               NestedProjectionUtils::GetPaimonFieldId(data_type->field(i)));
-        if (!data_field_indexes.emplace(field_id, i).second) {
-            return Status::Invalid(fmt::format("duplicate field id {} in stored schema", field_id));
-        }
-    }
-
-    std::unordered_map<int32_t, bool> requested_field_ids;
-    requested_field_ids.reserve(read_type->num_fields());
-    std::vector<std::shared_ptr<arrow::ArrayData>> children;
-    children.reserve(read_type->num_fields());
-    bool uses_arrow_pool = false;
-    for (const std::shared_ptr<arrow::Field>& read_field : read_type->fields()) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
-                               NestedProjectionUtils::GetPaimonFieldId(read_field));
-        if (!requested_field_ids.emplace(field_id, true).second) {
-            return Status::Invalid(
-                fmt::format("duplicate field id {} in requested schema", field_id));
-        }
-        const auto data_iter = data_field_indexes.find(field_id);
-        if (data_iter == data_field_indexes.end()) {
-            if (!read_field->nullable()) {
-                return Status::Invalid(fmt::format(
-                    "requested non-nullable field '{}' with id {} is absent from stored schema",
-                    read_field->name(), field_id));
-            }
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-                std::shared_ptr<arrow::Array> null_child,
-                arrow::MakeArrayOfNull(read_field->type(), array->offset() + array->length(),
-                                       pool));
-            children.push_back(null_child->data());
-            uses_arrow_pool = true;
-            continue;
-        }
-        std::shared_ptr<arrow::Array> child =
-            arrow::MakeArray(array->data()->child_data[data_iter->second]);
-        PAIMON_ASSIGN_OR_RAISE(AlignedArray aligned_child,
-                               AlignArrayByPaimonIds(child, read_field->type(), pool));
-        children.push_back(aligned_child.array->data());
-        uses_arrow_pool = uses_arrow_pool || aligned_child.uses_arrow_pool;
-    }
-
-    std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
-    aligned->type = read_type;
-    aligned->child_data = std::move(children);
-    return AlignedArray{arrow::MakeArray(std::move(aligned)), uses_arrow_pool};
-}
-
-Result<AlignedArray> AlignArrayByPaimonIds(const std::shared_ptr<arrow::Array>& array,
-                                           const std::shared_ptr<arrow::DataType>& read_type,
-                                           arrow::MemoryPool* pool) {
-    if (TypesExactlyEqual(array->type(), read_type)) {
-        return AlignedArray{array, false};
-    }
-    if (array->type_id() != read_type->id()) {
-        return Status::Invalid(fmt::format("stored value type {} does not match requested type {}",
-                                           array->type()->ToString(), read_type->ToString()));
-    }
-    switch (read_type->id()) {
-        case arrow::Type::STRUCT:
-            return AlignStructArrayByPaimonIds(checked_pointer_cast<arrow::StructArray>(array),
-                                               checked_pointer_cast<arrow::StructType>(read_type),
-                                               pool);
-        case arrow::Type::LIST: {
-            std::shared_ptr<arrow::Array> values =
-                checked_pointer_cast<arrow::ListArray>(array)->values();
-            PAIMON_ASSIGN_OR_RAISE(
-                AlignedArray aligned_values,
-                AlignArrayByPaimonIds(values, read_type->field(0)->type(), pool));
-            std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
-            aligned->type = read_type;
-            aligned->child_data = {aligned_values.array->data()};
-            return AlignedArray{arrow::MakeArray(std::move(aligned)),
-                                aligned_values.uses_arrow_pool};
-        }
-        case arrow::Type::MAP: {
-            const std::shared_ptr<arrow::MapArray> map =
-                checked_pointer_cast<arrow::MapArray>(array);
-            const std::shared_ptr<arrow::MapType> map_type =
-                checked_pointer_cast<arrow::MapType>(read_type);
-            std::shared_ptr<arrow::Array> keys = map->keys();
-            PAIMON_ASSIGN_OR_RAISE(AlignedArray aligned_keys,
-                                   AlignArrayByPaimonIds(keys, map_type->key_type(), pool));
-            std::shared_ptr<arrow::Array> items = map->items();
-            PAIMON_ASSIGN_OR_RAISE(AlignedArray aligned_items,
-                                   AlignArrayByPaimonIds(items, map_type->item_type(), pool));
-            std::shared_ptr<arrow::ArrayData> entries = array->data()->child_data[0]->Copy();
-            entries->type = arrow::struct_({map_type->key_field(), map_type->item_field()});
-            entries->child_data = {aligned_keys.array->data(), aligned_items.array->data()};
-            std::shared_ptr<arrow::ArrayData> aligned = array->data()->Copy();
-            aligned->type = read_type;
-            aligned->child_data = {std::move(entries)};
-            return AlignedArray{arrow::MakeArray(std::move(aligned)),
-                                aligned_keys.uses_arrow_pool || aligned_items.uses_arrow_pool};
-        }
-        default:
-            return Status::Invalid(
-                fmt::format("stored leaf type {} does not match requested type {}",
-                            array->type()->ToString(), read_type->ToString()));
-    }
-}
-
 struct StoredBatch {
     std::shared_ptr<arrow::StructArray> data;
     OffsetRange offset_range;
@@ -360,10 +225,14 @@ class PrimaryKeyRealtimeStore::Impl {
         for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
             for (const StoredBatch& batch : segment->Batches()) {
                 PAIMON_ASSIGN_OR_RAISE(
-                    AlignedArray projected,
-                    AlignArrayByPaimonIds(batch.data, arrow::struct_(read_schema->fields()),
-                                          arrow_pool_.get()));
-                StoredBatch query_batch{checked_pointer_cast<arrow::StructArray>(projected.array),
+                    std::shared_ptr<arrow::Array> projected,
+                    NestedProjectionUtils::AlignArrayToReadType(
+                        batch.data, arrow::struct_(read_schema->fields()), arrow_pool_.get()));
+                if (!projected || projected->type_id() != arrow::Type::STRUCT) {
+                    return Status::Invalid(
+                        "PK memory query projection did not produce a StructArray");
+                }
+                StoredBatch query_batch{checked_pointer_cast<arrow::StructArray>(projected),
                                         batch.offset_range, /*memory_usage=*/0};
                 readers.push_back(std::make_unique<StoredBatchReader>(query_batch, arrow_pool_));
             }
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 7b2e09245..5f2e46928 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -21,7 +21,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <memory>
-#include <new>
 #include <optional>
 #include <string>
 #include <utility>
@@ -84,16 +83,6 @@ std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) {
     return RecordBatchBuilder(c_array.get()).Finish().value();
 }
 
-std::unique_ptr<RecordBatch> MakeBatch(const std::shared_ptr<arrow::Schema>& schema,
-                                       const std::string& json) {
-    std::shared_ptr<arrow::Array> array =
-        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), json)
-            .ValueOrDie();
-    auto c_array = std::make_unique<ArrowArray>();
-    EXPECT_TRUE(arrow::ExportArray(*array, c_array.get()).ok());
-    return RecordBatchBuilder(c_array.get()).Finish().value();
-}
-
 std::unique_ptr<RecordBatch> MakeSlicedBatch(const std::shared_ptr<arrow::Schema>& schema,
                                              const std::string& json, int64_t offset,
                                              int64_t length) {
@@ -139,18 +128,10 @@ Result<std::string> ReadJson(const std::vector<std::unique_ptr<BatchReader>>& re
 class TestingMemoryPool final : public MemoryPool {
  public:
     void* Malloc(uint64_t size, uint64_t alignment) override {
-        ++allocation_count;
-        if (reject_allocations) {
-            throw std::bad_alloc();
-        }
         return delegate_->Malloc(size, alignment);
     }
 
     void* Realloc(void* pointer, size_t old_size, size_t new_size, uint64_t alignment) override {
-        ++allocation_count;
-        if (reject_allocations) {
-            throw std::bad_alloc();
-        }
         return delegate_->Realloc(pointer, old_size, new_size, alignment);
     }
 
@@ -170,9 +151,6 @@ class TestingMemoryPool final : public MemoryPool {
         return delegate_->MaxMemoryUsage();
     }
 
-    bool reject_allocations = false;
-    int64_t allocation_count = 0;
-
  private:
     std::unique_ptr<MemoryPool> delegate_ = GetMemoryPool();
 };
@@ -368,111 +346,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
     ASSERT_NE(std::string::npos, actual.find("\"two\""));
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsTopLevelFieldsById) {
-    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
-    ASSERT_OK(store->Write(RealtimeWriteBatch{
-        MakeSlicedBatch(stored_schema,
-                        R"([[0, 1, 0, 6, "six"], [0, 2, 1, 7, "seven"], [0, 3, 2, 8, "eight"]])", 1,
-                        1),
-        OffsetRange(0, 1)}));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-
-    arrow::FieldVector requested_fields(stored_schema->fields().begin(),
-                                        stored_schema->fields().begin() + 3);
-    requested_fields.push_back(FieldWithId("renamed_value", arrow::utf8(), 1));
-    requested_fields.push_back(FieldWithId("added", arrow::int32(), 2));
-    std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
-    auto c_schema = std::make_unique<ArrowSchema>();
-    ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
-    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
-    ASSERT_EQ(1, readers.size());
-    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
-    arrow::Result<std::shared_ptr<arrow::Array>> import_result =
-        arrow::ImportArray(batch.first.get(), batch.second.get());
-    ASSERT_TRUE(import_result.ok()) << import_result.status().ToString();
-    std::shared_ptr<arrow::Array> array = std::move(import_result).ValueOrDie();
-    ASSERT_TRUE(array->type()->Equals(arrow::struct_(requested_schema->fields())));
-    std::shared_ptr<arrow::StructArray> projected = checked_pointer_cast<arrow::StructArray>(array);
-    ASSERT_EQ(5, projected->num_fields());
-    ASSERT_EQ("seven", checked_pointer_cast<arrow::StringArray>(projected->field(3))->GetString(0));
-    ASSERT_TRUE(projected->field(4)->IsNull(0));
-}
-
-TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderRejectsMissingNonNullableTopLevelField) {
-    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
-    ASSERT_OK(store->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 7, "seven"]])"), OffsetRange(0, 1)}));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-
-    arrow::FieldVector requested_fields = stored_schema->fields();
-    requested_fields.push_back(
-        FieldWithId("required_added", arrow::int32(), 2, /*nullable=*/false));
-    auto c_schema = std::make_unique<ArrowSchema>();
-    ASSERT_TRUE(
-        arrow::ExportSchema(*arrow::schema(std::move(requested_fields)), c_schema.get()).ok());
-    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-    ASSERT_NOK_WITH_MSG(store->CreateQueryReaders(view, /*offset_begin=*/0, context),
-                        "requested non-nullable field 'required_added' with id 2 is absent");
-}
-
-TEST(PrimaryKeyRealtimeStoreTest, TestQuerySchemaAlignmentUsesCallerPool) {
-    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
-    std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
-    auto write_schema = std::make_unique<ArrowSchema>();
-    ASSERT_TRUE(arrow::ExportSchema(*stored_schema, write_schema.get()).ok());
-    RealtimeStoreCreateRequest request{std::move(write_schema),
-                                       /*options=*/{}, pool, RealtimeStoreMode::PRIMARY_KEY};
-    ArrowRealtimeStoreFactory factory;
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> store, factory.Create(std::move(request)));
-    ASSERT_OK(store->Write(
-        RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 7, "seven"]])"), OffsetRange(0, 1)}));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-
-    arrow::FieldVector requested_fields = stored_schema->fields();
-    requested_fields.push_back(FieldWithId("added", arrow::int32(), 2));
-    std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
-    auto c_schema = std::make_unique<ArrowSchema>();
-    ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
-    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
-                                 /*enable_predicate_pushdown=*/false};
-
-    std::vector<arrow::FieldVector> zero_copy_schemas;
-    zero_copy_schemas.push_back(stored_schema->fields());
-    arrow::FieldVector reordered_fields(stored_schema->fields().begin(),
-                                        stored_schema->fields().begin() + 3);
-    reordered_fields.push_back(FieldWithId("renamed_value", arrow::utf8(), 1));
-    reordered_fields.push_back(FieldWithId("renamed_id", arrow::int64(), 0));
-    zero_copy_schemas.push_back(std::move(reordered_fields));
-    pool->reject_allocations = true;
-    for (const arrow::FieldVector& fields : zero_copy_schemas) {
-        auto zero_copy_schema = std::make_unique<ArrowSchema>();
-        ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(fields), zero_copy_schema.get()).ok());
-        RealtimeQueryContext zero_copy_context{zero_copy_schema.get(), /*predicate=*/nullptr,
-                                               /*enable_predicate_pushdown=*/false};
-        const int64_t allocations_before_query = pool->allocation_count;
-        ASSERT_OK_AND_ASSIGN(
-            std::vector<std::unique_ptr<BatchReader>> readers,
-            store->CreateQueryReaders(view, /*offset_begin=*/0, zero_copy_context));
-        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
-        ASSERT_TRUE(arrow::ImportArray(batch.first.get(), batch.second.get()).ok());
-        ASSERT_EQ(allocations_before_query, pool->allocation_count);
-    }
-
-    const int64_t allocations_before_query = pool->allocation_count;
-    ASSERT_NOK_WITH_MSG(store->CreateQueryReaders(view, /*offset_begin=*/0, context),
-                        "Out of memory");
-    ASSERT_GT(pool->allocation_count, allocations_before_query);
-}
-
-TEST(PrimaryKeyRealtimeStoreTest, TestQueryAlignmentPoolOutlivesStoreReaderAndExport) {
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryPoolOutlivesStoreReaderAndExport) {
     const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
     std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
     std::weak_ptr<TestingMemoryPool> pool_lifetime = pool;
@@ -487,18 +361,13 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryAlignmentPoolOutlivesStoreReaderAndEx
         RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 7, "seven"]])"), OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
 
-    arrow::FieldVector requested_fields = stored_schema->fields();
-    requested_fields.push_back(FieldWithId("added", arrow::int32(), 2));
     auto c_schema = std::make_unique<ArrowSchema>();
-    ASSERT_TRUE(
-        arrow::ExportSchema(*arrow::schema(std::move(requested_fields)), c_schema.get()).ok());
+    ASSERT_TRUE(arrow::ExportSchema(*stored_schema, c_schema.get()).ok());
     RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
                          store->CreateQueryReaders(view, /*offset_begin=*/0, context));
     ASSERT_EQ(1, readers.size());
-    ASSERT_GT(pool->allocation_count, 0);
-
     view.reset();
     store.reset();
     pool.reset();
@@ -516,7 +385,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryAlignmentPoolOutlivesStoreReaderAndEx
     ASSERT_TRUE(pool_lifetime.expired());
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
+TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsNestedFields) {
     const std::shared_ptr<arrow::Field> stored_profile_a =
         FieldWithId("profile_a", arrow::int32(), 30);
     const std::shared_ptr<arrow::Field> stored_a = FieldWithId("a", arrow::int32(), 10);
@@ -543,30 +412,13 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
         OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
 
-    const std::shared_ptr<arrow::Field> requested_profile_missing =
-        FieldWithId("added_profile", arrow::int32(), 31);
-    const std::shared_ptr<arrow::Field> requested_profile_a =
-        FieldWithId("renamed_profile_a", arrow::int32(), 30);
-    const std::shared_ptr<arrow::Field> requested_b = FieldWithId("renamed_b", arrow::int32(), 11);
-    const std::shared_ptr<arrow::Field> requested_a = FieldWithId("renamed_a", arrow::int32(), 10);
-    const std::shared_ptr<arrow::Field> requested_item_missing =
-        FieldWithId("added_item", arrow::int32(), 12);
-    const std::shared_ptr<arrow::Field> requested_y = FieldWithId("renamed_y", arrow::int32(), 21);
-    const std::shared_ptr<arrow::Field> requested_x = FieldWithId("renamed_x", arrow::int32(), 20);
-    const std::shared_ptr<arrow::Field> requested_attr_missing =
-        FieldWithId("added_attr", arrow::int32(), 22);
     arrow::FieldVector requested_fields(stored_schema->fields().begin(),
                                         stored_schema->fields().begin() + 3);
-    requested_fields.push_back(FieldWithId(
-        "renamed_profile", arrow::struct_({requested_profile_missing, requested_profile_a}), 1));
-    requested_fields.push_back(FieldWithId(
-        "renamed_items",
-        arrow::list(arrow::struct_({requested_b, requested_item_missing, requested_a})), 2));
+    requested_fields.push_back(FieldWithId("profile", arrow::struct_({stored_profile_a}), 1));
+    requested_fields.push_back(
+        FieldWithId("items", arrow::list(arrow::struct_({stored_b, stored_a})), 2));
     requested_fields.push_back(
-        FieldWithId("renamed_attrs",
-                    arrow::map(arrow::utf8(),
-                               arrow::struct_({requested_y, requested_attr_missing, requested_x})),
-                    3));
+        FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_y, stored_x})), 3));
     std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
     auto c_schema = std::make_unique<ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
@@ -583,15 +435,13 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
     std::shared_ptr<arrow::StructArray> projected = checked_pointer_cast<arrow::StructArray>(array);
     const std::shared_ptr<arrow::StructArray> profile =
         checked_pointer_cast<arrow::StructArray>(projected->field(3));
-    ASSERT_TRUE(profile->field(0)->IsNull(0));
-    ASSERT_EQ(50, checked_pointer_cast<arrow::Int32Array>(profile->field(1))->Value(0));
+    ASSERT_EQ(50, checked_pointer_cast<arrow::Int32Array>(profile->field(0))->Value(0));
     const std::shared_ptr<arrow::ListArray> items =
         checked_pointer_cast<arrow::ListArray>(projected->field(4));
     const std::shared_ptr<arrow::StructArray> item_values =
         checked_pointer_cast<arrow::StructArray>(items->value_slice(0));
     ASSERT_EQ(200, checked_pointer_cast<arrow::Int32Array>(item_values->field(0))->Value(0));
-    ASSERT_TRUE(item_values->field(1)->IsNull(0));
-    ASSERT_EQ(100, checked_pointer_cast<arrow::Int32Array>(item_values->field(2))->Value(0));
+    ASSERT_EQ(100, checked_pointer_cast<arrow::Int32Array>(item_values->field(1))->Value(0));
     ASSERT_TRUE(item_values->IsNull(1));
 
     const std::shared_ptr<arrow::MapArray> attrs =
@@ -604,56 +454,9 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderAlignsNestedFieldsById) {
     const std::shared_ptr<arrow::StructArray> attr_values =
         checked_pointer_cast<arrow::StructArray>(attrs->items()->Slice(attr_offset, attr_length));
     ASSERT_EQ(8, checked_pointer_cast<arrow::Int32Array>(attr_values->field(0))->Value(0));
-    ASSERT_TRUE(attr_values->field(1)->IsNull(0));
-    ASSERT_EQ(7, checked_pointer_cast<arrow::Int32Array>(attr_values->field(2))->Value(0));
+    ASSERT_EQ(7, checked_pointer_cast<arrow::Int32Array>(attr_values->field(1))->Value(0));
     ASSERT_TRUE(attr_values->IsNull(1));
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderRejectsMissingNonNullableNestedFields) {
-    const std::shared_ptr<arrow::Field> stored_profile_a =
-        FieldWithId("profile_a", arrow::int32(), 30);
-    const std::shared_ptr<arrow::Field> stored_item_a = FieldWithId("item_a", arrow::int32(), 10);
-    const std::shared_ptr<arrow::Field> stored_attr_a = FieldWithId("attr_a", arrow::int32(), 20);
-    arrow::FieldVector stored_fields = {
-        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-            ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
-        FieldWithId("profile", arrow::struct_({stored_profile_a}), 1),
-        FieldWithId("items", arrow::list(arrow::struct_({stored_item_a})), 2),
-        FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_attr_a})), 3)};
-    const std::shared_ptr<arrow::Schema> stored_schema = arrow::schema(std::move(stored_fields));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
-    ASSERT_OK(store->Write(
-        RealtimeWriteBatch{MakeBatch(stored_schema, R"([[0, 1, 0, [5], [[10]], [["key", [20]]]]])"),
-                           OffsetRange(0, 1)}));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-
-    const std::shared_ptr<arrow::Field> required =
-        FieldWithId("required_nested", arrow::int32(), 99, /*nullable=*/false);
-    std::vector<arrow::FieldVector> requested_schemas;
-    arrow::FieldVector struct_fields = stored_schema->fields();
-    struct_fields[3] = FieldWithId("profile", arrow::struct_({stored_profile_a, required}), 1);
-    requested_schemas.push_back(std::move(struct_fields));
-    arrow::FieldVector list_fields = stored_schema->fields();
-    list_fields[4] =
-        FieldWithId("items", arrow::list(arrow::struct_({stored_item_a, required})), 2);
-    requested_schemas.push_back(std::move(list_fields));
-    arrow::FieldVector map_fields = stored_schema->fields();
-    map_fields[5] = FieldWithId(
-        "attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_attr_a, required})), 3);
-    requested_schemas.push_back(std::move(map_fields));
-
-    for (const arrow::FieldVector& fields : requested_schemas) {
-        auto c_schema = std::make_unique<ArrowSchema>();
-        ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(fields), c_schema.get()).ok());
-        RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
-                                     /*enable_predicate_pushdown=*/false};
-        ASSERT_NOK_WITH_MSG(store->CreateQueryReaders(view, /*offset_begin=*/0, context),
-                            "requested non-nullable field 'required_nested' with id 99 is absent");
-    }
-}
-
 }  // namespace
 }  // namespace paimon::test
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 85a9e83b3..66a3f7426 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -41,7 +41,6 @@
 #include "paimon/core/operation/data_evolution_file_store_scan.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
-#include "paimon/core/realtime/framework/primary_key_realtime_validator.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/schema_validation.h"
@@ -64,6 +63,7 @@
 #include "paimon/core/utils/field_mapping.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/index_file_path_factories.h"
+#include "paimon/core/utils/primary_key_table_utils.h"
 #include "paimon/core/utils/snapshot_manager.h"
 #include "paimon/format/file_format.h"
 #include "paimon/realtime/realtime_context.h"
@@ -238,7 +238,7 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions&
                 "PK real-time union read does not support read-optimized scans");
         }
         PAIMON_RETURN_NOT_OK(
-            PrimaryKeyRealtimeValidator::ValidateOptions(core_options, table_schema));
+            PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema));
     }
     if (context.IsStreamingMode()) {
         return Status::Invalid("real-time union read currently supports batch scans only");
diff --git a/src/paimon/core/utils/primary_key_table_utils.cpp b/src/paimon/core/utils/primary_key_table_utils.cpp
index cf72da4ae..446cdc4b3 100644
--- a/src/paimon/core/utils/primary_key_table_utils.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils.cpp
@@ -29,12 +29,14 @@
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/common/utils/object_utils.h"
 #include "paimon/core/core_options.h"
+#include "paimon/core/index/pk/primary_key_index_definitions.h"
 #include "paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/first_row_merge_function.h"
 #include "paimon/core/mergetree/compact/merge_function.h"
 #include "paimon/core/mergetree/compact/partial_update_merge_function.h"
 #include "paimon/core/options/merge_engine.h"
+#include "paimon/core/schema/table_schema.h"
 #include "paimon/status.h"
 
 namespace arrow {
@@ -96,4 +98,52 @@ Result<std::unique_ptr<FieldsComparator>> PrimaryKeyTableUtils::CreateSequenceFi
                                     options.SequenceFieldSortOrderIsAscending());
 }
 
+Status PrimaryKeyTableUtils::ValidateRealtimeOptions(const CoreOptions& options,
+                                                     const TableSchema& schema) {
+    if (options.GetBucket() <= 0) {
+        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
+    }
+    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
+        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
+    }
+    if (options.DataEvolutionEnabled()) {
+        return Status::NotImplemented("PK realtime v1 does not support data evolution");
+    }
+    if (!options.GetFieldsSequenceGroups().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
+    }
+    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
+        options.AggregationRemoveRecordOnDelete() ||
+        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
+        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
+    }
+    if (!options.GetSequenceField().empty()) {
+        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
+    }
+    if (!options.SequenceFieldSortOrderIsAscending()) {
+        return Status::NotImplemented(
+            "PK realtime v1 supports only ascending sequence.field.sort-order");
+    }
+    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
+        options.GetChangelogProducer() != ChangelogProducer::NONE) {
+        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> primary_key_fields,
+                           schema.TrimmedPrimaryKeyFields());
+    for (const DataField& field : primary_key_fields) {
+        if (field.Type()->id() == arrow::Type::FLOAT || field.Type()->id() == arrow::Type::DOUBLE) {
+            return Status::NotImplemented(
+                "PK realtime v1 does not support FLOAT or DOUBLE primary keys");
+        }
+    }
+    if (options.GlobalIndexEnabled()) {
+        PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions,
+                               PrimaryKeyIndexDefinitions::Create(schema));
+        if (!definitions.Definitions().empty()) {
+            return Status::NotImplemented("PK realtime v1 does not support global indexes");
+        }
+    }
+    return Status::OK();
+}
+
 }  // namespace paimon
diff --git a/src/paimon/core/utils/primary_key_table_utils.h b/src/paimon/core/utils/primary_key_table_utils.h
index 82a108ab7..114801cc1 100644
--- a/src/paimon/core/utils/primary_key_table_utils.h
+++ b/src/paimon/core/utils/primary_key_table_utils.h
@@ -36,6 +36,7 @@ class CoreOptions;
 class MemoryPool;
 class FieldsComparator;
 class DataField;
+class TableSchema;
 
 class PrimaryKeyTableUtils {
  public:
@@ -57,6 +58,8 @@ class PrimaryKeyTableUtils {
 
     static Result<std::unique_ptr<FieldsComparator>> CreateSequenceFieldsComparator(
         const std::vector<DataField>& value_fields, const CoreOptions& options);
+
+    static Status ValidateRealtimeOptions(const CoreOptions& options, const TableSchema& schema);
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/utils/primary_key_table_utils_test.cpp b/src/paimon/core/utils/primary_key_table_utils_test.cpp
index 1a7345fdf..5f35f78a2 100644
--- a/src/paimon/core/utils/primary_key_table_utils_test.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils_test.cpp
@@ -19,6 +19,7 @@
 #include "paimon/core/utils/primary_key_table_utils.h"
 
 #include <cstdint>
+#include <map>
 #include <memory>
 #include <optional>
 #include <string>
@@ -33,6 +34,7 @@
 #include "paimon/core/core_options.h"
 #include "paimon/core/key_value.h"
 #include "paimon/core/mergetree/compact/merge_function.h"
+#include "paimon/core/schema/table_schema.h"
 #include "paimon/defs.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/status.h"
@@ -41,6 +43,62 @@
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
+namespace {
+
+std::shared_ptr<TableSchema> PkSchema(
+    const std::shared_ptr<arrow::DataType>& key_type = arrow::int64(),
+    const std::map<std::string, std::string>& options = {}) {
+    return TableSchema::Create(
+               /*schema_id=*/0,
+               arrow::schema({arrow::field("id", key_type), arrow::field("value", arrow::utf8())}),
+               /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options)
+        .value();
+}
+
+}  // namespace
+
+TEST(PrimaryKeyTableUtilsTest, TestSupportedRealtimeOptions) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *PkSchema()));
+}
+
+TEST(PrimaryKeyTableUtilsTest, TestUnsupportedRealtimeOptions) {
+    const std::string sequence_group =
+        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
+    const std::vector<std::map<std::string, std::string>> unsupported_options = {
+        {{Options::BUCKET, "0"}},
+        {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
+        {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
+        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
+        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
+        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
+    };
+    for (const std::map<std::string, std::string>& option_map : unsupported_options) {
+        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+        ASSERT_NOK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *PkSchema()));
+    }
+}
+
+TEST(PrimaryKeyTableUtilsTest, TestRealtimeRejectsFloatingPrimaryKeys) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
+    ASSERT_NOK_WITH_MSG(
+        PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *PkSchema(arrow::float32())),
+        "FLOAT or DOUBLE primary keys");
+    ASSERT_NOK_WITH_MSG(
+        PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *PkSchema(arrow::float64())),
+        "FLOAT or DOUBLE primary keys");
+}
+
+TEST(PrimaryKeyTableUtilsTest, TestRealtimeRejectsEnabledGlobalIndex) {
+    const std::map<std::string, std::string> option_map = {{Options::BUCKET, "1"},
+                                                           {Options::PK_BTREE_INDEX_COLUMNS, "id"}};
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+    ASSERT_NOK_WITH_MSG(PrimaryKeyTableUtils::ValidateRealtimeOptions(
+                            options, *PkSchema(arrow::int64(), option_map)),
+                        "does not support global indexes");
+}
 
 TEST(PrimaryKeyTableUtilsTest, TestCreateSequenceFieldsComparator) {
     {
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 94bf81edc..62db90296 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -1882,45 +1882,6 @@ TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkMemoryReadAfterSchemaEvolution) {
-    CreatePkTable();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "old", "p0"}}, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-
-    std::shared_ptr<arrow::Field> renamed_payload = arrow::field("renamed_payload", arrow::utf8());
-    std::shared_ptr<arrow::Field> added = arrow::field("added", arrow::int32());
-    ASSERT_OK(TestHelper::WriteNextSchema(dir_->GetFileSystem(), table_path_,
-                                          {DataField(0, fields_[0]), DataField(1, renamed_payload),
-                                           DataField(2, fields_[2]), DataField(3, added)},
-                                          /*highest_field_id=*/3, options_));
-    fields_[1] = renamed_payload;
-    fields_.push_back(added);
-    schema_ = arrow::schema(fields_);
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
-                         CreatePlan(realtime_context, /*predicate=*/nullptr));
-    ASSERT_OK_AND_ASSIGN(CollectedReadResult result,
-                         ReadPlan(plan, realtime_context, {"id", "renamed_payload", "pt", "added"},
-                                  /*predicate=*/nullptr, /*enable_predicate_filter=*/false));
-    ASSERT_EQ(1, result.data->num_chunks());
-    std::shared_ptr<arrow::StructArray> row =
-        std::dynamic_pointer_cast<arrow::StructArray>(result.data->chunk(0));
-    ASSERT_NE(nullptr, row);
-    ASSERT_EQ(1, row->length());
-    std::shared_ptr<arrow::StringArray> renamed_values =
-        std::dynamic_pointer_cast<arrow::StringArray>(row->field(2));
-    ASSERT_NE(nullptr, renamed_values);
-    ASSERT_EQ("old", renamed_values->GetString(0));
-    ASSERT_TRUE(row->field(4)->IsNull(0));
-    result.reader->Close();
-    ASSERT_OK(writer->Close());
-}
-
 TEST_F(RealtimeWriteInteTest, TestPkCompositeMerge) {
     CreatePkTable(/*partition_keys=*/{}, /*primary_keys=*/{"id", "payload"});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,

From 32844c103a5e71cc542e42186db933f78dc81376 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 14:15:22 +0800
Subject: [PATCH 62/93] refactor(realtime): centralize Arrow array memory
 retention

---
 src/paimon/CMakeLists.txt                     |  1 -
 src/paimon/common/utils/arrow/mem_utils.cpp   | 41 +++++++++++
 src/paimon/common/utils/arrow/mem_utils.h     |  6 ++
 .../core/realtime/arrow_array_pool_holder.cpp | 69 -------------------
 .../core/realtime/arrow_array_pool_holder.h   | 36 ----------
 .../realtime/primary_key_realtime_store.cpp   |  1 -
 .../primary_key_realtime_store_test.cpp       | 13 +++-
 .../realtime/realtime_primary_key_writer.cpp  |  1 -
 8 files changed, 59 insertions(+), 109 deletions(-)
 delete mode 100644 src/paimon/core/realtime/arrow_array_pool_holder.cpp
 delete mode 100644 src/paimon/core/realtime/arrow_array_pool_holder.h

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 55d033570..0a78b0902 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -380,7 +380,6 @@ set(PAIMON_CORE_SRCS
     core/manifest/snapshot_live_manifest_entries.cpp
     core/operation/write_context.cpp
     core/operation/write_restore.cpp
-    core/realtime/arrow_array_pool_holder.cpp
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
     core/realtime/prepared_key_value_reader.cpp
diff --git a/src/paimon/common/utils/arrow/mem_utils.cpp b/src/paimon/common/utils/arrow/mem_utils.cpp
index 7e8986be5..1e9695332 100644
--- a/src/paimon/common/utils/arrow/mem_utils.cpp
+++ b/src/paimon/common/utils/arrow/mem_utils.cpp
@@ -24,12 +24,31 @@
 #include <new>
 #include <string>
 
+#include "arrow/c/abi.h"
+#include "arrow/c/helpers.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "fmt/format.h"
 #include "paimon/memory/memory_pool.h"
 
 namespace paimon {
+namespace {
+
+struct ArrowArrayPrivateData {
+    void (*release)(ArrowArray*);
+    void* private_data;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool;
+};
+
+void ReleaseArrowArray(ArrowArray* array) {
+    std::unique_ptr<ArrowArrayPrivateData> data(
+        static_cast<ArrowArrayPrivateData*>(array->private_data));
+    array->release = data->release;
+    array->private_data = data->private_data;
+    array->release(array);
+}
+
+}  // namespace
 
 class ArrowMemPoolAdaptor : public arrow::MemoryPool {
  public:
@@ -107,4 +126,26 @@ std::unique_ptr<arrow::MemoryPool> GetArrowPool(const std::shared_ptr<MemoryPool
     return std::make_unique<ArrowMemPoolAdaptor>(pool);
 }
 
+Status RetainArrowArrayMemoryPool(ArrowArray* array,
+                                  const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+    if (!array || !array->release) {
+        return Status::Invalid("cannot retain Arrow array memory pool");
+    }
+    if (!arrow_pool) {
+        ArrowArrayRelease(array);
+        return Status::Invalid("cannot retain Arrow array memory pool");
+    }
+    std::unique_ptr<ArrowArrayPrivateData> data;
+    try {
+        data = std::make_unique<ArrowArrayPrivateData>(
+            ArrowArrayPrivateData{array->release, array->private_data, arrow_pool});
+    } catch (const std::bad_alloc&) {
+        ArrowArrayRelease(array);
+        return Status::OutOfMemory("failed to retain Arrow array memory pool");
+    }
+    array->private_data = data.release();
+    array->release = ReleaseArrowArray;
+    return Status::OK();
+}
+
 }  // namespace paimon
diff --git a/src/paimon/common/utils/arrow/mem_utils.h b/src/paimon/common/utils/arrow/mem_utils.h
index 96b59e3e8..214bb4509 100644
--- a/src/paimon/common/utils/arrow/mem_utils.h
+++ b/src/paimon/common/utils/arrow/mem_utils.h
@@ -23,11 +23,17 @@
 
 #include "arrow/memory_pool.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/status.h"
 #include "paimon/visibility.h"
 
+struct ArrowArray;
+
 namespace paimon {
 
 PAIMON_EXPORT std::unique_ptr<arrow::MemoryPool> GetArrowPool(
     const std::shared_ptr<MemoryPool>& pool);
 
+Status RetainArrowArrayMemoryPool(ArrowArray* array,
+                                  const std::shared_ptr<arrow::MemoryPool>& arrow_pool);
+
 }  // namespace paimon
diff --git a/src/paimon/core/realtime/arrow_array_pool_holder.cpp b/src/paimon/core/realtime/arrow_array_pool_holder.cpp
deleted file mode 100644
index 97a01dd19..000000000
--- a/src/paimon/core/realtime/arrow_array_pool_holder.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "paimon/core/realtime/arrow_array_pool_holder.h"
-
-#include <memory>
-#include <new>
-
-#include "arrow/c/abi.h"
-#include "arrow/c/helpers.h"
-#include "arrow/memory_pool.h"
-
-namespace paimon {
-namespace {
-
-struct ArrowArrayPrivateData {
-    void (*release)(ArrowArray*);
-    void* private_data;
-    std::shared_ptr<arrow::MemoryPool> arrow_pool;
-};
-
-void ReleaseArrowArray(ArrowArray* array) {
-    std::unique_ptr<ArrowArrayPrivateData> data(
-        static_cast<ArrowArrayPrivateData*>(array->private_data));
-    array->release = data->release;
-    array->private_data = data->private_data;
-    array->release(array);
-}
-
-}  // namespace
-
-Status RetainArrowArrayMemoryPool(ArrowArray* array,
-                                  const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
-    if (!array || !array->release) {
-        return Status::Invalid("cannot retain Arrow array memory pool");
-    }
-    if (!arrow_pool) {
-        ArrowArrayRelease(array);
-        return Status::Invalid("cannot retain Arrow array memory pool");
-    }
-    std::unique_ptr<ArrowArrayPrivateData> data;
-    try {
-        data = std::make_unique<ArrowArrayPrivateData>(
-            ArrowArrayPrivateData{array->release, array->private_data, arrow_pool});
-    } catch (const std::bad_alloc&) {
-        ArrowArrayRelease(array);
-        return Status::OutOfMemory("failed to retain Arrow array memory pool");
-    }
-    array->private_data = data.release();
-    array->release = ReleaseArrowArray;
-    return Status::OK();
-}
-
-}  // namespace paimon
diff --git a/src/paimon/core/realtime/arrow_array_pool_holder.h b/src/paimon/core/realtime/arrow_array_pool_holder.h
deleted file mode 100644
index 05f9ed9dc..000000000
--- a/src/paimon/core/realtime/arrow_array_pool_holder.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include <memory>
-
-#include "paimon/status.h"
-
-struct ArrowArray;
-
-namespace arrow {
-class MemoryPool;
-}  // namespace arrow
-
-namespace paimon {
-
-Status RetainArrowArrayMemoryPool(ArrowArray* array,
-                                  const std::shared_ptr<arrow::MemoryPool>& arrow_pool);
-
-}  // namespace paimon
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 4e1defdf5..847347113 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -32,7 +32,6 @@
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/scope_guard.h"
-#include "paimon/core/realtime/arrow_array_pool_holder.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 5f2e46928..209d2ae7a 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -318,9 +318,20 @@ TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 0, 4, 1, "one"]])"), OffsetRange(4, 5)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
+    ASSERT_GT(store->GetMemoryUsage(), 0);
     ASSERT_OK(store->AdvanceCommittedOffset(5));
-    ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(4, 5)), view->GetOffsetRange());
+    ASSERT_EQ(0, store->GetMemoryUsage());
+
+    auto c_schema = std::make_unique<ArrowSchema>();
+    ASSERT_TRUE(arrow::ExportSchema(*PreparedSchema(), c_schema.get()).ok());
+    RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
+                                 /*enable_predicate_pushdown=*/false};
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+    ASSERT_OK_AND_ASSIGN(std::string actual, ReadJson(readers));
+    ASSERT_NE(std::string::npos, actual.find("\"one\""));
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index d2b2b86ce..89e86e787 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -36,7 +36,6 @@
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/mergetree/merge_tree_writer.h"
-#include "paimon/core/realtime/arrow_array_pool_holder.h"
 #include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
 #include "paimon/core/utils/commit_increment.h"

From 3476342bca2e3d8f93ed4cad78a23933b59c1202 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 15:45:55 +0800
Subject: [PATCH 63/93] fix(realtime): refine primary key option validation

---
 .../core/utils/primary_key_table_utils.cpp    | 18 +++---
 .../utils/primary_key_table_utils_test.cpp    | 60 +++++++++++++++++--
 2 files changed, 63 insertions(+), 15 deletions(-)

diff --git a/src/paimon/core/utils/primary_key_table_utils.cpp b/src/paimon/core/utils/primary_key_table_utils.cpp
index 446cdc4b3..b5efb5ded 100644
--- a/src/paimon/core/utils/primary_key_table_utils.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils.cpp
@@ -109,12 +109,7 @@ Status PrimaryKeyTableUtils::ValidateRealtimeOptions(const CoreOptions& options,
     if (options.DataEvolutionEnabled()) {
         return Status::NotImplemented("PK realtime v1 does not support data evolution");
     }
-    if (!options.GetFieldsSequenceGroups().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence groups");
-    }
-    if (options.IgnoreDelete() || options.PartialUpdateRemoveRecordOnDelete() ||
-        options.AggregationRemoveRecordOnDelete() ||
-        !options.GetPartialUpdateRemoveRecordOnSequenceGroup().empty()) {
+    if (options.IgnoreDelete()) {
         return Status::NotImplemented("PK realtime v1 requires default delete behavior");
     }
     if (!options.GetSequenceField().empty()) {
@@ -124,9 +119,14 @@ Status PrimaryKeyTableUtils::ValidateRealtimeOptions(const CoreOptions& options,
         return Status::NotImplemented(
             "PK realtime v1 supports only ascending sequence.field.sort-order");
     }
-    if (options.NeedLookup() || options.DeletionVectorsEnabled() ||
-        options.GetChangelogProducer() != ChangelogProducer::NONE) {
-        return Status::NotImplemented("PK realtime v1 does not support lookup or early MOR");
+    if (options.GetChangelogProducer() != ChangelogProducer::NONE) {
+        return Status::NotImplemented("PK realtime v1 supports only the NONE changelog producer");
+    }
+    if (options.DeletionVectorsEnabled()) {
+        return Status::NotImplemented("PK realtime v1 does not support deletion vectors");
+    }
+    if (options.NeedLookup()) {
+        return Status::NotImplemented("PK realtime v1 does not support lookup");
     }
     PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> primary_key_fields,
                            schema.TrimmedPrimaryKeyFields());
diff --git a/src/paimon/core/utils/primary_key_table_utils_test.cpp b/src/paimon/core/utils/primary_key_table_utils_test.cpp
index 5f35f78a2..8887102cd 100644
--- a/src/paimon/core/utils/primary_key_table_utils_test.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils_test.cpp
@@ -63,17 +63,11 @@ TEST(PrimaryKeyTableUtilsTest, TestSupportedRealtimeOptions) {
 }
 
 TEST(PrimaryKeyTableUtilsTest, TestUnsupportedRealtimeOptions) {
-    const std::string sequence_group =
-        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
     const std::vector<std::map<std::string, std::string>> unsupported_options = {
         {{Options::BUCKET, "0"}},
         {{Options::BUCKET, "1"}, {Options::MERGE_ENGINE, "partial-update"}},
         {{Options::BUCKET, "1"}, {Options::DATA_EVOLUTION_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {sequence_group, "seq"}},
         {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD, "seq"}},
-        {{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
-        {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
-        {{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
     };
     for (const std::map<std::string, std::string>& option_map : unsupported_options) {
         ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
@@ -81,6 +75,60 @@ TEST(PrimaryKeyTableUtilsTest, TestUnsupportedRealtimeOptions) {
     }
 }
 
+TEST(PrimaryKeyTableUtilsTest, TestRealtimeAcceptsInactiveMergeEngineOptions) {
+    const std::string sequence_group =
+        std::string(Options::FIELDS_PREFIX) + ".value." + Options::SEQUENCE_GROUP;
+    const std::map<std::string, std::string> option_map = {
+        {Options::BUCKET, "1"},
+        {sequence_group, "seq"},
+        {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"},
+        {Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, "true"},
+        {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP, "seq"},
+    };
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> schema,
+                         TableSchema::Create(0,
+                                             arrow::schema({arrow::field("id", arrow::int64()),
+                                                            arrow::field("value", arrow::utf8()),
+                                                            arrow::field("seq", arrow::int64())}),
+                                             {}, {"id"}, option_map));
+    ASSERT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema));
+}
+
+TEST(PrimaryKeyTableUtilsTest, TestRealtimeRejectsDeleteAndSequenceOrderingOptions) {
+    ASSERT_OK_AND_ASSIGN(
+        CoreOptions ignore_delete,
+        CoreOptions::FromMap({{Options::BUCKET, "1"}, {Options::IGNORE_DELETE, "true"}}));
+    ASSERT_NOK_WITH_MSG(PrimaryKeyTableUtils::ValidateRealtimeOptions(ignore_delete, *PkSchema()),
+                        "requires default delete behavior");
+
+    ASSERT_OK_AND_ASSIGN(
+        CoreOptions descending,
+        CoreOptions::FromMap(
+            {{Options::BUCKET, "1"}, {Options::SEQUENCE_FIELD_SORT_ORDER, "descending"}}));
+    ASSERT_NOK_WITH_MSG(PrimaryKeyTableUtils::ValidateRealtimeOptions(descending, *PkSchema()),
+                        "supports only ascending sequence.field.sort-order");
+}
+
+TEST(PrimaryKeyTableUtilsTest, TestRealtimeReportsSpecificLookupErrors) {
+    const std::vector<std::pair<std::map<std::string, std::string>, std::string>> cases = {
+        {{{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
+         "PK realtime v1 does not support lookup"},
+        {{{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
+         "PK realtime v1 does not support deletion vectors"},
+        {{{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
+         "PK realtime v1 supports only the NONE changelog producer"},
+        {{{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "lookup"}},
+         "PK realtime v1 supports only the NONE changelog producer"},
+    };
+    for (const auto& [option_map, expected_message] : cases) {
+        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
+        Status status = PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *PkSchema());
+        ASSERT_TRUE(status.IsNotImplemented());
+        ASSERT_EQ(status.message(), expected_message);
+    }
+}
+
 TEST(PrimaryKeyTableUtilsTest, TestRealtimeRejectsFloatingPrimaryKeys) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::BUCKET, "1"}}));
     ASSERT_NOK_WITH_MSG(

From 961aaaacbe814d00c1cedd66f9b1be77c14a8bfe Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 16:49:01 +0800
Subject: [PATCH 64/93] fix(realtime): validate plugin bitmap bounds

---
 .../merged_key_value_record_reader_test.cpp   | 52 +++++++++++++++++++
 .../realtime/prepared_key_value_reader.cpp    | 14 ++++-
 2 files changed, 64 insertions(+), 2 deletions(-)

diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index bf1aaec9e..1a94aee0e 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -98,6 +98,36 @@ class TrackingBatchReader : public BatchReader {
     int32_t* close_count_;
 };
 
+class MalformedBitmapBatchReader : public BatchReader {
+ public:
+    MalformedBitmapBatchReader(std::unique_ptr<BatchReader>&& delegate, int32_t row_id)
+        : delegate_(std::move(delegate)), row_id_(row_id) {}
+
+    Result<ReadBatch> NextBatch() override {
+        return delegate_->NextBatch();
+    }
+
+    Result<ReadBatchWithBitmap> NextBatchWithBitmap() override {
+        PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, delegate_->NextBatchWithBitmap());
+        if (!IsEofBatch(batch)) {
+            batch.second.Add(row_id_);
+        }
+        return batch;
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    int32_t row_id_;
+};
+
 }  // namespace
 
 class MergedKeyValueRecordReaderTest : public testing::Test {
@@ -259,6 +289,28 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsReversedVisibleO
     ASSERT_NOK_WITH_MSG(result, "prepared visible offset range begin exceeds end");
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderBitmapBounds) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+    auto batch_reader = std::make_unique<MalformedBitmapBatchReader>(
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1),
+        /*row_id=*/1);
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                                        value_schema, value_schema, pool_));
+    Result<std::vector<KeyValue>> result =
+        ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                   KeyValueRecordReader::Iterator>(reader.get());
+    ASSERT_TRUE(result.status().IsInvalid());
+    ASSERT_NOK_WITH_MSG(result, "selected row id 1 is out of bounds for prepared batch length 1");
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index ed1f9e158..2a2251784 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -321,7 +321,9 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             }
             key_ctx_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
             value_ctx_ = std::make_shared<ColumnarBatchContext>(value_fields, pool_);
-            if (!SelectRows(*offset_array, std::move(selection))) {
+            PAIMON_ASSIGN_OR_RAISE(bool has_selected_rows,
+                                   SelectRows(*offset_array, std::move(selection)));
+            if (!has_selected_rows) {
                 continue;
             }
             ArrowUtils::TraverseArray(data_batch);
@@ -362,7 +364,15 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         return Status::OK();
     }
 
-    bool SelectRows(const arrow::Int64Array& offsets, RoaringBitmap32&& selection) {
+    Result<bool> SelectRows(const arrow::Int64Array& offsets, RoaringBitmap32&& selection) {
+        for (auto iter = selection.Begin(); iter != selection.End(); ++iter) {
+            const int32_t row = *iter;
+            if (row < 0 || row >= offsets.length()) {
+                return Status::Invalid(
+                    fmt::format("selected row id {} is out of bounds for prepared batch length {}",
+                                row, offsets.length()));
+            }
+        }
         if (!visible_offsets_.has_value()) {
             selected_rows_.reserve(offsets.length());
             for (int64_t row = 0; row < offsets.length(); ++row) {

From 69d9693f6d1f72ddf059378db8a69e3f4793ddb5 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 18:41:09 +0800
Subject: [PATCH 65/93] fix(realtime): preserve primary keys in projected reads

---
 src/paimon/common/table/special_fields.h      | 15 +++++
 .../common/table/special_fields_test.cpp      | 21 ++++++
 .../realtime/prepared_key_value_reader.cpp    | 65 ++++++++++---------
 .../realtime/realtime_primary_key_writer.cpp  | 15 ++---
 .../table/source/key_value_table_read.cpp     | 28 +++++---
 test/inte/realtime_write_inte_test.cpp        | 54 +++++++++++++++
 6 files changed, 148 insertions(+), 50 deletions(-)

diff --git a/src/paimon/common/table/special_fields.h b/src/paimon/common/table/special_fields.h
index 9771ac232..0e07882a8 100644
--- a/src/paimon/common/table/special_fields.h
+++ b/src/paimon/common/table/special_fields.h
@@ -21,6 +21,7 @@
 #include <cstdint>
 #include <limits>
 #include <string>
+#include <utility>
 
 #include "arrow/type_fwd.h"
 #include "paimon/common/types/data_field.h"
@@ -35,6 +36,10 @@ struct SpecialFields {
 
     static constexpr char KEY_FIELD_PREFIX[] = "_KEY_";
     static constexpr int32_t KEY_VALUE_SPECIAL_FIELD_COUNT = 2;
+    static constexpr int32_t kPreparedKeyValueValueKindIndex = 0;
+    static constexpr int32_t kPreparedKeyValueSequenceNumberIndex = 1;
+    static constexpr int32_t kPreparedKeyValueRealtimeOffsetIndex = 2;
+    static constexpr int32_t kPreparedKeyValueValueStartIndex = 3;
 
     static const DataField& SequenceNumber() {
         static const DataField data_field = DataField(
@@ -92,6 +97,16 @@ struct SpecialFields {
         target_fields.insert(target_fields.end(), schema->fields().begin(), schema->fields().end());
         return arrow::schema(target_fields);
     }
+
+    static std::shared_ptr<arrow::Schema> PreparedKeyValueSchema(
+        const arrow::FieldVector& value_fields) {
+        arrow::FieldVector fields = {
+            DataField::ConvertDataFieldToArrowField(ValueKind())->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(SequenceNumber())->WithNullable(false),
+            DataField::ConvertDataFieldToArrowField(RealtimeOffset())};
+        fields.insert(fields.end(), value_fields.begin(), value_fields.end());
+        return arrow::schema(std::move(fields));
+    }
 };
 
 }  // namespace paimon
diff --git a/src/paimon/common/table/special_fields_test.cpp b/src/paimon/common/table/special_fields_test.cpp
index 58a025ba2..a0a0980a5 100644
--- a/src/paimon/common/table/special_fields_test.cpp
+++ b/src/paimon/common/table/special_fields_test.cpp
@@ -66,6 +66,27 @@ TEST(SpecialFieldsTest, TestKeyValueSpecialFieldCount) {
     ASSERT_EQ(SpecialFields::KEY_VALUE_SPECIAL_FIELD_COUNT, 2);
 }
 
+TEST(SpecialFieldsTest, TestPreparedKeyValueSchema) {
+    arrow::FieldVector value_fields = {arrow::field("key", arrow::int64(), false),
+                                       arrow::field("value", arrow::utf8())};
+    std::shared_ptr<arrow::Schema> schema = SpecialFields::PreparedKeyValueSchema(value_fields);
+
+    ASSERT_EQ(SpecialFields::kPreparedKeyValueValueKindIndex, 0);
+    ASSERT_EQ(SpecialFields::kPreparedKeyValueSequenceNumberIndex, 1);
+    ASSERT_EQ(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex, 2);
+    ASSERT_EQ(SpecialFields::kPreparedKeyValueValueStartIndex, 3);
+    ASSERT_EQ(schema->field(0)->name(), "_VALUE_KIND");
+    ASSERT_EQ(schema->field(1)->name(), "_SEQUENCE_NUMBER");
+    ASSERT_EQ(schema->field(2)->name(), "_REALTIME_OFFSET");
+    ASSERT_EQ(schema->field(3)->name(), "key");
+    ASSERT_EQ(schema->field(4)->name(), "value");
+    ASSERT_FALSE(schema->field(0)->nullable());
+    ASSERT_FALSE(schema->field(1)->nullable());
+    ASSERT_EQ(schema->field(2)->nullable(), SpecialFields::RealtimeOffset().Nullable());
+    ASSERT_FALSE(schema->field(3)->nullable());
+    ASSERT_TRUE(schema->field(4)->nullable());
+}
+
 TEST(SpecialFieldsTest, TestIsSystemField) {
     ASSERT_TRUE(SpecialFields::IsSystemField("_SEQUENCE_NUMBER"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_VALUE_KIND"));
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 2a2251784..4a0a52920 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -48,11 +48,6 @@ namespace paimon {
 
 namespace {
 
-constexpr int32_t kValueKindIndex = 0;
-constexpr int32_t kSequenceNumberIndex = 1;
-constexpr int32_t kRealtimeOffsetIndex = 2;
-constexpr int32_t kPreparedValueStartIndex = 3;
-
 template <typename Reader>
 void CloseReaders(const std::vector<std::unique_ptr<Reader>>& readers) {
     for (const std::unique_ptr<Reader>& reader : readers) {
@@ -151,7 +146,7 @@ Result<std::vector<int32_t>> ResolveFieldIndexes(
     const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::shared_ptr<arrow::Schema>& row_schema) {
     arrow::FieldVector prepared_value_fields(
-        prepared_schema->fields().begin() + kPreparedValueStartIndex,
+        prepared_schema->fields().begin() + SpecialFields::kPreparedKeyValueValueStartIndex,
         prepared_schema->fields().end());
     std::vector<int32_t> result;
     result.reserve(row_schema->num_fields());
@@ -167,18 +162,19 @@ Result<std::vector<int32_t>> ResolveFieldIndexes(
                 "type {}",
                 field_id, prepared_field->type()->ToString(), row_field->type()->ToString()));
         }
-        result.push_back(value_index + kPreparedValueStartIndex);
+        result.push_back(value_index + SpecialFields::kPreparedKeyValueValueStartIndex);
     }
     return result;
 }
 
 Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_schema,
                                  const std::shared_ptr<arrow::Schema>& value_schema) {
-    if (prepared_schema->num_fields() != value_schema->num_fields() + kPreparedValueStartIndex) {
+    if (prepared_schema->num_fields() !=
+        value_schema->num_fields() + SpecialFields::kPreparedKeyValueValueStartIndex) {
         return Status::Invalid("commit requires the exact prepared writer schema");
     }
     for (int32_t i = 0; i < value_schema->num_fields(); ++i) {
-        if (!prepared_schema->field(i + kPreparedValueStartIndex)
+        if (!prepared_schema->field(i + SpecialFields::kPreparedKeyValueValueStartIndex)
                  ->Equals(value_schema->field(i), true)) {
             return Status::Invalid("commit requires the exact prepared writer schema");
         }
@@ -300,15 +296,15 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
             std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> offset_array =
                 checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
-                    data_batch->field(kRealtimeOffsetIndex));
+                    data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex));
             if (offset_coverage_) {
                 PAIMON_RETURN_NOT_OK(offset_coverage_->Add(*offset_array));
             }
 
             row_kind_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int8Type>>(
-                data_batch->field(kValueKindIndex));
+                data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex));
             sequence_number_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
-                data_batch->field(kSequenceNumberIndex));
+                data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex));
             arrow::ArrayVector key_fields;
             key_fields.reserve(key_field_indexes_.size());
             for (int32_t index : key_field_indexes_) {
@@ -344,21 +340,26 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                     "prepared batch field {} does not match declared prepared schema", i));
             }
         }
-        if (!data_batch->field(kValueKindIndex) ||
-            data_batch->field(kValueKindIndex)->type_id() != arrow::Type::INT8) {
+        if (!data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex) ||
+            data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex)->type_id() !=
+                arrow::Type::INT8) {
             return Status::Invalid("cannot cast VALUE_KIND column to int8 arrow array");
         }
-        if (!data_batch->field(kSequenceNumberIndex) ||
-            data_batch->field(kSequenceNumberIndex)->type_id() != arrow::Type::INT64) {
+        if (!data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex) ||
+            data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex)->type_id() !=
+                arrow::Type::INT64) {
             return Status::Invalid("cannot cast SEQUENCE_NUMBER column to int64 arrow array");
         }
-        if (!data_batch->field(kRealtimeOffsetIndex) ||
-            data_batch->field(kRealtimeOffsetIndex)->type_id() != arrow::Type::INT64) {
+        if (!data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex) ||
+            data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex)->type_id() !=
+                arrow::Type::INT64) {
             return Status::Invalid("cannot cast REALTIME_OFFSET column to int64 arrow array");
         }
-        if (data_batch->field(kValueKindIndex)->null_count() != 0 ||
-            data_batch->field(kSequenceNumberIndex)->null_count() != 0 ||
-            data_batch->field(kRealtimeOffsetIndex)->null_count() != 0) {
+        if (data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex)->null_count() != 0 ||
+            data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex)->null_count() !=
+                0 ||
+            data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex)->null_count() !=
+                0) {
             return Status::Invalid("prepared transport columns must not contain nulls");
         }
         return Status::OK();
@@ -428,15 +429,19 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
 Status PreparedKeyValueReaderFactory::ValidateTransportSchema(
     const std::shared_ptr<arrow::Schema>& prepared_schema) {
-    if (!prepared_schema || prepared_schema->num_fields() < kPreparedValueStartIndex) {
+    if (!prepared_schema ||
+        prepared_schema->num_fields() < SpecialFields::kPreparedKeyValueValueStartIndex) {
         return Status::Invalid("prepared schema must contain realtime transport fields");
     }
-    PAIMON_RETURN_NOT_OK(
-        CheckPreparedField(prepared_schema, kValueKindIndex, SpecialFields::ValueKind()));
-    PAIMON_RETURN_NOT_OK(
-        CheckPreparedField(prepared_schema, kSequenceNumberIndex, SpecialFields::SequenceNumber()));
-    PAIMON_RETURN_NOT_OK(
-        CheckPreparedField(prepared_schema, kRealtimeOffsetIndex, SpecialFields::RealtimeOffset()));
+    PAIMON_RETURN_NOT_OK(CheckPreparedField(prepared_schema,
+                                            SpecialFields::kPreparedKeyValueValueKindIndex,
+                                            SpecialFields::ValueKind()));
+    PAIMON_RETURN_NOT_OK(CheckPreparedField(prepared_schema,
+                                            SpecialFields::kPreparedKeyValueSequenceNumberIndex,
+                                            SpecialFields::SequenceNumber()));
+    PAIMON_RETURN_NOT_OK(CheckPreparedField(prepared_schema,
+                                            SpecialFields::kPreparedKeyValueRealtimeOffsetIndex,
+                                            SpecialFields::RealtimeOffset()));
     return Status::OK();
 }
 
@@ -474,9 +479,9 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
                            ResolveFieldIndexes(prepared_schema, key_schema));
     PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> value_field_indexes,
                            ResolveFieldIndexes(prepared_schema, value_schema));
-    std::unique_ptr<KeyValueRecordReader> result(new PreparedKeyValueReader(
+    std::unique_ptr<KeyValueRecordReader> result = std::make_unique<PreparedKeyValueReader>(
         std::move(owned_reader), prepared_schema, visible_offsets, std::move(key_field_indexes),
-        std::move(value_field_indexes), memory_pool, offset_coverage));
+        std::move(value_field_indexes), memory_pool, offset_coverage);
     close_guard.Release();
     return result;
 }
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 89e86e787..ae123a643 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -27,7 +27,6 @@
 #include "arrow/c/bridge.h"
 #include "arrow/compute/api.h"
 #include "paimon/common/table/special_fields.h"
-#include "paimon/common/types/data_field.h"
 #include "paimon/common/types/row_kind.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
@@ -139,22 +138,16 @@ Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Crea
         }
         key_fields.push_back(std::move(field));
     }
-    arrow::FieldVector prepared_fields = {
-        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-            ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
-    prepared_fields.insert(prepared_fields.end(), write_schema->fields().begin(),
-                           write_schema->fields().end());
+    std::shared_ptr<arrow::Schema> prepared_schema =
+        SpecialFields::PreparedKeyValueSchema(write_schema->fields());
     const RealtimePartitionBucket partition_bucket(partition, bucket);
     PAIMON_ASSIGN_OR_RAISE(int64_t initial_max_sequence_number,
                            realtime_context->AdvanceMaterializedMaxSequenceNumber(
                                partition_bucket, restored_max_sequence_number));
     return std::shared_ptr<RealtimePrimaryKeyWriter>(new RealtimePrimaryKeyWriter(
         store_state.store, merge_tree_writer, realtime_context, partition_bucket, write_schema,
-        arrow::schema(std::move(prepared_fields)), arrow::schema(std::move(key_fields)),
-        trimmed_primary_keys, key_comparator, store_state.initial_offset,
-        initial_max_sequence_number, memory_pool));
+        prepared_schema, arrow::schema(std::move(key_fields)), trimmed_primary_keys, key_comparator,
+        store_state.initial_offset, initial_max_sequence_number, memory_pool));
 }
 
 RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index e13140f54..e4225587d 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -19,6 +19,7 @@
 
 #include "paimon/core/table/source/key_value_table_read.h"
 
+#include <unordered_set>
 #include <utility>
 #include <vector>
 
@@ -26,7 +27,6 @@
 #include "arrow/c/bridge.h"
 #include "paimon/common/reader/concat_batch_reader.h"
 #include "paimon/common/table/special_fields.h"
-#include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/global_index/indexed_split_impl.h"
@@ -42,6 +42,7 @@
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/table/source/pk_count_reader.h"
 #include "paimon/core/table/source/realtime_split.h"
+#include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/core/utils/primary_key_table_utils.h"
 #include "paimon/status.h"
 
@@ -62,14 +63,23 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
     const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<InternalReadContext>& context,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    arrow::FieldVector prepared_fields = {
-        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-            ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
-    prepared_fields.insert(prepared_fields.end(), value_schema->fields().begin(),
-                           value_schema->fields().end());
-    std::shared_ptr<arrow::Schema> prepared_schema = arrow::schema(std::move(prepared_fields));
+    arrow::FieldVector prepared_value_fields;
+    prepared_value_fields.reserve(key_schema->num_fields() + value_schema->num_fields());
+    std::unordered_set<int32_t> field_ids;
+    for (const std::shared_ptr<arrow::Field>& field : key_schema->fields()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(field));
+        if (field_ids.insert(field_id).second) {
+            prepared_value_fields.push_back(field);
+        }
+    }
+    for (const std::shared_ptr<arrow::Field>& field : value_schema->fields()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(field));
+        if (field_ids.insert(field_id).second) {
+            prepared_value_fields.push_back(field);
+        }
+    }
+    std::shared_ptr<arrow::Schema> prepared_schema =
+        SpecialFields::PreparedKeyValueSchema(prepared_value_fields);
     auto c_schema = std::make_unique<ArrowSchema>();
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
     ScopeGuard schema_guard([schema = c_schema.get()]() { ArrowSchemaRelease(schema); });
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 62db90296..73ddf050b 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -1882,6 +1882,60 @@ TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) {
     ASSERT_OK(writer->Close());
 }
 
+TEST_F(RealtimeWriteInteTest, TestPkKeylessProjection) {
+    CreatePkTable();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
+                         MakeBatch({Row{1, "disk", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(disk_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(disk_progress, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
+                         MakeBatch({Row{1, "memory", "p0"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(memory_batch)));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ReadPlanWithSchemaAndCheck(plan, realtime_context,
+                               arrow::schema({arrow::field("payload", arrow::utf8())}), R"([
+        [0, "memory"]
+    ])");
+    ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestCompositePkKeylessProjection) {
+    CreatePkTable(/*partition_keys=*/{}, /*primary_keys=*/{"id", "payload"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
+                         MakeBatch({Row{1, "key", "disk"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(disk_batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_progress,
+                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(disk_progress, /*commit_identifier=*/0));
+    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
+                         MakeBatch({Row{1, "key", "memory"}}, /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(memory_batch)));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
+                         CreatePlan(realtime_context, /*predicate=*/nullptr));
+    ReadPlanWithSchemaAndCheck(plan, realtime_context,
+                               arrow::schema({arrow::field("pt", arrow::utf8())}), R"([
+        [0, "memory"]
+    ])");
+    ASSERT_OK(writer->Close());
+}
+
 TEST_F(RealtimeWriteInteTest, TestPkCompositeMerge) {
     CreatePkTable(/*partition_keys=*/{}, /*primary_keys=*/{"id", "payload"});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,

From 09c53c905c70946faf02f628cb562280624b75e0 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 18:41:27 +0800
Subject: [PATCH 66/93] fix(read): bound realtime merge fan-in

---
 .../core/operation/merge_file_split_read.cpp  | 104 ++++++++++++++++--
 .../core/operation/merge_file_split_read.h    |  10 ++
 .../operation/merge_file_split_read_test.cpp  |  85 +++++++++++++-
 3 files changed, 184 insertions(+), 15 deletions(-)

diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 02a7fac20..6c8bbecad 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -36,6 +36,7 @@
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/object_utils.h"
+#include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/deletionvectors/apply_deletion_vector_batch_reader.h"
 #include "paimon/core/deletionvectors/bitmap_deletion_vector.h"
@@ -78,6 +79,53 @@ struct KeyValue;
 template <typename T>
 class MergeFunctionWrapper;
 
+namespace {
+
+class SortMergeKeyValueRecordReader : public KeyValueRecordReader {
+ public:
+    explicit SortMergeKeyValueRecordReader(std::unique_ptr<SortMergeReader>&& reader)
+        : reader_(std::move(reader)) {}
+
+    class Iterator : public KeyValueRecordReader::Iterator {
+     public:
+        explicit Iterator(std::unique_ptr<SortMergeReader::Iterator>&& iterator)
+            : iterator_(std::move(iterator)) {}
+
+        Result<bool> HasNext() const override {
+            return iterator_->HasNext();
+        }
+
+        Result<KeyValue> Next() override {
+            return std::move(iterator_->Next());
+        }
+
+     private:
+        std::unique_ptr<SortMergeReader::Iterator> iterator_;
+    };
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader::Iterator> iterator,
+                               reader_->NextBatch());
+        if (!iterator) {
+            return std::unique_ptr<KeyValueRecordReader::Iterator>();
+        }
+        return std::make_unique<Iterator>(std::move(iterator));
+    }
+
+    void Close() override {
+        reader_->Close();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return reader_->GetReaderMetrics();
+    }
+
+ private:
+    std::unique_ptr<SortMergeReader> reader_;
+};
+
+}  // namespace
+
 class MergeFileSplitRead::RealtimeReaderBuilder {
  public:
     static Result<std::unique_ptr<BatchReader>> Create(
@@ -145,15 +193,34 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
         std::vector<std::vector<SortedRun>> disk_sections;
         PAIMON_RETURN_NOT_OK(
             owner_->CreateDiskSections(data_files, deletion_files, &dv_factory, &disk_sections));
+        std::vector<std::unique_ptr<KeyValueRecordReader>> section_readers;
+        ScopeGuard section_readers_guard([&section_readers]() {
+            for (const std::unique_ptr<KeyValueRecordReader>& reader : section_readers) {
+                reader->Close();
+            }
+        });
+        section_readers.reserve(disk_sections.size());
+        std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper;
+        if (!disk_sections.empty()) {
+            PAIMON_ASSIGN_OR_RAISE(merge_function_wrapper,
+                                   MergeFileSplitRead::CreateMergeFunctionWrapper(
+                                       owner_->options_, owner_->context_->GetTableSchema(),
+                                       owner_->value_schema_, owner_->pool_));
+        }
         for (const std::vector<SortedRun>& section : disk_sections) {
             PAIMON_ASSIGN_OR_RAISE(
-                std::vector<std::unique_ptr<KeyValueRecordReader>> section_readers,
-                owner_->CreateRecordReadersForSection(section, partition, dv_factory,
-                                                      owner_->predicate_for_keys_,
-                                                      data_file_path_factory));
-            for (std::unique_ptr<KeyValueRecordReader>& reader : section_readers) {
-                readers->push_back(std::move(reader));
-            }
+                std::unique_ptr<SortMergeReader> section_reader,
+                owner_->CreateSortMergeReaderForSection(
+                    section, partition, dv_factory, owner_->predicate_for_keys_,
+                    data_file_path_factory, /*drop_delete=*/false, merge_function_wrapper));
+            section_readers.push_back(
+                std::make_unique<SortMergeKeyValueRecordReader>(std::move(section_reader)));
+        }
+        if (!section_readers.empty()) {
+            std::unique_ptr<KeyValueRecordReader> concat_reader =
+                std::make_unique<ConcatKeyValueRecordReader>(std::move(section_readers));
+            section_readers_guard.Release();
+            readers->push_back(std::move(concat_reader));
         }
         return Status::OK();
     }
@@ -618,11 +685,24 @@ Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeRead
     const std::vector<SortedRun>& section, const BinaryRow& partition,
     DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
     const std::shared_ptr<DataFilePathFactory>& data_file_path_factory, bool drop_delete) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper,
+                           GetMergeFunctionWrapper());
+    return CreateSortMergeReaderForSection(section, partition, dv_factory, predicate,
+                                           data_file_path_factory, drop_delete,
+                                           merge_function_wrapper);
+}
+
+Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeReaderForSection(
+    const std::vector<SortedRun>& section, const BinaryRow& partition,
+    DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
+    const std::shared_ptr<DataFilePathFactory>& data_file_path_factory, bool drop_delete,
+    const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper) {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> record_readers,
                            CreateRecordReadersForSection(section, partition, dv_factory, predicate,
                                                          data_file_path_factory));
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> sort_merge_reader,
-                           CreateSortMergeReader(std::move(record_readers)));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<SortMergeReader> sort_merge_reader,
+        CreateSortMergeReader(std::move(record_readers), merge_function_wrapper));
     if (drop_delete) {
         sort_merge_reader = std::make_unique<DropDeleteReader>(std::move(sort_merge_reader));
     }
@@ -657,6 +737,12 @@ Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeRead
     std::vector<std::unique_ptr<KeyValueRecordReader>>&& record_readers) {
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper,
                            GetMergeFunctionWrapper());
+    return CreateSortMergeReader(std::move(record_readers), merge_function_wrapper);
+}
+
+Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeReader(
+    std::vector<std::unique_ptr<KeyValueRecordReader>>&& record_readers,
+    const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper) const {
     auto sort_engine = options_.GetSortEngine();
     if (sort_engine == SortEngine::MIN_HEAP) {
         return std::make_unique<SortMergeReaderWithMinHeap>(
diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h
index ad45e4cf5..b54c96331 100644
--- a/src/paimon/core/operation/merge_file_split_read.h
+++ b/src/paimon/core/operation/merge_file_split_read.h
@@ -157,6 +157,12 @@ class MergeFileSplitRead : public AbstractSplitRead {
         std::unique_ptr<SortMergeReader>&& sort_merge_reader,
         const std::shared_ptr<Predicate>& predicate, bool complete_row_kind);
 
+    Result<std::unique_ptr<SortMergeReader>> CreateSortMergeReaderForSection(
+        const std::vector<SortedRun>& section, const BinaryRow& partition,
+        DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
+        const std::shared_ptr<DataFilePathFactory>& data_file_path_factory, bool drop_delete,
+        const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper);
+
     Result<std::unique_ptr<KeyValueRecordReader>> CreateReaderForRun(
         const BinaryRow& partition, const SortedRun& sorted_run, DeletionVector::Factory dv_factory,
         const std::shared_ptr<Predicate>& predicate,
@@ -165,6 +171,10 @@ class MergeFileSplitRead : public AbstractSplitRead {
     Result<std::unique_ptr<SortMergeReader>> CreateSortMergeReader(
         std::vector<std::unique_ptr<KeyValueRecordReader>>&& record_readers);
 
+    Result<std::unique_ptr<SortMergeReader>> CreateSortMergeReader(
+        std::vector<std::unique_ptr<KeyValueRecordReader>>&& record_readers,
+        const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper) const;
+
     Result<std::shared_ptr<MergeFunctionWrapper<KeyValue>>> GetMergeFunctionWrapper();
 
     MergeFileSplitRead(const std::shared_ptr<FileStorePathFactory>& path_factory,
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index 911cf7961..a899fd904 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -40,6 +40,7 @@
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/key_value_in_memory_record_reader.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/operation/internal_read_context.h"
 #include "paimon/core/schema/schema_manager.h"
@@ -328,9 +329,8 @@ class MergeFileSplitReadTest : public ::testing::Test,
         return {data_split1};
     }
 
-    Result<std::unique_ptr<BatchReader>> CreateReader(
-        const std::shared_ptr<InternalReadContext>& internal_context,
-        const std::vector<std::shared_ptr<DataSplit>>& data_splits) {
+    Result<std::unique_ptr<MergeFileSplitRead>> CreateMergeFileSplitRead(
+        const std::shared_ptr<InternalReadContext>& internal_context) {
         const auto& core_options = internal_context->GetCoreOptions();
         const auto& table_schema = internal_context->GetTableSchema();
         auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
@@ -347,9 +347,14 @@ class MergeFileSplitReadTest : public ::testing::Test,
                 core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(),
                 external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(),
                 pool_));
-        PAIMON_ASSIGN_OR_RAISE(auto split_read,
-                               MergeFileSplitRead::Create(path_factory, std::move(internal_context),
-                                                          pool_, executor_));
+        return MergeFileSplitRead::Create(path_factory, internal_context, pool_, executor_);
+    }
+
+    Result<std::unique_ptr<BatchReader>> CreateReader(
+        const std::shared_ptr<InternalReadContext>& internal_context,
+        const std::vector<std::shared_ptr<DataSplit>>& data_splits) {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<MergeFileSplitRead> split_read,
+                               CreateMergeFileSplitRead(internal_context));
         std::vector<std::unique_ptr<BatchReader>> batch_readers;
         batch_readers.reserve(data_splits.size());
         for (const auto& split : data_splits) {
@@ -666,6 +671,74 @@ TEST_P(MergeFileSplitReadTest, TestSimple) {
     CheckResult(result_array, expected_array, read_schema);
 }
 
+TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) {
+    std::string path =
+        paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
+    ReadContextBuilder context_builder(path);
+    std::vector<DataField> raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())),
+                                              DataField(1, arrow::field("k1", arrow::int32())),
+                                              DataField(5, arrow::field("s1", arrow::utf8())),
+                                              DataField(6, arrow::field("v0", arrow::float64()))};
+    std::shared_ptr<arrow::Schema> read_schema =
+        DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
+    ASSERT_TRUE(read_schema);
+
+    context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"});
+    context_builder.SetOptions(
+        {{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}});
+    AddOptions(&context_builder);
+    context_builder.EnableMultiThreadRowToBatch(false);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
+    std::shared_ptr<InternalReadContext> internal_context = CreateInternalReadContext(read_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<MergeFileSplitRead> split_read,
+                         CreateMergeFileSplitRead(internal_context));
+
+    std::shared_ptr<arrow::DataType> memory_type =
+        arrow::struct_(split_read->GetValueSchema()->fields());
+    std::shared_ptr<arrow::StructArray> memory_array =
+        std::dynamic_pointer_cast<arrow::StructArray>(
+            arrow::ipc::internal::json::ArrayFromJSON(memory_type, R"([
+                [100, 200, "memory-late",   10000.0, "zzzz"],
+                [1,     1, "memory-delete",  1100.0, "zzzz"],
+                [0,     0, "memory-first",   1000.0, "zzzz"],
+                [50,    0, "memory-middle",  5000.0, "zzzz"]
+            ])")
+                .ValueOrDie());
+    std::vector<std::unique_ptr<KeyValueRecordReader>> memory_readers;
+    memory_readers.push_back(std::make_unique<KeyValueInMemoryRecordReader>(
+        /*last_sequence_num=*/9, memory_array,
+        std::vector<RecordBatch::RowKind>(
+            {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE,
+             RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::INSERT}),
+        std::vector<std::string>({"k0", "k1"}), std::vector<std::string>({"s0", "s1"}),
+        /*sequence_fields_ascending=*/true, split_read->GetKeyComparator(), pool_));
+
+    std::vector<std::shared_ptr<Split>> disk_splits = {PrepareDataSplit().front()};
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> batch_reader,
+                         split_read->CreateRealtimeReader(disk_splits, std::move(memory_readers)));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result_array,
+                         ReadResultCollector::CollectResult(batch_reader.get()));
+
+    arrow::FieldVector fields_with_row_kind = read_schema->fields();
+    fields_with_row_kind.insert(fields_with_row_kind.begin(),
+                                arrow::field("_VALUE_KIND", arrow::int8()));
+    std::shared_ptr<arrow::ChunkedArray> expected_array;
+    auto expected_status =
+        arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow::struct_(fields_with_row_kind), {R"([
+            [0,   0,   0, "memory-first",  1000.0],
+            [0,   0,   1, "you",             11.1],
+            [0,   1,   0, "later",           12.2],
+            [0,   1,   2, "!",               13.3],
+            [0,  50,   0, "memory-middle", 5000.0],
+            [0, 100, 200, "memory-late", 10000.0]
+        ])"},
+                                                         &expected_array);
+    ASSERT_TRUE(expected_status.ok());
+    CheckResult(result_array, expected_array, read_schema);
+    ASSERT_TRUE(batch_reader->GetReaderMetrics());
+    batch_reader->Close();
+}
+
 TEST_P(MergeFileSplitReadTest, TestLookUp) {
     std::string path =
         paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";

From 712b1e7b6e4806666c3b5bac01d513f5b3039968 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 18:41:57 +0800
Subject: [PATCH 67/93] refactor(realtime): simplify primary key maintenance

---
 .../core/operation/file_store_write.cpp       |  5 ++-
 .../operation/key_value_file_store_write.cpp  | 19 ++++-------
 .../realtime/primary_key_realtime_store.cpp   | 11 ++++---
 .../primary_key_realtime_store_test.cpp       | 32 +++++++++++++++++--
 .../core/realtime/realtime_context_impl.cpp   | 15 +++++----
 .../realtime/realtime_primary_key_writer.h    |  1 +
 .../core/utils/primary_key_table_utils.cpp    | 22 ++++++-------
 .../utils/primary_key_table_utils_test.cpp    |  8 ++---
 8 files changed, 68 insertions(+), 45 deletions(-)

diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 9710b57f0..fa1294d1b 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -212,11 +212,10 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
             PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema));
             if (ignore_previous_files) {
                 return Status::NotImplemented(
-                    "PK realtime v1 requires restore from the latest snapshot");
+                    "PK realtime requires restore from the latest snapshot");
             }
             if (!ctx->GetWriteSchema().empty()) {
-                return Status::NotImplemented(
-                    "PK realtime v1 does not support a custom write schema");
+                return Status::NotImplemented("PK realtime does not support a custom write schema");
             }
             PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(),
                                                                   snapshot_manager, options));
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index c6a62c107..f431597a7 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -24,7 +24,6 @@
 #include "arrow/c/bridge.h"
 #include "paimon/common/data/binary_row.h"
 #include "paimon/common/table/special_fields.h"
-#include "paimon/common/types/data_field.h"
 #include "paimon/core/compact/noop_compact_manager.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/io/data_file_meta.h"
@@ -121,9 +120,6 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
                            file_store_path_factory_->CreateDataFilePathFactory(partition, bucket));
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> trimmed_primary_keys,
                            table_schema_->TrimmedPrimaryKeys());
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<Levels> levels,
-        Levels::Create(key_comparator_, restore_data_files, options_.GetNumLevels()));
     std::map<std::string, std::string> partition_map;
     std::shared_ptr<CompactManager> compact_manager;
     std::shared_ptr<RealtimeContextImpl> realtime_context_impl;
@@ -139,17 +135,11 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
             return Status::Invalid("PK real-time write schema contains reserved transport field " +
                                    SpecialFields::RealtimeOffset().Name());
         }
-        arrow::FieldVector prepared_fields = {
-            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
-                ->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-                ->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
-        prepared_fields.insert(prepared_fields.end(), schema_->fields().begin(),
-                               schema_->fields().end());
+        std::shared_ptr<arrow::Schema> prepared_schema =
+            SpecialFields::PreparedKeyValueSchema(schema_->fields());
         auto c_write_schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(
-            arrow::ExportSchema(*arrow::schema(std::move(prepared_fields)), c_write_schema.get()));
+            arrow::ExportSchema(*prepared_schema, c_write_schema.get()));
         PAIMON_ASSIGN_OR_RAISE(
             RealtimeStoreState store_state,
             realtime_context_impl->GetOrCreateRealtimeStore(
@@ -159,6 +149,9 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         realtime_store_state = std::move(store_state);
         compact_manager = std::make_shared<NoopCompactManager>();
     } else {
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr<Levels> levels,
+            Levels::Create(key_comparator_, restore_data_files, options_.GetNumLevels()));
         auto compact_strategy = compact_manager_factory_->CreateCompactStrategy();
         PAIMON_ASSIGN_OR_RAISE(compact_manager, compact_manager_factory_->CreateCompactManager(
                                                     partition, bucket, compact_strategy,
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index 847347113..dc3498424 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -18,6 +18,7 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
+#include <algorithm>
 #include <mutex>
 #include <optional>
 #include <utility>
@@ -204,7 +205,7 @@ class PrimaryKeyRealtimeStore::Impl {
             segments.push_back(
                 std::make_shared<Segment>(range, std::vector<StoredBatch>(building_)));
         }
-        return std::shared_ptr<RealtimeReadView>(new ReadView(std::move(segments)));
+        return std::make_shared<ReadView>(std::move(segments));
     }
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
@@ -241,9 +242,11 @@ class PrimaryKeyRealtimeStore::Impl {
 
     Status AdvanceCommittedOffset(int64_t committed_end_offset) {
         std::lock_guard<std::mutex> lock(mutex_);
-        while (!sealed_.empty() && sealed_.front()->GetOffsetRange().end <= committed_end_offset) {
-            sealed_.erase(sealed_.begin());
-        }
+        auto first_retained = std::find_if(
+            sealed_.begin(), sealed_.end(), [committed_end_offset](const auto& segment) {
+                return segment->GetOffsetRange().end > committed_end_offset;
+            });
+        sealed_.erase(sealed_.begin(), first_retained);
         return Status::OK();
     }
 
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 209d2ae7a..46d9a8e7d 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -319,19 +319,45 @@ TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
     ASSERT_TRUE(segment.has_value());
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
-    ASSERT_GT(store->GetMemoryUsage(), 0);
+    ASSERT_OK(
+        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 1, 5, 2, "two"]])"), OffsetRange(5, 6)}));
+    ASSERT_OK_AND_ASSIGN(segment, store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK(store->Write(
+        RealtimeWriteBatch{MakeBatch(R"([[0, 2, 6, 3, "three"]])"), OffsetRange(6, 7)}));
+    ASSERT_OK_AND_ASSIGN(segment, store->SealForCommit());
+    ASSERT_TRUE(segment.has_value());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> retained_view, store->AcquireReadView());
+    ASSERT_EQ(OffsetRange(4, 7), retained_view->GetOffsetRange());
+
+    const uint64_t initial_memory_usage = store->GetMemoryUsage();
+    ASSERT_GT(initial_memory_usage, 0);
+    ASSERT_OK(store->AdvanceCommittedOffset(4));
+    ASSERT_EQ(initial_memory_usage, store->GetMemoryUsage());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> current_view, store->AcquireReadView());
+    ASSERT_EQ(OffsetRange(4, 7), current_view->GetOffsetRange());
     ASSERT_OK(store->AdvanceCommittedOffset(5));
+    ASSERT_LT(store->GetMemoryUsage(), initial_memory_usage);
+    ASSERT_OK_AND_ASSIGN(current_view, store->AcquireReadView());
+    ASSERT_EQ(OffsetRange(5, 7), current_view->GetOffsetRange());
+    ASSERT_OK(store->AdvanceCommittedOffset(6));
+    ASSERT_OK_AND_ASSIGN(current_view, store->AcquireReadView());
+    ASSERT_EQ(OffsetRange(6, 7), current_view->GetOffsetRange());
+    ASSERT_OK(store->AdvanceCommittedOffset(7));
     ASSERT_EQ(0, store->GetMemoryUsage());
+    ASSERT_OK_AND_ASSIGN(current_view, store->AcquireReadView());
+    ASSERT_FALSE(current_view->GetOffsetRange().has_value());
 
     auto c_schema = std::make_unique<ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*PreparedSchema(), c_schema.get()).ok());
     RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateQueryReaders(view, /*offset_begin=*/0, context));
+                         store->CreateQueryReaders(retained_view, /*offset_begin=*/0, context));
     ASSERT_OK_AND_ASSIGN(std::string actual, ReadJson(readers));
     ASSERT_NE(std::string::npos, actual.find("\"one\""));
+    ASSERT_NE(std::string::npos, actual.find("\"two\""));
+    ASSERT_NE(std::string::npos, actual.find("\"three\""));
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index ad17ee695..2b9d7d07c 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -37,6 +37,7 @@
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
+#include "fmt/format.h"
 #include "paimon/arrow/abi.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/scope_guard.h"
@@ -119,10 +120,10 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
     if (iter != stores_.end()) {
         if (iter->second.mode != request.mode ||
             !iter->second.write_schema->Equals(*requested_schema, /*check_metadata=*/true)) {
-            return Status::Invalid("real-time store schema or mode mismatch for partition " +
-                                   PartitionToString(partition_bucket.partition) + ", bucket " +
-                                   std::to_string(partition_bucket.bucket) +
-                                   "; recreate the RealtimeContext");
+            return Status::Invalid(fmt::format(
+                "real-time store schema or mode mismatch for partition {}, bucket {}; recreate "
+                "the RealtimeContext",
+                PartitionToString(partition_bucket.partition), partition_bucket.bucket));
         }
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
                                iter->second.store->AcquireReadView());
@@ -165,9 +166,9 @@ Result<int64_t> RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
     std::lock_guard<std::mutex> lock(mutex_);
     auto iter = stores_.find(partition_bucket);
     if (iter == stores_.end()) {
-        return Status::KeyError("real-time store not found for partition " +
-                                PartitionToString(partition_bucket.partition) + ", bucket " +
-                                std::to_string(partition_bucket.bucket));
+        return Status::KeyError(fmt::format("real-time store not found for partition {}, bucket {}",
+                                            PartitionToString(partition_bucket.partition),
+                                            partition_bucket.bucket));
     }
     StoreEntry& entry = iter->second;
     if (max_sequence_number > entry.materialized_max_sequence_number) {
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index d65c7e533..5cae9ca47 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -20,6 +20,7 @@
 #pragma once
 
 #include <cstdint>
+#include <map>
 #include <memory>
 #include <mutex>
 #include <string>
diff --git a/src/paimon/core/utils/primary_key_table_utils.cpp b/src/paimon/core/utils/primary_key_table_utils.cpp
index b5efb5ded..2ca444a7a 100644
--- a/src/paimon/core/utils/primary_key_table_utils.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils.cpp
@@ -101,46 +101,46 @@ Result<std::unique_ptr<FieldsComparator>> PrimaryKeyTableUtils::CreateSequenceFi
 Status PrimaryKeyTableUtils::ValidateRealtimeOptions(const CoreOptions& options,
                                                      const TableSchema& schema) {
     if (options.GetBucket() <= 0) {
-        return Status::NotImplemented("PK realtime v1 requires fixed buckets");
+        return Status::NotImplemented("PK realtime requires fixed buckets");
     }
     if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
-        return Status::NotImplemented("PK realtime v1 supports only the DEDUPLICATE merge engine");
+        return Status::NotImplemented("PK realtime supports only the DEDUPLICATE merge engine");
     }
     if (options.DataEvolutionEnabled()) {
-        return Status::NotImplemented("PK realtime v1 does not support data evolution");
+        return Status::NotImplemented("PK realtime does not support data evolution");
     }
     if (options.IgnoreDelete()) {
-        return Status::NotImplemented("PK realtime v1 requires default delete behavior");
+        return Status::NotImplemented("PK realtime requires default delete behavior");
     }
     if (!options.GetSequenceField().empty()) {
-        return Status::NotImplemented("PK realtime v1 does not support sequence.field");
+        return Status::NotImplemented("PK realtime does not support sequence.field");
     }
     if (!options.SequenceFieldSortOrderIsAscending()) {
         return Status::NotImplemented(
-            "PK realtime v1 supports only ascending sequence.field.sort-order");
+            "PK realtime supports only ascending sequence.field.sort-order");
     }
     if (options.GetChangelogProducer() != ChangelogProducer::NONE) {
-        return Status::NotImplemented("PK realtime v1 supports only the NONE changelog producer");
+        return Status::NotImplemented("PK realtime supports only the NONE changelog producer");
     }
     if (options.DeletionVectorsEnabled()) {
-        return Status::NotImplemented("PK realtime v1 does not support deletion vectors");
+        return Status::NotImplemented("PK realtime does not support deletion vectors");
     }
     if (options.NeedLookup()) {
-        return Status::NotImplemented("PK realtime v1 does not support lookup");
+        return Status::NotImplemented("PK realtime does not support lookup");
     }
     PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> primary_key_fields,
                            schema.TrimmedPrimaryKeyFields());
     for (const DataField& field : primary_key_fields) {
         if (field.Type()->id() == arrow::Type::FLOAT || field.Type()->id() == arrow::Type::DOUBLE) {
             return Status::NotImplemented(
-                "PK realtime v1 does not support FLOAT or DOUBLE primary keys");
+                "PK realtime does not support FLOAT or DOUBLE primary keys");
         }
     }
     if (options.GlobalIndexEnabled()) {
         PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions,
                                PrimaryKeyIndexDefinitions::Create(schema));
         if (!definitions.Definitions().empty()) {
-            return Status::NotImplemented("PK realtime v1 does not support global indexes");
+            return Status::NotImplemented("PK realtime does not support global indexes");
         }
     }
     return Status::OK();
diff --git a/src/paimon/core/utils/primary_key_table_utils_test.cpp b/src/paimon/core/utils/primary_key_table_utils_test.cpp
index 8887102cd..796922336 100644
--- a/src/paimon/core/utils/primary_key_table_utils_test.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils_test.cpp
@@ -113,13 +113,13 @@ TEST(PrimaryKeyTableUtilsTest, TestRealtimeRejectsDeleteAndSequenceOrderingOptio
 TEST(PrimaryKeyTableUtilsTest, TestRealtimeReportsSpecificLookupErrors) {
     const std::vector<std::pair<std::map<std::string, std::string>, std::string>> cases = {
         {{{Options::BUCKET, "1"}, {Options::FORCE_LOOKUP, "true"}},
-         "PK realtime v1 does not support lookup"},
+         "PK realtime does not support lookup"},
         {{{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}},
-         "PK realtime v1 does not support deletion vectors"},
+         "PK realtime does not support deletion vectors"},
         {{{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "input"}},
-         "PK realtime v1 supports only the NONE changelog producer"},
+         "PK realtime supports only the NONE changelog producer"},
         {{{Options::BUCKET, "1"}, {Options::CHANGELOG_PRODUCER, "lookup"}},
-         "PK realtime v1 supports only the NONE changelog producer"},
+         "PK realtime supports only the NONE changelog producer"},
     };
     for (const auto& [option_map, expected_message] : cases) {
         ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));

From 3392f393b30360651f3c3ec1e55e843112312896 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 22:30:20 +0800
Subject: [PATCH 68/93] refactor(realtime): simplify primary key schema and
 reader setup

---
 .../core/io/single_file_writer_test.cpp       |   4 +-
 .../operation/key_value_file_store_write.cpp  |  10 +-
 .../key_value_file_store_write_test.cpp       |  22 --
 .../realtime/prepared_key_value_reader.cpp    | 246 +++++++++++-------
 .../core/realtime/prepared_key_value_reader.h |   7 +
 .../realtime/primary_key_realtime_store.cpp   |   9 +-
 .../realtime/realtime_primary_key_writer.cpp  |  16 +-
 .../realtime/realtime_primary_key_writer.h    |   1 +
 .../table/source/key_value_table_read.cpp     |  64 ++---
 .../core/table/source/key_value_table_read.h  |   3 +
 10 files changed, 202 insertions(+), 180 deletions(-)

diff --git a/src/paimon/core/io/single_file_writer_test.cpp b/src/paimon/core/io/single_file_writer_test.cpp
index 4136702e8..78fce54c7 100644
--- a/src/paimon/core/io/single_file_writer_test.cpp
+++ b/src/paimon/core/io/single_file_writer_test.cpp
@@ -18,10 +18,8 @@
 
 #include "paimon/core/io/single_file_writer.h"
 
-#include <functional>
 #include <map>
-#include <memory>
-#include <string>
+#include <utility>
 
 #include "arrow/api.h"
 #include "arrow/c/abi.h"
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index f431597a7..86f432998 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -124,6 +124,7 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
     std::shared_ptr<CompactManager> compact_manager;
     std::shared_ptr<RealtimeContextImpl> realtime_context_impl;
     std::optional<RealtimeStoreState> realtime_store_state;
+    std::shared_ptr<arrow::Schema> prepared_schema;
     if (realtime_context_) {
         std::vector<std::pair<std::string, std::string>> partition_values;
         PAIMON_ASSIGN_OR_RAISE(partition_values,
@@ -131,12 +132,7 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         partition_map =
             std::map<std::string, std::string>(partition_values.begin(), partition_values.end());
         PAIMON_ASSIGN_OR_RAISE(realtime_context_impl, RealtimeContextImpl::Cast(realtime_context_));
-        if (schema_->GetFieldByName(SpecialFields::RealtimeOffset().Name())) {
-            return Status::Invalid("PK real-time write schema contains reserved transport field " +
-                                   SpecialFields::RealtimeOffset().Name());
-        }
-        std::shared_ptr<arrow::Schema> prepared_schema =
-            SpecialFields::PreparedKeyValueSchema(schema_->fields());
+        prepared_schema = SpecialFields::PreparedKeyValueSchema(schema_->fields());
         auto c_write_schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(
             arrow::ExportSchema(*prepared_schema, c_write_schema.get()));
@@ -169,7 +165,7 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         return std::shared_ptr<BatchWriter>(std::move(writer));
     }
     return RealtimePrimaryKeyWriter::Create(
-        partition_map, bucket, schema_, trimmed_primary_keys, key_comparator_,
+        partition_map, bucket, schema_, prepared_schema, trimmed_primary_keys, key_comparator_,
         realtime_context_impl, realtime_store_state.value(), restore_max_seq_number, writer, pool_);
 }
 
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index d52841f95..fc6bfff11 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -479,28 +479,6 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(KeyValueFileStoreWriteTest, TestRealtimeOffsetCollision) {
-    const std::map<std::string, std::string> options = {{Options::BUCKET, "1"},
-                                                        {Options::REALTIME_ENABLED, "true"}};
-    const std::shared_ptr<arrow::Schema> schema = arrow::schema({
-        arrow::field("id", arrow::int64(), false),
-        arrow::field("_REALTIME_OFFSET", arrow::int64()),
-    });
-    std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
-    ASSERT_TRUE(dir);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<Catalog> catalog, Catalog::Create(dir->Str(), options));
-    ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false));
-    ArrowSchema c_schema;
-    ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
-    Status create_status =
-        catalog->CreateTable(Identifier("foo", "bar"), &c_schema,
-                             /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options,
-                             /*ignore_if_exists=*/false);
-    ArrowSchemaRelease(&c_schema);
-    ASSERT_NOK_WITH_MSG(create_status,
-                        "field name '_REALTIME_OFFSET' in schema cannot be special field");
-}
-
 TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
     const std::map<std::string, std::string> options = {{Options::BUCKET, "1"},
                                                         {Options::REALTIME_ENABLED, "true"}};
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 4a0a52920..40b54897a 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -21,6 +21,7 @@
 #include <cstdint>
 #include <memory>
 #include <optional>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -123,46 +124,29 @@ Status CheckPreparedField(const std::shared_ptr<arrow::Schema>& schema, int32_t
     return Status::OK();
 }
 
-Result<int32_t> FindFieldIndexByPaimonId(const arrow::FieldVector& fields, int32_t field_id) {
-    std::optional<int32_t> matching_index;
-    for (int32_t i = 0; i < static_cast<int32_t>(fields.size()); ++i) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t candidate_id,
-                               NestedProjectionUtils::GetPaimonFieldId(fields[i]));
-        if (candidate_id == field_id) {
-            if (matching_index.has_value()) {
-                return Status::Invalid(
-                    fmt::format("duplicate field id {} in prepared schema", field_id));
-            }
-            matching_index = i;
-        }
-    }
-    if (matching_index.has_value()) {
-        return matching_index.value();
-    }
-    return Status::Invalid(fmt::format("cannot find field id {} in prepared schema", field_id));
-}
-
 Result<std::vector<int32_t>> ResolveFieldIndexes(
     const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::unordered_map<int32_t, int32_t>& field_indexes,
     const std::shared_ptr<arrow::Schema>& row_schema) {
-    arrow::FieldVector prepared_value_fields(
-        prepared_schema->fields().begin() + SpecialFields::kPreparedKeyValueValueStartIndex,
-        prepared_schema->fields().end());
     std::vector<int32_t> result;
     result.reserve(row_schema->num_fields());
     for (const std::shared_ptr<arrow::Field>& row_field : row_schema->fields()) {
         PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
                                NestedProjectionUtils::GetPaimonFieldId(row_field));
-        PAIMON_ASSIGN_OR_RAISE(int32_t value_index,
-                               FindFieldIndexByPaimonId(prepared_value_fields, field_id));
-        const std::shared_ptr<arrow::Field>& prepared_field = prepared_value_fields[value_index];
+        auto field_index = field_indexes.find(field_id);
+        if (field_index == field_indexes.end()) {
+            return Status::Invalid(
+                fmt::format("cannot find field id {} in prepared schema", field_id));
+        }
+        const std::shared_ptr<arrow::Field>& prepared_field =
+            prepared_schema->field(field_index->second);
         if (!prepared_field->type()->Equals(row_field->type())) {
             return Status::Invalid(fmt::format(
                 "prepared field id {} type {} does not match row "
                 "type {}",
                 field_id, prepared_field->type()->ToString(), row_field->type()->ToString()));
         }
-        result.push_back(value_index + SpecialFields::kPreparedKeyValueValueStartIndex);
+        result.push_back(field_index->second);
     }
     return result;
 }
@@ -182,20 +166,81 @@ Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_
     return Status::OK();
 }
 
+class PreparedReaderPlan {
+ public:
+    static Result<std::shared_ptr<const PreparedReaderPlan>> Create(
+        const std::shared_ptr<arrow::Schema>& prepared_schema,
+        const std::shared_ptr<arrow::Schema>& key_schema,
+        const std::shared_ptr<arrow::Schema>& value_schema,
+        const std::shared_ptr<MemoryPool>& memory_pool, bool exact_commit_schema) {
+        PAIMON_RETURN_NOT_OK(
+            PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
+        if (!key_schema) {
+            return Status::Invalid("prepared key schema cannot be null");
+        }
+        if (!value_schema) {
+            return Status::Invalid("prepared value schema cannot be null");
+        }
+        if (!memory_pool) {
+            return Status::Invalid("prepared reader memory pool cannot be null");
+        }
+        if (exact_commit_schema) {
+            PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
+        }
+        std::unordered_map<int32_t, int32_t> field_indexes;
+        field_indexes.reserve(prepared_schema->num_fields() -
+                              SpecialFields::kPreparedKeyValueValueStartIndex);
+        for (int32_t i = SpecialFields::kPreparedKeyValueValueStartIndex;
+             i < prepared_schema->num_fields(); ++i) {
+            PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(
+                                                         prepared_schema->field(i)));
+            if (!field_indexes.emplace(field_id, i).second) {
+                return Status::Invalid(
+                    fmt::format("duplicate field id {} in prepared schema", field_id));
+            }
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> key_field_indexes,
+                               ResolveFieldIndexes(prepared_schema, field_indexes, key_schema));
+        PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> value_field_indexes,
+                               ResolveFieldIndexes(prepared_schema, field_indexes, value_schema));
+        return std::shared_ptr<const PreparedReaderPlan>(new PreparedReaderPlan(
+            prepared_schema, std::move(key_field_indexes), std::move(value_field_indexes)));
+    }
+
+    const std::shared_ptr<arrow::Schema>& PreparedSchema() const {
+        return prepared_schema_;
+    }
+
+    const std::vector<int32_t>& KeyFieldIndexes() const {
+        return key_field_indexes_;
+    }
+
+    const std::vector<int32_t>& ValueFieldIndexes() const {
+        return value_field_indexes_;
+    }
+
+ private:
+    PreparedReaderPlan(const std::shared_ptr<arrow::Schema>& schema,
+                       std::vector<int32_t>&& key_indexes, std::vector<int32_t>&& value_indexes)
+        : prepared_schema_(schema),
+          key_field_indexes_(std::move(key_indexes)),
+          value_field_indexes_(std::move(value_indexes)) {}
+
+    const std::shared_ptr<arrow::Schema> prepared_schema_;
+    const std::vector<int32_t> key_field_indexes_;
+    const std::vector<int32_t> value_field_indexes_;
+};
+
 class PreparedKeyValueReader final : public KeyValueRecordReader {
  public:
     PreparedKeyValueReader(std::unique_ptr<BatchReader>&& reader,
-                           const std::shared_ptr<arrow::Schema>& prepared_schema,
+                           const std::shared_ptr<const PreparedReaderPlan>& plan,
                            const std::optional<OffsetRange>& visible_offsets,
-                           std::vector<int32_t>&& key_field_indexes,
-                           std::vector<int32_t>&& value_field_indexes,
                            const std::shared_ptr<MemoryPool>& pool,
                            const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage)
         : reader_(std::move(reader)),
-          prepared_schema_(prepared_schema),
+          plan_(plan),
           visible_offsets_(visible_offsets),
-          key_field_indexes_(std::move(key_field_indexes)),
-          value_field_indexes_(std::move(value_field_indexes)),
           pool_(pool),
           offset_coverage_(offset_coverage) {}
 
@@ -284,14 +329,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             }
             std::shared_ptr<arrow::StructArray> data_batch =
                 checked_pointer_cast<arrow::StructArray>(arrow_array);
-            Status transport_status = PreparedKeyValueReaderFactory::ValidateTransportSchema(
-                arrow::schema(data_batch->type()->fields()));
-            if (!transport_status.ok()) {
-                return Status::Invalid(
-                    "prepared batch field does not match prepared transport "
-                    "schema: ",
-                    transport_status.ToString());
-            }
             PAIMON_RETURN_NOT_OK(ValidatePreparedBatch(data_batch));
 
             std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> offset_array =
@@ -306,13 +343,13 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             sequence_number_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
                 data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex));
             arrow::ArrayVector key_fields;
-            key_fields.reserve(key_field_indexes_.size());
-            for (int32_t index : key_field_indexes_) {
+            key_fields.reserve(plan_->KeyFieldIndexes().size());
+            for (int32_t index : plan_->KeyFieldIndexes()) {
                 key_fields.push_back(data_batch->field(index));
             }
             arrow::ArrayVector value_fields;
-            value_fields.reserve(value_field_indexes_.size());
-            for (int32_t index : value_field_indexes_) {
+            value_fields.reserve(plan_->ValueFieldIndexes().size());
+            for (int32_t index : plan_->ValueFieldIndexes()) {
                 value_fields.push_back(data_batch->field(index));
             }
             key_ctx_ = std::make_shared<ColumnarBatchContext>(key_fields, pool_);
@@ -328,33 +365,18 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     }
 
     Status ValidatePreparedBatch(const std::shared_ptr<arrow::StructArray>& data_batch) const {
-        if (data_batch->num_fields() != prepared_schema_->num_fields()) {
+        if (data_batch->num_fields() != plan_->PreparedSchema()->num_fields()) {
             return Status::Invalid(fmt::format(
                 "prepared batch field count {} does not match prepared schema field count {}",
-                data_batch->num_fields(), prepared_schema_->num_fields()));
+                data_batch->num_fields(), plan_->PreparedSchema()->num_fields()));
         }
         const arrow::FieldVector& batch_fields = data_batch->type()->fields();
         for (int32_t i = 0; i < data_batch->num_fields(); ++i) {
-            if (!batch_fields[i]->Equals(prepared_schema_->field(i), true)) {
+            if (!batch_fields[i]->Equals(plan_->PreparedSchema()->field(i), true)) {
                 return Status::Invalid(fmt::format(
                     "prepared batch field {} does not match declared prepared schema", i));
             }
         }
-        if (!data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex) ||
-            data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex)->type_id() !=
-                arrow::Type::INT8) {
-            return Status::Invalid("cannot cast VALUE_KIND column to int8 arrow array");
-        }
-        if (!data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex) ||
-            data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex)->type_id() !=
-                arrow::Type::INT64) {
-            return Status::Invalid("cannot cast SEQUENCE_NUMBER column to int64 arrow array");
-        }
-        if (!data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex) ||
-            data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex)->type_id() !=
-                arrow::Type::INT64) {
-            return Status::Invalid("cannot cast REALTIME_OFFSET column to int64 arrow array");
-        }
         if (data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex)->null_count() != 0 ||
             data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex)->null_count() !=
                 0 ||
@@ -411,10 +433,8 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     bool closed_ = false;
     std::optional<Status> first_error_;
     std::unique_ptr<BatchReader> reader_;
-    std::shared_ptr<arrow::Schema> prepared_schema_;
+    std::shared_ptr<const PreparedReaderPlan> plan_;
     std::optional<OffsetRange> visible_offsets_;
-    std::vector<int32_t> key_field_indexes_;
-    std::vector<int32_t> value_field_indexes_;
     std::shared_ptr<MemoryPool> pool_;
     std::shared_ptr<RealtimeOffsetCoverage> offset_coverage_;
     bool offset_coverage_finished_ = false;
@@ -448,10 +468,8 @@ Status PreparedKeyValueReaderFactory::ValidateTransportSchema(
 namespace {
 
 Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<const PreparedReaderPlan>& plan,
     const std::optional<OffsetRange>& visible_offsets,
-    const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool,
     const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage) {
     std::unique_ptr<BatchReader> owned_reader = std::move(reader);
@@ -462,26 +480,8 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
     if (visible_offsets.has_value() && visible_offsets->begin > visible_offsets->end) {
         return Status::Invalid("prepared visible offset range begin exceeds end");
     }
-    PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
-    if (!key_schema) {
-        return Status::Invalid("prepared key schema cannot be null");
-    }
-    if (!value_schema) {
-        return Status::Invalid("prepared value schema cannot be null");
-    }
-    if (!memory_pool) {
-        return Status::Invalid("prepared reader memory pool cannot be null");
-    }
-    if (!visible_offsets.has_value()) {
-        PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> key_field_indexes,
-                           ResolveFieldIndexes(prepared_schema, key_schema));
-    PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> value_field_indexes,
-                           ResolveFieldIndexes(prepared_schema, value_schema));
     std::unique_ptr<KeyValueRecordReader> result = std::make_unique<PreparedKeyValueReader>(
-        std::move(owned_reader), prepared_schema, visible_offsets, std::move(key_field_indexes),
-        std::move(value_field_indexes), memory_pool, offset_coverage);
+        std::move(owned_reader), plan, visible_offsets, memory_pool, offset_coverage);
     close_guard.Release();
     return result;
 }
@@ -494,9 +494,61 @@ Result<std::unique_ptr<KeyValueRecordReader>> PreparedKeyValueReaderFactory::Cre
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    return AdaptPreparedBatchReaderImpl(std::move(reader), prepared_schema, visible_offsets,
-                                        key_schema, value_schema, memory_pool,
-                                        /*offset_coverage=*/nullptr);
+    std::unique_ptr<BatchReader> owned_reader = std::move(reader);
+    ScopeGuard reader_guard([&owned_reader]() {
+        if (owned_reader) {
+            owned_reader->Close();
+        }
+    });
+    if (visible_offsets.has_value() && visible_offsets->begin > visible_offsets->end) {
+        return Status::Invalid("prepared visible offset range begin exceeds end");
+    }
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<const PreparedReaderPlan> plan,
+        PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema, memory_pool,
+                                   /*exact_commit_schema=*/!visible_offsets.has_value()));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<KeyValueRecordReader> result,
+        AdaptPreparedBatchReaderImpl(std::move(owned_reader), plan, visible_offsets, memory_pool,
+                                     /*offset_coverage=*/nullptr));
+    reader_guard.Release();
+    return result;
+}
+
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
+PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchReader>>&& readers,
+                                              const std::shared_ptr<arrow::Schema>& prepared_schema,
+                                              const OffsetRange& visible_offsets,
+                                              const std::shared_ptr<arrow::Schema>& key_schema,
+                                              const std::shared_ptr<arrow::Schema>& value_schema,
+                                              const std::shared_ptr<MemoryPool>& memory_pool) {
+    std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
+    ScopeGuard readers_guard([&readers, &adapted_readers]() {
+        CloseReaders(readers);
+        CloseReaders(adapted_readers);
+    });
+    if (visible_offsets.begin > visible_offsets.end) {
+        return Status::Invalid("prepared visible offset range begin exceeds end");
+    }
+    for (const std::unique_ptr<BatchReader>& reader : readers) {
+        if (!reader) {
+            return Status::Invalid("PK real-time store returned a null query reader");
+        }
+    }
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<const PreparedReaderPlan> plan,
+        PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema, memory_pool,
+                                   /*exact_commit_schema=*/false));
+    adapted_readers.reserve(readers.size());
+    for (std::unique_ptr<BatchReader>& reader : readers) {
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<KeyValueRecordReader> adapted_reader,
+            AdaptPreparedBatchReaderImpl(std::move(reader), plan, visible_offsets, memory_pool,
+                                         /*offset_coverage=*/nullptr));
+        adapted_readers.push_back(std::move(adapted_reader));
+    }
+    readers_guard.Release();
+    return adapted_readers;
 }
 
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
@@ -511,22 +563,22 @@ PreparedKeyValueReaderFactory::CreateForCommit(
         CloseReaders(readers);
         CloseReaders(adapted_readers);
     });
-    if (!memory_pool) {
-        return Status::Invalid("prepared reader memory pool cannot be null");
-    }
     for (const std::unique_ptr<BatchReader>& reader : readers) {
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null commit reader");
         }
     }
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<const PreparedReaderPlan> plan,
+        PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema, memory_pool,
+                                   /*exact_commit_schema=*/true));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
                            RealtimeOffsetCoverage::Create(sealed_offsets, readers.size()));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        PAIMON_ASSIGN_OR_RAISE(
-            std::unique_ptr<KeyValueRecordReader> adapted_reader,
-            AdaptPreparedBatchReaderImpl(std::move(reader), prepared_schema, std::nullopt,
-                                         key_schema, value_schema, memory_pool, offset_coverage));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> adapted_reader,
+                               AdaptPreparedBatchReaderImpl(std::move(reader), plan, std::nullopt,
+                                                            memory_pool, offset_coverage));
         adapted_readers.push_back(std::move(adapted_reader));
     }
     readers_guard.Release();
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
index 389c75f9a..658ffec08 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/prepared_key_value_reader.h
@@ -47,6 +47,13 @@ class PreparedKeyValueReaderFactory {
         const std::shared_ptr<arrow::Schema>& value_schema,
         const std::shared_ptr<MemoryPool>& memory_pool);
 
+    static Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateForQuery(
+        std::vector<std::unique_ptr<BatchReader>>&& readers,
+        const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& visible_offsets,
+        const std::shared_ptr<arrow::Schema>& key_schema,
+        const std::shared_ptr<arrow::Schema>& value_schema,
+        const std::shared_ptr<MemoryPool>& memory_pool);
+
     static Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateForCommit(
         std::vector<std::unique_ptr<BatchReader>>&& readers,
         const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index dc3498424..a22ba8fed 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -138,11 +138,9 @@ class StoredBatchReader final : public BatchReader {
 
 class PrimaryKeyRealtimeStore::Impl {
  public:
-    Impl(std::shared_ptr<arrow::Schema> prepared_schema, std::shared_ptr<MemoryPool> memory_pool,
+    Impl(std::shared_ptr<arrow::Schema> prepared_schema,
          std::shared_ptr<arrow::MemoryPool> arrow_pool)
-        : prepared_schema_(std::move(prepared_schema)),
-          memory_pool_(std::move(memory_pool)),
-          arrow_pool_(std::move(arrow_pool)) {}
+        : prepared_schema_(std::move(prepared_schema)), arrow_pool_(std::move(arrow_pool)) {}
 
     Status Write(RealtimeWriteBatch&& write_batch) {
         if (!write_batch.batch || !write_batch.batch->GetData()) {
@@ -263,7 +261,6 @@ class PrimaryKeyRealtimeStore::Impl {
 
  private:
     std::shared_ptr<arrow::Schema> prepared_schema_;
-    std::shared_ptr<MemoryPool> memory_pool_;
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     mutable std::mutex mutex_;
     std::vector<StoredBatch> building_;
@@ -284,7 +281,7 @@ Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create
     }
     std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
     return std::shared_ptr<PrimaryKeyRealtimeStore>(new PrimaryKeyRealtimeStore(
-        std::make_unique<Impl>(prepared_schema, memory_pool, std::move(arrow_pool))));
+        std::make_unique<Impl>(prepared_schema, std::move(arrow_pool))));
 }
 Status PrimaryKeyRealtimeStore::Write(RealtimeWriteBatch&& batch) {
     return impl_->Write(std::move(batch));
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index ae123a643..58103c599 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -100,11 +100,7 @@ Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
         arrow::Datum sorted,
         arrow::compute::Take(arrow::Datum(prepared), indices,
                              arrow::compute::TakeOptions::NoBoundsCheck(), &context));
-    std::shared_ptr<arrow::Array> sorted_array = sorted.make_array();
-    if (!sorted_array || sorted_array->type_id() != arrow::Type::STRUCT) {
-        return Status::Invalid("PK real-time sorted batch is not a StructArray");
-    }
-    return checked_pointer_cast<arrow::StructArray>(std::move(sorted_array));
+    return checked_pointer_cast<arrow::StructArray>(sorted.make_array());
 }
 
 }  // namespace
@@ -112,19 +108,13 @@ Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
 Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Create(
     const std::map<std::string, std::string>& partition, int32_t bucket,
     const std::shared_ptr<arrow::Schema>& write_schema,
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
     const std::vector<std::string>& trimmed_primary_keys,
     const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<RealtimeContextImpl>& realtime_context,
     const RealtimeStoreState& store_state, int64_t restored_max_sequence_number,
     const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    if (!store_state.store || !merge_tree_writer || !write_schema || !key_comparator ||
-        !realtime_context || !memory_pool) {
-        return Status::Invalid("PK real-time writer received a null dependency");
-    }
-    if (trimmed_primary_keys.empty()) {
-        return Status::Invalid("PK real-time writer requires at least one primary key");
-    }
     if (restored_max_sequence_number < -1 ||
         restored_max_sequence_number == std::numeric_limits<int64_t>::max()) {
         return Status::Invalid("PK restored sequence number is invalid");
@@ -138,8 +128,6 @@ Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Crea
         }
         key_fields.push_back(std::move(field));
     }
-    std::shared_ptr<arrow::Schema> prepared_schema =
-        SpecialFields::PreparedKeyValueSchema(write_schema->fields());
     const RealtimePartitionBucket partition_bucket(partition, bucket);
     PAIMON_ASSIGN_OR_RAISE(int64_t initial_max_sequence_number,
                            realtime_context->AdvanceMaterializedMaxSequenceNumber(
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index 5cae9ca47..4a26f930d 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -48,6 +48,7 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
     static Result<std::shared_ptr<RealtimePrimaryKeyWriter>> Create(
         const std::map<std::string, std::string>& partition, int32_t bucket,
         const std::shared_ptr<arrow::Schema>& write_schema,
+        const std::shared_ptr<arrow::Schema>& prepared_schema,
         const std::vector<std::string>& trimmed_primary_keys,
         const std::shared_ptr<FieldsComparator>& key_comparator,
         const std::shared_ptr<RealtimeContextImpl>& realtime_context,
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index e4225587d..110994331 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -56,13 +56,9 @@ struct ColumnarBatchContext;
 
 namespace {
 
-Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
-    const std::shared_ptr<RealtimeSplit>& split, const RealtimePartitionBucketView& memory,
+Result<std::shared_ptr<arrow::Schema>> CreatePreparedQuerySchema(
     const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
-    const std::shared_ptr<InternalReadContext>& context,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
+    const std::shared_ptr<arrow::Schema>& value_schema) {
     arrow::FieldVector prepared_value_fields;
     prepared_value_fields.reserve(key_schema->num_fields() + value_schema->num_fields());
     std::unordered_set<int32_t> field_ids;
@@ -78,33 +74,31 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
             prepared_value_fields.push_back(field);
         }
     }
-    std::shared_ptr<arrow::Schema> prepared_schema =
-        SpecialFields::PreparedKeyValueSchema(prepared_value_fields);
+    return SpecialFields::PreparedKeyValueSchema(prepared_value_fields);
+}
+
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
+    const std::shared_ptr<RealtimeSplit>& split, const RealtimePartitionBucketView& memory,
+    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<InternalReadContext>& context,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
     auto c_schema = std::make_unique<ArrowSchema>();
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
     ScopeGuard schema_guard([schema = c_schema.get()]() { ArrowSchemaRelease(schema); });
     RealtimeQueryContext query_context{c_schema.get(), nullptr, false};
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> batch_readers,
                            memory.store->CreateQueryReaders(memory.read_view, 0, query_context));
-    ScopeGuard batch_readers_guard([&batch_readers]() {
-        for (const std::unique_ptr<BatchReader>& reader : batch_readers) {
-            if (reader) {
-                reader->Close();
-            }
-        }
-    });
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
+                           PreparedKeyValueReaderFactory::CreateForQuery(
+                               std::move(batch_readers), prepared_schema,
+                               OffsetRange(split->CommittedEndOffset(), split->MemoryEndOffset()),
+                               key_schema, value_schema, memory_pool));
     std::vector<std::unique_ptr<KeyValueRecordReader>> result;
-    result.reserve(batch_readers.size());
-    for (std::unique_ptr<BatchReader>& reader : batch_readers) {
-        if (!reader) {
-            return Status::Invalid("PK real-time store returned a null query reader");
-        }
-        PAIMON_ASSIGN_OR_RAISE(
-            std::unique_ptr<KeyValueRecordReader> prepared_reader,
-            PreparedKeyValueReaderFactory::Create(
-                std::move(reader), prepared_schema,
-                OffsetRange(split->CommittedEndOffset(), split->MemoryEndOffset()), key_schema,
-                value_schema, memory_pool));
+    result.reserve(prepared_readers.size());
+    for (std::unique_ptr<KeyValueRecordReader>& prepared_reader : prepared_readers) {
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<MergeFunction> merge,
                                PrimaryKeyTableUtils::CreateMergeFunction(
                                    value_schema, context->GetTableSchema()->PrimaryKeys(),
@@ -113,7 +107,6 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
             std::move(prepared_reader), key_comparator,
             std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge))));
     }
-    batch_readers_guard.Release();
     return result;
 }
 
@@ -122,12 +115,14 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
 KeyValueTableRead::KeyValueTableRead(std::vector<std::unique_ptr<SplitRead>>&& split_reads,
                                      const std::shared_ptr<FileStorePathFactory>& path_factory,
                                      const std::shared_ptr<InternalReadContext>& context,
+                                     const std::shared_ptr<arrow::Schema>& prepared_query_schema,
                                      const std::shared_ptr<MemoryPool>& memory_pool,
                                      const std::shared_ptr<Executor>& executor)
     : TableRead(memory_pool),
       split_reads_(std::move(split_reads)),
       path_factory_(path_factory),
       context_(context),
+      prepared_query_schema_(prepared_query_schema),
       executor_(executor) {}
 
 Result<std::unique_ptr<TableRead>> KeyValueTableRead::Create(
@@ -141,10 +136,17 @@ Result<std::unique_ptr<TableRead>> KeyValueTableRead::Create(
     PAIMON_ASSIGN_OR_RAISE(
         std::unique_ptr<MergeFileSplitRead> merge_file_split_read,
         MergeFileSplitRead::Create(path_factory, context, memory_pool, executor));
+    std::shared_ptr<arrow::Schema> prepared_query_schema;
+    if (context->GetRealtimeContext()) {
+        PAIMON_ASSIGN_OR_RAISE(prepared_query_schema,
+                               CreatePreparedQuerySchema(merge_file_split_read->GetKeySchema(),
+                                                         merge_file_split_read->GetValueSchema()));
+    }
     split_reads.emplace_back(std::move(merge_file_split_read));
 
     return std::unique_ptr<TableRead>(new KeyValueTableRead(std::move(split_reads), path_factory,
-                                                            context, memory_pool, executor));
+                                                            context, prepared_query_schema,
+                                                            memory_pool, executor));
 }
 
 void KeyValueTableRead::ForceKeepDelete(bool force_keep_delete) {
@@ -287,9 +289,9 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
         if (merge_read) {
             PAIMON_ASSIGN_OR_RAISE(
                 std::vector<std::unique_ptr<KeyValueRecordReader>> memory_readers,
-                CreateMemoryReaders(realtime_split, memory, merge_read->GetKeySchema(),
-                                    merge_read->GetValueSchema(), merge_read->GetKeyComparator(),
-                                    context_, GetMemoryPool()));
+                CreateMemoryReaders(realtime_split, memory, prepared_query_schema_,
+                                    merge_read->GetKeySchema(), merge_read->GetValueSchema(),
+                                    merge_read->GetKeyComparator(), context_, GetMemoryPool()));
             PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
                                    merge_read->CreateRealtimeReader(realtime_split->DiskSplits(),
                                                                     std::move(memory_readers)));
diff --git a/src/paimon/core/table/source/key_value_table_read.h b/src/paimon/core/table/source/key_value_table_read.h
index 6824ae59e..54f802cf6 100644
--- a/src/paimon/core/table/source/key_value_table_read.h
+++ b/src/paimon/core/table/source/key_value_table_read.h
@@ -22,6 +22,7 @@
 #include <memory>
 #include <vector>
 
+#include "arrow/type_fwd.h"
 #include "paimon/core/operation/internal_read_context.h"
 #include "paimon/core/operation/split_read.h"
 #include "paimon/core/utils/file_store_path_factory.h"
@@ -58,6 +59,7 @@ class KeyValueTableRead : public TableRead {
     KeyValueTableRead(std::vector<std::unique_ptr<SplitRead>>&& split_reads,
                       const std::shared_ptr<FileStorePathFactory>& path_factory,
                       const std::shared_ptr<InternalReadContext>& context,
+                      const std::shared_ptr<arrow::Schema>& prepared_query_schema,
                       const std::shared_ptr<MemoryPool>& memory_pool,
                       const std::shared_ptr<Executor>& executor);
 
@@ -67,6 +69,7 @@ class KeyValueTableRead : public TableRead {
     std::vector<std::unique_ptr<SplitRead>> split_reads_;
     std::shared_ptr<FileStorePathFactory> path_factory_;
     std::shared_ptr<InternalReadContext> context_;
+    std::shared_ptr<arrow::Schema> prepared_query_schema_;
     std::shared_ptr<Executor> executor_;
     bool force_keep_delete_ = false;
 };

From dfb332948580ffb35838060ae269274c8600e76f Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Wed, 26 Aug 2026 22:54:32 +0800
Subject: [PATCH 69/93] refactor(realtime): remove duplicate offset validation

---
 src/paimon/core/realtime/prepared_key_value_reader.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 40b54897a..8620ee872 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -477,9 +477,6 @@ Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
         return Status::Invalid("prepared batch reader cannot be null");
     }
     ScopeGuard close_guard([&owned_reader]() -> void { owned_reader->Close(); });
-    if (visible_offsets.has_value() && visible_offsets->begin > visible_offsets->end) {
-        return Status::Invalid("prepared visible offset range begin exceeds end");
-    }
     std::unique_ptr<KeyValueRecordReader> result = std::make_unique<PreparedKeyValueReader>(
         std::move(owned_reader), plan, visible_offsets, memory_pool, offset_coverage);
     close_guard.Release();

From c73f0087cea6174fbffe813601ed8c7a904599f8 Mon Sep 17 00:00:00 2001
From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com>
Date: Thu, 27 Aug 2026 11:22:08 +0800
Subject: [PATCH 70/93] feat(reader): support late materialization with
 probe/payload two-phase reads (#243)

---
 include/paimon/read_context.h                 |  15 +-
 .../reader/prefetch_file_batch_reader.h       |   2 +-
 src/paimon/CMakeLists.txt                     |   2 +
 .../late_materializing_file_batch_reader.cpp  | 369 ++++++++++
 .../late_materializing_file_batch_reader.h    | 184 +++++
 ...e_materializing_file_batch_reader_test.cpp | 670 ++++++++++++++++++
 .../late_materializing_reader_builder.h       |  70 ++
 .../prefetch_file_batch_reader_impl.cpp       |   9 +-
 .../reader/prefetch_file_batch_reader_impl.h  |   2 +-
 src/paimon/common/utils/arrow/arrow_utils.cpp |  11 +-
 src/paimon/common/utils/arrow/arrow_utils.h   |   3 +
 .../core/operation/abstract_split_read.cpp    |  15 +-
 .../core/operation/abstract_split_read.h      |   4 +-
 .../operation/data_evolution_split_read.h     |   3 +-
 .../core/operation/internal_read_context.h    |   3 +
 .../core/operation/merge_file_split_read.h    |   3 +-
 .../operation/merge_file_split_read_test.cpp  |  58 ++
 .../core/operation/raw_file_split_read.h      |   3 +-
 src/paimon/core/operation/read_context.cpp    |  27 +-
 src/paimon/format/orc/orc_file_batch_reader.h |   2 +-
 .../parquet/parquet_file_batch_reader.cpp     |   2 +
 .../parquet/parquet_file_batch_reader.h       |   5 +-
 .../testing/mock/mock_file_batch_reader.h     |  87 ++-
 test/inte/blob_table_inte_test.cpp            |   1 +
 test/inte/global_index_test.cpp               |   3 +-
 test/inte/read_inte_test.cpp                  | 170 +++++
 test/inte/read_inte_with_index_test.cpp       |  71 +-
 test/inte/scan_and_read_inte_test.cpp         |  92 ++-
 28 files changed, 1841 insertions(+), 45 deletions(-)
 create mode 100644 src/paimon/common/reader/late_materializing_file_batch_reader.cpp
 create mode 100644 src/paimon/common/reader/late_materializing_file_batch_reader.h
 create mode 100644 src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp
 create mode 100644 src/paimon/common/reader/late_materializing_reader_builder.h

diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h
index 3e58b1c45..e645645ba 100644
--- a/include/paimon/read_context.h
+++ b/include/paimon/read_context.h
@@ -51,7 +51,7 @@ class PAIMON_EXPORT ReadContext {
                 const std::vector<std::string>& read_field_names,
                 const std::vector<int32_t>& read_field_ids,
                 const std::shared_ptr<Predicate>& predicate, bool enable_predicate_filter,
-                bool enable_prefetch, uint32_t prefetch_batch_count,
+                bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count,
                 uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch,
                 uint32_t row_to_batch_thread_number, const std::optional<std::string>& table_schema,
                 const std::shared_ptr<MemoryPool>& memory_pool,
@@ -97,6 +97,9 @@ class PAIMON_EXPORT ReadContext {
     bool EnablePrefetch() const {
         return enable_prefetch_;
     }
+    bool EnableLateMaterializing() const {
+        return enable_late_materializing_;
+    }
     uint32_t GetPrefetchBatchCount() const {
         return prefetch_batch_count_;
     }
@@ -163,6 +166,7 @@ class PAIMON_EXPORT ReadContext {
     std::shared_ptr<Predicate> predicate_;
     bool enable_predicate_filter_;
     bool enable_prefetch_;
+    bool enable_late_materializing_;
     uint32_t prefetch_batch_count_;
     uint32_t prefetch_max_parallel_num_;
     bool enable_multi_thread_row_to_batch_;
@@ -306,6 +310,15 @@ class PAIMON_EXPORT ReadContextBuilder {
     /// @return Reference to this builder for method chaining.
     ReadContextBuilder& EnablePrefetch(bool enabled);
 
+    /// Enable or disable late materialization (probe/payload two-phase reads). When enabled,
+    /// each parallel reader under the prefetch layer performs a probe read of predicate
+    /// columns first and only materializes payload columns for matched rows.
+    /// @param enabled Whether to enable late materialization (default: false)
+    /// @return Reference to this builder for method chaining.
+    /// @note Without a pushed-down predicate the late-materializing reader degrades to a
+    /// plain passthrough.
+    ReadContextBuilder& EnableLateMaterializing(bool enabled);
+
     /// Enable or disable the read-ahead cache for read operations.
     ///
     /// A read-ahead cache is used to prebuffer data ranges before they are needed,
diff --git a/include/paimon/reader/prefetch_file_batch_reader.h b/include/paimon/reader/prefetch_file_batch_reader.h
index acc7d0bbd..5e6313e8f 100644
--- a/include/paimon/reader/prefetch_file_batch_reader.h
+++ b/include/paimon/reader/prefetch_file_batch_reader.h
@@ -40,7 +40,7 @@ class PAIMON_EXPORT PrefetchFileBatchReader : public FileBatchReader {
     /// Retrieves the row number of the next row to be read.
     /// This method indicates the current read position within the file.
     /// @return The row number of the next row to read.
-    virtual uint64_t GetNextRowToRead() const = 0;
+    virtual Result<uint64_t> GetNextRowToRead() const = 0;
 
     /// Generates a list of row ranges to be read in batches.
     /// Each range specifies the start and end row numbers for a batch,
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 61fd7c96d..8fb6c17d1 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -134,6 +134,7 @@ set(PAIMON_COMMON_SRCS
     common/predicate/starts_with.cpp
     common/reader/batch_reader.cpp
     common/reader/concat_batch_reader.cpp
+    common/reader/late_materializing_file_batch_reader.cpp
     common/reader/predicate_batch_reader.cpp
     common/reader/prefetch_file_batch_reader_impl.cpp
     common/reader/reader_utils.cpp
@@ -606,6 +607,7 @@ if(PAIMON_BUILD_TESTS)
                     common/predicate/predicate_utils_test.cpp
                     common/predicate/predicate_validator_test.cpp
                     common/reader/concat_batch_reader_test.cpp
+                    common/reader/late_materializing_file_batch_reader_test.cpp
                     common/reader/predicate_batch_reader_test.cpp
                     common/reader/prefetch_file_batch_reader_impl_test.cpp
                     common/reader/reader_utils_test.cpp
diff --git a/src/paimon/common/reader/late_materializing_file_batch_reader.cpp b/src/paimon/common/reader/late_materializing_file_batch_reader.cpp
new file mode 100644
index 000000000..ed7fa304d
--- /dev/null
+++ b/src/paimon/common/reader/late_materializing_file_batch_reader.cpp
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/reader/late_materializing_file_batch_reader.h"
+
+#include <cassert>
+#include <map>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/array/concatenate.h"
+#include "arrow/array/util.h"
+#include "arrow/c/bridge.h"
+#include "arrow/memory_pool.h"
+#include "arrow/type.h"
+#include "arrow/util/checked_cast.h"
+#include "fmt/format.h"
+#include "paimon/common/predicate/predicate_filter.h"
+#include "paimon/common/predicate/predicate_validator.h"
+#include "paimon/common/reader/reader_utils.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/predicate/predicate_utils.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+Result<std::unique_ptr<LateMaterializingFileBatchReader>> LateMaterializingFileBatchReader::Create(
+    std::unique_ptr<FileBatchReader> inner, std::shared_ptr<MemoryPool> pool) {
+    // The reader's own compaction allocations go through an arrow pool; bridge the paimon pool
+    // once here so the accounting matches the rest of the read path.
+    if (pool == nullptr) {
+        return Status::Invalid("pool could not be nullptr.");
+    }
+    if (inner == nullptr) {
+        return Status::Invalid("inner could not be nullptr.");
+    }
+    auto* prefetch_inner = dynamic_cast<PrefetchFileBatchReader*>(inner.get());
+    std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(pool);
+    auto reader =
+        std::unique_ptr<LateMaterializingFileBatchReader>(new LateMaterializingFileBatchReader(
+            std::move(inner), prefetch_inner, std::move(arrow_pool)));
+    return reader;
+}
+
+Result<FileBatchReader::ReadBatch> LateMaterializingFileBatchReader::NextBatch() {
+    if (state_ == kInit) {
+        // SetReadSchema has not been called: read with the file schema, matching the
+        // FileBatchReader contract for schema-less reads.
+        state_ = kNoLatMat;
+    }
+    if (state_ == kProbing) {
+        PAIMON_RETURN_NOT_OK(ReadAndFilterProbeData());
+        if (matched_bitmap_.IsEmpty()) {
+            state_ = kEOF;
+        } else {
+            // payload pass reads only the matched rows (matched_bitmap_ is non-empty here).
+            PAIMON_RETURN_NOT_OK(
+                SetInnerReadSchema(payload_schema_, /*predicate=*/nullptr, matched_bitmap_));
+            state_ = kRunning;
+        }
+    }
+
+    if (state_ == kNoLatMat) {
+        return inner_->NextBatch();
+    } else if (state_ == kRunning) {
+        return ReadPayloadBatch();
+    } else if (state_ == kEOF) {
+        return MakeEofBatch();
+    }
+    return Status::Invalid("invalid state when calling NextBatch: " + std::to_string(state_));
+}
+
+Result<RoaringBitmap32> LateMaterializingFileBatchReader::FilterProbeBatch(
+    const std::shared_ptr<arrow::Array>& array,
+    const std::shared_ptr<PredicateFilter>& bound_filter) {
+    // TODO(zhouhonfeng.zhf): use arrow::compute::Filter instead of PredicateFilter
+    PAIMON_ASSIGN_OR_RAISE(std::vector<char> results, bound_filter->Test(*array));
+    if (results.size() != static_cast<size_t>(array->length())) {
+        return Status::Invalid(
+            fmt::format("predicate result size {} does not match probe batch length {}",
+                        results.size(), array->length()));
+    }
+    // batch-local offsets of the rows passing both the predicate and the selection
+    RoaringBitmap32 batch_matched;
+    for (int64_t i = 0; i < array->length(); ++i) {
+        if (!results[static_cast<size_t>(i)]) {
+            continue;
+        }
+        // map batch offset to file row id
+        PAIMON_ASSIGN_OR_RAISE(uint64_t file_row,
+                               inner_->GetPreviousBatchFileRowId(static_cast<uint64_t>(i)));
+        if (selection_ && !selection_->Contains(static_cast<int32_t>(file_row))) {
+            continue;
+        }
+        batch_matched.Add(static_cast<uint32_t>(i));
+        matched_bitmap_.Add(file_row);
+    }
+    return batch_matched;
+}
+
+Status LateMaterializingFileBatchReader::ReadAndFilterProbeData() {
+    matched_bitmap_ = RoaringBitmap32();
+    probe_cursor_ = 0;
+    arrow::ArrayVector probe_arrays;
+    while (true) {
+        PAIMON_ASSIGN_OR_RAISE(FileBatchReader::ReadBatch batch, inner_->NextBatch());
+        if (BatchReader::IsEofBatch(batch)) {
+            break;
+        }
+        auto& [c_array, c_schema] = batch;
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
+                                          arrow::ImportArray(c_array.get(), c_schema.get()));
+        PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 batch_matched,
+                               FilterProbeBatch(array, probe_filter_));
+        // Compact each probe batch down to its matched rows so probe_data_ aligns row-for-row
+        // (ascending file order) with matched_bitmap_ and the later payload output.
+        if (!batch_matched.IsEmpty()) {
+            PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector matched_slices,
+                                   ReaderUtils::GenerateFilteredArrayVector(array, batch_matched));
+            probe_arrays.insert(probe_arrays.end(), std::make_move_iterator(matched_slices.begin()),
+                                std::make_move_iterator(matched_slices.end()));
+        }
+    }
+
+    std::shared_ptr<arrow::Array> probe_array;
+    if (probe_arrays.empty()) {
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            probe_array, arrow::MakeEmptyArray(arrow::struct_(probe_schema_->fields())));
+    } else {
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(probe_array,
+                                          arrow::Concatenate(probe_arrays, arrow_pool_.get()));
+    }
+    probe_data_ = arrow::internal::checked_pointer_cast<arrow::StructArray>(probe_array);
+    return Status::OK();
+}
+
+Result<FileBatchReader::ReadBatch> LateMaterializingFileBatchReader::ReadPayloadBatch() {
+    while (true) {
+        PAIMON_ASSIGN_OR_RAISE(FileBatchReader::ReadBatchWithBitmap batch_with_bitmap,
+                               inner_->NextBatchWithBitmap());
+        if (BatchReader::IsEofBatch(batch_with_bitmap)) {
+            state_ = kEOF;
+            if (probe_cursor_ != probe_data_->length()) {
+                return Status::Invalid(
+                    fmt::format("probe cursor {} does not match probe data length {}",
+                                probe_cursor_, probe_data_->length()));
+            }
+            return MakeEofBatch();
+        }
+        auto& [batch, bitmap] = batch_with_bitmap;
+        if (bitmap.IsEmpty()) {
+            ReaderUtils::ReleaseReadBatch(std::move(batch));
+            return Status::Invalid("inner read bitmap is empty.");
+        }
+        auto& [c_array, c_schema] = batch;
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> payload_array,
+                                          arrow::ImportArray(c_array.get(), c_schema.get()));
+
+        // Generate the valid bitmap and row_mapping_
+        RoaringBitmap32 valid;
+        row_mapping_.clear();
+        for (auto it = bitmap.Begin(); it != bitmap.End(); ++it) {
+            auto offset = static_cast<uint64_t>(*it);
+            PAIMON_ASSIGN_OR_RAISE(uint64_t file_row, inner_->GetPreviousBatchFileRowId(offset));
+            if (!matched_bitmap_.Contains(file_row)) {
+                continue;
+            }
+            valid.Add(static_cast<uint32_t>(offset));
+            row_mapping_.push_back(file_row);
+        }
+        if (valid.IsEmpty()) {
+            ReaderUtils::ReleaseReadBatch(std::move(batch));
+            continue;
+        }
+
+        // Compact the payload superset down to the matched rows (ascending file row order).
+        PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector payload_slices,
+                               ReaderUtils::GenerateFilteredArrayVector(payload_array, valid));
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> payload_compacted,
+                                          arrow::Concatenate(payload_slices, arrow_pool_.get()));
+
+        auto card = static_cast<int64_t>(valid.Cardinality());
+        if (probe_cursor_ + card > probe_data_->length()) {
+            return Status::Invalid(
+                fmt::format("probe cache underflow: cursor {} + {} exceeds probe rows {}",
+                            probe_cursor_, card, probe_data_->length()));
+        }
+        std::shared_ptr<arrow::Array> probe_selected = probe_data_->Slice(probe_cursor_, card);
+        PAIMON_ASSIGN_OR_RAISE(
+            probe_selected, ArrowUtils::NormalizeArrayOffsets(probe_selected, arrow_pool_.get()));
+        probe_cursor_ += card;
+
+        PAIMON_ASSIGN_OR_RAISE(FileBatchReader::ReadBatch assembled,
+                               AssembleFullBatch(payload_compacted, probe_selected));
+        return assembled;
+    }
+}
+
+Result<FileBatchReader::ReadBatch> LateMaterializingFileBatchReader::AssembleFullBatch(
+    const std::shared_ptr<arrow::Array>& payload_array,
+    const std::shared_ptr<arrow::Array>& probe_array) {
+    auto payload_struct = arrow::internal::checked_pointer_cast<arrow::StructArray>(payload_array);
+    auto probe_struct = arrow::internal::checked_pointer_cast<arrow::StructArray>(probe_array);
+    arrow::ArrayVector children;
+    children.reserve(full_schema_->num_fields());
+    for (const auto& field : full_schema_->fields()) {
+        std::shared_ptr<arrow::Array> col = payload_struct->GetFieldByName(field->name());
+        if (!col) {
+            col = probe_struct->GetFieldByName(field->name());
+        }
+        if (!col) {
+            return Status::Invalid(
+                fmt::format("field {} missing in both payload and probe columns", field->name()));
+        }
+        PAIMON_ASSIGN_OR_RAISE(col, ArrowUtils::NormalizeArrayOffsets(col, arrow_pool_.get()));
+        children.push_back(std::move(col));
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> full_struct,
+                                      arrow::StructArray::Make(children, full_schema_->fields()));
+    std::unique_ptr<::ArrowArray> c_array = std::make_unique<::ArrowArray>();
+    std::unique_ptr<::ArrowSchema> c_schema = std::make_unique<::ArrowSchema>();
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(
+        arrow::ExportArray(*full_struct, c_array.get(), c_schema.get()));
+    return std::make_pair(std::move(c_array), std::move(c_schema));
+}
+
+Status LateMaterializingFileBatchReader::SetInnerReadSchema(
+    const std::shared_ptr<arrow::Schema>& read_schema, const std::shared_ptr<Predicate>& predicate,
+    const std::optional<RoaringBitmap32>& selection) {
+    ::ArrowSchema c_read_schema;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, &c_read_schema));
+    PAIMON_RETURN_NOT_OK(inner_->SetReadSchema(&c_read_schema, predicate, selection));
+    return Status::OK();
+}
+
+Status LateMaterializingFileBatchReader::SetReadSchema(
+    ::ArrowSchema* read_schema, const std::shared_ptr<Predicate>& predicate,
+    const std::optional<RoaringBitmap32>& selection_bitmap) {
+    Reset();
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(full_schema_, arrow::ImportSchema(read_schema));
+    predicate_ = predicate;
+    selection_ = selection_bitmap;
+    if (predicate_ != nullptr) {
+        std::set<std::string> probe_names;
+        PAIMON_RETURN_NOT_OK(PredicateUtils::GetAllNames(predicate_, &probe_names));
+        arrow::FieldVector probe_fields;
+        arrow::FieldVector payload_fields;
+        for (const auto& field : full_schema_->fields()) {
+            if (probe_names.count(field->name()) > 0) {
+                probe_fields.push_back(field);
+            } else {
+                payload_fields.push_back(field);
+            }
+        }
+        // probing only pays off when the predicate fields are a strict subset of the read schema
+        if (!probe_fields.empty() && !payload_fields.empty()) {
+            probe_schema_ = arrow::schema(probe_fields, full_schema_->metadata());
+            payload_schema_ = arrow::schema(payload_fields, full_schema_->metadata());
+            PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema(
+                *probe_schema_, predicate_, /*validate_field_idx=*/false));
+            std::map<std::string, int32_t> name_to_idx;
+            for (int32_t i = 0; i < probe_schema_->num_fields(); ++i) {
+                name_to_idx.emplace(probe_schema_->field(i)->name(), i);
+            }
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr<Predicate> bound_predicate,
+                PredicateUtils::CreatePickedFieldFilter(predicate_, name_to_idx));
+            probe_filter_ = std::dynamic_pointer_cast<PredicateFilter>(bound_predicate);
+            if (!probe_filter_) {
+                return Status::Invalid("failed to bind predicate to probe schema");
+            }
+        }
+    }
+
+    if (predicate_ == nullptr || probe_schema_ == nullptr) {
+        PAIMON_RETURN_NOT_OK(SetInnerReadSchema(full_schema_, predicate_, selection_));
+        state_ = kNoLatMat;
+    } else {
+        PAIMON_RETURN_NOT_OK(SetInnerReadSchema(probe_schema_, predicate_, selection_));
+        state_ = kProbing;
+    }
+    return Status::OK();
+}
+
+Result<uint64_t> LateMaterializingFileBatchReader::GetPreviousBatchFileRowId(
+    uint64_t batch_row_id) const {
+    if (state_ == kNoLatMat) {
+        return inner_->GetPreviousBatchFileRowId(batch_row_id);
+    }
+    // In kRunning the emitted batch is compacted/reassembled, so row ids come from row_mapping_
+    // instead of the inner reader.
+    if (batch_row_id >= row_mapping_.size()) {
+        return Status::Invalid(
+            fmt::format("batch_row_id {} is out of range, last batch row count is {}", batch_row_id,
+                        row_mapping_.size()));
+    }
+    return row_mapping_[batch_row_id];
+}
+
+Status LateMaterializingFileBatchReader::SeekToRow(uint64_t row_number) {
+    PAIMON_ASSIGN_OR_RAISE(PrefetchFileBatchReader * prefetch_reader,
+                           GetPrefetchReaderOrRaise("SeekToRow"));
+    PAIMON_RETURN_NOT_OK(prefetch_reader->SeekToRow(row_number));
+    if (state_ == kRunning || state_ == kEOF) {
+        if (matched_bitmap_.IsEmpty()) {
+            state_ = kEOF;
+            return Status::OK();
+        }
+        int64_t cursor = 0;
+        for (auto it = matched_bitmap_.Begin(); it != matched_bitmap_.End(); ++it) {
+            if (static_cast<uint64_t>(*it) >= row_number) {
+                break;
+            }
+            ++cursor;
+        }
+        probe_cursor_ = cursor;
+        // a seek after EOF re-activates payload reading
+        state_ = kRunning;
+    }
+    return Status::OK();
+}
+
+Status LateMaterializingFileBatchReader::SetReadRanges(
+    const std::vector<std::pair<uint64_t, uint64_t>>& read_ranges) {
+    if (prefetch_inner_ == nullptr) {
+        // Only the format reader can act on this hint, and the PrefetchFileBatchReader contract
+        // lets an implementation that cannot honor it ignore the hint.
+        return Status::OK();
+    }
+    return prefetch_inner_->SetReadRanges(read_ranges);
+}
+
+void LateMaterializingFileBatchReader::Reset() {
+    state_ = kInit;
+    matched_bitmap_ = RoaringBitmap32();
+    probe_data_.reset();
+    probe_cursor_ = 0;
+    row_mapping_.clear();
+    probe_schema_.reset();
+    payload_schema_.reset();
+    full_schema_.reset();
+    probe_filter_.reset();
+    predicate_.reset();
+    selection_.reset();
+    probe_cursor_ = 0;
+    row_mapping_.clear();
+}
+
+}  // namespace paimon
diff --git a/src/paimon/common/reader/late_materializing_file_batch_reader.h b/src/paimon/common/reader/late_materializing_file_batch_reader.h
new file mode 100644
index 000000000..231625db6
--- /dev/null
+++ b/src/paimon/common/reader/late_materializing_file_batch_reader.h
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/array/array_nested.h>
+#include <arrow/c/abi.h>
+
+#include <cstdint>
+#include <memory>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+#include "fmt/format.h"
+#include "paimon/reader/prefetch_file_batch_reader.h"
+
+namespace paimon {
+
+class PredicateFilter;
+
+// For convenience, we abbreviate `Later Materializing` as `LatMat`.
+// This reader is installed below the prefetch layer (see
+// AbstractSplitRead::CreateFileBatchReader) and performs probe/payload two-phase reads when a
+// predicate is pushed down through SetReadSchema; without a predicate it is a plain passthrough.
+class LateMaterializingFileBatchReader : public PrefetchFileBatchReader {
+ public:
+    static Result<std::unique_ptr<LateMaterializingFileBatchReader>> Create(
+        std::unique_ptr<FileBatchReader> inner, std::shared_ptr<MemoryPool> pool);
+
+    Result<FileBatchReader::ReadBatch> NextBatch() override;
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return inner_->GetReaderMetrics();
+    };
+
+    void Close() override {
+        Reset();
+        inner_->Close();
+    }
+
+    Result<std::unique_ptr<::ArrowSchema>> GetFileSchema() const override {
+        return inner_->GetFileSchema();
+    }
+
+    Status SetReadSchema(::ArrowSchema* read_schema, const std::shared_ptr<Predicate>& predicate,
+                         const std::optional<RoaringBitmap32>& selection_bitmap) override;
+
+    Result<uint64_t> GetPreviousBatchFileRowId(uint64_t batch_row_id) const override;
+
+    Result<uint64_t> GetNumberOfRows() const override {
+        return inner_->GetNumberOfRows();
+    }
+
+    bool SupportPreciseBitmapSelection() const override {
+        // When probe_schema_ or payload_schema_ is null, lat-mat does not take effect.
+        // Here we simply pass through the inner reader's support.
+        return inner_->SupportPreciseBitmapSelection();
+    }
+
+    Status SeekToRow(uint64_t row_number) override;
+
+    Result<uint64_t> GetNextRowToRead() const override {
+        PAIMON_ASSIGN_OR_RAISE(PrefetchFileBatchReader * prefetch_reader,
+                               GetPrefetchReaderOrRaise("GetNextRowToRead"));
+        return prefetch_reader->GetNextRowToRead();
+    }
+
+    Result<std::vector<std::pair<uint64_t, uint64_t>>> GenReadRanges(
+        bool* need_prefetch) const override {
+        PAIMON_ASSIGN_OR_RAISE(PrefetchFileBatchReader * prefetch_reader,
+                               GetPrefetchReaderOrRaise("GenReadRanges"));
+        return prefetch_reader->GenReadRanges(need_prefetch);
+    }
+
+    Status SetReadRanges(const std::vector<std::pair<uint64_t, uint64_t>>& read_ranges) override;
+
+    Result<std::vector<std::pair<uint64_t, uint64_t>>> PreBufferRange() override {
+        // TODO(zhouhongfeng.zhf): PrebufferRange (called by PrefetchFileBatchReader) only read the
+        // probe data, consider read the payload data as well.
+        if (prefetch_inner_ == nullptr) {
+            return std::vector<std::pair<uint64_t, uint64_t>>{};
+        }
+        return prefetch_inner_->PreBufferRange();
+    }
+
+ private:
+    LateMaterializingFileBatchReader(std::unique_ptr<FileBatchReader> inner,
+                                     PrefetchFileBatchReader* prefetch_inner,
+                                     std::shared_ptr<arrow::MemoryPool> arrow_pool)
+        : inner_(std::move(inner)),
+          prefetch_inner_(prefetch_inner),
+          arrow_pool_(std::move(arrow_pool)) {}
+
+    /// Reset the state of the late materializing reader, does not close inner reader.
+    void Reset();
+
+    enum LatMatState {
+        kInit,
+        kProbing,   // schema is set, probing is in progress
+        kNoLatMat,  // no need to late materialization
+        kRunning,   // Lat-mat is enabled and the payload reader is reading data
+        kEOF
+    };
+
+    /// Read the probe projection once (whole file) and evaluating the predicate batch by batch.
+    /// This function updates matched_bitmap_ and probe_data_.
+    /// TODO(zhouhongfeng.zhf): Read the probe data batch by batch to save memory.
+    Status ReadAndFilterProbeData();
+
+    Result<RoaringBitmap32> FilterProbeBatch(const std::shared_ptr<arrow::Array>& array,
+                                             const std::shared_ptr<PredicateFilter>& bound_filter);
+
+    /// Read one payload batch with bitmap (matched rows only)
+    Result<FileBatchReader::ReadBatch> ReadPayloadBatch();
+
+    /// Combine the compacted payload columns and the selected probe columns into a single struct
+    /// array following full_schema_'s field order.
+    Result<FileBatchReader::ReadBatch> AssembleFullBatch(
+        const std::shared_ptr<arrow::Array>& payload_array,
+        const std::shared_ptr<arrow::Array>& probe_array);
+
+    Status SetInnerReadSchema(const std::shared_ptr<arrow::Schema>& read_schema,
+                              const std::shared_ptr<Predicate>& predicate,
+                              const std::optional<RoaringBitmap32>& selection);
+
+    /// Returns the inner reader's prefetch interface, or an error when the format reader does not
+    /// implement it (avro and blob do not).
+    Result<PrefetchFileBatchReader*> GetPrefetchReaderOrRaise(
+        std::string_view function_name) const {
+        if (prefetch_inner_ == nullptr) {
+            return Status::NotImplemented(
+                fmt::format("format reader is not a prefetch reader, function {} not supported",
+                            function_name));
+        }
+        return prefetch_inner_;
+    }
+
+    /// The probe/payload logic needs nothing beyond FileBatchReader, so the inner reader is held as
+    /// the base type: parquet and orc readers implement PrefetchFileBatchReader, while avro and
+    /// blob readers only implement FileBatchReader. The prefetch-only methods are rejected for the
+    /// latter; AbstractSplitRead::CreateFileBatchReader keeps those formats out of the prefetch
+    /// layer so nothing calls them.
+    std::unique_ptr<FileBatchReader> inner_;
+    /// Non-owning view of inner_ when it implements the prefetch interface, nullptr otherwise.
+    /// inner_ is never reassigned, so the cast is resolved once in Create().
+    PrefetchFileBatchReader* prefetch_inner_ = nullptr;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    LatMatState state_ = kInit;
+    std::shared_ptr<arrow::Schema> full_schema_;
+    // projection holding only the predicate fields; nullptr when probing is not applicable
+    std::shared_ptr<arrow::Schema> probe_schema_;
+    // projection holding the payload (non-probe) fields; nullptr when probing is not applicable
+    std::shared_ptr<arrow::Schema> payload_schema_;
+    std::shared_ptr<Predicate> predicate_;
+    // predicate bound to probe_schema_'s field indices; null when probing is not applicable
+    std::shared_ptr<PredicateFilter> probe_filter_;
+    std::optional<RoaringBitmap32> selection_;
+    // the probe_data_ is sliced and compacted with the matched_bitmap_
+    std::shared_ptr<arrow::StructArray> probe_data_;
+    RoaringBitmap32 matched_bitmap_;
+    // read cursor into probe_data_ for the payload phase
+    int64_t probe_cursor_ = 0;
+    // to support GetPreviousBatchFileRowId
+    std::vector<uint64_t> row_mapping_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp b/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp
new file mode 100644
index 000000000..ff571a284
--- /dev/null
+++ b/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp
@@ -0,0 +1,670 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/reader/late_materializing_file_batch_reader.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <limits>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/array/builder_nested.h"
+#include "arrow/c/bridge.h"
+#include "gtest/gtest.h"
+#include "paimon/common/reader/late_materializing_reader_builder.h"
+#include "paimon/common/reader/prefetch_file_batch_reader_impl.h"
+#include "paimon/common/reader/reader_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/read_ahead_cache.h"
+#include "paimon/executor.h"
+#include "paimon/format/reader_builder.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/reader/prefetch_file_batch_reader.h"
+#include "paimon/status.h"
+#include "paimon/testing/mock/mock_file_batch_reader.h"
+#include "paimon/testing/mock/mock_file_system.h"
+#include "paimon/testing/mock/mock_format_reader_builder.h"
+#include "paimon/testing/utils/read_result_collector.h"
+#include "paimon/testing/utils/testharness.h"
+#include "paimon/utils/roaring_bitmap32.h"
+
+namespace paimon::test {
+
+class LateMaterializingFileBatchReaderTest : public ::testing::Test {
+ public:
+    void SetUp() override {
+        k_field_ = arrow::field("k", arrow::int64());
+        v_field_ = arrow::field("v", arrow::utf8());
+        full_fields_ = {k_field_, v_field_};
+        full_type_ = arrow::struct_(full_fields_);
+    }
+
+    // Build a struct array with column k (int64, values = ks) and column v (utf8, "v_<index>").
+    std::shared_ptr<arrow::Array> BuildData(const std::vector<int64_t>& ks) {
+        arrow::StructBuilder builder(
+            full_type_, arrow::default_memory_pool(),
+            {std::make_shared<arrow::Int64Builder>(), std::make_shared<arrow::StringBuilder>()});
+        auto* k_builder = checked_cast<arrow::Int64Builder*>(builder.field_builder(0));
+        auto* v_builder = checked_cast<arrow::StringBuilder*>(builder.field_builder(1));
+        for (size_t i = 0; i < ks.size(); ++i) {
+            EXPECT_TRUE(builder.Append().ok());
+            EXPECT_TRUE(k_builder->Append(ks[i]).ok());
+            EXPECT_TRUE(v_builder->Append("v_" + std::to_string(i)).ok());
+        }
+        std::shared_ptr<arrow::Array> array;
+        EXPECT_TRUE(builder.Finish(&array).ok());
+        return array;
+    }
+
+    struct Row {
+        int64_t k;
+        std::string v;
+        uint64_t file_row;
+    };
+
+    // Drive the reader through NextBatchWithBitmap to EOF, decoding the full-schema output rows.
+    Result<std::vector<Row>> Collect(LateMaterializingFileBatchReader* reader) {
+        std::vector<Row> rows;
+        while (true) {
+            PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap,
+                                   reader->NextBatchWithBitmap());
+            if (BatchReader::IsEofBatch(batch_with_bitmap)) {
+                break;
+            }
+            auto& [batch, bitmap] = batch_with_bitmap;
+            auto& [c_array, c_schema] = batch;
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
+                                              arrow::ImportArray(c_array.get(), c_schema.get()));
+            auto struct_array = arrow::internal::checked_pointer_cast<arrow::StructArray>(array);
+            EXPECT_EQ(bitmap.Cardinality(), static_cast<int32_t>(struct_array->length()));
+            auto k_array = arrow::internal::checked_pointer_cast<arrow::Int64Array>(
+                struct_array->GetFieldByName("k"));
+            if (!k_array) {
+                return Status::Invalid("output batch missing k column");
+            }
+            // v is only present when it belongs to the read schema (payload projection).
+            auto v_array = arrow::internal::checked_pointer_cast<arrow::StringArray>(
+                struct_array->GetFieldByName("v"));
+            for (int64_t i = 0; i < struct_array->length(); ++i) {
+                PAIMON_ASSIGN_OR_RAISE(uint64_t file_row,
+                                       reader->GetPreviousBatchFileRowId(static_cast<uint64_t>(i)));
+                rows.push_back(Row{k_array->Value(i),
+                                   v_array ? v_array->GetString(i) : std::string(), file_row});
+            }
+        }
+        return rows;
+    }
+
+    Status SetReadSchema(LateMaterializingFileBatchReader* reader,
+                         const std::shared_ptr<arrow::Schema>& schema,
+                         const std::shared_ptr<Predicate>& predicate,
+                         const std::optional<RoaringBitmap32>& selection) {
+        ::ArrowSchema c_schema;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema));
+        return reader->SetReadSchema(&c_schema, predicate, selection);
+    }
+
+    // Collect all output rows as a single concatenated struct array (for schema/nested checks),
+    // reusing the shared collector so the batch-offset and bitmap contracts are checked too.
+    Result<std::shared_ptr<arrow::StructArray>> CollectStruct(FileBatchReader* reader) {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::ChunkedArray> chunked,
+                               ReadResultCollector::CollectResult(reader));
+        if (chunked == nullptr) {
+            return std::shared_ptr<arrow::StructArray>();
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> combined,
+                                          arrow::Concatenate(chunked->chunks()));
+        return arrow::internal::checked_pointer_cast<arrow::StructArray>(combined);
+    }
+
+    // Build a struct with 5 columns [a:int64, b:utf8, c:int64, d:utf8, e:int64], each carrying a
+    // distinct value pattern so any column reordering is detected.
+    std::shared_ptr<arrow::Array> BuildMultiFieldData(int32_t n) {
+        auto type =
+            arrow::struct_({arrow::field("a", arrow::int64()), arrow::field("b", arrow::utf8()),
+                            arrow::field("c", arrow::int64()), arrow::field("d", arrow::utf8()),
+                            arrow::field("e", arrow::int64())});
+        arrow::StructBuilder builder(
+            type, arrow::default_memory_pool(),
+            {std::make_shared<arrow::Int64Builder>(), std::make_shared<arrow::StringBuilder>(),
+             std::make_shared<arrow::Int64Builder>(), std::make_shared<arrow::StringBuilder>(),
+             std::make_shared<arrow::Int64Builder>()});
+        auto* a = checked_cast<arrow::Int64Builder*>(builder.field_builder(0));
+        auto* b = checked_cast<arrow::StringBuilder*>(builder.field_builder(1));
+        auto* c = checked_cast<arrow::Int64Builder*>(builder.field_builder(2));
+        auto* d = checked_cast<arrow::StringBuilder*>(builder.field_builder(3));
+        auto* e = checked_cast<arrow::Int64Builder*>(builder.field_builder(4));
+        for (int32_t i = 0; i < n; ++i) {
+            EXPECT_TRUE(builder.Append().ok());
+            EXPECT_TRUE(a->Append(i).ok());
+            EXPECT_TRUE(b->Append("b_" + std::to_string(i)).ok());
+            EXPECT_TRUE(c->Append(static_cast<int64_t>(i) * 100).ok());
+            EXPECT_TRUE(d->Append("d_" + std::to_string(i)).ok());
+            EXPECT_TRUE(e->Append(static_cast<int64_t>(i) * 10000).ok());
+        }
+        std::shared_ptr<arrow::Array> array;
+        EXPECT_TRUE(builder.Finish(&array).ok());
+        return array;
+    }
+
+    // Build a struct with a nested payload column [k:int64, arr:list<int64>, tag:utf8].
+    std::shared_ptr<arrow::Array> BuildNestedData(int32_t n) {
+        auto type = arrow::struct_({arrow::field("k", arrow::int64()),
+                                    arrow::field("arr", arrow::list(arrow::int64())),
+                                    arrow::field("tag", arrow::utf8())});
+        auto arr_value_builder = std::make_shared<arrow::Int64Builder>();
+        arrow::StructBuilder builder(
+            type, arrow::default_memory_pool(),
+            {std::make_shared<arrow::Int64Builder>(),
+             std::make_shared<arrow::ListBuilder>(arrow::default_memory_pool(), arr_value_builder),
+             std::make_shared<arrow::StringBuilder>()});
+        auto* k = checked_cast<arrow::Int64Builder*>(builder.field_builder(0));
+        auto* arr = checked_cast<arrow::ListBuilder*>(builder.field_builder(1));
+        auto* arr_values = checked_cast<arrow::Int64Builder*>(arr->value_builder());
+        auto* tag = checked_cast<arrow::StringBuilder*>(builder.field_builder(2));
+        for (int32_t i = 0; i < n; ++i) {
+            EXPECT_TRUE(builder.Append().ok());
+            EXPECT_TRUE(k->Append(i).ok());
+            EXPECT_TRUE(arr->Append().ok());
+            EXPECT_TRUE(arr_values->Append(i).ok());
+            EXPECT_TRUE(arr_values->Append(i + 1).ok());
+            EXPECT_TRUE(tag->Append("t_" + std::to_string(i)).ok());
+        }
+        std::shared_ptr<arrow::Array> array;
+        EXPECT_TRUE(builder.Finish(&array).ok());
+        return array;
+    }
+
+ protected:
+    std::shared_ptr<arrow::Field> k_field_;
+    std::shared_ptr<arrow::Field> v_field_;
+    arrow::FieldVector full_fields_;
+    std::shared_ptr<arrow::DataType> full_type_;
+};
+
+// No predicate: the reader must pass through the inner reader unchanged (all rows, all columns).
+TEST_F(LateMaterializingFileBatchReaderTest, PassThroughWhenNoPredicate) {
+    auto data = BuildData({0, 1, 2, 3, 4});
+    auto mock = std::make_unique<MockFileBatchReader>(data, full_type_, /*batch_size=*/2);
+    ASSERT_OK_AND_ASSIGN(
+        auto reader, LateMaterializingFileBatchReader::Create(std::move(mock), GetDefaultPool()));
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), /*predicate=*/nullptr,
+                            std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, Collect(reader.get()));
+    ASSERT_EQ(rows.size(), 5u);
+    for (int64_t i = 0; i < 5; ++i) {
+        EXPECT_EQ(rows[i].k, i);
+        EXPECT_EQ(rows[i].v, "v_" + std::to_string(i));
+        EXPECT_EQ(rows[i].file_row, static_cast<uint64_t>(i));
+    }
+}
+
+// The predicate references every projected column, so the payload set is empty: no late
+// materialization, plain pass-through.
+TEST_F(LateMaterializingFileBatchReaderTest, PassThroughWhenPayloadEmpty) {
+    auto data = BuildData({0, 1, 2, 3, 4});
+    auto mock = std::make_unique<MockFileBatchReader>(data, full_type_, /*batch_size=*/2);
+    ASSERT_OK_AND_ASSIGN(
+        auto reader, LateMaterializingFileBatchReader::Create(std::move(mock), GetDefaultPool()));
+    // read schema is just {k}; the predicate on k covers all columns -> payload empty
+    auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"k",
+                                                      FieldType::BIGINT, Literal(0l));
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema({k_field_}), predicate, std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, Collect(reader.get()));
+    ASSERT_EQ(rows.size(), 5u);
+    for (size_t idx = 0; idx < rows.size(); ++idx) {
+        EXPECT_EQ(rows[idx].k, static_cast<int64_t>(idx));
+        // v is outside the read schema, so the pass-through output must not carry it
+        EXPECT_EQ(rows[idx].v, "");
+        EXPECT_EQ(rows[idx].file_row, static_cast<uint64_t>(idx));
+    }
+}
+
+// Contiguous matched subset spanning multiple batches.
+TEST_F(LateMaterializingFileBatchReaderTest, ContiguousSubsetAcrossBatches) {
+    auto data = BuildData({0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+    auto mock = std::make_unique<MockFileBatchReader>(data, full_type_, /*batch_size=*/3);
+    ASSERT_OK_AND_ASSIGN(
+        auto reader, LateMaterializingFileBatchReader::Create(std::move(mock), GetDefaultPool()));
+    auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"k",
+                                                   FieldType::BIGINT, Literal(4l));
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, Collect(reader.get()));
+    ASSERT_EQ(rows.size(), 5u);  // k = 5..9
+    for (size_t idx = 0; idx < rows.size(); ++idx) {
+        int64_t expected = 5 + static_cast<int64_t>(idx);
+        EXPECT_EQ(rows[idx].k, expected);
+        EXPECT_EQ(rows[idx].v, "v_" + std::to_string(expected));
+        EXPECT_EQ(rows[idx].file_row, static_cast<uint64_t>(expected));
+    }
+}
+
+// Scattered (alternating) matched rows: predicate matches every other row.
+TEST_F(LateMaterializingFileBatchReaderTest, ScatteredAlternatingMatch) {
+    // k = 0,1,0,1,... ; predicate k == 1 matches all odd file rows.
+    std::vector<int64_t> ks;
+    for (int i = 0; i < 12; ++i) {
+        ks.push_back(i % 2);
+    }
+    auto data = BuildData(ks);
+    auto mock = std::make_unique<MockFileBatchReader>(data, full_type_, /*batch_size=*/3);
+    ASSERT_OK_AND_ASSIGN(
+        auto reader, LateMaterializingFileBatchReader::Create(std::move(mock), GetDefaultPool()));
+    auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k",
+                                             FieldType::BIGINT, Literal(1l));
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, Collect(reader.get()));
+    ASSERT_EQ(rows.size(), 6u);  // odd rows 1,3,5,7,9,11
+    for (size_t idx = 0; idx < rows.size(); ++idx) {
+        uint64_t expected_row = 2 * idx + 1;
+        EXPECT_EQ(rows[idx].k, 1);
+        EXPECT_EQ(rows[idx].v, "v_" + std::to_string(expected_row));
+        EXPECT_EQ(rows[idx].file_row, expected_row);
+    }
+}
+
+// The selection bitmap further restricts the matched rows: matched must be a subset of selection.
+TEST_F(LateMaterializingFileBatchReaderTest, MatchedIntersectsSelection) {
+    std::vector<int64_t> ks;
+    for (int i = 0; i < 12; ++i) {
+        ks.push_back(i % 2);
+    }
+    auto data = BuildData(ks);
+    auto mock = std::make_unique<MockFileBatchReader>(data, full_type_, /*batch_size=*/4);
+    ASSERT_OK_AND_ASSIGN(
+        auto reader, LateMaterializingFileBatchReader::Create(std::move(mock), GetDefaultPool()));
+    auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k",
+                                             FieldType::BIGINT, Literal(1l));
+    // predicate hits {1,3,5,7,9,11}; selection keeps only {1,5,9}
+    RoaringBitmap32 selection;
+    selection.Add(1);
+    selection.Add(5);
+    selection.Add(9);
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate,
+                            std::optional<RoaringBitmap32>(selection)));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, Collect(reader.get()));
+    ASSERT_EQ(rows.size(), 3u);
+    const std::vector<uint64_t> expected_rows = {1u, 5u, 9u};
+    for (size_t idx = 0; idx < rows.size(); ++idx) {
+        EXPECT_EQ(rows[idx].k, 1);
+        EXPECT_EQ(rows[idx].v, "v_" + std::to_string(expected_rows[idx]));
+        EXPECT_EQ(rows[idx].file_row, expected_rows[idx]);
+    }
+}
+
+// No matched rows: the reader returns EOF immediately.
+TEST_F(LateMaterializingFileBatchReaderTest, EmptyMatchReturnsEof) {
+    auto data = BuildData({0, 1, 2, 3, 4});
+    auto mock = std::make_unique<MockFileBatchReader>(data, full_type_, /*batch_size=*/2);
+    ASSERT_OK_AND_ASSIGN(
+        auto reader, LateMaterializingFileBatchReader::Create(std::move(mock), GetDefaultPool()));
+    auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"k",
+                                                   FieldType::BIGINT, Literal(100l));
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, Collect(reader.get()));
+    ASSERT_TRUE(rows.empty());
+}
+
+// SeekToRow during payload emission must re-align the probe cursor so probe/payload stay matched.
+TEST_F(LateMaterializingFileBatchReaderTest, SeekToRowRealignsProbeCursor) {
+    auto data = BuildData({0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+    auto mock = std::make_unique<MockFileBatchReader>(data, full_type_, /*batch_size=*/4);
+    ASSERT_OK_AND_ASSIGN(
+        auto reader, LateMaterializingFileBatchReader::Create(std::move(mock), GetDefaultPool()));
+    auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"k",
+                                                      FieldType::BIGINT, Literal(5l));
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt));
+
+    // First payload batch triggers the probe scan; matched rows are 5..9.
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap first, reader->NextBatchWithBitmap());
+    ASSERT_FALSE(BatchReader::IsEofBatch(first));
+    ReaderUtils::ReleaseReadBatch(std::move(first.first));
+
+    // Seek forward to file row 8: subsequent output must be exactly rows 8 and 9, correctly paired.
+    ASSERT_OK(reader->SeekToRow(8));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, Collect(reader.get()));
+    ASSERT_EQ(rows.size(), 2u);
+    EXPECT_EQ(rows[0].k, 8);
+    EXPECT_EQ(rows[0].v, "v_8");
+    EXPECT_EQ(rows[0].file_row, 8u);
+    EXPECT_EQ(rows[1].k, 9);
+    EXPECT_EQ(rows[1].v, "v_9");
+    EXPECT_EQ(rows[1].file_row, 9u);
+}
+
+// SetReadRanges must be cached and re-forwarded to the inner reader across the probe/payload
+// schema switches (SetReadSchema resets the inner reader's ranges).
+TEST_F(LateMaterializingFileBatchReaderTest, ReadRangesForwardedAcrossPhases) {
+    auto data = BuildData({0, 1, 2, 3, 4, 5, 6, 7});
+    auto mock = std::make_unique<MockFileBatchReader>(data, full_type_, /*batch_size=*/4);
+    auto* mock_ptr = mock.get();
+    ASSERT_OK_AND_ASSIGN(
+        auto reader, LateMaterializingFileBatchReader::Create(std::move(mock), GetDefaultPool()));
+    auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"k",
+                                                      FieldType::BIGINT, Literal(2l));
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt));
+
+    std::vector<std::pair<uint64_t, uint64_t>> ranges = {{0, 8}};
+    ASSERT_OK(reader->SetReadRanges(ranges));
+
+    // Drive to EOF; this performs the probe pass and the payload schema switch.
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, Collect(reader.get()));
+    ASSERT_EQ(rows.size(), 6u);  // k = 2..7
+    for (size_t idx = 0; idx < rows.size(); ++idx) {
+        int64_t expected = 2 + static_cast<int64_t>(idx);
+        EXPECT_EQ(rows[idx].k, expected);
+        EXPECT_EQ(rows[idx].v, "v_" + std::to_string(expected));
+        EXPECT_EQ(rows[idx].file_row, static_cast<uint64_t>(expected));
+    }
+
+    // The inner reader must have received the cached ranges again after the payload switch.
+    // SetReadSchema (invoked on the payload switch) clears the inner reader's ranges, so the
+    // cached ranges still being present at the end proves LM re-forwarded them after the switch.
+    ASSERT_EQ(mock_ptr->GetReadRanges(), ranges);
+}
+
+// SetReadSchema is re-entrant: a second call with a different predicate resets probe state.
+TEST_F(LateMaterializingFileBatchReaderTest, ReentrantSetReadSchema) {
+    auto data = BuildData({0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+    auto mock = std::make_unique<MockFileBatchReader>(data, full_type_, /*batch_size=*/3);
+    ASSERT_OK_AND_ASSIGN(
+        auto reader, LateMaterializingFileBatchReader::Create(std::move(mock), GetDefaultPool()));
+
+    auto predicate1 = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"k",
+                                                    FieldType::BIGINT, Literal(7l));
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate1, std::nullopt));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows1, Collect(reader.get()));
+    ASSERT_EQ(rows1.size(), 2u);  // k = 8,9
+    for (size_t idx = 0; idx < rows1.size(); ++idx) {
+        int64_t expected = 8 + static_cast<int64_t>(idx);
+        EXPECT_EQ(rows1[idx].k, expected);
+        EXPECT_EQ(rows1[idx].v, "v_" + std::to_string(expected));
+        EXPECT_EQ(rows1[idx].file_row, static_cast<uint64_t>(expected));
+    }
+
+    auto predicate2 = PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"k",
+                                                 FieldType::BIGINT, Literal(3l));
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate2, std::nullopt));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows2, Collect(reader.get()));
+    ASSERT_EQ(rows2.size(), 3u);  // k = 0,1,2
+    for (size_t idx = 0; idx < rows2.size(); ++idx) {
+        EXPECT_EQ(rows2[idx].k, static_cast<int64_t>(idx));
+        EXPECT_EQ(rows2[idx].v, "v_" + std::to_string(idx));
+        EXPECT_EQ(rows2[idx].file_row, static_cast<uint64_t>(idx));
+    }
+}
+
+// Forwarded metadata accessors should reflect the inner reader.
+TEST_F(LateMaterializingFileBatchReaderTest, ForwardsRowCountAndFileSchema) {
+    auto data = BuildData({0, 1, 2, 3});
+    auto mock = std::make_unique<MockFileBatchReader>(data, full_type_, /*batch_size=*/2);
+    ASSERT_OK_AND_ASSIGN(
+        auto reader, LateMaterializingFileBatchReader::Create(std::move(mock), GetDefaultPool()));
+
+    ASSERT_OK_AND_ASSIGN(uint64_t num_rows, reader->GetNumberOfRows());
+    EXPECT_EQ(num_rows, 4u);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<::ArrowSchema> c_file_schema, reader->GetFileSchema());
+    auto import_result = arrow::ImportType(c_file_schema.get());
+    ASSERT_TRUE(import_result.ok());
+    EXPECT_TRUE(import_result.ValueOrDie()->Equals(full_type_));
+}
+
+// With many columns and a predicate over two non-adjacent probe columns, the output must keep the
+// full read-schema field order (and each probe/payload column's values must not be scrambled).
+TEST_F(LateMaterializingFileBatchReaderTest, MultiFieldPreservesColumnOrder) {
+    auto data = BuildMultiFieldData(10);
+    auto type = data->type();
+    auto mock = std::make_unique<MockFileBatchReader>(data, type, /*batch_size=*/3);
+    ASSERT_OK_AND_ASSIGN(
+        auto reader, LateMaterializingFileBatchReader::Create(std::move(mock), GetDefaultPool()));
+    // probe columns = {a (idx0), c (idx2)}; payload columns = {b, d, e}
+    auto pred_a =
+        PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "a", FieldType::BIGINT, Literal(3l));
+    auto pred_c =
+        PredicateBuilder::LessThan(/*field_index=*/2, "c", FieldType::BIGINT, Literal(700l));
+    ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({pred_a, pred_c}));
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(type->fields()), predicate, std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::StructArray> result, CollectStruct(reader.get()));
+    ASSERT_TRUE(result);
+    // a >= 3 and c(=i*100) < 700 -> i in {3,4,5,6}
+    ASSERT_EQ(result->length(), 4);
+    // output field order must equal the requested full schema order
+    ASSERT_EQ(result->num_fields(), 5);
+    auto out_type = arrow::internal::checked_pointer_cast<arrow::StructType>(result->type());
+    EXPECT_EQ(out_type->field(0)->name(), "a");
+    EXPECT_EQ(out_type->field(1)->name(), "b");
+    EXPECT_EQ(out_type->field(2)->name(), "c");
+    EXPECT_EQ(out_type->field(3)->name(), "d");
+    EXPECT_EQ(out_type->field(4)->name(), "e");
+    auto a = arrow::internal::checked_pointer_cast<arrow::Int64Array>(result->GetFieldByName("a"));
+    auto b = arrow::internal::checked_pointer_cast<arrow::StringArray>(result->GetFieldByName("b"));
+    auto c = arrow::internal::checked_pointer_cast<arrow::Int64Array>(result->GetFieldByName("c"));
+    auto d = arrow::internal::checked_pointer_cast<arrow::StringArray>(result->GetFieldByName("d"));
+    auto e = arrow::internal::checked_pointer_cast<arrow::Int64Array>(result->GetFieldByName("e"));
+    const std::vector<int64_t> expected = {3, 4, 5, 6};
+    for (size_t j = 0; j < expected.size(); ++j) {
+        int64_t i = expected[j];
+        EXPECT_EQ(a->Value(j), i);
+        EXPECT_EQ(b->GetString(j), "b_" + std::to_string(i));
+        EXPECT_EQ(c->Value(j), i * 100);
+        EXPECT_EQ(d->GetString(j), "d_" + std::to_string(i));
+        EXPECT_EQ(e->Value(j), i * 10000);
+    }
+}
+
+// A nested (list) payload column must round-trip unchanged for the matched rows.
+TEST_F(LateMaterializingFileBatchReaderTest, NestedPayloadColumn) {
+    auto data = BuildNestedData(8);
+    auto type = data->type();
+    auto mock = std::make_unique<MockFileBatchReader>(data, type, /*batch_size=*/3);
+    ASSERT_OK_AND_ASSIGN(
+        auto reader, LateMaterializingFileBatchReader::Create(std::move(mock), GetDefaultPool()));
+    // probe = {k}; payload = {arr (list<int64>), tag}
+    auto predicate =
+        PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, Literal(5l));
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(type->fields()), predicate, std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::StructArray> result, CollectStruct(reader.get()));
+    ASSERT_TRUE(result);
+    ASSERT_EQ(result->length(), 3);  // k = 5,6,7
+    auto k = arrow::internal::checked_pointer_cast<arrow::Int64Array>(result->GetFieldByName("k"));
+    auto arr =
+        arrow::internal::checked_pointer_cast<arrow::ListArray>(result->GetFieldByName("arr"));
+    auto tag =
+        arrow::internal::checked_pointer_cast<arrow::StringArray>(result->GetFieldByName("tag"));
+    ASSERT_TRUE(k && arr && tag);
+    for (int64_t j = 0; j < result->length(); ++j) {
+        int64_t i = 5 + j;
+        EXPECT_EQ(k->Value(j), i);
+        EXPECT_EQ(tag->GetString(j), "t_" + std::to_string(i));
+        auto sub = arrow::internal::checked_pointer_cast<arrow::Int64Array>(arr->value_slice(j));
+        ASSERT_EQ(sub->length(), 2);
+        EXPECT_EQ(sub->Value(0), i);
+        EXPECT_EQ(sub->Value(1), i + 1);
+    }
+}
+
+// The late-materialization reader must work correctly as an inner reader driven by
+// PrefetchFileBatchReaderImpl (schema broadcast, range dispatch, seek, row-id tracking).
+TEST_F(LateMaterializingFileBatchReaderTest, WorksAsInnerOfPrefetchReader) {
+    auto data = BuildData({0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+    LateMaterializingReaderBuilder builder(
+        std::make_unique<MockFormatReaderBuilder>(data, full_type_, /*batch_size=*/3),
+        GetDefaultPool());
+    auto mock_fs = std::make_shared<MockFileSystem>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Executor> executor, CreateDefaultExecutor(2));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<PrefetchFileBatchReaderImpl> impl,
+        PrefetchFileBatchReaderImpl::Create(
+            /*data_file_path=*/"", /*data_file_size=*/0, &builder, mock_fs,
+            /*prefetch_max_parallel_num=*/1, /*batch_size=*/3, /*prefetch_batch_count=*/2,
+            /*enable_adaptive_prefetch_strategy=*/false, executor,
+            /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/false, CacheConfig(),
+            GetDefaultPool()));
+    auto predicate =
+        PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, Literal(4l));
+    ::ArrowSchema c_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(full_fields_), &c_schema).ok());
+    ASSERT_OK(impl->SetReadSchema(&c_schema, predicate, std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::StructArray> result, CollectStruct(impl.get()));
+    ASSERT_TRUE(result);
+    ASSERT_EQ(result->length(), 6);  // k = 4..9
+    auto k = arrow::internal::checked_pointer_cast<arrow::Int64Array>(result->GetFieldByName("k"));
+    auto v = arrow::internal::checked_pointer_cast<arrow::StringArray>(result->GetFieldByName("v"));
+    ASSERT_TRUE(k && v);
+    for (int64_t j = 0; j < result->length(); ++j) {
+        EXPECT_EQ(k->Value(j), 4 + j);
+        EXPECT_EQ(v->GetString(j), "v_" + std::to_string(4 + j));
+    }
+    impl->Close();
+}
+
+// Re-setting the read schema on the prefetch impl (which re-broadcasts to the inner LM readers and
+// re-plans ranges) must reset the probe state and produce correct results for the new predicate.
+TEST_F(LateMaterializingFileBatchReaderTest, PrefetchInnerReentrantSetReadSchema) {
+    auto data = BuildData({0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+    LateMaterializingReaderBuilder builder(
+        std::make_unique<MockFormatReaderBuilder>(data, full_type_, /*batch_size=*/3),
+        GetDefaultPool());
+    auto mock_fs = std::make_shared<MockFileSystem>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Executor> executor, CreateDefaultExecutor(2));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<PrefetchFileBatchReaderImpl> impl,
+        PrefetchFileBatchReaderImpl::Create(
+            /*data_file_path=*/"", /*data_file_size=*/0, &builder, mock_fs,
+            /*prefetch_max_parallel_num=*/1, /*batch_size=*/3, /*prefetch_batch_count=*/2,
+            /*enable_adaptive_prefetch_strategy=*/false, executor,
+            /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/false, CacheConfig(),
+            GetDefaultPool()));
+
+    auto full_schema = arrow::schema(full_fields_);
+    auto predicate1 =
+        PredicateBuilder::GreaterThan(/*field_index=*/0, "k", FieldType::BIGINT, Literal(6l));
+    ::ArrowSchema c_schema1;
+    ASSERT_TRUE(arrow::ExportSchema(*full_schema, &c_schema1).ok());
+    ASSERT_OK(impl->SetReadSchema(&c_schema1, predicate1, std::nullopt));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::StructArray> result1, CollectStruct(impl.get()));
+    ASSERT_TRUE(result1);
+    ASSERT_EQ(result1->length(), 3);  // k = 7,8,9
+    auto k1 =
+        arrow::internal::checked_pointer_cast<arrow::Int64Array>(result1->GetFieldByName("k"));
+    auto v1 =
+        arrow::internal::checked_pointer_cast<arrow::StringArray>(result1->GetFieldByName("v"));
+    ASSERT_TRUE(k1 && v1);
+    for (int64_t j = 0; j < result1->length(); ++j) {
+        EXPECT_EQ(k1->Value(j), 7 + j);
+        EXPECT_EQ(v1->GetString(j), "v_" + std::to_string(7 + j));
+    }
+
+    auto predicate2 =
+        PredicateBuilder::LessThan(/*field_index=*/0, "k", FieldType::BIGINT, Literal(3l));
+    ::ArrowSchema c_schema2;
+    ASSERT_TRUE(arrow::ExportSchema(*full_schema, &c_schema2).ok());
+    ASSERT_OK(impl->SetReadSchema(&c_schema2, predicate2, std::nullopt));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::StructArray> result2, CollectStruct(impl.get()));
+    ASSERT_TRUE(result2);
+    ASSERT_EQ(result2->length(), 3);  // k = 0,1,2
+    auto k2 =
+        arrow::internal::checked_pointer_cast<arrow::Int64Array>(result2->GetFieldByName("k"));
+    auto v2 =
+        arrow::internal::checked_pointer_cast<arrow::StringArray>(result2->GetFieldByName("v"));
+    for (int64_t j = 0; j < result2->length(); ++j) {
+        EXPECT_EQ(k2->Value(j), j);
+        EXPECT_EQ(v2->GetString(j), "v_" + std::to_string(j));
+    }
+    impl->Close();
+}
+
+// With multiple parallel inner readers and per-batch ranges, the prefetch impl dispatches disjoint
+// ranges to each LM reader and drives them via EnsureReaderPosition/SeekToRow. The merged output
+// must still be exactly the matched rows in ascending file order.
+TEST_F(LateMaterializingFileBatchReaderTest, PrefetchInnerParallelReadersWithSeek) {
+    std::vector<int64_t> ks;
+    for (int i = 0; i < 20; ++i) {
+        ks.push_back(i);
+    }
+    auto data = BuildData(ks);
+    // Per-batch ranges (the mock's default) let the impl split work across the parallel readers,
+    // and each range-honoring reader only reads its assigned slice.
+    LateMaterializingReaderBuilder builder(
+        std::make_unique<MockFormatReaderBuilder>(data, full_type_, /*batch_size=*/3),
+        GetDefaultPool());
+    auto mock_fs = std::make_shared<MockFileSystem>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Executor> executor, CreateDefaultExecutor(3));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<PrefetchFileBatchReaderImpl> impl,
+        PrefetchFileBatchReaderImpl::Create(
+            /*data_file_path=*/"", /*data_file_size=*/0, &builder, mock_fs,
+            /*prefetch_max_parallel_num=*/3, /*batch_size=*/3, /*prefetch_batch_count=*/6,
+            /*enable_adaptive_prefetch_strategy=*/false, executor,
+            /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/false, CacheConfig(),
+            GetDefaultPool()));
+    auto predicate =
+        PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, Literal(5l));
+    ::ArrowSchema c_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(full_fields_), &c_schema).ok());
+    ASSERT_OK(impl->SetReadSchema(&c_schema, predicate, std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::StructArray> result, CollectStruct(impl.get()));
+    ASSERT_TRUE(result);
+    ASSERT_EQ(result->length(), 15);  // k = 5..19
+    auto k = arrow::internal::checked_pointer_cast<arrow::Int64Array>(result->GetFieldByName("k"));
+    auto v = arrow::internal::checked_pointer_cast<arrow::StringArray>(result->GetFieldByName("v"));
+    ASSERT_TRUE(k && v);
+    for (int64_t j = 0; j < result->length(); ++j) {
+        EXPECT_EQ(k->Value(j), 5 + j);
+        EXPECT_EQ(v->GetString(j), "v_" + std::to_string(5 + j));
+    }
+    impl->Close();
+}
+
+// When the predicate's field type does not match the probe schema, the
+// ValidatePredicateWithSchema check must fail with a clear error instead
+// of silently producing incorrect results.
+TEST_F(LateMaterializingFileBatchReaderTest, FailsOnPredicateTypeMismatch) {
+    auto data = BuildData({0, 1, 2, 3, 4});
+    auto mock = std::make_unique<MockFileBatchReader>(data, full_type_, /*batch_size=*/2);
+    ASSERT_OK_AND_ASSIGN(
+        auto reader, LateMaterializingFileBatchReader::Create(std::move(mock), GetDefaultPool()));
+    // k is int64 in the schema, but the predicate claims FieldType::INT (int32).
+    auto predicate =
+        PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k", FieldType::INT, Literal(10));
+    ASSERT_NOK_WITH_MSG(
+        SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt),
+        "mismatches");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/common/reader/late_materializing_reader_builder.h b/src/paimon/common/reader/late_materializing_reader_builder.h
new file mode 100644
index 000000000..5c7be5077
--- /dev/null
+++ b/src/paimon/common/reader/late_materializing_reader_builder.h
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <utility>
+
+#include "paimon/common/reader/late_materializing_file_batch_reader.h"
+#include "paimon/format/reader_builder.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/reader/prefetch_file_batch_reader.h"
+#include "paimon/result.h"
+
+namespace paimon {
+
+class LateMaterializingReaderBuilder : public ReaderBuilder {
+ public:
+    LateMaterializingReaderBuilder(std::unique_ptr<ReaderBuilder> inner,
+                                   std::shared_ptr<MemoryPool> pool)
+        : inner_(std::move(inner)), pool_(std::move(pool)) {}
+
+    ReaderBuilder* WithMemoryPool(const std::shared_ptr<MemoryPool>& pool) override {
+        pool_ = pool;
+        inner_->WithMemoryPool(pool);
+        return this;
+    }
+
+    ReaderBuilder* WithCache(const std::shared_ptr<Cache>& cache) override {
+        inner_->WithCache(cache);
+        return this;
+    }
+
+    ReaderBuilder* WithReadHints(const std::optional<ReadHints>& hints) override {
+        inner_->WithReadHints(hints);
+        return this;
+    }
+
+    Result<std::unique_ptr<FileBatchReader>> Build(
+        const std::shared_ptr<InputStream>& stream) const override {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileBatchReader> format_reader,
+                               inner_->Build(stream));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<LateMaterializingFileBatchReader> reader,
+            LateMaterializingFileBatchReader::Create(std::move(format_reader), pool_));
+        return std::unique_ptr<FileBatchReader>(std::move(reader));
+    }
+
+ private:
+    std::unique_ptr<ReaderBuilder> inner_;
+    std::shared_ptr<MemoryPool> pool_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp
index 12e38966d..369bb58fe 100644
--- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp
+++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp
@@ -411,7 +411,8 @@ std::optional<std::pair<uint64_t, uint64_t>> PrefetchFileBatchReaderImpl::GetCur
 Status PrefetchFileBatchReaderImpl::EnsureReaderPosition(
     size_t reader_idx, const std::pair<uint64_t, uint64_t>& current_read_range) const {
     uint64_t pos = std::max(readers_pos_[reader_idx]->load(), current_read_range.first);
-    if (readers_[reader_idx]->GetNextRowToRead() != pos) {
+    PAIMON_ASSIGN_OR_RAISE(uint64_t next_row_to_read, readers_[reader_idx]->GetNextRowToRead());
+    if (next_row_to_read != pos) {
         return readers_[reader_idx]->SeekToRow(pos);
     }
     return Status::OK();
@@ -480,7 +481,9 @@ Status PrefetchFileBatchReaderImpl::HandleReadResult(
         } else {
             // all within the range, data before readers_[reader_idx]->GetNextRowToRead() has been
             // effectively consumed
-            readers_pos_[reader_idx]->store(readers_[reader_idx]->GetNextRowToRead());
+            PAIMON_ASSIGN_OR_RAISE(uint64_t next_row_to_read,
+                                   readers_[reader_idx]->GetNextRowToRead());
+            readers_pos_[reader_idx]->store(next_row_to_read);
         }
         if (bitmap.IsEmpty()) {
             ReaderUtils::ReleaseReadBatch(std::move(read_batch));
@@ -646,7 +649,7 @@ Result<uint64_t> PrefetchFileBatchReaderImpl::GetNumberOfRows() const {
     return readers_[0]->GetNumberOfRows();
 }
 
-uint64_t PrefetchFileBatchReaderImpl::GetNextRowToRead() const {
+Result<uint64_t> PrefetchFileBatchReaderImpl::GetNextRowToRead() const {
     assert(false);
     return -1;
 }
diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h
index c21856d09..4750501e2 100644
--- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h
+++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h
@@ -80,7 +80,7 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader {
     Status SeekToRow(uint64_t row_number) override;
     Result<uint64_t> GetPreviousBatchFileRowId(uint64_t batch_row_id) const override;
     Result<uint64_t> GetNumberOfRows() const override;
-    uint64_t GetNextRowToRead() const override;
+    Result<uint64_t> GetNextRowToRead() const override;
     void Close() override;
     Status SetReadRanges(const std::vector<std::pair<uint64_t, uint64_t>>& read_ranges) override;
 
diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp b/src/paimon/common/utils/arrow/arrow_utils.cpp
index f29e1d11e..c6a07b2bf 100644
--- a/src/paimon/common/utils/arrow/arrow_utils.cpp
+++ b/src/paimon/common/utils/arrow/arrow_utils.cpp
@@ -449,9 +449,7 @@ Result<std::shared_ptr<arrow::RecordBatch>> ArrowUtils::NormalizeRecordBatchOffs
         if (normalized_columns.empty()) {
             normalized_columns = record_batch->columns();
         }
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::ArrayData> normalized_data,
-                               RebaseToZeroOffset(column->data(), pool));
-        normalized_columns[i] = arrow::MakeArray(normalized_data);
+        PAIMON_ASSIGN_OR_RAISE(normalized_columns[i], NormalizeArrayOffsets(column, pool));
     }
     if (normalized_columns.empty()) {
         return record_batch;
@@ -460,6 +458,13 @@ Result<std::shared_ptr<arrow::RecordBatch>> ArrowUtils::NormalizeRecordBatchOffs
                                     std::move(normalized_columns));
 }
 
+Result<std::shared_ptr<arrow::Array>> ArrowUtils::NormalizeArrayOffsets(
+    const std::shared_ptr<arrow::Array>& array, arrow::MemoryPool* pool) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::ArrayData> normalized_data,
+                           RebaseToZeroOffset(array->data(), pool));
+    return arrow::MakeArray(normalized_data);
+}
+
 Result<arrow::Compression::type> ArrowUtils::GetCompressionType(const std::string& compression) {
     std::string normalized = StringUtils::ToLowerCase(compression);
     if (normalized.empty() || normalized == "none") {
diff --git a/src/paimon/common/utils/arrow/arrow_utils.h b/src/paimon/common/utils/arrow/arrow_utils.h
index 326b3889e..13bd81549 100644
--- a/src/paimon/common/utils/arrow/arrow_utils.h
+++ b/src/paimon/common/utils/arrow/arrow_utils.h
@@ -57,6 +57,9 @@ class PAIMON_EXPORT ArrowUtils {
     static Result<std::shared_ptr<arrow::RecordBatch>> NormalizeRecordBatchOffsets(
         const std::shared_ptr<arrow::RecordBatch>& record_batch, arrow::MemoryPool* pool);
 
+    static Result<std::shared_ptr<arrow::Array>> NormalizeArrayOffsets(
+        const std::shared_ptr<arrow::Array>& array, arrow::MemoryPool* pool);
+
     static bool EqualsIgnoreNullable(const std::shared_ptr<arrow::DataType>& type,
                                      const std::shared_ptr<arrow::DataType>& other_type);
 
diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp
index d057e1d72..89f5071e9 100644
--- a/src/paimon/core/operation/abstract_split_read.cpp
+++ b/src/paimon/core/operation/abstract_split_read.cpp
@@ -33,6 +33,7 @@
 #include "paimon/common/data/shredding/shredding_file_reader.h"
 #include "paimon/common/data/variant/variant_shredding_read_plan_factory.h"
 #include "paimon/common/reader/delegating_prefetch_reader.h"
+#include "paimon/common/reader/late_materializing_reader_builder.h"
 #include "paimon/common/reader/predicate_batch_reader.h"
 #include "paimon/common/reader/prefetch_file_batch_reader_impl.h"
 #include "paimon/common/table/special_fields.h"
@@ -96,7 +97,7 @@ Result<std::vector<std::unique_ptr<FileBatchReader>>> AbstractSplitRead::CreateR
                                PrepareReaderBuilder(data_file_identifier, extra_format_options));
         PAIMON_ASSIGN_OR_RAISE(
             std::unique_ptr<FileBatchReader> file_reader,
-            CreateFieldMappingReader(data_file_path, file, partition, reader_builder.get(),
+            CreateFieldMappingReader(data_file_path, file, partition, std::move(reader_builder),
                                      field_mapping_builder.get(), dv_factory, row_ranges,
                                      data_file_path_factory));
         if (file_reader) {
@@ -151,13 +152,17 @@ Result<std::unique_ptr<ReaderBuilder>> AbstractSplitRead::PrepareReaderBuilder(
 
 Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::CreateFileBatchReader(
     const std::string& file_format_identifier, const std::string& data_file_path,
-    int64_t data_file_size, const ReaderBuilder* reader_builder) const {
+    int64_t data_file_size, std::unique_ptr<ReaderBuilder> reader_builder) const {
+    if (context_->EnableLateMaterializing()) {
+        reader_builder =
+            std::make_unique<LateMaterializingReaderBuilder>(std::move(reader_builder), pool_);
+    }
     if (context_->EnablePrefetch() && file_format_identifier != "blob" &&
         file_format_identifier != "avro") {
         PAIMON_ASSIGN_OR_RAISE(
             std::unique_ptr<PrefetchFileBatchReaderImpl> prefetch_reader,
             PrefetchFileBatchReaderImpl::Create(
-                data_file_path, data_file_size, reader_builder, options_.GetFileSystem(),
+                data_file_path, data_file_size, reader_builder.get(), options_.GetFileSystem(),
                 context_->GetPrefetchMaxParallelNum(), options_.GetReadBatchSize(),
                 context_->GetPrefetchBatchCount(), options_.EnableAdaptivePrefetchStrategy(),
                 executor_,
@@ -174,7 +179,7 @@ Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::CreateFileBatchReade
 
 Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::CreateFieldMappingReader(
     const std::string& data_file_path, const std::shared_ptr<DataFileMeta>& file_meta,
-    const BinaryRow& partition, const ReaderBuilder* reader_builder,
+    const BinaryRow& partition, std::unique_ptr<ReaderBuilder> reader_builder,
     const FieldMappingBuilder* field_mapping_builder, DeletionVector::Factory dv_factory,
     const std::optional<std::vector<Range>>& row_ranges,
     const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) const {
@@ -214,7 +219,7 @@ Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::CreateFieldMappingRe
     PAIMON_ASSIGN_OR_RAISE(std::string file_format_identifier, file_meta->FileFormat());
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileBatchReader> file_reader,
                            CreateFileBatchReader(file_format_identifier, data_file_path,
-                                                 file_meta->file_size, reader_builder));
+                                                 file_meta->file_size, std::move(reader_builder)));
     if (VectorFileBatchReader::ContainsVector(read_schema)) {
         file_reader = std::make_unique<VectorFileBatchReader>(std::move(file_reader), pool_);
     }
diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h
index a56b48fdf..a02ed5fb2 100644
--- a/src/paimon/core/operation/abstract_split_read.h
+++ b/src/paimon/core/operation/abstract_split_read.h
@@ -107,12 +107,12 @@ class AbstractSplitRead : public SplitRead {
 
     Result<std::unique_ptr<FileBatchReader>> CreateFileBatchReader(
         const std::string& file_format_identifier, const std::string& data_file_path,
-        int64_t data_file_size, const ReaderBuilder* reader_builder) const;
+        int64_t data_file_size, std::unique_ptr<ReaderBuilder> reader_builder) const;
 
     // return nullptr if data file is skipped by index or dv
     Result<std::unique_ptr<FileBatchReader>> CreateFieldMappingReader(
         const std::string& data_file_path, const std::shared_ptr<DataFileMeta>& file_meta,
-        const BinaryRow& partition, const ReaderBuilder* reader_builder,
+        const BinaryRow& partition, std::unique_ptr<ReaderBuilder> reader_builder,
         const FieldMappingBuilder* field_mapping_builder, DeletionVector::Factory dv_factory,
         const std::optional<std::vector<Range>>& row_ranges,
         const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) const;
diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h
index fad0e6742..2568eae46 100644
--- a/src/paimon/core/operation/data_evolution_split_read.h
+++ b/src/paimon/core/operation/data_evolution_split_read.h
@@ -64,7 +64,8 @@ struct DeletionFile;
 /// ->(ConcatBatchReader across blob files | BlobFallbackBatchReader across blob sequence layers)
 /// ->FieldMappingReader->(ApplyDeletionVectorBatchReader)->(ApplyBitmapIndexBatchReader)
 /// ->(CompleteRowTrackingFieldsBatchReader)->(ShreddingFileReader)
-/// ->(VectorFileBatchReader)->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader
+/// ->(VectorFileBatchReader)->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)
+/// ->(LateMaterializingFileBatchReader)->FormatReader
 ///
 ///
 /// A union `SplitRead` to read multiple inner files to merge columns. A single-file row range
diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h
index 8ef9f2d26..8e773cdcd 100644
--- a/src/paimon/core/operation/internal_read_context.h
+++ b/src/paimon/core/operation/internal_read_context.h
@@ -74,6 +74,9 @@ class InternalReadContext {
     bool EnablePrefetch() const {
         return read_context_->EnablePrefetch();
     }
+    bool EnableLateMaterializing() const {
+        return read_context_->EnableLateMaterializing();
+    }
     uint32_t GetPrefetchBatchCount() const {
         return read_context_->GetPrefetchBatchCount();
     }
diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h
index 5003cb55a..11dcd0b37 100644
--- a/src/paimon/core/operation/merge_file_split_read.h
+++ b/src/paimon/core/operation/merge_file_split_read.h
@@ -74,7 +74,8 @@ class MergeFunctionWrapper;
 /// files->KeyValueProjectionReader/AsyncKeyValueProjectionReader
 /// ->DropDeleteReader->SortMergeReader->ConcatKeyValueRecordReader->KeyValueDataFileRecordReader
 /// ->FieldMappingReader->(ApplyDeletionVectorBatchReader)->(ShreddingFileReader)
-/// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader
+/// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)
+/// ->(LateMaterializingFileBatchReader)->FormatReader
 class MergeFileSplitRead : public AbstractSplitRead {
  public:
     static Result<std::unique_ptr<MergeFileSplitRead>> Create(
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index 911cf7961..19cd96d67 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -798,6 +798,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) {
     context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"},
                                 {Options::MERGE_ENGINE, "deduplicate"},
                                 {Options::IGNORE_DELETE, "true"}});
+    context_builder.EnableLateMaterializing(false);
     AddOptions(&context_builder);
 
     // less_than will be ignore as it is partition predicate
@@ -842,6 +843,63 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) {
     CheckResult(result_array, expected_array, read_schema);
 }
 
+TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) {
+    std::string path =
+        paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
+    ReadContextBuilder context_builder(path);
+
+    std::vector<DataField> raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())),
+                                              DataField(3, arrow::field("p1", arrow::int32())),
+                                              DataField(5, arrow::field("s1", arrow::utf8())),
+                                              DataField(4, arrow::field("s0", arrow::utf8())),
+                                              DataField(6, arrow::field("v0", arrow::float64())),
+                                              DataField(7, arrow::field("v1", arrow::boolean()))};
+    auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
+    ASSERT_TRUE(read_schema);
+
+    context_builder.SetReadFieldNames({"k1", "p1", "s1", "s0", "v0", "v1"});
+    context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"},
+                                {Options::MERGE_ENGINE, "deduplicate"},
+                                {Options::IGNORE_DELETE, "true"}});
+    AddOptions(&context_builder);
+    context_builder.EnableLateMaterializing(true);
+    // key predicate, always pushed down into the data files
+    auto greater_or_equal = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"k1",
+                                                             FieldType::INT, Literal(1));
+    // value predicate, only pushed down when a section holds a single sorted run
+    auto greater_than = PredicateBuilder::GreaterThan(/*field_index=*/4, /*field_name=*/"v0",
+                                                      FieldType::DOUBLE, Literal(12.0));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> predicate_result,
+                         PredicateBuilder::And({greater_or_equal, greater_than}));
+    context_builder.SetPredicate(predicate_result);
+    context_builder.EnablePredicateFilter(true).EnableLateMaterializing(true);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
+
+    auto internal_context = CreateInternalReadContext(read_context);
+    ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit()));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result_array,
+                         ReadResultCollector::CollectResult(batch_reader.get()));
+
+    auto fields_with_row_kind = read_schema->fields();
+    fields_with_row_kind.insert(fields_with_row_kind.begin(),
+                                arrow::field("_VALUE_KIND", arrow::int8()));
+
+    // Only the merged rows with k1 >= 1 and v0 > 12.0 remain.
+    std::shared_ptr<arrow::ChunkedArray> expected_array;
+    auto array_status =
+        arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow::struct_(fields_with_row_kind), {R"([
+                        [0, 1, 0, "!",      "driver", 13.3, false],
+                        [0, 2, 0, "!",      "driver", 13.3, false],
+                        [0, 200, 0, "number", "max",  140.4, false],
+                        [0, 1, 1, "you",    "zoo",    130.0, false]
+
+    ])"},
+                                                         &expected_array);
+    ASSERT_TRUE(array_status.ok());
+    CheckResult(result_array, expected_array, read_schema);
+}
+
 TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) {
     std::string path =
         paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h
index 93eab5509..646f24ac7 100644
--- a/src/paimon/core/operation/raw_file_split_read.h
+++ b/src/paimon/core/operation/raw_file_split_read.h
@@ -55,7 +55,8 @@ struct DeletionFile;
 /// ->ConcatBatchReader across
 /// files->FieldMappingReader->(ApplyBitmapIndexBatchReader)->(CompleteRowTrackingFieldsBatchReader)
 /// ->(ShreddingFileReader)->(VectorFileBatchReader)
-/// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader
+/// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)
+/// ->(LateMaterializingFileBatchReader)->FormatReader
 
 class RawFileSplitRead : public AbstractSplitRead {
  public:
diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp
index 08a854d84..deacfa78b 100644
--- a/src/paimon/core/operation/read_context.cpp
+++ b/src/paimon/core/operation/read_context.cpp
@@ -35,10 +35,10 @@ ReadContext::ReadContext(
     const std::string& path, const std::string& branch,
     const std::vector<std::string>& read_field_names, const std::vector<int32_t>& read_field_ids,
     const std::shared_ptr<Predicate>& predicate, bool enable_predicate_filter, bool enable_prefetch,
-    uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num,
-    bool enable_multi_thread_row_to_batch, uint32_t row_to_batch_thread_number,
-    const std::optional<std::string>& table_schema, const std::shared_ptr<MemoryPool>& memory_pool,
-    const std::shared_ptr<Executor>& executor,
+    bool enable_late_materializing, uint32_t prefetch_batch_count,
+    uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch,
+    uint32_t row_to_batch_thread_number, const std::optional<std::string>& table_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool, const std::shared_ptr<Executor>& executor,
     const std::shared_ptr<FileSystem>& specific_file_system,
     const std::map<std::string, std::string>& fs_scheme_to_identifier_map,
     const std::shared_ptr<RealtimeContext>& realtime_context,
@@ -51,6 +51,7 @@ ReadContext::ReadContext(
       predicate_(predicate),
       enable_predicate_filter_(enable_predicate_filter),
       enable_prefetch_(enable_prefetch),
+      enable_late_materializing_(enable_late_materializing),
       prefetch_batch_count_(prefetch_batch_count),
       prefetch_max_parallel_num_(prefetch_max_parallel_num),
       enable_multi_thread_row_to_batch_(enable_multi_thread_row_to_batch),
@@ -97,6 +98,7 @@ class ReadContextBuilder::Impl {
         predicate_.reset();
         enable_predicate_filter_ = false;
         enable_prefetch_ = false;
+        enable_late_materializing_ = false;
         read_ahead_cache_enabled_ = true;
         prefetch_batch_count_ = 600;
         prefetch_max_parallel_num_ = 3;
@@ -122,6 +124,7 @@ class ReadContextBuilder::Impl {
     std::shared_ptr<Predicate> predicate_;
     bool enable_predicate_filter_ = false;
     bool enable_prefetch_ = false;
+    bool enable_late_materializing_ = false;
     uint32_t prefetch_batch_count_ = 600;
     uint32_t prefetch_max_parallel_num_ = 3;
     bool enable_multi_thread_row_to_batch_ = false;
@@ -191,6 +194,11 @@ ReadContextBuilder& ReadContextBuilder::EnablePrefetch(bool enabled) {
     return *this;
 }
 
+ReadContextBuilder& ReadContextBuilder::EnableLateMaterializing(bool enabled) {
+    impl_->enable_late_materializing_ = enabled;
+    return *this;
+}
+
 ReadContextBuilder& ReadContextBuilder::SetPrefetchBatchCount(uint32_t batch_count) {
     impl_->prefetch_batch_count_ = batch_count;
     return *this;
@@ -297,11 +305,12 @@ Result<std::unique_ptr<ReadContext>> ReadContextBuilder::Finish() {
     auto ctx = std::make_unique<ReadContext>(
         impl_->path_, impl_->branch_, impl_->read_field_names_, impl_->read_field_ids_,
         impl_->predicate_, impl_->enable_predicate_filter_, impl_->enable_prefetch_,
-        impl_->prefetch_batch_count_, impl_->prefetch_max_parallel_num_,
-        impl_->enable_multi_thread_row_to_batch_, impl_->row_to_batch_thread_number_,
-        impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_,
-        impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_,
-        impl_->read_ahead_cache_enabled_, impl_->cache_config_, impl_->cache_);
+        impl_->enable_late_materializing_, impl_->prefetch_batch_count_,
+        impl_->prefetch_max_parallel_num_, impl_->enable_multi_thread_row_to_batch_,
+        impl_->row_to_batch_thread_number_, impl_->table_schema_, impl_->memory_pool_,
+        impl_->executor_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_,
+        impl_->realtime_context_, impl_->options_, impl_->read_ahead_cache_enabled_,
+        impl_->cache_config_, impl_->cache_);
     if (impl_->read_schema_ && impl_->read_schema_->release) {
         ctx->SetReadSchema(std::move(impl_->read_schema_));
     }
diff --git a/src/paimon/format/orc/orc_file_batch_reader.h b/src/paimon/format/orc/orc_file_batch_reader.h
index 85673a93e..b48f7cdb0 100644
--- a/src/paimon/format/orc/orc_file_batch_reader.h
+++ b/src/paimon/format/orc/orc_file_batch_reader.h
@@ -85,7 +85,7 @@ class OrcFileBatchReader : public PrefetchFileBatchReader {
         return reader_->GetNumberOfRows();
     }
 
-    uint64_t GetNextRowToRead() const override {
+    Result<uint64_t> GetNextRowToRead() const override {
         return reader_->GetNextRowToRead();
     }
 
diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp
index 0c9d065e5..7605c4242 100644
--- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp
+++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp
@@ -154,6 +154,7 @@ ParquetFileBatchReader::ParquetFileBatchReader(
       arrow_pool_(arrow_pool),
       input_stream_(std::move(input_stream)),
       reader_(std::move(reader)),
+      read_ranges_(reader_->GetAllRowGroupRanges()),
       metrics_(std::make_shared<MetricsImpl>()),
       storage_read_bytes_(std::move(storage_read_bytes)),
       logger_(Logger::GetLogger("ParquetFileBatchReader")) {}
@@ -309,6 +310,7 @@ Status ParquetFileBatchReader::SetReadSchema(
 
         PAIMON_RETURN_NOT_OK(UpdateAllTargetRowRanges(target_row_groups));
         PAIMON_RETURN_NOT_OK(reader_->PrepareForReadingLazy(target_row_groups, column_indices));
+        PAIMON_RETURN_NOT_OK(reader_->ApplyReadRanges(read_ranges_));
     }
     PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::SetReadSchema")
     return Status::OK();
diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h
index daa18f040..2b1097fb4 100644
--- a/src/paimon/format/parquet/parquet_file_batch_reader.h
+++ b/src/paimon/format/parquet/parquet_file_batch_reader.h
@@ -128,12 +128,13 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader {
         return reader_->GetNumberOfRows();
     }
 
-    uint64_t GetNextRowToRead() const override {
+    Result<uint64_t> GetNextRowToRead() const override {
         assert(reader_);
         return reader_->GetNextRowToRead();
     }
 
     Status SetReadRanges(const std::vector<std::pair<uint64_t, uint64_t>>& read_ranges) override {
+        read_ranges_ = read_ranges;
         return reader_->ApplyReadRanges(read_ranges);
     }
 
@@ -261,6 +262,8 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader {
 
     std::shared_ptr<arrow::DataType> read_data_type_;
 
+    std::vector<std::pair<uint64_t, uint64_t>> read_ranges_;
+
     std::shared_ptr<Metrics> metrics_;
     // storageReadBytes counter shared with the underlying ArrowInputStreamAdapter.
     std::shared_ptr<std::atomic<uint64_t>> storage_read_bytes_;
diff --git a/src/paimon/testing/mock/mock_file_batch_reader.h b/src/paimon/testing/mock/mock_file_batch_reader.h
index f05a2347b..4566289f4 100644
--- a/src/paimon/testing/mock/mock_file_batch_reader.h
+++ b/src/paimon/testing/mock/mock_file_batch_reader.h
@@ -29,6 +29,7 @@
 
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
+#include "arrow/util/checked_cast.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/reader/reader_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
@@ -78,11 +79,14 @@ class MockFileBatchReader : public PrefetchFileBatchReader {
 
     Status SetReadSchema(::ArrowSchema* read_schema, const std::shared_ptr<Predicate>& predicate,
                          const std::optional<RoaringBitmap32>& selection_bitmap) override {
-        // Noted that SetReadSchema only change inner read_schema_, but take no effective on
-        // NextBatch
         PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> arrow_schema,
                                           arrow::ImportSchema(read_schema));
         read_schema_ = arrow_schema;
+        // A real FileBatchReader restarts from the first row and drops its assigned read ranges
+        // when the read schema is (re)set. Readers that switch schemas mid-file, such as the
+        // late-materialization reader moving from its probe pass to its payload pass, rely on it.
+        current_pos_ = 0;
+        previous_batch_first_row_num_ = std::numeric_limits<uint64_t>::max();
         return Status::OK();
     }
 
@@ -119,8 +123,29 @@ class MockFileBatchReader : public PrefetchFileBatchReader {
     Result<ReadBatchWithBitmap> NextBatchWithBitmap() override {
         while (true) {
             PAIMON_RETURN_NOT_OK(next_batch_status_);
-            if (current_pos_ >= read_end_pos_) {
-                previous_batch_first_row_num_ = current_pos_;
+            int32_t begin_pos = current_pos_;
+            int32_t range_end_pos = read_end_pos_;
+            if (!read_ranges_.empty()) {
+                // Reading is restricted to the assigned ranges (ascending and half-open), like a
+                // real format reader, so that a prefetch reader may dispatch disjoint ranges to
+                // parallel readers. An empty range set means the whole file may be read.
+                const std::pair<uint64_t, uint64_t>* selected = nullptr;
+                for (const auto& range : read_ranges_) {
+                    if (static_cast<int64_t>(range.second) > begin_pos) {
+                        selected = &range;
+                        break;
+                    }
+                }
+                if (selected == nullptr) {
+                    previous_batch_first_row_num_ = ToReaderRowNumber(begin_pos);
+                    return BatchReader::MakeEofBatchWithBitmap();
+                }
+                // Skip the gap in front of the first range that has not been read yet.
+                begin_pos = std::max(begin_pos, static_cast<int32_t>(selected->first));
+                range_end_pos = std::min(range_end_pos, static_cast<int32_t>(selected->second));
+            }
+            if (begin_pos >= read_end_pos_) {
+                previous_batch_first_row_num_ = ToReaderRowNumber(begin_pos);
                 return BatchReader::MakeEofBatchWithBitmap();
             }
             int32_t actual_batch_size = batch_size_;
@@ -128,21 +153,23 @@ class MockFileBatchReader : public PrefetchFileBatchReader {
                 std::uniform_int_distribution<int32_t> distribution(1, batch_size_);
                 actual_batch_size = distribution(random_engine_);
             }
-            int32_t batch_end_pos = std::min(read_end_pos_, current_pos_ + actual_batch_size);
-            auto slice = data_->Slice(current_pos_, batch_end_pos - current_pos_);
+            int32_t batch_end_pos =
+                std::min({read_end_pos_, range_end_pos, begin_pos + actual_batch_size});
+            auto slice = data_->Slice(begin_pos, batch_end_pos - begin_pos);
             PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
                 std::shared_ptr<arrow::Array> concat_slice,
                 arrow::Concatenate({slice}, arrow::default_memory_pool()));
             RoaringBitmap32 bitmap;
-            for (auto iter = bitmap_.EqualOrLarger(current_pos_);
+            for (auto iter = bitmap_.EqualOrLarger(begin_pos);
                  iter != bitmap_.End() && *iter < batch_end_pos; ++iter) {
-                bitmap.Add(*iter - current_pos_);
+                bitmap.Add(*iter - begin_pos);
             }
-            previous_batch_first_row_num_ = current_pos_;
+            previous_batch_first_row_num_ = ToReaderRowNumber(begin_pos);
             current_pos_ = batch_end_pos;
             if (bitmap.IsEmpty()) {
                 continue;
             }
+            PAIMON_ASSIGN_OR_RAISE(concat_slice, ProjectBatch(concat_slice));
             std::unique_ptr<ArrowArray> c_array = std::make_unique<ArrowArray>();
             std::unique_ptr<ArrowSchema> c_schema = std::make_unique<ArrowSchema>();
             PAIMON_RETURN_NOT_OK_FROM_ARROW(
@@ -168,7 +195,7 @@ class MockFileBatchReader : public PrefetchFileBatchReader {
     Result<uint64_t> GetNumberOfRows() const override {
         return ToReaderRowNumber(read_end_pos_);
     }
-    uint64_t GetNextRowToRead() const override {
+    Result<uint64_t> GetNextRowToRead() const override {
         return ToReaderRowNumber(current_pos_);
     }
     void Close() override {}
@@ -181,7 +208,7 @@ class MockFileBatchReader : public PrefetchFileBatchReader {
         return false;
     }
 
- private:
+ protected:
     static uint64_t ToReaderRowNumber(int32_t row_number) {
         if (row_number < 0) {
             return std::numeric_limits<uint64_t>::max();
@@ -189,6 +216,44 @@ class MockFileBatchReader : public PrefetchFileBatchReader {
         return static_cast<uint64_t>(row_number);
     }
 
+    /// Pick the columns requested by `read_schema_` out of `batch`, in the requested order.
+    ///
+    /// `batch` is returned as is unless the requested schema is a genuine re-selection of the
+    /// columns this file has. Requesting a field the file does not have means the read schema is a
+    /// logical view over some other physical layout, as the shredding and the row tracking readers
+    /// do, and those map the raw batch themselves.
+    /// `batch` is expected to have a zero offset, so its validity buffer can be reused as is.
+    Result<std::shared_ptr<arrow::Array>> ProjectBatch(
+        const std::shared_ptr<arrow::Array>& batch) const {
+        auto struct_batch = std::dynamic_pointer_cast<arrow::StructArray>(batch);
+        if (struct_batch == nullptr) {
+            return batch;
+        }
+        arrow::ArrayVector children;
+        arrow::FieldVector fields;
+        for (const auto& field : read_schema_->fields()) {
+            std::shared_ptr<arrow::Array> column = struct_batch->GetFieldByName(field->name());
+            if (column == nullptr) {
+                return batch;
+            }
+            children.push_back(column);
+            fields.push_back(field);
+        }
+        const arrow::FieldVector& batch_fields = struct_batch->type()->fields();
+        bool keeps_every_column = fields.size() == batch_fields.size();
+        for (size_t i = 0; keeps_every_column && i < fields.size(); i++) {
+            keeps_every_column = fields[i]->name() == batch_fields[i]->name();
+        }
+        if (keeps_every_column) {
+            return batch;
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::StructArray> projected,
+            arrow::StructArray::Make(children, fields, struct_batch->null_bitmap(),
+                                     struct_batch->null_count()));
+        return projected;
+    }
+
     std::shared_ptr<arrow::Array> data_;
     std::shared_ptr<arrow::DataType> file_schema_;
     std::shared_ptr<arrow::Schema> read_schema_;
diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp
index ea951a311..b3c5cb237 100644
--- a/test/inte/blob_table_inte_test.cpp
+++ b/test/inte/blob_table_inte_test.cpp
@@ -299,6 +299,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter
         auto splits = plan->Splits();
         ReadContextBuilder read_context_builder(table_path);
         read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate);
+        read_context_builder.EnableLateMaterializing(false);
         if (!options.empty()) {
             read_context_builder.SetOptions(options);
         }
diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp
index ccb9ce324..84ab22df8 100644
--- a/test/inte/global_index_test.cpp
+++ b/test/inte/global_index_test.cpp
@@ -209,7 +209,8 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter
         ReadContextBuilder read_context_builder(table_path);
         read_context_builder.SetReadFieldNames(read_schema)
             .SetPredicate(predicate)
-            .WithFileSystem(fs_);
+            .WithFileSystem(fs_)
+            .EnableLateMaterializing(false);
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context,
                                read_context_builder.Finish());
         PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context)));
diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp
index 1e0952e08..a2a52d343 100644
--- a/test/inte/read_inte_test.cpp
+++ b/test/inte/read_inte_test.cpp
@@ -2209,6 +2209,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateOnlyPushdown) {
         .AddOption("test.enable-adaptive-prefetch-strategy",
                    param.enable_adaptive_prefetch_strategy)
         .SetPredicate(predicate)
+        .EnableLateMaterializing(false)
         .EnablePrefetch(param.enable_prefetch);
 
     ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
@@ -2265,6 +2266,89 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateOnlyPushdown) {
     ASSERT_TRUE(result_array->Equals(*expected_array)) << result_array->ToString();
 }
 
+// Late materialization reads the predicate columns first and only materializes the remaining
+// columns for matched rows. Combined with the top-level predicate filter, the read path returns
+// the exact user-predicate match set.
+TEST_P(ReadInteTest, TestAppendReadWithLateMaterializing) {
+    std::vector<DataField> read_fields = {DataField(3, arrow::field("f3", arrow::float64())),
+                                          DataField(0, arrow::field("f0", arrow::utf8())),
+                                          DataField(1, arrow::field("f1", arrow::int32()))};
+    ASSERT_OK_AND_ASSIGN(
+        auto predicate,
+        PredicateBuilder::Or(
+            {PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f3",
+                                           FieldType::DOUBLE, Literal(static_cast<double>(15.0))),
+             PredicateBuilder::IsNull(/*field_index=*/0, /*field_name=*/"f3", FieldType::DOUBLE)}));
+
+    auto param = GetParam();
+    std::string path =
+        paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09";
+
+    ReadContextBuilder context_builder(path);
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
+    context_builder.SetReadFieldNames({"f3", "f0", "f1"});
+    context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
+        .AddOption("read.batch-size", "2")
+        .AddOption("test.enable-adaptive-prefetch-strategy",
+                   param.enable_adaptive_prefetch_strategy)
+        .SetPredicate(predicate)
+        .EnableLateMaterializing(true)
+        .EnablePrefetch(param.enable_prefetch);
+    ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context)));
+
+    std::vector<std::string> file_list_0;
+    std::vector<std::string> file_list_1;
+    std::vector<std::string> file_list_2;
+    if (param.file_format == "orc") {
+        file_list_0 = {"data-d41fd7d1-b3e4-4905-aad9-b20a780e90a2-0.orc"};
+        file_list_1 = {"data-4e30d6c0-f109-4300-a010-4ba03047dd9d-0.orc",
+                       "data-10b9eea8-241d-4e4b-8ab8-2a82d72d79a2-0.orc",
+                       "data-e2bb59ee-ae25-4e5b-9bcc-257250bc5fdd-0.orc",
+                       "data-2d5ea1ea-77c1-47ff-bb87-19a509962a37-0.orc"};
+        file_list_2 = {"data-db2b44c0-0d73-449d-82a0-4075bd2cb6e3-0.orc",
+                       "data-b913a160-a4d1-4084-af2a-18333c35668e-0.orc"};
+    } else if (param.file_format == "parquet") {
+        file_list_0 = {"data-46e27d5b-4850-4d1e-abb6-b3aabbbc08cb-0.parquet"};
+        file_list_1 = {"data-864a052b-a938-4e04-b32c-6c72699a0c92-0.parquet",
+                       "data-c0401350-64a3-4a54-a143-dd125ad9a8e5-0.parquet",
+                       "data-7a912f84-04b7-4bbb-8dc6-53f4a292ea25-0.parquet",
+                       "data-bb891df7-ea12-4b7e-9017-41aabe08c8ec-0.parquet"};
+        file_list_2 = {"data-b446f78a-2cfb-4b3b-add8-31295d24a277-0.parquet",
+                       "data-fd72a479-53ae-42f7-aec0-e982ee555928-0.parquet"};
+    }
+
+    DataSplitsSimple input_data_splits = {
+        {paimon::test::GetDataDir() + "/" + param.file_format +
+             "/append_09.db/append_09/f1=10/bucket-0",
+         BinaryRowGenerator::GenerateRow({10}, pool_.get()), file_list_0},
+        {paimon::test::GetDataDir() + "/" + param.file_format +
+             "/append_09.db/append_09/f1=10/bucket-1",
+         BinaryRowGenerator::GenerateRow({10}, pool_.get()), file_list_1},
+        {paimon::test::GetDataDir() + "/" + param.file_format +
+             "/append_09.db/append_09/f1=20/bucket-0",
+         BinaryRowGenerator::GenerateRow({20}, pool_.get()), file_list_2}};
+
+    auto data_splits = CreateDataSplits(input_data_splits, /*snapshot_id=*/4);
+    ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits));
+    ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(batch_reader.get()));
+
+    auto fields_with_row_kind = read_fields;
+    fields_with_row_kind.insert(fields_with_row_kind.begin(), SpecialFields::ValueKind());
+    std::shared_ptr<arrow::DataType> arrow_data_type =
+        DataField::ConvertDataFieldsToArrowStructType(fields_with_row_kind);
+
+    // "Bob" (f3 = 12.1) is the only row that does not match the predicate.
+    std::shared_ptr<arrow::ChunkedArray> expected_array;
+    auto array_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([
+        [0, 15.1, "Emily", 10], [0, 16.1, "Alex", 10],  [0, 17.1, "David", 10],
+        [0, 17.1, "Lily", 10],  [0, null, "Paul", 20]
+    ])"},
+                                                                         &expected_array);
+    ASSERT_TRUE(array_status.ok());
+    ASSERT_TRUE(result_array->Equals(*expected_array)) << result_array->ToString();
+}
+
 TEST_P(ReadInteTest, TestAppendReadWithPredicateAllFiltered) {
     std::vector<DataField> read_fields = {DataField(3, arrow::field("f3", arrow::float64())),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
@@ -3109,6 +3193,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateOnlyPush
     context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
     context_builder.SetPredicate(predicate);
     context_builder.EnablePrefetch(param.enable_prefetch)
+        .EnableLateMaterializing(false)
         .AddOption("test.enable-adaptive-prefetch-strategy",
                    param.enable_adaptive_prefetch_strategy);
     ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
@@ -3166,6 +3251,91 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateOnlyPush
     ASSERT_TRUE(result_array->Equals(*expected_array));
 }
 
+TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithLateMaterializing) {
+    std::vector<DataField> read_fields = {DataField(1, arrow::field("key1", arrow::int32())),
+                                          DataField(7, arrow::field("k", arrow::utf8())),
+                                          DataField(2, arrow::field("key_2", arrow::int32())),
+                                          DataField(4, arrow::field("c", arrow::int32())),
+                                          DataField(8, arrow::field("d", arrow::int32())),
+                                          DataField(6, arrow::field("a", arrow::int32())),
+                                          DataField(0, arrow::field("key0", arrow::int32())),
+                                          DataField(9, arrow::field("e", arrow::int32()))};
+    auto param = GetParam();
+    std::string path = paimon::test::GetDataDir() + "/" + param.file_format +
+                       "/pk_table_with_alter_table.db/pk_table_with_alter_table/";
+    // equal is a partition filter and is not pushed into the data files; less_than is pushed down
+    // and only matches the column added by schema evolution, where the older files yield nulls.
+    auto equal = PredicateBuilder::Equal(/*field_index=*/6, /*field_name=*/"key0", FieldType::INT,
+                                         Literal(0));
+    auto less_than = PredicateBuilder::LessThan(/*field_index=*/7, /*field_name=*/"e",
+                                                FieldType::INT, Literal(510));
+    ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than}));
+
+    ReadContextBuilder context_builder(path);
+    context_builder.SetReadFieldNames({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}});
+    context_builder.AddOption(Options::FILE_FORMAT, param.file_format)
+        .AddOption("read.batch-size", "2");
+    context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled);
+    context_builder.SetPredicate(predicate);
+    context_builder.EnableLateMaterializing(true)
+        .EnablePrefetch(param.enable_prefetch)
+        .AddOption("test.enable-adaptive-prefetch-strategy",
+                   param.enable_adaptive_prefetch_strategy);
+    ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context)));
+
+    std::vector<std::string> file_list_0;
+    std::vector<std::string> file_list_1;
+    std::string deletion_file;
+    if (param.file_format == "orc") {
+        file_list_0 = {"data-3842c1d6-6b34-4b2c-a648-9e95b4fb941b-0.orc",
+                       "data-d6d370f3-242b-45c9-8739-44bf31b2b449-0.orc"};
+        file_list_1 = {"data-7b538b91-5dbb-4e16-a639-1b5c0696db8c-0.orc"};
+        deletion_file = "index-51804749-ed6c-4e7b-b3e9-337cfe38499c-1";
+    } else if (param.file_format == "parquet") {
+        file_list_0 = {"data-8969384c-d715-4113-b663-2248c9a8c8d9-0.parquet",
+                       "data-f2f38e80-7d28-4d51-90b3-c28951e5cdc0-0.parquet"};
+        file_list_1 = {"data-d7a33230-223e-4d65-8e39-bc7ed26bdd32-0.parquet"};
+        deletion_file = "index-c93829f3-1a72-4d88-8401-70663ce46426-1";
+    }
+
+    DataSplitsSchemaDv input_data_splits = {
+        {path + "key0=1/key1=1/bucket-0",
+         BinaryRowGenerator::GenerateRow({1, 1}, pool_.get()),
+         file_list_0,
+         /*schema ids*/ {0, 1},
+         /*deletion file*/
+         {DeletionFile(path + "index/" + deletion_file,
+                       /*offset=*/1, /*length=*/26, /*cardinality=*/std::nullopt),
+          std::nullopt}},
+        {path + "key0=0/key1=1/bucket-0", BinaryRowGenerator::GenerateRow({0, 1}, pool_.get()),
+         file_list_1,
+         /*schema ids*/ {1},
+         /*deletion file*/ {std::nullopt}}};
+
+    auto data_splits = CreateDataSplits(input_data_splits, /*snapshot_id=*/6);
+    ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits));
+    ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(batch_reader.get()));
+
+    auto fields_with_row_kind = read_fields;
+    fields_with_row_kind.insert(fields_with_row_kind.begin(), SpecialFields::ValueKind());
+    std::shared_ptr<arrow::DataType> arrow_data_type =
+        DataField::ConvertDataFieldsToArrowStructType(fields_with_row_kind);
+
+    // "Paul" is the only row in partition key0 = 0 whose e is not null and matches e < 510.
+    std::shared_ptr<arrow::ChunkedArray> expected_array;
+    auto array_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([
+      [0, 1, "Bob", 22, 24, null, 26, 1, null],
+      [0, 1, "Emily", 32, 34, null, 36, 1, null],
+      [0, 1, "David", 62, 64, null, 66, 1, null],
+      [0, 1, "Whether I shall turn out to be the hero of my own life.", 72, 74, null, 76, 1, null],
+      [0, 1, "Paul", 502, 504, 508, 506, 0, 509]
+])"},
+                                                                         &expected_array);
+    ASSERT_TRUE(array_status.ok());
+    ASSERT_TRUE(result_array->Equals(*expected_array)) << result_array->ToString();
+}
+
 TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateFilter) {
     std::vector<DataField> read_fields = {DataField(1, arrow::field("key1", arrow::int32())),
                                           DataField(7, arrow::field("k", arrow::utf8())),
diff --git a/test/inte/read_inte_with_index_test.cpp b/test/inte/read_inte_with_index_test.cpp
index f1316c6ce..7a4439734 100644
--- a/test/inte/read_inte_with_index_test.cpp
+++ b/test/inte/read_inte_with_index_test.cpp
@@ -84,7 +84,8 @@ class ReadInteWithIndexTest : public testing::Test,
         ReadContextBuilder context_builder(table_path);
         context_builder.AddOption("read.batch-size", "2")
             .AddOption("test.enable-adaptive-prefetch-strategy", "false")
-            .SetPredicate(predicate);
+            .SetPredicate(predicate)
+            .EnableLateMaterializing(false);
         if (enable_prefetch) {
             context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3);
         }
@@ -1232,6 +1233,74 @@ TEST_P(ReadInteWithIndexTest, TestNoEmbeddingBitmapIndex) {
     CheckResultForBitmapWithSingleRowGroup(path, arrow_data_type, split);
 }
 
+TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) {
+    auto [file_format, enable_prefetch] = GetParam();
+    std::string path = GetDataDir() + "/" + file_format +
+                       "/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/";
+    std::string file_name;
+    if (file_format == "orc") {
+        file_name = "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc";
+    } else if (file_format == "parquet") {
+        file_name = "data-783929b2-49d4-4006-a898-194a62e3278d-0.parquet";
+    }
+
+    std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
+                                          DataField(0, arrow::field("f0", arrow::utf8())),
+                                          DataField(1, arrow::field("f1", arrow::int32())),
+                                          DataField(2, arrow::field("f2", arrow::int32())),
+                                          DataField(3, arrow::field("f3", arrow::float64()))};
+    std::shared_ptr<arrow::DataType> arrow_data_type =
+        DataField::ConvertDataFieldsToArrowStructType(read_fields);
+
+    auto data_file_meta = std::make_shared<DataFileMeta>(
+        file_name, /*file_size=*/689,
+        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+        /*max_sequence_number=*/7, /*schema_id=*/0,
+        /*level=*/0,
+        /*extra_files=*/
+        std::vector<std::optional<std::string>>({file_name + ".index"}),
+        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+        /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt,
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+    ASSERT_OK_AND_ASSIGN(auto split,
+                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+
+    std::string literal_str = "Bob";
+    auto predicate = PredicateBuilder::Equal(
+        /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING,
+        Literal(FieldType::STRING, literal_str.data(), literal_str.size()));
+
+    ReadContextBuilder context_builder(path);
+    context_builder.AddOption("read.batch-size", "2")
+        .AddOption("test.enable-adaptive-prefetch-strategy", "false")
+        .SetPredicate(predicate)
+        .EnablePredicateFilter(true)
+        .EnableLateMaterializing(true);
+    if (enable_prefetch) {
+        context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3);
+    }
+    ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context)));
+    ASSERT_OK_AND_ASSIGN(auto batch_reader,
+                         table_read->CreateReader(std::vector<std::shared_ptr<Split>>{split}));
+    ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(batch_reader.get()));
+
+    // Only the two "Bob" rows match the predicate.
+    std::shared_ptr<arrow::ChunkedArray> expected_array;
+    auto array_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([
+[0, "Bob", 10, 1, 12.1],
+[0, "Bob", 10, 1, 16.1]
+    ])"},
+                                                                         &expected_array);
+    ASSERT_TRUE(array_status.ok());
+    ASSERT_TRUE(result_array->Equals(*expected_array)) << result_array->ToString();
+}
+
 TEST_P(ReadInteWithIndexTest, TestNoEmbeddingBitmapIndexWithExternalPath) {
     auto [file_format, enable_prefetch] = GetParam();
     std::string path = GetDataDir() + "/" + file_format +
diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp
index 96a7c9a19..537a4cb36 100644
--- a/test/inte/scan_and_read_inte_test.cpp
+++ b/test/inte/scan_and_read_inte_test.cpp
@@ -724,7 +724,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) {
 
     ReadContextBuilder read_context_builder(table_path);
     AddReadOptionsForPrefetch(&read_context_builder);
-    read_context_builder.SetPredicate(predicate);
+    read_context_builder.SetPredicate(predicate).EnableLateMaterializing(false);
     ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish());
     ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context)));
 
@@ -744,6 +744,45 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) {
     ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString();
 }
 
+TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLateMaterializing) {
+    auto file_format = FileFormat();
+    std::string table_path = paimon::test::GetDataDir() + file_format +
+                             "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/";
+    ScanContextBuilder scan_context_builder(table_path);
+    scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6");
+
+    std::string literal_str = "Alice";
+    auto not_equal = PredicateBuilder::NotEqual(
+        /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING,
+        Literal(FieldType::STRING, literal_str.data(), literal_str.size()));
+    auto greater_than = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3",
+                                                      FieldType::DOUBLE, Literal(18.0));
+    ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, greater_than}));
+    scan_context_builder.SetPredicate(predicate);
+    ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder));
+    ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context)));
+
+    ReadContextBuilder read_context_builder(table_path);
+    AddReadOptionsForPrefetch(&read_context_builder);
+    read_context_builder.SetPredicate(predicate).EnableLateMaterializing(true);
+    ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context)));
+
+    ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan());
+    ASSERT_EQ(result_plan->SnapshotId().value(), 6);
+    ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits()));
+    ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get()));
+
+    // check result: "Lucy" (f3 = 14.1) does not match f3 > 18 and is filtered out.
+    auto expected = std::make_shared<arrow::ChunkedArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type_, R"([
+[0, "Paul", 20, 1, 18.1]
+   ])")
+            .ValueOrDie());
+    ASSERT_TRUE(expected);
+    ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString();
+}
+
 TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot4WithPredicate) {
     auto file_format = FileFormat();
     std::string table_path = paimon::test::GetDataDir() + file_format +
@@ -1251,7 +1290,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) {
 
     ReadContextBuilder read_context_builder(table_path);
     AddReadOptionsForPrefetch(&read_context_builder);
-    read_context_builder.SetPredicate(predicate);
+    read_context_builder.SetPredicate(predicate).EnableLateMaterializing(false);
     ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish());
     ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context)));
 
@@ -1279,6 +1318,55 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) {
     ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString();
 }
 
+// Same coverage as the deletion-vector case above, for the merge-on-read path where only the
+// key part of the predicate is pushed down into the data files.
+TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLateMaterializing) {
+    auto file_format = FileFormat();
+    std::string table_path = paimon::test::GetDataDir() + file_format +
+                             "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/";
+
+    ScanContextBuilder scan_context_builder(table_path);
+    scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5");
+
+    std::string literal_str = "Alice";
+    auto not_equal = PredicateBuilder::NotEqual(
+        /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING,
+        Literal(FieldType::STRING, literal_str.data(), literal_str.size()));
+    std::string literal_str2 = "Lucy";
+    auto less_than = PredicateBuilder::LessThan(
+        /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING,
+        Literal(FieldType::STRING, literal_str2.data(), literal_str2.size()));
+    auto less_or_equal = PredicateBuilder::LessOrEqual(/*field_index=*/3, /*field_name=*/"f3",
+                                                       FieldType::DOUBLE, Literal(30.0));
+    ASSERT_OK_AND_ASSIGN(auto predicate,
+                         PredicateBuilder::And({not_equal, less_than, less_or_equal}));
+    scan_context_builder.SetPredicate(predicate);
+    ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder));
+    ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context)));
+
+    ReadContextBuilder read_context_builder(table_path);
+    AddReadOptionsForPrefetch(&read_context_builder);
+    read_context_builder.SetPredicate(predicate).EnableLateMaterializing(true);
+    ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context)));
+
+    ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan());
+    ASSERT_EQ(result_plan->SnapshotId().value(), 5);
+    ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits()));
+    ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get()));
+
+    // check result: only the rows before "Lucy" with f3 <= 30.0 remain.
+    auto expected = std::make_shared<arrow::ChunkedArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type_, R"([
+[0, "Bob", 10, 0, 12.1],
+[0, "David", 10, 0, 17.1],
+[0, "Emily", 10, 0, 13.1]
+   ])")
+            .ValueOrDie());
+    ASSERT_TRUE(expected);
+    ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString();
+}
+
 TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot3WithPredicate) {
     auto file_format = FileFormat();
     std::string table_path = paimon::test::GetDataDir() + file_format +

From b2e5fba43536c39e1d155ed7e2cd2d42e6eb1073 Mon Sep 17 00:00:00 2001
From: Zouxxyy <zouxinyu.zxy@alibaba-inc.com>
Date: Thu, 27 Aug 2026 14:10:07 +0800
Subject: [PATCH 71/93] feat(core): support per-column maximum sequence numbers
 (#247)

---
 src/paimon/CMakeLists.txt                     |   1 +
 .../bucketed_append_compact_manager_test.cpp  |   4 +-
 .../deletionvectors/deletion_vector_test.cpp  |   2 +-
 .../core/global_index/indexed_split_test.cpp  |  35 ++--
 .../pk_sorted_bucket_index_state_test.cpp     |   2 +-
 src/paimon/core/io/data_file_meta.cpp         |  41 +++--
 src/paimon/core/io/data_file_meta.h           |  13 +-
 .../core/io/data_file_meta_09_serializer.cpp  |   2 +-
 .../core/io/data_file_meta_10_serializer.cpp  |   3 +-
 .../core/io/data_file_meta_12_serializer.cpp  |   3 +-
 ...le_meta_first_row_id_legacy_serializer.cpp |   3 +-
 .../core/io/data_file_meta_serializer.cpp     |  20 ++-
 .../io/data_file_meta_serializer_test.cpp     |  20 ++-
 src/paimon/core/io/data_file_meta_test.cpp    |  29 ++--
 ...file_meta_write_cols_legacy_serializer.cpp | 156 ++++++++++++++++++
 ...a_file_meta_write_cols_legacy_serializer.h |  51 ++++++
 .../core/io/data_file_path_factory_test.cpp   |   7 +-
 .../core/io/file_index_evaluator_test.cpp     |  12 +-
 .../core/io/key_value_data_file_writer.cpp    |   3 +-
 .../core/io/rolling_blob_file_writer_test.cpp |   9 +-
 .../manifest_entry_serializer_test.cpp        |   3 +-
 .../manifest/manifest_entry_writer_test.cpp   |   8 +-
 .../core/manifest/manifest_file_test.cpp      |  18 +-
 .../core/manifest/partition_entry_test.cpp    |   2 +-
 .../compact/compact_strategy_test.cpp         |   2 +-
 .../compact/early_full_compaction_test.cpp    |   2 +-
 .../force_up_level0_compaction_test.cpp       |   2 +-
 .../compact/interval_partition_test.cpp       |   2 +-
 ...ookup_merge_tree_compact_rewriter_test.cpp |   9 +-
 ...erge_tree_compact_manager_factory_test.cpp |   3 +-
 .../merge_tree_compact_manager_test.cpp       |   4 +-
 .../merge_tree_compact_rewriter_test.cpp      |   6 +-
 .../compact/universal_compaction_test.cpp     |   2 +-
 src/paimon/core/mergetree/levels_test.cpp     |   5 +-
 .../core/mergetree/merge_tree_writer_test.cpp |  22 +--
 src/paimon/core/mergetree/sorted_run_test.cpp |   2 +-
 .../core/migrate/file_meta_utils_test.cpp     |  14 +-
 .../commit/commit_changes_provider_test.cpp   |   2 +-
 .../commit/conflict_detection_test.cpp        |   4 +-
 .../commit/manifest_entry_changes_test.cpp    |   4 +-
 .../overwrite_changes_provider_test.cpp       |   2 +-
 .../row_id_column_conflict_checker_test.cpp   |   3 +-
 .../commit/row_tracking_commit_utils_test.cpp |   6 +-
 .../sequence_snapshot_properties_test.cpp     |   2 +-
 .../core/operation/commit_metrics_test.cpp    |   2 +-
 .../data_evolution_file_store_scan_test.cpp   |   5 +-
 .../core/operation/expire_snapshots_test.cpp  |   2 +-
 .../operation/file_store_commit_impl_test.cpp |   8 +-
 .../operation/internal_read_context_test.cpp  |   1 +
 .../key_value_file_store_scan_test.cpp        |  13 +-
 .../operation/manifest_file_merger_test.cpp   |   2 +-
 .../operation/merge_file_split_read_test.cpp  |  22 +--
 .../operation/metrics/commit_stats_test.cpp   |   2 +-
 .../operation/raw_file_split_read_test.cpp    |   8 +-
 .../core/operation/write_restore_test.cpp     |   3 +-
 .../postpone/postpone_bucket_writer_test.cpp  |  10 +-
 .../table/sink/commit_message_serializer.cpp  |  20 ++-
 .../core/table/sink/commit_message_test.cpp   | 112 ++++++++++---
 .../source/data_evolution_batch_scan_test.cpp |   2 +-
 .../core/table/source/data_split_impl.cpp     |   2 +
 .../core/table/source/data_split_impl.h       |   3 +-
 .../core/table/source/data_split_test.cpp     | 146 +++++++++++-----
 .../table/source/fallback_data_split_test.cpp |  12 +-
 .../primary_key_sorted_index_scan_test.cpp    |   3 +-
 .../source/snapshot/snapshot_reader_test.cpp  |   2 +-
 .../table/source/split_generator_test.cpp     |   8 +-
 test/inte/read_inte_test.cpp                  |   2 +-
 test/inte/read_inte_with_index_test.cpp       |  49 ++++--
 test/inte/scan_inte_test.cpp                  |  61 ++++---
 test/inte/write_inte_test.cpp                 |  96 ++++++-----
 .../compatibility/commit_message-v13          | Bin 0 -> 693 bytes
 test/test_data/compatibility/data_split-v9    | Bin 0 -> 738 bytes
 72 files changed, 817 insertions(+), 324 deletions(-)
 create mode 100644 src/paimon/core/io/data_file_meta_write_cols_legacy_serializer.cpp
 create mode 100644 src/paimon/core/io/data_file_meta_write_cols_legacy_serializer.h
 create mode 100644 test/test_data/compatibility/commit_message-v13
 create mode 100644 test/test_data/compatibility/data_split-v9

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 8fb6c17d1..3de2b667e 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -275,6 +275,7 @@ set(PAIMON_CORE_SRCS
     core/io/data_file_meta_first_row_id_legacy_serializer.cpp
     core/io/data_file_meta.cpp
     core/io/data_file_meta_serializer.cpp
+    core/io/data_file_meta_write_cols_legacy_serializer.cpp
     core/io/data_file_path_factory.cpp
     core/io/data_file_index_writer.cpp
     core/io/file_index_options.cpp
diff --git a/src/paimon/core/append/bucketed_append_compact_manager_test.cpp b/src/paimon/core/append/bucketed_append_compact_manager_test.cpp
index ee56825b5..712dd3667 100644
--- a/src/paimon/core/append/bucketed_append_compact_manager_test.cpp
+++ b/src/paimon/core/append/bucketed_append_compact_manager_test.cpp
@@ -115,7 +115,7 @@ class BucketedAppendCompactManagerTest : public testing::Test {
             /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     std::shared_ptr<DataFileMeta> NewNamedFile(const std::string& file_name, int64_t file_size,
@@ -135,7 +135,7 @@ class BucketedAppendCompactManagerTest : public testing::Test {
             /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     std::shared_ptr<BucketedDvMaintainer> CreateTestDvMaintainer(
diff --git a/src/paimon/core/deletionvectors/deletion_vector_test.cpp b/src/paimon/core/deletionvectors/deletion_vector_test.cpp
index daf2ecf1c..bff4b46a3 100644
--- a/src/paimon/core/deletionvectors/deletion_vector_test.cpp
+++ b/src/paimon/core/deletionvectors/deletion_vector_test.cpp
@@ -54,7 +54,7 @@ std::shared_ptr<DataFileMeta> CreateDataFileMeta(const std::string& file_name) {
         /*min_sequence_number=*/0, /*max_sequence_number=*/0, /*schema_id=*/0,
         DataFileMeta::DUMMY_LEVEL, std::vector<std::optional<std::string>>{}, Timestamp(0, 0),
         std::nullopt, nullptr, FileSource::Append(), std::nullopt, std::nullopt, std::nullopt,
-        std::nullopt);
+        std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 }
 
 }  // namespace
diff --git a/src/paimon/core/global_index/indexed_split_test.cpp b/src/paimon/core/global_index/indexed_split_test.cpp
index 857f50bfa..7cd121257 100644
--- a/src/paimon/core/global_index/indexed_split_test.cpp
+++ b/src/paimon/core/global_index/indexed_split_test.cpp
@@ -56,17 +56,20 @@ TEST(IndexedSplitTest, TestSimple) {
         "file1.orc", 100l, 200l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(),
         SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 50l, 249l, 0, 0,
         std::vector<std::optional<std::string>>(), Timestamp(1765535214349l, 0), 0, nullptr,
-        FileSource::Append(), std::nullopt, std::nullopt, 50l, std::nullopt);
+        FileSource::Append(), std::nullopt, std::nullopt, 50l, std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     auto meta2 = std::make_shared<DataFileMeta>(
         "file2.orc", 101l, 100l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(),
         SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 250l, 349l, 0, 0,
         std::vector<std::optional<std::string>>(), Timestamp(1765535214349l, 0), 0, nullptr,
-        FileSource::Append(), std::nullopt, std::nullopt, 250l, std::nullopt);
+        FileSource::Append(), std::nullopt, std::nullopt, 250l, std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     auto meta3 = std::make_shared<DataFileMeta>(
         "file3.orc", 102l, 200l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(),
         SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 1000l, 1199l, 0, 0,
         std::vector<std::optional<std::string>>(), Timestamp(1765535214349l, 0), 0, nullptr,
-        FileSource::Append(), std::nullopt, std::nullopt, 1000l, std::nullopt);
+        FileSource::Append(), std::nullopt, std::nullopt, 1000l, std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRow::EmptyRow(),
@@ -82,8 +85,11 @@ TEST(IndexedSplitTest, TestSimple) {
 
     ASSERT_EQ(*result_indexed_split, *expected_indexed_split) << result_indexed_split->ToString();
     ASSERT_OK_AND_ASSIGN(std::string serialize_bytes, Split::Serialize(result_indexed_split, pool));
-    ASSERT_EQ(serialize_bytes,
-              std::string(reinterpret_cast<char*>(split_bytes.data()), split_bytes.size()));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Split> roundtrip,
+                         Split::Deserialize(serialize_bytes.data(), serialize_bytes.size(), pool));
+    auto roundtrip_indexed_split = std::dynamic_pointer_cast<IndexedSplitImpl>(roundtrip);
+    ASSERT_EQ(*roundtrip_indexed_split, *expected_indexed_split)
+        << roundtrip_indexed_split->ToString();
 }
 
 TEST(IndexedSplitTest, TestIndexedSplitWithScore) {
@@ -107,17 +113,20 @@ TEST(IndexedSplitTest, TestIndexedSplitWithScore) {
         "file1.orc", 100l, 200l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(),
         SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 50l, 249l, 0, 0,
         std::vector<std::optional<std::string>>(), Timestamp(1765549435648l, 0), 0, nullptr,
-        FileSource::Append(), std::nullopt, std::nullopt, 50l, std::nullopt);
+        FileSource::Append(), std::nullopt, std::nullopt, 50l, std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     auto meta2 = std::make_shared<DataFileMeta>(
         "file2.orc", 101l, 100l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(),
         SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 250l, 349l, 0, 0,
         std::vector<std::optional<std::string>>(), Timestamp(1765549435649l, 0), 0, nullptr,
-        FileSource::Append(), std::nullopt, std::nullopt, 250l, std::nullopt);
+        FileSource::Append(), std::nullopt, std::nullopt, 250l, std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     auto meta3 = std::make_shared<DataFileMeta>(
         "file3.orc", 102l, 200l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(),
         SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 1000l, 1199l, 0, 0,
         std::vector<std::optional<std::string>>(), Timestamp(1765549435649l, 0), 0, nullptr,
-        FileSource::Append(), std::nullopt, std::nullopt, 1000l, std::nullopt);
+        FileSource::Append(), std::nullopt, std::nullopt, 1000l, std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRow::EmptyRow(),
@@ -139,8 +148,11 @@ TEST(IndexedSplitTest, TestIndexedSplitWithScore) {
             "rowRanges=[[55, 56],[270, 270],[1001, 1002]], scores=[1.01,2.1,-1.32,4.23,50.74]") !=
         std::string::npos);
     ASSERT_OK_AND_ASSIGN(std::string serialize_bytes, Split::Serialize(result_indexed_split, pool));
-    ASSERT_EQ(serialize_bytes,
-              std::string(reinterpret_cast<char*>(split_bytes.data()), split_bytes.size()));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Split> roundtrip,
+                         Split::Deserialize(serialize_bytes.data(), serialize_bytes.size(), pool));
+    auto roundtrip_indexed_split = std::dynamic_pointer_cast<IndexedSplitImpl>(roundtrip);
+    ASSERT_EQ(*roundtrip_indexed_split, *expected_indexed_split)
+        << roundtrip_indexed_split->ToString();
 }
 
 TEST(IndexedSplitTest, TestValidate) {
@@ -148,7 +160,8 @@ TEST(IndexedSplitTest, TestValidate) {
         "file.orc", 1l, 200l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(),
         SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 1000l, 1199l, 0, 0,
         std::vector<std::optional<std::string>>(), Timestamp(0l, 0), 0, nullptr,
-        FileSource::Append(), std::nullopt, std::nullopt, 1000l, std::nullopt);
+        FileSource::Append(), std::nullopt, std::nullopt, 1000l, std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRow::EmptyRow(),
diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp
index d01f18c44..ec65f4ff1 100644
--- a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp
+++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp
@@ -53,7 +53,7 @@ class PkSortedBucketIndexStateTest : public ::testing::Test {
             /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/std::nullopt,
             /*embedded_index=*/nullptr, file_source, /*value_stats_cols=*/std::nullopt,
             /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     /// Builds a payload whose source metadata lists the given sources in the given order.
diff --git a/src/paimon/core/io/data_file_meta.cpp b/src/paimon/core/io/data_file_meta.cpp
index 7ce767f4b..cd8666098 100644
--- a/src/paimon/core/io/data_file_meta.cpp
+++ b/src/paimon/core/io/data_file_meta.cpp
@@ -72,7 +72,8 @@ Result<std::shared_ptr<DataFileMeta>> DataFileMeta::ForAppend(
         file_name, file_size, row_count, EmptyMinKey(), EmptyMaxKey(), SimpleStats::EmptyStats(),
         row_stats, min_sequence_number, max_sequence_number, schema_id, DUMMY_LEVEL, extra_files,
         Timestamp(/*millisecond=*/local_micro / 1000, /*nano_of_millisecond=*/0), 0ll,
-        embedded_index, file_source, value_stats_cols, external_path, first_row_id, write_cols);
+        embedded_index, file_source, value_stats_cols, external_path, first_row_id, write_cols,
+        /*column_max_sequence_numbers=*/std::nullopt);
 }
 
 Result<std::shared_ptr<DataFileMeta>> DataFileMeta::Upgrade(int32_t new_level) const {
@@ -84,7 +85,7 @@ Result<std::shared_ptr<DataFileMeta>> DataFileMeta::Upgrade(int32_t new_level) c
         file_name, file_size, row_count, min_key, max_key, key_stats, value_stats,
         min_sequence_number, max_sequence_number, schema_id, new_level, extra_files, creation_time,
         delete_row_count, embedded_index, file_source, value_stats_cols, external_path,
-        first_row_id, write_cols);
+        first_row_id, write_cols, column_max_sequence_numbers);
 }
 
 std::shared_ptr<DataFileMeta> DataFileMeta::CopyWithExtraFiles(
@@ -93,7 +94,16 @@ std::shared_ptr<DataFileMeta> DataFileMeta::CopyWithExtraFiles(
         file_name, file_size, row_count, min_key, max_key, key_stats, value_stats,
         min_sequence_number, max_sequence_number, schema_id, level, new_extra_files, creation_time,
         delete_row_count, embedded_index, file_source, value_stats_cols, external_path,
-        first_row_id, write_cols);
+        first_row_id, write_cols, column_max_sequence_numbers);
+}
+
+std::shared_ptr<DataFileMeta> DataFileMeta::CopyWithColumnMaxSequenceNumbers(
+    const std::optional<std::vector<int64_t>>& new_column_max_sequence_numbers) const {
+    return std::make_shared<DataFileMeta>(
+        file_name, file_size, row_count, min_key, max_key, key_stats, value_stats,
+        min_sequence_number, max_sequence_number, schema_id, level, extra_files, creation_time,
+        delete_row_count, embedded_index, file_source, value_stats_cols, external_path,
+        first_row_id, write_cols, new_column_max_sequence_numbers);
 }
 
 std::shared_ptr<DataFileMeta> DataFileMeta::CopyWithoutStats() const {
@@ -101,7 +111,7 @@ std::shared_ptr<DataFileMeta> DataFileMeta::CopyWithoutStats() const {
         file_name, file_size, row_count, min_key, max_key, key_stats, SimpleStats::EmptyStats(),
         min_sequence_number, max_sequence_number, schema_id, level, extra_files, creation_time,
         delete_row_count, embedded_index, file_source, std::vector<std::string>(), external_path,
-        first_row_id, write_cols);
+        first_row_id, write_cols, column_max_sequence_numbers);
 }
 
 DataFileMeta::DataFileMeta(
@@ -113,7 +123,8 @@ DataFileMeta::DataFileMeta(
     const std::shared_ptr<Bytes>& _embedded_index, const std::optional<FileSource>& _file_source,
     const std::optional<std::vector<std::string>>& _value_stats_cols,
     const std::optional<std::string>& _external_path, const std::optional<int64_t>& _first_row_id,
-    const std::optional<std::vector<std::string>>& _write_cols)
+    const std::optional<std::vector<std::string>>& _write_cols,
+    const std::optional<std::vector<int64_t>>& _column_max_sequence_numbers)
     : file_name(_file_name),
       file_size(_file_size),
       row_count(_row_count),
@@ -133,7 +144,8 @@ DataFileMeta::DataFileMeta(
       value_stats_cols(_value_stats_cols),
       external_path(_external_path),
       first_row_id(_first_row_id),
-      write_cols(_write_cols) {}
+      write_cols(_write_cols),
+      column_max_sequence_numbers(_column_max_sequence_numbers) {}
 
 Result<std::string> DataFileMeta::FileFormat() const {
     size_t last_dot_index = file_name.find_last_of(".");
@@ -198,7 +210,8 @@ bool DataFileMeta::operator==(const DataFileMeta& other) const {
            creation_time == other.creation_time && delete_row_count == other.delete_row_count &&
            file_source == other.file_source && value_stats_cols == other.value_stats_cols &&
            external_path == other.external_path && first_row_id == other.first_row_id &&
-           write_cols == other.write_cols;
+           write_cols == other.write_cols &&
+           column_max_sequence_numbers == other.column_max_sequence_numbers;
 }
 
 bool DataFileMeta::operator!=(const DataFileMeta& other) const {
@@ -243,7 +256,8 @@ bool DataFileMeta::TEST_Equal(const DataFileMeta& other) const {
            level == other.level && delete_row_count == other.delete_row_count &&
            file_source == other.file_source && value_stats_cols == other.value_stats_cols &&
            compare_optional_ignore_name(external_path, other.external_path) &&
-           first_row_id == other.first_row_id && write_cols == other.write_cols;
+           first_row_id == other.first_row_id && write_cols == other.write_cols &&
+           column_max_sequence_numbers == other.column_max_sequence_numbers;
 }
 
 std::string DataFileMeta::ToString() const {
@@ -261,7 +275,8 @@ std::string DataFileMeta::ToString() const {
         "{}, "
         "keyStats: {}, valueStats: {}, minSequenceNumber: {}, maxSequenceNumber: {}, schemaId: "
         "{}, level: {}, extraFiles: {}, creationTime: {}, deleteRowCount: {}, fileSource: {}, "
-        "valueStatsCols: {}, externalPath: {}, firstRowId: {}, writeCols: {}}}",
+        "valueStatsCols: {}, externalPath: {}, firstRowId: {}, writeCols: {}, "
+        "columnMaxSequenceNumbers: {}}}",
         file_name, file_size, row_count,
         embedded_index == nullptr ? "null"
                                   : std::string(embedded_index->data(), embedded_index->size()),
@@ -275,7 +290,10 @@ std::string DataFileMeta::ToString() const {
             : fmt::format("{}", fmt::join(value_stats_cols.value(), ", ")),
         external_path == std::nullopt ? "null" : external_path.value(),
         first_row_id == std::nullopt ? "null" : std::to_string(first_row_id.value()),
-        write_cols == std::nullopt ? "null" : fmt::format("{}", write_cols.value()));
+        write_cols == std::nullopt ? "null" : fmt::format("{}", write_cols.value()),
+        column_max_sequence_numbers == std::nullopt
+            ? "null"
+            : fmt::format("{}", column_max_sequence_numbers.value()));
 }
 
 int64_t DataFileMeta::GetMaxSequenceNumber(
@@ -315,6 +333,9 @@ const std::shared_ptr<arrow::DataType>& DataFileMeta::DataType() {
          arrow::field("_FIRST_ROW_ID", arrow::int64(), /*nullable=*/true),
          arrow::field("_WRITE_COLS",
                       arrow::list(arrow::field("item", arrow::utf8(), /*nullable=*/false)),
+                      /*nullable=*/true),
+         arrow::field("_WRITE_COLS_SEQUENCES",
+                      arrow::list(arrow::field("item", arrow::int64(), /*nullable=*/false)),
                       /*nullable=*/true)});
     return schema;
 }
diff --git a/src/paimon/core/io/data_file_meta.h b/src/paimon/core/io/data_file_meta.h
index 98aaa0ee7..436d73785 100644
--- a/src/paimon/core/io/data_file_meta.h
+++ b/src/paimon/core/io/data_file_meta.h
@@ -58,7 +58,8 @@ struct DataFileMeta {
                  const std::optional<std::vector<std::string>>& _value_stats_cols,
                  const std::optional<std::string>& _external_path,
                  const std::optional<int64_t>& _first_row_id,
-                 const std::optional<std::vector<std::string>>& _write_cols);
+                 const std::optional<std::vector<std::string>>& _write_cols,
+                 const std::optional<std::vector<int64_t>>& _column_max_sequence_numbers);
 
     static Result<std::shared_ptr<DataFileMeta>> ForAppend(
         const std::string& file_name, int64_t file_size, int64_t row_count,
@@ -83,6 +84,9 @@ struct DataFileMeta {
     std::shared_ptr<DataFileMeta> CopyWithExtraFiles(
         const std::vector<std::optional<std::string>>& new_extra_files) const;
 
+    std::shared_ptr<DataFileMeta> CopyWithColumnMaxSequenceNumbers(
+        const std::optional<std::vector<int64_t>>& new_column_max_sequence_numbers) const;
+
     /// Create a copy without value statistics. All other metadata is preserved.
     ///
     /// @return A new metadata object with empty value statistics and value-stat columns.
@@ -167,5 +171,12 @@ struct DataFileMeta {
     std::optional<int64_t> first_row_id;
 
     std::optional<std::vector<std::string>> write_cols;
+
+    /// Maximum sequence number per physical table field after data-evolution compaction.
+    ///
+    /// Values follow the table-field order selected by `write_cols` when it is non-null (system
+    /// fields are ignored), or the file schema field order otherwise. A null value means that only
+    /// the file-level sequence range is available.
+    std::optional<std::vector<int64_t>> column_max_sequence_numbers;
 };
 }  // namespace paimon
diff --git a/src/paimon/core/io/data_file_meta_09_serializer.cpp b/src/paimon/core/io/data_file_meta_09_serializer.cpp
index 7eaffc911..7eae41d13 100644
--- a/src/paimon/core/io/data_file_meta_09_serializer.cpp
+++ b/src/paimon/core/io/data_file_meta_09_serializer.cpp
@@ -114,7 +114,7 @@ Result<std::shared_ptr<DataFileMeta>> DataFileMeta09Serializer::FromRow(
         embedded_file_index, file_source,
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/io/data_file_meta_10_serializer.cpp b/src/paimon/core/io/data_file_meta_10_serializer.cpp
index fea60eeb5..bf99f99d8 100644
--- a/src/paimon/core/io/data_file_meta_10_serializer.cpp
+++ b/src/paimon/core/io/data_file_meta_10_serializer.cpp
@@ -123,7 +123,8 @@ Result<std::shared_ptr<DataFileMeta>> DataFileMeta10Serializer::FromRow(
         min_sequence_number, max_sequence_number, schema_id, level,
         InternalRowUtils::FromStringArrayData(extra_files.get()), creation_time, delete_row_count,
         embedded_file_index, file_source, std::optional<std::vector<std::string>>(value_stats_cols),
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/io/data_file_meta_12_serializer.cpp b/src/paimon/core/io/data_file_meta_12_serializer.cpp
index fe46a85c8..07fa1353a 100644
--- a/src/paimon/core/io/data_file_meta_12_serializer.cpp
+++ b/src/paimon/core/io/data_file_meta_12_serializer.cpp
@@ -128,7 +128,8 @@ Result<std::shared_ptr<DataFileMeta>> DataFileMeta12Serializer::FromRow(
         min_sequence_number, max_sequence_number, schema_id, level,
         InternalRowUtils::FromStringArrayData(extra_files.get()), creation_time, delete_row_count,
         embedded_file_index, file_source, std::optional<std::vector<std::string>>(value_stats_cols),
-        external_path, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        external_path, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/io/data_file_meta_first_row_id_legacy_serializer.cpp b/src/paimon/core/io/data_file_meta_first_row_id_legacy_serializer.cpp
index a834e47f0..3e82290d6 100644
--- a/src/paimon/core/io/data_file_meta_first_row_id_legacy_serializer.cpp
+++ b/src/paimon/core/io/data_file_meta_first_row_id_legacy_serializer.cpp
@@ -134,7 +134,8 @@ Result<std::shared_ptr<DataFileMeta>> DataFileMetaFirstRowIdLegacySerializer::Fr
         min_sequence_number, max_sequence_number, schema_id, level,
         InternalRowUtils::FromStringArrayData(extra_files.get()), creation_time, delete_row_count,
         embedded_file_index, file_source, std::optional<std::vector<std::string>>(value_stats_cols),
-        external_path, first_row_id, /*write_cols=*/std::nullopt);
+        external_path, first_row_id, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/io/data_file_meta_serializer.cpp b/src/paimon/core/io/data_file_meta_serializer.cpp
index 560299684..9f88ebec0 100644
--- a/src/paimon/core/io/data_file_meta_serializer.cpp
+++ b/src/paimon/core/io/data_file_meta_serializer.cpp
@@ -24,6 +24,7 @@
 #include <string>
 #include <utility>
 
+#include "paimon/common/data/binary_array.h"
 #include "paimon/common/data/binary_row_writer.h"
 #include "paimon/common/data/binary_string.h"
 #include "paimon/common/data/internal_row.h"
@@ -41,7 +42,7 @@ class InternalArray;
 class MemoryPool;
 
 Result<BinaryRow> DataFileMetaSerializer::ToRow(const std::shared_ptr<DataFileMeta>& meta) const {
-    BinaryRow row(20);
+    BinaryRow row(NumFields());
     BinaryRowWriter writer(&row, 32 * 1024, pool_.get());
     writer.WriteString(0, BinaryString::FromString(meta->file_name, pool_.get()));
     writer.WriteLong(1, meta->file_size);
@@ -95,6 +96,12 @@ Result<BinaryRow> DataFileMetaSerializer::ToRow(const std::shared_ptr<DataFileMe
         writer.WriteArray(
             19, InternalRowUtils::ToNotNullStringArrayData(meta->write_cols.value(), pool_));
     }
+    if (meta->column_max_sequence_numbers == std::nullopt) {
+        writer.SetNullAt(20);
+    } else {
+        writer.WriteArray(
+            20, BinaryArray::FromLongArray(meta->column_max_sequence_numbers.value(), pool_.get()));
+    }
     writer.Complete();
     return row;
 }
@@ -160,6 +167,15 @@ Result<std::shared_ptr<DataFileMeta>> DataFileMetaSerializer::FromRow(
         }
         write_cols = InternalRowUtils::FromNotNullStringArrayData(array.get());
     }
+
+    std::optional<std::vector<int64_t>> column_max_sequence_numbers;
+    if (!row.IsNullAt(20)) {
+        std::shared_ptr<InternalArray> array = row.GetArray(20);
+        if (array == nullptr) {
+            return Status::Invalid("invalid column max sequence numbers");
+        }
+        PAIMON_ASSIGN_OR_RAISE(column_max_sequence_numbers, array->ToLongArray());
+    }
     PAIMON_ASSIGN_OR_RAISE(BinaryRow min_values, SerializationUtils::DeserializeBinaryRow(min_key));
     PAIMON_ASSIGN_OR_RAISE(BinaryRow max_values, SerializationUtils::DeserializeBinaryRow(max_key));
     PAIMON_ASSIGN_OR_RAISE(SimpleStats key_stats,
@@ -171,7 +187,7 @@ Result<std::shared_ptr<DataFileMeta>> DataFileMetaSerializer::FromRow(
         min_sequence_number, max_sequence_number, schema_id, level,
         InternalRowUtils::FromStringArrayData(extra_files.get()), creation_time, delete_row_count,
         embedded_file_index, file_source, std::optional<std::vector<std::string>>(value_stats_cols),
-        external_path, first_row_id, write_cols);
+        external_path, first_row_id, write_cols, column_max_sequence_numbers);
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/io/data_file_meta_serializer_test.cpp b/src/paimon/core/io/data_file_meta_serializer_test.cpp
index 18ba25788..5fdc452b3 100644
--- a/src/paimon/core/io/data_file_meta_serializer_test.cpp
+++ b/src/paimon/core/io/data_file_meta_serializer_test.cpp
@@ -21,13 +21,16 @@
 #include <cstdint>
 #include <optional>
 #include <string>
+#include <vector>
 
 #include "arrow/api.h"
 #include "arrow/array/builder_base.h"
 #include "gtest/gtest.h"
 #include "paimon/common/io/memory_segment_output_stream.h"
 #include "paimon/common/memory/memory_segment_utils.h"
+#include "paimon/common/utils/checked_cast.h"
 #include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/data_file_meta_write_cols_legacy_serializer.h"
 #include "paimon/core/stats/simple_stats.h"
 #include "paimon/data/timestamp.h"
 #include "paimon/io/byte_array_input_stream.h"
@@ -49,7 +52,9 @@ class DataFileMetaSerializerTest : public testing::Test {
     }
 
  private:
-    std::shared_ptr<DataFileMeta> GetDataFileMeta() {
+    std::shared_ptr<DataFileMeta> GetDataFileMeta(
+        const std::optional<std::vector<int64_t>>& column_max_sequence_numbers =
+            std::vector<int64_t>{16, 32}) {
         return std::make_shared<DataFileMeta>(
             "some_file_name", 1024, 8, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(),
             SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16,
@@ -58,7 +63,8 @@ class DataFileMetaSerializerTest : public testing::Test {
             /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/3,
             /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::optional<std::string>(),
-            /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+            /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+            column_max_sequence_numbers);
     }
 
     const int32_t TRIES = 100;
@@ -68,6 +74,7 @@ class DataFileMetaSerializerTest : public testing::Test {
 
 TEST_F(DataFileMetaSerializerTest, TestToFromRow) {
     DataFileMetaSerializer serializer(memory_pool_);
+    ASSERT_EQ(serializer.NumFields(), 21);
     auto expected = GetDataFileMeta();
     for (int32_t i = 0; i < TRIES; i++) {
         ASSERT_OK_AND_ASSIGN(auto row, serializer.ToRow(expected));
@@ -76,6 +83,15 @@ TEST_F(DataFileMetaSerializerTest, TestToFromRow) {
     }
 }
 
+TEST_F(DataFileMetaSerializerTest, TestLegacySerializerSchema) {
+    DataFileMetaWriteColsLegacySerializer serializer(memory_pool_);
+    ASSERT_EQ(serializer.NumFields(), 20);
+    auto legacy_type =
+        checked_pointer_cast<arrow::StructType>(DataFileMetaWriteColsLegacySerializer::DataType());
+    ASSERT_EQ(legacy_type->num_fields(), 20);
+    ASSERT_EQ(legacy_type->field(19)->name(), "_WRITE_COLS");
+}
+
 TEST_F(DataFileMetaSerializerTest, TestSerialize) {
     DataFileMetaSerializer serializer(memory_pool_);
     auto expected = GetDataFileMeta();
diff --git a/src/paimon/core/io/data_file_meta_test.cpp b/src/paimon/core/io/data_file_meta_test.cpp
index f7283f158..f76bc0ae8 100644
--- a/src/paimon/core/io/data_file_meta_test.cpp
+++ b/src/paimon/core/io/data_file_meta_test.cpp
@@ -37,7 +37,8 @@ TEST(DataFileMetaTest, TestCopyWithoutStats) {
         /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::vector<std::string>({"f0", "f1"}),
         /*external_path=*/"file:/tmp/bucket-0/data-0.orc", /*first_row_id=*/100,
-        /*write_cols=*/std::vector<std::string>({"f0"}));
+        /*write_cols=*/std::vector<std::string>({"f0"}),
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     std::shared_ptr<DataFileMeta> result = file_meta->CopyWithoutStats();
 
@@ -68,7 +69,8 @@ TEST(DataFileMetaTest, TestAddRowCount) {
                            /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
                            /*value_stats_cols=*/std::nullopt,
                            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-                           /*write_cols=*/std::nullopt);
+                           /*write_cols=*/std::nullopt,
+                           /*column_max_sequence_numbers=*/std::nullopt);
     ASSERT_EQ(3, file_meta.AddRowCount().value());
     // test null delete row count
     file_meta.delete_row_count = std::nullopt;
@@ -85,7 +87,8 @@ TEST(DataFileMetaTest, TestFileFormat) {
                            /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
                            /*value_stats_cols=*/std::nullopt,
                            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-                           /*write_cols=*/std::nullopt);
+                           /*write_cols=*/std::nullopt,
+                           /*column_max_sequence_numbers=*/std::nullopt);
     ASSERT_OK_AND_ASSIGN(auto file_format, file_meta.FileFormat());
     ASSERT_EQ("orc", file_format);
     file_meta.file_name = "data-80110e15-97b5-4bcf-ac09-6ca2659a4950-0.parquet";
@@ -107,7 +110,8 @@ TEST(DataFileMetaTest, TestExternalPathDir) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
         /*external_path=*/"file:/tmp/bucket-0/data-80110e15-97b5-4bcf-ac09-6ca2659a4950-0.orc",
-        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     ASSERT_EQ("file:/tmp/bucket-0", file_meta.ExternalPathDir().value());
     file_meta.external_path = std::nullopt;
     ASSERT_EQ(std::nullopt, file_meta.ExternalPathDir());
@@ -123,7 +127,8 @@ TEST(DataFileMetaTest, TestGetMaxSequenceNumber) {
         /*creation_time=*/Timestamp(1737111915429ll, 0),
         /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-80110e15-97b5-4bcf-ac09-6ca2659a4950-1.orc", /*file_size=*/645,
         /*row_count=*/5, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(), SimpleStats::EmptyStats(),
@@ -133,7 +138,8 @@ TEST(DataFileMetaTest, TestGetMaxSequenceNumber) {
         /*creation_time=*/Timestamp(1737111915429ll, 0),
         /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     ASSERT_EQ(4, DataFileMeta::GetMaxSequenceNumber({file_meta1}));
     ASSERT_EQ(10, DataFileMeta::GetMaxSequenceNumber({file_meta1, file_meta2}));
     ASSERT_EQ(-1, DataFileMeta::GetMaxSequenceNumber({}));
@@ -152,7 +158,8 @@ TEST(DataFileMetaTest, TestNonNullFirstRowId) {
             /*creation_time=*/Timestamp(1737111915429ll, 0),
             /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt,
-            /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt);
+            /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
         ASSERT_OK_AND_ASSIGN(int64_t first_row_id, file_meta->NonNullFirstRowId());
         ASSERT_EQ(100, first_row_id);
     }
@@ -167,7 +174,7 @@ TEST(DataFileMetaTest, TestNonNullFirstRowId) {
             /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt,
             /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         ASSERT_NOK_WITH_MSG(file_meta->NonNullFirstRowId(),
                             "First row id of data-1.orc should not be null.");
     }
@@ -184,7 +191,8 @@ TEST(DataFileMetaTest, TestToFileSelection) {
         /*creation_time=*/Timestamp(1737111915429ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     {
         ASSERT_OK_AND_ASSIGN(std::optional<RoaringBitmap32> result,
@@ -223,7 +231,8 @@ TEST(DataFileMetaTest, TestUpgrade) {
         /*creation_time=*/Timestamp(1737111915429ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     // test normal upgrade
     ASSERT_OK_AND_ASSIGN(auto new_file_meta, file_meta->Upgrade(10));
     ASSERT_EQ(new_file_meta->level, 10);
diff --git a/src/paimon/core/io/data_file_meta_write_cols_legacy_serializer.cpp b/src/paimon/core/io/data_file_meta_write_cols_legacy_serializer.cpp
new file mode 100644
index 000000000..ac2d480b9
--- /dev/null
+++ b/src/paimon/core/io/data_file_meta_write_cols_legacy_serializer.cpp
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/io/data_file_meta_write_cols_legacy_serializer.h"
+
+#include <cassert>
+#include <cstdint>
+#include <optional>
+#include <string>
+
+#include "arrow/type.h"
+#include "paimon/common/data/internal_row.h"
+#include "paimon/common/utils/internal_row_utils.h"
+#include "paimon/common/utils/serialization_utils.h"
+#include "paimon/core/manifest/file_source.h"
+#include "paimon/core/stats/simple_stats.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+class Bytes;
+class InternalArray;
+
+const std::shared_ptr<arrow::DataType>& DataFileMetaWriteColsLegacySerializer::DataType() {
+    static std::shared_ptr<arrow::DataType> schema = arrow::struct_(
+        {arrow::field("_FILE_NAME", arrow::utf8(), /*nullable=*/false),
+         arrow::field("_FILE_SIZE", arrow::int64(), /*nullable=*/false),
+         arrow::field("_ROW_COUNT", arrow::int64(), /*nullable=*/false),
+         arrow::field("_MIN_KEY", arrow::binary(), /*nullable=*/false),
+         arrow::field("_MAX_KEY", arrow::binary(), /*nullable=*/false),
+         arrow::field("_KEY_STATS", SimpleStats::DataType(), /*nullable=*/false),
+         arrow::field("_VALUE_STATS", SimpleStats::DataType(), /*nullable=*/false),
+         arrow::field("_MIN_SEQUENCE_NUMBER", arrow::int64(), /*nullable=*/false),
+         arrow::field("_MAX_SEQUENCE_NUMBER", arrow::int64(), /*nullable=*/false),
+         arrow::field("_SCHEMA_ID", arrow::int64(), /*nullable=*/false),
+         arrow::field("_LEVEL", arrow::int32(), /*nullable=*/false),
+         arrow::field("_EXTRA_FILES",
+                      arrow::list(arrow::field("item", arrow::utf8(), /*nullable=*/false)),
+                      /*nullable=*/false),
+         arrow::field("_CREATION_TIME", arrow::timestamp(arrow::TimeUnit::MILLI),
+                      /*nullable=*/true),
+         arrow::field("_DELETE_ROW_COUNT", arrow::int64(), /*nullable=*/true),
+         arrow::field("_EMBEDDED_FILE_INDEX", arrow::binary(), /*nullable=*/true),
+         arrow::field("_FILE_SOURCE", arrow::int8(), /*nullable=*/true),
+         arrow::field("_VALUE_STATS_COLS",
+                      arrow::list(arrow::field("item", arrow::utf8(), /*nullable=*/false)),
+                      /*nullable=*/true),
+         arrow::field("_EXTERNAL_PATH", arrow::utf8(), /*nullable=*/true),
+         arrow::field("_FIRST_ROW_ID", arrow::int64(), /*nullable=*/true),
+         arrow::field("_WRITE_COLS",
+                      arrow::list(arrow::field("item", arrow::utf8(), /*nullable=*/false)),
+                      /*nullable=*/true)});
+    return schema;
+}
+
+Result<BinaryRow> DataFileMetaWriteColsLegacySerializer::ToRow(
+    const std::shared_ptr<DataFileMeta>&) const {
+    assert(false);
+    return Status::Invalid("to row for DataFileMetaWriteColsLegacySerializer is invalid");
+}
+
+Result<std::shared_ptr<DataFileMeta>> DataFileMetaWriteColsLegacySerializer::FromRow(
+    const InternalRow& row) const {
+    auto file_name = row.GetString(0);
+    auto file_size = row.GetLong(1);
+    auto row_count = row.GetLong(2);
+    auto min_key = row.GetBinary(3);
+    auto max_key = row.GetBinary(4);
+    auto key_stats_row = row.GetRow(5, 3);
+    auto value_stats_row = row.GetRow(6, 3);
+    auto min_sequence_number = row.GetLong(7);
+    auto max_sequence_number = row.GetLong(8);
+    auto schema_id = row.GetLong(9);
+    auto level = row.GetInt(10);
+    std::shared_ptr<InternalArray> extra_files = row.GetArray(11);
+    auto creation_time = row.GetTimestamp(12, 3);
+
+    assert(min_key && max_key && key_stats_row && value_stats_row);
+    if (extra_files == nullptr) {
+        return Status::Invalid("extra files is empty");
+    }
+
+    std::optional<int64_t> delete_row_count;
+    if (!row.IsNullAt(13)) {
+        delete_row_count = row.GetLong(13);
+    }
+    std::shared_ptr<Bytes> embedded_file_index;
+    if (!row.IsNullAt(14)) {
+        embedded_file_index = row.GetBinary(14);
+    }
+
+    std::optional<FileSource> file_source;
+    if (!row.IsNullAt(15)) {
+        PAIMON_ASSIGN_OR_RAISE(file_source, FileSource::FromByteValue(row.GetByte(15)));
+    }
+
+    std::optional<std::vector<std::string>> value_stats_cols;
+    if (!row.IsNullAt(16)) {
+        std::shared_ptr<InternalArray> array = row.GetArray(16);
+        if (array == nullptr) {
+            return Status::Invalid("invalid value stats cols");
+        }
+        value_stats_cols = InternalRowUtils::FromNotNullStringArrayData(array.get());
+    }
+
+    std::optional<std::string> external_path;
+    if (!row.IsNullAt(17)) {
+        external_path = row.GetString(17).ToString();
+    }
+    std::optional<int64_t> first_row_id;
+    if (!row.IsNullAt(18)) {
+        first_row_id = row.GetLong(18);
+    }
+
+    std::optional<std::vector<std::string>> write_cols;
+    if (!row.IsNullAt(19)) {
+        std::shared_ptr<InternalArray> array = row.GetArray(19);
+        if (array == nullptr) {
+            return Status::Invalid("invalid write cols");
+        }
+        write_cols = InternalRowUtils::FromNotNullStringArrayData(array.get());
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(BinaryRow min_values, SerializationUtils::DeserializeBinaryRow(min_key));
+    PAIMON_ASSIGN_OR_RAISE(BinaryRow max_values, SerializationUtils::DeserializeBinaryRow(max_key));
+    PAIMON_ASSIGN_OR_RAISE(SimpleStats key_stats,
+                           SimpleStats::FromRow(key_stats_row.get(), pool_.get()));
+    PAIMON_ASSIGN_OR_RAISE(SimpleStats value_stats,
+                           SimpleStats::FromRow(value_stats_row.get(), pool_.get()));
+    return std::make_shared<DataFileMeta>(
+        file_name.ToString(), file_size, row_count, min_values, max_values, key_stats, value_stats,
+        min_sequence_number, max_sequence_number, schema_id, level,
+        InternalRowUtils::FromStringArrayData(extra_files.get()), creation_time, delete_row_count,
+        embedded_file_index, file_source, std::optional<std::vector<std::string>>(value_stats_cols),
+        external_path, first_row_id, write_cols,
+        /*column_max_sequence_numbers=*/std::nullopt);
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/data_file_meta_write_cols_legacy_serializer.h b/src/paimon/core/io/data_file_meta_write_cols_legacy_serializer.h
new file mode 100644
index 000000000..0a08d28cc
--- /dev/null
+++ b/src/paimon/core/io/data_file_meta_write_cols_legacy_serializer.h
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "paimon/common/data/binary_row.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/utils/object_serializer.h"
+#include "paimon/result.h"
+
+namespace arrow {
+class DataType;
+}  // namespace arrow
+
+namespace paimon {
+class InternalRow;
+class MemoryPool;
+
+/// Legacy serializer for `DataFileMeta` before column sequence numbers were introduced.
+class DataFileMetaWriteColsLegacySerializer
+    : public ObjectSerializer<std::shared_ptr<DataFileMeta>> {
+ public:
+    static const std::shared_ptr<arrow::DataType>& DataType();
+
+    explicit DataFileMetaWriteColsLegacySerializer(const std::shared_ptr<MemoryPool>& pool)
+        : ObjectSerializer<std::shared_ptr<DataFileMeta>>(DataType(), pool) {}
+
+    Result<BinaryRow> ToRow(const std::shared_ptr<DataFileMeta>& meta) const override;
+
+    Result<std::shared_ptr<DataFileMeta>> FromRow(const InternalRow& row) const override;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/data_file_path_factory_test.cpp b/src/paimon/core/io/data_file_path_factory_test.cpp
index 12618a2ec..6283876df 100644
--- a/src/paimon/core/io/data_file_path_factory_test.cpp
+++ b/src/paimon/core/io/data_file_path_factory_test.cpp
@@ -97,7 +97,7 @@ TEST_F(DataFilePathFactoryTest, TestToPath) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
         /*external_path=*/"file:/test/bucket-0/example.txt", /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     ASSERT_EQ(factory_.ToPath(file_meta), "file:/test/bucket-0/example.txt");
 }
 
@@ -118,7 +118,7 @@ TEST_F(DataFilePathFactoryTest, TestToAlignedPath) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
         /*external_path=*/"file:/test/bucket-0/data-0.txt", /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     ASSERT_EQ(factory_.ToAlignedPath("index-0", file_meta), "file:/test/bucket-0/index-0");
 
@@ -135,7 +135,8 @@ TEST_F(DataFilePathFactoryTest, TestCollectFiles) {
         /*creation_time=*/Timestamp(1737111915429ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     ASSERT_EQ(factory_.CollectFiles(file_meta), std::vector<std::string>({"/tmp/data-0.txt"}));
 
     file_meta->extra_files = {"data-0.index", "data-1.index"};
diff --git a/src/paimon/core/io/file_index_evaluator_test.cpp b/src/paimon/core/io/file_index_evaluator_test.cpp
index d9730ade1..1824564e6 100644
--- a/src/paimon/core/io/file_index_evaluator_test.cpp
+++ b/src/paimon/core/io/file_index_evaluator_test.cpp
@@ -319,7 +319,8 @@ TEST_F(FileIndexEvaluatorTest, TestEvaluateEmbeddingIndex) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/embedded_index, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     CheckResult(/*data_file_path_factory=*/nullptr, data_file_meta);
 }
 
@@ -339,7 +340,8 @@ TEST_F(FileIndexEvaluatorTest, TestEvaluateNoEmbeddingIndex) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     auto data_file_path_factory = std::make_shared<DataFilePathFactory>();
     ASSERT_OK(data_file_path_factory->Init(path + "/bucket-0/", /*format_identifier=*/"orc",
                                            /*data_file_prefix=*/"data-", nullptr));
@@ -374,7 +376,8 @@ TEST_F(FileIndexEvaluatorTest, TestTimestampType) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     auto data_file_path_factory = std::make_shared<DataFilePathFactory>();
     ASSERT_OK(data_file_path_factory->Init(path + "/bucket-0/", /*format_identifier=*/"orc",
                                            /*data_file_prefix=*/"data-", nullptr));
@@ -408,7 +411,8 @@ TEST_F(FileIndexEvaluatorTest, TestInvalidEvaluate) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     auto predicate =
         PredicateBuilder::IsNull(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT);
     ASSERT_NOK_WITH_MSG(
diff --git a/src/paimon/core/io/key_value_data_file_writer.cpp b/src/paimon/core/io/key_value_data_file_writer.cpp
index 9c32e0674..881712ebe 100644
--- a/src/paimon/core/io/key_value_data_file_writer.cpp
+++ b/src/paimon/core/io/key_value_data_file_writer.cpp
@@ -110,7 +110,8 @@ Result<std::shared_ptr<DataFileMeta>> KeyValueDataFileWriter::GetResult() {
         file_index.extra_files,
         Timestamp(/*millisecond=*/local_micro / 1000, /*nano_of_millisecond=*/0), delete_row_count_,
         file_index.embedded_index, file_source_, /*value_stats_cols=*/std::nullopt, final_path,
-        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 }
 
 Status KeyValueDataFileWriter::GenerateMinMaxKey(BinaryRow* min_key, BinaryRow* max_key) const {
diff --git a/src/paimon/core/io/rolling_blob_file_writer_test.cpp b/src/paimon/core/io/rolling_blob_file_writer_test.cpp
index 654c91266..b88f4a329 100644
--- a/src/paimon/core/io/rolling_blob_file_writer_test.cpp
+++ b/src/paimon/core/io/rolling_blob_file_writer_test.cpp
@@ -57,7 +57,8 @@ TEST_F(RollingBlobFileWriterTest, ValidateFileConsistency) {
         /*creation_time=*/Timestamp(1724090888706ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0,
-        /*write_cols=*/std::vector<std::string>({"f0", "f1"}));
+        /*write_cols=*/std::vector<std::string>({"f0", "f1"}),
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-xxx.blob", /*file_size=*/764, /*row_count=*/3,
@@ -70,7 +71,8 @@ TEST_F(RollingBlobFileWriterTest, ValidateFileConsistency) {
         /*creation_time=*/Timestamp(1724090888706ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0,
-        /*write_cols=*/std::vector<std::string>({"blob"}));
+        /*write_cols=*/std::vector<std::string>({"blob"}),
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     auto file_meta3 = std::make_shared<DataFileMeta>(
         "data-xxx.blob", /*file_size=*/3023, /*row_count=*/1,
@@ -83,7 +85,8 @@ TEST_F(RollingBlobFileWriterTest, ValidateFileConsistency) {
         /*creation_time=*/Timestamp(1724090888706ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/3,
-        /*write_cols=*/std::vector<std::string>({"blob"}));
+        /*write_cols=*/std::vector<std::string>({"blob"}),
+        /*column_max_sequence_numbers=*/std::nullopt);
     ASSERT_OK(RollingBlobFileWriter::ValidateFileConsistency(file_meta1, {file_meta2, file_meta3}));
     ASSERT_NOK_WITH_MSG(RollingBlobFileWriter::ValidateFileConsistency(file_meta1, {file_meta2}),
                         "This is a bug: The row count of main file and blob file does not match.");
diff --git a/src/paimon/core/manifest/manifest_entry_serializer_test.cpp b/src/paimon/core/manifest/manifest_entry_serializer_test.cpp
index 2d8cffc37..a31f4bf73 100644
--- a/src/paimon/core/manifest/manifest_entry_serializer_test.cpp
+++ b/src/paimon/core/manifest/manifest_entry_serializer_test.cpp
@@ -43,7 +43,8 @@ class ManifestEntrySerializerTest : public testing::Test {
             /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/3,
             /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::optional<std::string>(),
-            /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+            /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
     }
 };
 TEST_F(ManifestEntrySerializerTest, TestToFromRow) {
diff --git a/src/paimon/core/manifest/manifest_entry_writer_test.cpp b/src/paimon/core/manifest/manifest_entry_writer_test.cpp
index 3c3d00d7e..6c9291b71 100644
--- a/src/paimon/core/manifest/manifest_entry_writer_test.cpp
+++ b/src/paimon/core/manifest/manifest_entry_writer_test.cpp
@@ -64,7 +64,7 @@ class ManifestEntryWriterTest : public ::testing::Test {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt,
             /*external_path=*/std::nullopt, first_row_id,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         return {FileKind::Add(), BinaryRowGenerator::GenerateRow({10}, pool_.get()), /*bucket=*/0,
                 /*total_buckets=*/-1, meta};
     }
@@ -126,7 +126,8 @@ TEST_F(ManifestEntryWriterTest, TestSimple) {
         /*creation_time=*/Timestamp(1743525392885ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     auto meta2 = std::make_shared<DataFileMeta>(
         "data-5858a84b-7081-4618-b828-ae3918c5e1f6-0.orc", /*file_size=*/943, /*row_count=*/4,
@@ -144,7 +145,8 @@ TEST_F(ManifestEntryWriterTest, TestSimple) {
         /*creation_time=*/Timestamp(1743525392921ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     auto entry1 = ManifestEntry(FileKind::Add(), BinaryRowGenerator::GenerateRow({10}, pool_.get()),
                                 0, 2, meta1);
diff --git a/src/paimon/core/manifest/manifest_file_test.cpp b/src/paimon/core/manifest/manifest_file_test.cpp
index a34f41524..df78fc00d 100644
--- a/src/paimon/core/manifest/manifest_file_test.cpp
+++ b/src/paimon/core/manifest/manifest_file_test.cpp
@@ -153,7 +153,7 @@ TEST_F(ManifestFileTest, TestSimple) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto manifest_entry1 =
         ManifestEntry(FileKind::Delete(), BinaryRowGenerator::GenerateRow({10}, pool.get()),
                       /*bucket=*/1, /*total_buckets=*/2, file_meta1);
@@ -171,7 +171,7 @@ TEST_F(ManifestFileTest, TestSimple) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto manifest_entry2 =
         ManifestEntry(FileKind::Delete(), BinaryRowGenerator::GenerateRow({10}, pool.get()),
                       /*bucket=*/1, /*total_buckets=*/2, file_meta2);
@@ -189,7 +189,7 @@ TEST_F(ManifestFileTest, TestSimple) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto manifest_entry3 =
         ManifestEntry(FileKind::Delete(), BinaryRowGenerator::GenerateRow({10}, pool.get()),
                       /*bucket=*/1, /*total_buckets=*/2, file_meta3);
@@ -207,7 +207,7 @@ TEST_F(ManifestFileTest, TestSimple) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto manifest_entry4 =
         ManifestEntry(FileKind::Delete(), BinaryRowGenerator::GenerateRow({10}, pool.get()),
                       /*bucket=*/1, /*total_buckets=*/2, file_meta4);
@@ -225,7 +225,7 @@ TEST_F(ManifestFileTest, TestSimple) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto manifest_entry5 =
         ManifestEntry(FileKind::Add(), BinaryRowGenerator::GenerateRow({10}, pool.get()),
                       /*bucket=*/1, /*total_buckets=*/2, file_meta5);
@@ -437,7 +437,7 @@ TEST_F(ManifestFileTest, TestWithNullCount) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto manifest_entry1 =
         ManifestEntry(FileKind::Add(), BinaryRowGenerator::GenerateRow({10}, pool.get()),
                       /*bucket=*/1, /*total_buckets=*/2, file_meta1);
@@ -464,7 +464,7 @@ TEST_F(ManifestFileTest, TestWithNullCount) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto manifest_entry2 =
         ManifestEntry(FileKind::Add(), BinaryRowGenerator::GenerateRow({20}, pool.get()),
                       /*bucket=*/0, /*total_buckets=*/2, file_meta2);
@@ -501,7 +501,7 @@ TEST_F(ManifestFileTest, TestManifestFileCompatibleWithJavaPaimon09) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto manifest_entry = ManifestEntry(FileKind::Add(), /*partition=*/BinaryRow::EmptyRow(),
                                         /*bucket=*/0, /*total_buckets=*/-1, file_meta);
 
@@ -540,7 +540,7 @@ TEST_F(ManifestFileTest, TestManifestFileCompatibleWithJavaPaimon11) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto manifest_entry = ManifestEntry(FileKind::Add(), /*partition=*/BinaryRow::EmptyRow(),
                                         /*bucket=*/0, /*total_buckets=*/-1, file_meta);
 
diff --git a/src/paimon/core/manifest/partition_entry_test.cpp b/src/paimon/core/manifest/partition_entry_test.cpp
index 8b7dd6269..213e0dbaa 100644
--- a/src/paimon/core/manifest/partition_entry_test.cpp
+++ b/src/paimon/core/manifest/partition_entry_test.cpp
@@ -50,7 +50,7 @@ class PartitionEntryTest : public testing::Test {
             /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt,
             /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 };
 
diff --git a/src/paimon/core/mergetree/compact/compact_strategy_test.cpp b/src/paimon/core/mergetree/compact/compact_strategy_test.cpp
index a0abebe33..e8508a794 100644
--- a/src/paimon/core/mergetree/compact/compact_strategy_test.cpp
+++ b/src/paimon/core/mergetree/compact/compact_strategy_test.cpp
@@ -38,7 +38,7 @@ class CompactStrategyTest : public testing::Test {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         return {level, SortedRun::FromSingle(file_meta)};
     }
 
diff --git a/src/paimon/core/mergetree/compact/early_full_compaction_test.cpp b/src/paimon/core/mergetree/compact/early_full_compaction_test.cpp
index a4cc9c663..37cc061f4 100644
--- a/src/paimon/core/mergetree/compact/early_full_compaction_test.cpp
+++ b/src/paimon/core/mergetree/compact/early_full_compaction_test.cpp
@@ -55,7 +55,7 @@ class EarlyFullCompactionTest : public testing::Test {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         return {level, SortedRun::FromSingle(file_meta)};
     }
 
diff --git a/src/paimon/core/mergetree/compact/force_up_level0_compaction_test.cpp b/src/paimon/core/mergetree/compact/force_up_level0_compaction_test.cpp
index 18a11229c..c1d26a569 100644
--- a/src/paimon/core/mergetree/compact/force_up_level0_compaction_test.cpp
+++ b/src/paimon/core/mergetree/compact/force_up_level0_compaction_test.cpp
@@ -39,7 +39,7 @@ class ForceUpLevel0CompactionTest : public testing::Test {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         return {level, SortedRun::FromSingle(file_meta)};
     }
 
diff --git a/src/paimon/core/mergetree/compact/interval_partition_test.cpp b/src/paimon/core/mergetree/compact/interval_partition_test.cpp
index 861a35c4c..9c68000ca 100644
--- a/src/paimon/core/mergetree/compact/interval_partition_test.cpp
+++ b/src/paimon/core/mergetree/compact/interval_partition_test.cpp
@@ -125,7 +125,7 @@ class IntervalPartitionTest : public testing::Test {
             /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt,
             /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
  private:
diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
index eeb7e0ce2..169170da1 100644
--- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
+++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
@@ -460,7 +460,8 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowRewrite) {
         BinaryRowGenerator::GenerateStats({1, 5}, {5, 33}, {0, 0}, pool_.get()),
         /*min_sequence_number=*/0l, /*max_sequence_number=*/3l, /*schema_id=*/0, /*level=*/5,
         std::vector<std::optional<std::string>>(), Timestamp(0l, 0), /*delete_row_count=*/0,
-        nullptr, FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt);
+        nullptr, FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     ASSERT_TRUE(expected_file_meta->TEST_Equal(*compact_file_meta));
 
     // check compact file exist
@@ -564,7 +565,8 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowUpgrade) {
         BinaryRowGenerator::GenerateStats({1, 5}, {5, 33}, {0, 0}, pool_.get()),
         /*min_sequence_number=*/0l, /*max_sequence_number=*/2l, /*schema_id=*/0, /*level=*/5,
         std::vector<std::optional<std::string>>(), Timestamp(0l, 0), /*delete_row_count=*/0,
-        nullptr, FileSource::Append(), std::nullopt, std::nullopt, std::nullopt, std::nullopt);
+        nullptr, FileSource::Append(), std::nullopt, std::nullopt, std::nullopt, std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     ASSERT_TRUE(expected_file_meta->TEST_Equal(*compact_file_meta))
         << compact_file_meta->ToString();
 }
@@ -1108,7 +1110,8 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) {
             BinaryRowGenerator::GenerateStats({1, 5}, {5, 33}, {0, 0}, pool_.get()),
             /*min_sequence_number=*/0l, /*max_sequence_number=*/3l, /*schema_id=*/0, level,
             std::vector<std::optional<std::string>>(), Timestamp(0l, 0), delete_row_count, nullptr,
-            FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt);
+            FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
     };
     {
         std::map<std::string, std::string> options = {};
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp
index 7473b7c0b..91e3bb1cc 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp
@@ -60,7 +60,8 @@ class MergeTreeCompactManagerFactoryStrategyTest : public ::testing::Test {
             /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
             /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
-            /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+            /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
         return {level, SortedRun::FromSingle(file_meta)};
     }
 
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp
index 19b8bc363..9186eb1c4 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp
@@ -142,7 +142,7 @@ class TestRewriter final : public CompactRewriter {
             /*max_sequence_number=*/max_sequence,
             /*schema_id=*/0, output_level, std::vector<std::optional<std::string>>(),
             Timestamp(1, 0), std::nullopt, nullptr, FileSource::Append(), std::nullopt,
-            std::nullopt, std::nullopt, std::nullopt);
+            std::nullopt, std::nullopt, std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
         return CompactResult(before, {after});
     }
@@ -208,7 +208,7 @@ class MergeTreeCompactManagerTest : public testing::Test {
             /*max_sequence_number=*/max_sequence,
             /*schema_id=*/0, minmax.level, std::vector<std::optional<std::string>>(),
             Timestamp(1, 0), std::nullopt, nullptr, FileSource::Append(), std::nullopt,
-            std::nullopt, std::nullopt, std::nullopt);
+            std::nullopt, std::nullopt, std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     StrategyFn TestStrategy() {
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp
index cb26540aa..dc19c4726 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp
@@ -148,7 +148,8 @@ TEST_F(MergeTreeCompactRewriterTest, TestSimple) {
                                           pool_.get()),
         /*min_sequence_number=*/0l, /*max_sequence_number=*/10l, /*schema_id=*/0, /*level=*/5,
         std::vector<std::optional<std::string>>(), Timestamp(0l, 0), /*delete_row_count=*/0,
-        nullptr, FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt);
+        nullptr, FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     ASSERT_TRUE(expected_file_meta->TEST_Equal(*compact_file_meta));
     // check compact file exist
     std::string compact_file_name =
@@ -245,7 +246,8 @@ TEST_F(MergeTreeCompactRewriterTest, TestNotDropDelete) {
                                           pool_.get()),
         /*min_sequence_number=*/0l, /*max_sequence_number=*/11l, /*schema_id=*/0, /*level=*/5,
         std::vector<std::optional<std::string>>(), Timestamp(0l, 0), /*delete_row_count=*/2,
-        nullptr, FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt);
+        nullptr, FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     ASSERT_TRUE(expected_file_meta->TEST_Equal(*compact_file_meta));
 
     std::string compact_file_name =
diff --git a/src/paimon/core/mergetree/compact/universal_compaction_test.cpp b/src/paimon/core/mergetree/compact/universal_compaction_test.cpp
index 74abacaef..9b2e6cd1f 100644
--- a/src/paimon/core/mergetree/compact/universal_compaction_test.cpp
+++ b/src/paimon/core/mergetree/compact/universal_compaction_test.cpp
@@ -56,7 +56,7 @@ class UniversalCompactionTest : public testing::Test {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         return {level, SortedRun::FromSingle(file_meta)};
     }
 
diff --git a/src/paimon/core/mergetree/levels_test.cpp b/src/paimon/core/mergetree/levels_test.cpp
index 37a347f92..d31a31c92 100644
--- a/src/paimon/core/mergetree/levels_test.cpp
+++ b/src/paimon/core/mergetree/levels_test.cpp
@@ -45,7 +45,7 @@ class LevelsTest : public testing::Test {
             max_sequence_number,
             /*schema_id=*/0, level, std::vector<std::optional<std::string>>(),
             Timestamp(ts_second, 0l), std::nullopt, nullptr, FileSource::Append(), std::nullopt,
-            std::nullopt, std::nullopt, std::nullopt);
+            std::nullopt, std::nullopt, std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     std::shared_ptr<FieldsComparator> CreateComparator() const {
@@ -261,7 +261,8 @@ TEST_F(LevelsTest, TestUpdateDropFileCallbackExcludesUpgradeFiles) {
         file_level0->key_stats, file_level0->value_stats, file_level0->min_sequence_number,
         file_level0->max_sequence_number, file_level0->schema_id, /*level=*/1,
         file_level0->extra_files, file_level0->creation_time, std::nullopt, nullptr,
-        FileSource::Append(), std::nullopt, std::nullopt, std::nullopt, std::nullopt);
+        FileSource::Append(), std::nullopt, std::nullopt, std::nullopt, std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     std::vector<std::shared_ptr<DataFileMeta>> before = {file_level0};
     std::vector<std::shared_ptr<DataFileMeta>> after = {upgraded_file};
diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 2155647a1..c5a114f36 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -192,7 +192,8 @@ class MergeTreeWriterTest : public ::testing::TestWithParam<bool> {
             /*delete_row_count=*/0,
             /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
-            /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+            /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     Result<std::shared_ptr<MergeTreeWriter>> CreateMergeWriter(
@@ -289,7 +290,7 @@ TEST_P(MergeTreeWriterTest, TestSimple) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{},
                                           /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
@@ -370,7 +371,7 @@ TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{},
                                           /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
@@ -478,7 +479,8 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) {
         /*creation_time=*/actual_meta->creation_time, /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
-        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     ASSERT_TRUE(expected_data_file_meta->TEST_Equal(*actual_meta));
 }
 
@@ -658,7 +660,7 @@ TEST_P(MergeTreeWriterTest, TestWriteWithDeleteRow) {
         /*delete_row_count=*/1, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{},
                                           /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
@@ -769,7 +771,7 @@ TEST_P(MergeTreeWriterTest, TestMultiplePrepareCommit) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     auto expected_data_file_meta2 = std::make_shared<DataFileMeta>(
         expected_data_file_name2, /*file_size=*/data_file_status2.GetLen(), /*row_count=*/3,
@@ -788,7 +790,7 @@ TEST_P(MergeTreeWriterTest, TestMultiplePrepareCommit) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement expected_data_increment1({expected_data_file_meta1},
                                            /*deleted_files=*/{},
                                            /*changelog_files=*/{});
@@ -991,7 +993,7 @@ TEST_P(MergeTreeWriterTest, TestAutoFlush) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     auto expected_data_file_meta2 = std::make_shared<DataFileMeta>(
         expected_data_file_name2, /*file_size=*/data_file_status2.GetLen(), /*row_count=*/3,
@@ -1010,7 +1012,7 @@ TEST_P(MergeTreeWriterTest, TestAutoFlush) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement expected_data_increment({expected_data_file_meta1, expected_data_file_meta2},
                                           /*deleted_files=*/{},
                                           /*changelog_files=*/{});
@@ -1124,7 +1126,7 @@ TEST_P(MergeTreeWriterTest, TestBulkData) {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         ASSERT_EQ(*commit_increment.GetNewFilesIncrement().NewFiles()[i], *expected_data_file_meta);
     }
 }
diff --git a/src/paimon/core/mergetree/sorted_run_test.cpp b/src/paimon/core/mergetree/sorted_run_test.cpp
index 7a19fac36..fc4dd08d6 100644
--- a/src/paimon/core/mergetree/sorted_run_test.cpp
+++ b/src/paimon/core/mergetree/sorted_run_test.cpp
@@ -53,7 +53,7 @@ class SortedRunTest : public testing::Test {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 };
 
diff --git a/src/paimon/core/migrate/file_meta_utils_test.cpp b/src/paimon/core/migrate/file_meta_utils_test.cpp
index ee30680d8..caca9d2fc 100644
--- a/src/paimon/core/migrate/file_meta_utils_test.cpp
+++ b/src/paimon/core/migrate/file_meta_utils_test.cpp
@@ -132,7 +132,7 @@ TEST_F(FileMetaUtilsTest, TestSimple) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-db2b44c0-0d73-449d-82a0-4075bd2cb6e3-0.orc", /*file_size=*/541,
@@ -148,7 +148,7 @@ TEST_F(FileMetaUtilsTest, TestSimple) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     CommitMessageImpl expected(BinaryRow::EmptyRow(), /*bucket=*/0, /*total_buckets=*/-1,
                                DataIncrement({file_meta1, file_meta2}, {}, {}),
                                CompactIncrement({}, {}, {}));
@@ -211,7 +211,7 @@ TEST_F(FileMetaUtilsTest, TestFailover) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-db2b44c0-0d73-449d-82a0-4075bd2cb6e3-0.orc", /*file_size=*/541,
@@ -227,7 +227,7 @@ TEST_F(FileMetaUtilsTest, TestFailover) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     CommitMessageImpl expected(BinaryRow::EmptyRow(), /*bucket=*/0, /*total_buckets=*/-1,
                                DataIncrement({file_meta1, file_meta2}, {}, {}),
                                CompactIncrement({}, {}, {}));
@@ -291,7 +291,7 @@ TEST_F(FileMetaUtilsTest, TestWithPartition) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-1c547e5f-48b2-4917-a996-71d306377661-0.orc", /*file_size=*/589,
@@ -307,7 +307,7 @@ TEST_F(FileMetaUtilsTest, TestWithPartition) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     CommitMessageImpl expected(BinaryRowGenerator::GenerateRow({10, 0}, pool_.get()), /*bucket=*/0,
                                /*total_buckets=*/-1,
                                DataIncrement({file_meta1, file_meta2}, {}, {}),
@@ -370,7 +370,7 @@ TEST_F(FileMetaUtilsTest, TestWithNestedType) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     CommitMessageImpl expected(BinaryRow::EmptyRow(), /*bucket=*/0, /*total_buckets=*/-1,
                                DataIncrement({file_meta1}, {}, {}), CompactIncrement({}, {}, {}));
diff --git a/src/paimon/core/operation/commit/commit_changes_provider_test.cpp b/src/paimon/core/operation/commit/commit_changes_provider_test.cpp
index edbb1a48b..f47e11262 100644
--- a/src/paimon/core/operation/commit/commit_changes_provider_test.cpp
+++ b/src/paimon/core/operation/commit/commit_changes_provider_test.cpp
@@ -63,7 +63,7 @@ ManifestEntry CreateManifestEntry(const std::string& file_name, const FileKind&
         /*value_stats_cols=*/std::nullopt,
         /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     return ManifestEntry(kind, CreateIntRow(partition_value), /*bucket=*/0, /*total_buckets=*/1,
                          file_meta);
diff --git a/src/paimon/core/operation/commit/conflict_detection_test.cpp b/src/paimon/core/operation/commit/conflict_detection_test.cpp
index 66d57894b..3fb751acf 100644
--- a/src/paimon/core/operation/commit/conflict_detection_test.cpp
+++ b/src/paimon/core/operation/commit/conflict_detection_test.cpp
@@ -175,7 +175,7 @@ class ConflictDetectionTest : public testing::Test {
             /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
             /*external_path=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         return ManifestEntry(kind, partition, bucket, total_buckets, data_file_meta);
     }
 
@@ -194,7 +194,7 @@ class ConflictDetectionTest : public testing::Test {
             /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
             /*external_path=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt, first_row_id,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         return ManifestEntry(kind, partition, bucket, /*total_buckets=*/2, data_file_meta);
     }
 
diff --git a/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp b/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp
index 8680ff2c3..5eb1608c4 100644
--- a/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp
+++ b/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp
@@ -67,7 +67,7 @@ class ManifestEntryChangesTest : public testing::Test {
             /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
             /*external_path=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     std::shared_ptr<IndexFileMeta> CreateIndexFileMeta(
@@ -140,7 +140,7 @@ TEST_F(ManifestEntryChangesTest, TestDropStatsOnlyForDeleteEntries) {
         /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
         /*value_stats_cols=*/std::vector<std::string>({"f0"}),
         /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<DataFileMeta> after, before->Upgrade(/*new_level=*/1));
 
     CompactIncrement compact_increment(/*compact_before=*/{before}, /*compact_after=*/{after},
diff --git a/src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp b/src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp
index 8130fe65c..3552b6da1 100644
--- a/src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp
+++ b/src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp
@@ -64,7 +64,7 @@ ManifestEntry CreateManifestEntry(const std::string& file_name, const FileKind&
         /*value_stats_cols=*/std::nullopt,
         /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     return ManifestEntry(kind, CreateIntRow(partition_value), /*bucket=*/0, /*total_buckets=*/1,
                          file_meta);
diff --git a/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp b/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp
index e9091b0fc..907aea44c 100644
--- a/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp
+++ b/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp
@@ -57,7 +57,8 @@ class RowIdColumnConflictCheckerTest : public testing::Test {
             /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt,
             /*external_path=*/std::nullopt,
-            /*first_row_id=*/first_row_id, write_cols);
+            /*first_row_id=*/first_row_id, write_cols,
+            /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     Result<std::shared_ptr<RowIdColumnConflictChecker>> CreateChecker(
diff --git a/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp b/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp
index 02e79fc56..7d2b0fb02 100644
--- a/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp
+++ b/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp
@@ -61,7 +61,8 @@ class RowTrackingCommitUtilsTest : public testing::Test {
             /*embedded_index=*/nullptr, file_source,
             /*value_stats_cols=*/std::nullopt,
             /*external_path=*/std::nullopt,
-            /*first_row_id=*/std::nullopt, write_cols);
+            /*first_row_id=*/std::nullopt, write_cols,
+            /*column_max_sequence_numbers=*/std::nullopt);
         return ManifestEntry(FileKind::Add(), CreateIntRow(1), /*bucket=*/0, /*total_buckets=*/1,
                              file_meta);
     }
@@ -81,7 +82,8 @@ class RowTrackingCommitUtilsTest : public testing::Test {
             /*delete_row_count=*/std::nullopt,
             /*embedded_index=*/nullptr, file_source,
             /*value_stats_cols=*/std::nullopt,
-            /*external_path=*/std::nullopt, first_row_id, write_cols);
+            /*external_path=*/std::nullopt, first_row_id, write_cols,
+            /*column_max_sequence_numbers=*/std::nullopt);
         return ManifestEntry(FileKind::Add(), CreateIntRow(1), /*bucket=*/0, /*total_buckets=*/1,
                              file_meta);
     }
diff --git a/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp b/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp
index af572b720..5975eb1e8 100644
--- a/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp
+++ b/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp
@@ -74,7 +74,7 @@ class SequenceSnapshotPropertiesTest : public testing::Test {
             /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
             /*external_path=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     ManifestEntry CreateEntry(const FileKind& kind, int64_t max_sequence_number) const {
diff --git a/src/paimon/core/operation/commit_metrics_test.cpp b/src/paimon/core/operation/commit_metrics_test.cpp
index 240d91345..9d1cf9cf2 100644
--- a/src/paimon/core/operation/commit_metrics_test.cpp
+++ b/src/paimon/core/operation/commit_metrics_test.cpp
@@ -62,7 +62,7 @@ ManifestEntry CreateEntry(const FileKind& kind, int32_t partition, int32_t bucke
         /*value_stats_cols=*/std::nullopt,
         /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     return ManifestEntry(kind, part, bucket, /*total_buckets=*/10, file_meta);
 }
 
diff --git a/src/paimon/core/operation/data_evolution_file_store_scan_test.cpp b/src/paimon/core/operation/data_evolution_file_store_scan_test.cpp
index b5581f85c..7d042814b 100644
--- a/src/paimon/core/operation/data_evolution_file_store_scan_test.cpp
+++ b/src/paimon/core/operation/data_evolution_file_store_scan_test.cpp
@@ -554,7 +554,8 @@ TEST_F(DataEvolutionFileStoreScanTest, TestFilterEntryByRowRanges) {
         /*creation_time=*/Timestamp(1737111915429ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     ManifestEntry entry(FileKind::Add(), BinaryRow::EmptyRow(), /*bucket=*/0, /*total_buckets=*/1,
                         file);
     {
@@ -574,7 +575,7 @@ TEST_F(DataEvolutionFileStoreScanTest, TestFilterEntryByRowRanges) {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt,
             /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         ManifestEntry entry_without_first_row_id(FileKind::Add(), BinaryRow::EmptyRow(),
                                                  /*bucket=*/0, /*total_buckets=*/1,
                                                  file_without_first_row_id);
diff --git a/src/paimon/core/operation/expire_snapshots_test.cpp b/src/paimon/core/operation/expire_snapshots_test.cpp
index 6a9e46ba7..ffdd30feb 100644
--- a/src/paimon/core/operation/expire_snapshots_test.cpp
+++ b/src/paimon/core/operation/expire_snapshots_test.cpp
@@ -148,7 +148,7 @@ class ExpireSnapshotsTest : public testing::Test {
             /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
             /*external_path=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         return ManifestEntry(kind, row, bucket, /*total_buckets=*/3, data_file_meta);
     }
 
diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp
index f32d86fac..b1b8677f8 100644
--- a/src/paimon/core/operation/file_store_commit_impl_test.cpp
+++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp
@@ -198,7 +198,7 @@ class FileStoreCommitImplTest : public testing::Test {
             /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
             /*external_path=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         return ManifestEntry(kind, partition, bucket, total_buckets, data_file_meta);
     }
 
@@ -223,7 +223,7 @@ class FileStoreCommitImplTest : public testing::Test {
             /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
             /*external_path=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         return ManifestEntry(kind, BinaryRow::EmptyRow(), 0, 2, data_file_meta);
     }
 
@@ -278,7 +278,7 @@ class FileStoreCommitImplTest : public testing::Test {
             /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt,
             /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     std::shared_ptr<DataFileMeta> CreateAppendDataFileMeta(const std::string& file_name,
@@ -294,7 +294,7 @@ class FileStoreCommitImplTest : public testing::Test {
             /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt,
             /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     bool IsStringInSet(const std::set<std::string>& strSet, const std::string& target) {
diff --git a/src/paimon/core/operation/internal_read_context_test.cpp b/src/paimon/core/operation/internal_read_context_test.cpp
index 286188570..20838122b 100644
--- a/src/paimon/core/operation/internal_read_context_test.cpp
+++ b/src/paimon/core/operation/internal_read_context_test.cpp
@@ -20,6 +20,7 @@
 
 #include <utility>
 
+#include "arrow/c/bridge.h"
 #include "arrow/type.h"
 #include "gtest/gtest.h"
 #include "paimon/common/table/special_fields.h"
diff --git a/src/paimon/core/operation/key_value_file_store_scan_test.cpp b/src/paimon/core/operation/key_value_file_store_scan_test.cpp
index b221715f5..703d7e493 100644
--- a/src/paimon/core/operation/key_value_file_store_scan_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_scan_test.cpp
@@ -353,7 +353,8 @@ TEST_F(KeyValueFileStoreScanTest, TestNoOverlapping) {
                     /*embedded_index=*/nullptr,
                     /*file_source=*/FileSource::Append(), /*external_path=*/std::nullopt,
                     /*value_stats_cols=*/std::nullopt,
-                    /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt));
+                    /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+                    /*column_max_sequence_numbers=*/std::nullopt));
         }
         return entries;
     };
@@ -405,7 +406,7 @@ TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithValueStatsCols) {
             /*value_stats_cols=*/value_stats_cols,
             /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt));
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt));
 
     // max(v0)=50 > 30.1, should be kept.
     SimpleStats value_stats_keep = BinaryRowGenerator::GenerateStats(
@@ -430,7 +431,7 @@ TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithValueStatsCols) {
             /*value_stats_cols=*/value_stats_cols,
             /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt));
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt));
 
     // max(v0)=20 <= 30.1, should be filtered out.
     ASSERT_OK_AND_ASSIGN(bool keep, scan->FilterByStats(entry));
@@ -482,7 +483,7 @@ TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithSchemaEvolution) {
             /*value_stats_cols=*/value_stats_cols,
             /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt));
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt));
 
     SimpleStats value_stats_keep = BinaryRowGenerator::GenerateStats(
         /*min=*/{40}, /*max=*/{50}, /*null=*/{0}, pool.get());
@@ -506,7 +507,7 @@ TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithSchemaEvolution) {
             /*value_stats_cols=*/value_stats_cols,
             /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt));
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt));
 
     ASSERT_OK_AND_ASSIGN(bool keep, scan->FilterByStats(entry));
     ASSERT_FALSE(keep);
@@ -555,7 +556,7 @@ TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithNewFieldUsesNullSta
             /*value_stats_cols=*/value_stats_cols,
             /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt));
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt));
 
     ASSERT_OK_AND_ASSIGN(bool keep, scan->FilterByStats(old_schema_entry));
     ASSERT_FALSE(keep);
diff --git a/src/paimon/core/operation/manifest_file_merger_test.cpp b/src/paimon/core/operation/manifest_file_merger_test.cpp
index bff1659e3..d2617a654 100644
--- a/src/paimon/core/operation/manifest_file_merger_test.cpp
+++ b/src/paimon/core/operation/manifest_file_merger_test.cpp
@@ -101,7 +101,7 @@ class ManifestFileMergerTest : public testing::Test {
                 nullptr,  // not used
                 FileSource::Append(), /*value_stats_cols=*/std::nullopt,
                 /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-                /*write_cols=*/std::nullopt));
+                /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt));
     }
 
     ManifestFileMeta MakeManifest(const std::vector<ManifestEntry>& entries) {
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index 19cd96d67..f857d4876 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -137,7 +137,7 @@ class MergeFileSplitReadTest : public ::testing::Test,
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         auto meta1_2 = std::make_shared<DataFileMeta>(
             "data-c80ccf0f-6387-4cbc-8889-ade8cef54c43-1.parquet", /*file_size=*/3370,
             /*row_count=*/4,
@@ -154,7 +154,7 @@ class MergeFileSplitReadTest : public ::testing::Test,
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         auto meta1_3 = std::make_shared<DataFileMeta>(
             "data-c80ccf0f-6387-4cbc-8889-ade8cef54c43-2.parquet", /*file_size=*/3252,
             /*row_count=*/1,
@@ -173,7 +173,7 @@ class MergeFileSplitReadTest : public ::testing::Test,
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         DataSplitImpl::Builder builder1(BinaryRowGenerator::GenerateRow({0, 0}, pool_.get()),
                                         /*bucket=*/0, /*bucket_path=*/
                                         paimon::test::GetDataDir() +
@@ -200,7 +200,7 @@ class MergeFileSplitReadTest : public ::testing::Test,
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         auto meta2_2 = std::make_shared<DataFileMeta>(
             "data-24f8588c-d950-4e44-9d99-a023ea65a136-1.parquet", /*file_size=*/3229,
             /*row_count=*/1,
@@ -218,7 +218,7 @@ class MergeFileSplitReadTest : public ::testing::Test,
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         DataSplitImpl::Builder builder2(BinaryRowGenerator::GenerateRow({0, 1}, pool_.get()),
                                         /*bucket=*/0, /*bucket_path=*/
                                         paimon::test::GetDataDir() +
@@ -246,7 +246,7 @@ class MergeFileSplitReadTest : public ::testing::Test,
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         auto meta3_2 = std::make_shared<DataFileMeta>(
             "data-184f2304-49fd-4916-ba07-037757e904eb-1.parquet", /*file_size=*/3259,
             /*row_count=*/1,
@@ -264,7 +264,7 @@ class MergeFileSplitReadTest : public ::testing::Test,
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         DataSplitImpl::Builder builder3(BinaryRowGenerator::GenerateRow({1, 0}, pool_.get()),
                                         /*bucket=*/0, /*bucket_path=*/
                                         paimon::test::GetDataDir() +
@@ -296,7 +296,7 @@ class MergeFileSplitReadTest : public ::testing::Test,
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         auto meta1_2 = std::make_shared<DataFileMeta>(
             "data-d03e13e5-5e2e-463a-b53a-8d44e4dc9141-1.parquet",
             /*file_size=*/2623, /*row_count=*/
@@ -314,7 +314,7 @@ class MergeFileSplitReadTest : public ::testing::Test,
             /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         DataSplitImpl::Builder builder1(
             /*partition=*/BinaryRow::EmptyRow(),
             /*bucket=*/0, /*bucket_path=*/
@@ -1293,7 +1293,7 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) {
         /*delete_row_count=*/1, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto meta2 = std::make_shared<DataFileMeta>(
         "data-6871b960-edd9-40fc-9859-aaca9ea205cf-0.orc", /*file_size=*/887, /*row_count=*/5,
         /*min_key=*/BinaryRowGenerator::GenerateRow({std::string("Alex"), 0}, pool_.get()),
@@ -1311,7 +1311,7 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(
         BinaryRowGenerator::GenerateRow({10}, pool_.get()),
         /*bucket=*/1, /*bucket_path=*/
diff --git a/src/paimon/core/operation/metrics/commit_stats_test.cpp b/src/paimon/core/operation/metrics/commit_stats_test.cpp
index a18f88a61..b22f7a60f 100644
--- a/src/paimon/core/operation/metrics/commit_stats_test.cpp
+++ b/src/paimon/core/operation/metrics/commit_stats_test.cpp
@@ -59,7 +59,7 @@ ManifestEntry CreateEntry(const FileKind& kind, int32_t partition, int32_t bucke
         /*value_stats_cols=*/std::nullopt,
         /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     return ManifestEntry(kind, part, bucket, /*total_buckets=*/10, file_meta);
 }
 
diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp
index 6f7ae9781..51c478325 100644
--- a/src/paimon/core/operation/raw_file_split_read_test.cpp
+++ b/src/paimon/core/operation/raw_file_split_read_test.cpp
@@ -67,7 +67,7 @@ class RawFileSplitReadTest : public ::testing::Test {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         DataSplitImpl::Builder builder1(BinaryRowGenerator::GenerateRow({10, 0}, pool_.get()),
                                         /*bucket=*/0, /*bucket_path=*/
                                         paimon::test::GetDataDir() +
@@ -91,7 +91,7 @@ class RawFileSplitReadTest : public ::testing::Test {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         DataSplitImpl::Builder builder2(BinaryRowGenerator::GenerateRow({20, 1}, pool_.get()),
                                         /*bucket=*/0, /*bucket_path=*/
                                         paimon::test::GetDataDir() +
@@ -115,7 +115,7 @@ class RawFileSplitReadTest : public ::testing::Test {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         DataSplitImpl::Builder builder3(BinaryRowGenerator::GenerateRow({10, 1}, pool_.get()),
                                         /*bucket=*/0, /*bucket_path=*/
                                         paimon::test::GetDataDir() +
@@ -462,7 +462,7 @@ TEST_F(RawFileSplitReadTest, TestMatch) {
             /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt,
             /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         DataSplitImpl::Builder builder(BinaryRowGenerator::GenerateRow({10, 0}, pool_.get()),
                                        /*bucket=*/0, /*bucket_path=*/
                                        paimon::test::GetDataDir() +
diff --git a/src/paimon/core/operation/write_restore_test.cpp b/src/paimon/core/operation/write_restore_test.cpp
index 43f070154..754dc6959 100644
--- a/src/paimon/core/operation/write_restore_test.cpp
+++ b/src/paimon/core/operation/write_restore_test.cpp
@@ -40,7 +40,8 @@ std::shared_ptr<DataFileMeta> CreateDataFileMeta(const std::string& file_name) {
         /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/std::nullopt,
         /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
-        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 }
 
 ManifestEntry CreateManifestEntry(int32_t total_buckets, const std::string& file_name) {
diff --git a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp
index e932d40da..30357327a 100644
--- a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp
+++ b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp
@@ -226,7 +226,7 @@ TEST_P(PostponeBucketWriterTest, TestSimple) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{},
                                           /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
@@ -308,7 +308,7 @@ TEST_P(PostponeBucketWriterTest, TestNestedType) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{},
                                           /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
@@ -474,7 +474,7 @@ TEST_P(PostponeBucketWriterTest, TestWriteMultiBatch) {
         /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{},
                                           /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
@@ -624,7 +624,7 @@ TEST_P(PostponeBucketWriterTest, TestMultiplePrepareCommit) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement expected_data_increment1({expected_data_file_meta1}, /*deleted_files=*/{},
                                            /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment1, commit_increment1.GetNewFilesIncrement());
@@ -644,7 +644,7 @@ TEST_P(PostponeBucketWriterTest, TestMultiplePrepareCommit) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement expected_data_increment2({expected_data_file_meta2}, /*deleted_files=*/{},
                                            /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment2, commit_increment2.GetNewFilesIncrement());
diff --git a/src/paimon/core/table/sink/commit_message_serializer.cpp b/src/paimon/core/table/sink/commit_message_serializer.cpp
index 1b0645f4e..e3424d4a2 100644
--- a/src/paimon/core/table/sink/commit_message_serializer.cpp
+++ b/src/paimon/core/table/sink/commit_message_serializer.cpp
@@ -38,6 +38,7 @@
 #include "paimon/core/io/data_file_meta_12_serializer.h"
 #include "paimon/core/io/data_file_meta_first_row_id_legacy_serializer.h"
 #include "paimon/core/io/data_file_meta_serializer.h"
+#include "paimon/core/io/data_file_meta_write_cols_legacy_serializer.h"
 #include "paimon/core/io/data_increment.h"
 #include "paimon/core/table/sink/commit_message_impl.h"
 #include "paimon/core/utils/object_serializer.h"
@@ -46,7 +47,7 @@
 namespace paimon {
 class MemoryPool;
 
-const int32_t CommitMessageSerializer::CURRENT_VERSION = 12;
+const int32_t CommitMessageSerializer::CURRENT_VERSION = 13;
 
 CommitMessageSerializer::CommitMessageSerializer(const std::shared_ptr<MemoryPool>& pool)
     : memory_pool_(pool),
@@ -204,16 +205,25 @@ Result<std::shared_ptr<CommitMessage>> CommitMessageSerializer::Deserialize(int3
                                                                             DataInputStream* in) {
     if (version == CURRENT_VERSION) {
         return Deserialize(version, data_file_serializer_.get(), index_entry_serializer_.get(), in);
+    } else if (version == 12) {
+        auto data_file_meta_write_cols_legacy_serializer =
+            std::make_unique<DataFileMetaWriteColsLegacySerializer>(memory_pool_);
+        return Deserialize(version, data_file_meta_write_cols_legacy_serializer.get(),
+                           index_entry_serializer_.get(), in);
     } else if (version == 11) {
+        auto data_file_meta_write_cols_legacy_serializer =
+            std::make_unique<DataFileMetaWriteColsLegacySerializer>(memory_pool_);
         auto index_entry_v4_deserializer =
             std::make_unique<IndexFileMetaV4Deserializer>(memory_pool_);
-        return Deserialize(version, data_file_serializer_.get(), index_entry_v4_deserializer.get(),
-                           in);
+        return Deserialize(version, data_file_meta_write_cols_legacy_serializer.get(),
+                           index_entry_v4_deserializer.get(), in);
     } else if (version == 9 || version == 10) {
+        auto data_file_meta_write_cols_legacy_serializer =
+            std::make_unique<DataFileMetaWriteColsLegacySerializer>(memory_pool_);
         auto index_entry_v3_deserializer =
             std::make_unique<IndexFileMetaV3Deserializer>(memory_pool_);
-        return Deserialize(version, data_file_serializer_.get(), index_entry_v3_deserializer.get(),
-                           in);
+        return Deserialize(version, data_file_meta_write_cols_legacy_serializer.get(),
+                           index_entry_v3_deserializer.get(), in);
     } else if (version == 8) {
         auto data_file_meta_first_row_id_legacy_serializer =
             std::make_unique<DataFileMetaFirstRowIdLegacySerializer>(memory_pool_);
diff --git a/src/paimon/core/table/sink/commit_message_test.cpp b/src/paimon/core/table/sink/commit_message_test.cpp
index d602da227..f372361cb 100644
--- a/src/paimon/core/table/sink/commit_message_test.cpp
+++ b/src/paimon/core/table/sink/commit_message_test.cpp
@@ -65,6 +65,44 @@ TEST(CommitMessageTest, TestCurrentVersion) {
     ASSERT_EQ(CommitMessageSerializer::CURRENT_VERSION, CommitMessage::CurrentVersion());
 }
 
+TEST(CommitMessageTest, TestDeserializeVersion13GeneratedByJava) {
+    // Generated by CommitMessageSerializer::serialize from Apache Paimon Java master.
+    std::string data_path = paimon::test::GetDataDir() + "/compatibility/commit_message-v13";
+    auto file_system = std::make_shared<LocalFileSystem>();
+    auto buffer_length = file_system->GetFileStatus(data_path).value().GetLen();
+    ASSERT_GT(buffer_length, 0);
+
+    std::vector<char> buffer(buffer_length, 0);
+    ASSERT_OK_AND_ASSIGN(auto in_stream, file_system->Open(data_path));
+    ASSERT_OK(in_stream->Read(buffer.data(), buffer.size()));
+    ASSERT_OK(in_stream->Close());
+
+    auto pool = GetDefaultPool();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<CommitMessage> result,
+                         CommitMessage::Deserialize(CommitMessage::CurrentVersion(), buffer.data(),
+                                                    buffer.size(), pool));
+    auto result_message = std::dynamic_pointer_cast<CommitMessageImpl>(result);
+    ASSERT_NE(result_message, nullptr);
+    ASSERT_EQ(result_message->Partition(),
+              BinaryRowGenerator::GenerateRow({std::string("aaaaa")}, pool.get()));
+    ASSERT_EQ(result_message->Bucket(), 20);
+    ASSERT_EQ(result_message->TotalBuckets(), std::optional<int32_t>(32));
+    ASSERT_EQ(result_message->GetNewFilesIncrement().NewFiles().size(), 1);
+    ASSERT_TRUE(result_message->GetCompactIncrement().IsEmpty());
+
+    const std::shared_ptr<DataFileMeta>& data_file =
+        result_message->GetNewFilesIncrement().NewFiles()[0];
+    ASSERT_EQ(data_file->file_name, "my_file");
+    ASSERT_TRUE(data_file->write_cols.has_value());
+    ASSERT_EQ(data_file->write_cols.value(), (std::vector<std::string>{"a", "b", "c", "f"}));
+    ASSERT_TRUE(data_file->column_max_sequence_numbers.has_value());
+    ASSERT_EQ(data_file->column_max_sequence_numbers.value(),
+              (std::vector<int64_t>{15, 100, 150, 200}));
+
+    ASSERT_OK_AND_ASSIGN(std::string serialized_bytes, CommitMessage::Serialize(result, pool));
+    ASSERT_EQ(serialized_bytes, std::string(buffer.data(), buffer.size()));
+}
+
 TEST(CommitMessageTest, TestCompatibleWithVersion12) {
     // index file meta: add global index meta source meta
     int32_t version = 12;
@@ -95,7 +133,13 @@ TEST(CommitMessageTest, TestCompatibleWithVersion12) {
 
     // check result
     ASSERT_OK_AND_ASSIGN(std::string serialized_bytes, CommitMessage::Serialize(ret, pool));
-    ASSERT_EQ(serialized_bytes, std::string(reinterpret_cast<char*>(buffer.data()), buffer.size()));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<CommitMessage> roundtrip,
+        CommitMessage::Deserialize(CommitMessage::CurrentVersion(), serialized_bytes.data(),
+                                   serialized_bytes.size(), pool));
+    auto roundtrip_message = std::dynamic_pointer_cast<CommitMessageImpl>(roundtrip);
+    ASSERT_NE(roundtrip_message, nullptr);
+    ASSERT_EQ(*roundtrip_message, *res_msg);
 }
 
 TEST(CommitMessageTest, TestCompatibleWithVersion11) {
@@ -185,7 +229,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion10) {
         /*creation_time=*/Timestamp(1761242383412ll, 0),
         /*delete_row_count=*/1, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     LinkedHashMap<std::string, DeletionVectorMeta> dv_ranges;
     dv_ranges.insert_or_assign(
@@ -249,7 +294,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion9) {
         /*creation_time=*/Timestamp(1757349273600ll, 0),
         /*delete_row_count=*/1, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     LinkedHashMap<std::string, DeletionVectorMeta> dv_ranges;
     dv_ranges.insert_or_assign(
@@ -316,7 +362,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion9WithExternalPathForIndex) {
         /*value_stats_cols=*/std::nullopt,
         /*external_path=*/
         "FILE:/tmp/external/f1=10/bucket-1/data-72b62a5f-d698-4db5-b51a-04c0dc027702-1.orc",
-        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     LinkedHashMap<std::string, DeletionVectorMeta> dv_ranges;
     dv_ranges.insert_or_assign(
@@ -379,7 +426,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion8) {
         /*creation_time=*/Timestamp(1754068646844ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     expected_msgs.emplace_back(/*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
                                /*bucket=*/0, /*total_bucket=*/2, DataIncrement({file_meta}, {}, {}),
                                CompactIncrement({}, {}, {}));
@@ -398,7 +446,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion8) {
         /*creation_time=*/Timestamp(1754068646864ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     expected_msgs.emplace_back(/*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
                                /*bucket=*/1, /*total_bucket=*/2,
                                DataIncrement({file_meta2}, {}, {}), CompactIncrement({}, {}, {}));
@@ -448,7 +497,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion7) {
         /*creation_time=*/Timestamp(1743525392885ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     expected_msgs.emplace_back(/*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
                                /*bucket=*/0, /*total_bucket=*/2, DataIncrement({file_meta}, {}, {}),
                                CompactIncrement({}, {}, {}));
@@ -469,7 +519,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion7) {
         /*creation_time=*/Timestamp(1743525392921ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     expected_msgs.emplace_back(/*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
                                /*bucket=*/1, /*total_bucket=*/2,
                                DataIncrement({file_meta2}, {}, {}), CompactIncrement({}, {}, {}));
@@ -515,7 +566,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion6) {
         /*creation_time=*/Timestamp(1737052260143ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     expected_msgs.emplace_back(BinaryRow::EmptyRow(), /*bucket=*/0, /*total_bucket=*/std::nullopt,
                                DataIncrement({file_meta}, {}, {}), CompactIncrement({}, {}, {}));
     // check result
@@ -562,7 +614,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion5) {
         /*creation_time=*/Timestamp(1734707236040ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::optional<std::vector<std::string>>({"f0", "f1", "f2"}),
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta1_after_compact = std::make_shared<DataFileMeta>(
         "data-0d0f29cc-63c6-4fab-a594-71bd7d06fcde-1.orc", /*file_size=*/859, /*row_count=*/1,
         BinaryRowGenerator::GenerateRow({std::string("Alice"), 1}, pool.get()),
@@ -576,7 +629,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion5) {
         /*creation_time=*/Timestamp(1734707236040ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::optional<std::vector<std::string>>({"f0", "f1", "f2"}),
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataIncrement data_increment1({file_meta1}, {}, {}, {}, {});
     LinkedHashMap<std::string, DeletionVectorMeta> dv_metas1;
@@ -605,7 +659,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion5) {
         /*creation_time=*/Timestamp(1734707236109ll, 0),
         /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::optional<std::vector<std::string>>({"f0", "f1", "f2"}),
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment2({file_meta2}, {}, {}, {}, {});
     LinkedHashMap<std::string, DeletionVectorMeta> dv_metas2;
     dv_metas2.insert_or_assign(
@@ -663,7 +718,7 @@ TEST(CommitMessageTest, TestCompatibleWithVersion4) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment1({file_meta1}, {}, {});
     expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/0,
                                /*total_bucket=*/std::nullopt, data_increment1,
@@ -682,7 +737,7 @@ TEST(CommitMessageTest, TestCompatibleWithVersion4) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment2({file_meta2}, {}, {});
     expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1,
                                /*total_bucket=*/std::nullopt, data_increment2,
@@ -701,7 +756,7 @@ TEST(CommitMessageTest, TestCompatibleWithVersion4) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment3({file_meta3}, {}, {});
     expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({20}, pool.get()), /*bucket=*/0,
                                /*total_bucket=*/std::nullopt, data_increment3,
@@ -751,7 +806,8 @@ TEST(CommitMessageTest, TestCompatibleWithJavaPaimon10WithStatsDenseStore) {
         /*creation_time=*/Timestamp(1731412938869ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::optional<std::vector<std::string>>({"f0", "f1", "f2"}),
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment1({file_meta1}, {}, {});
     expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/0,
                                /*total_bucket=*/std::nullopt, data_increment1,
@@ -768,7 +824,8 @@ TEST(CommitMessageTest, TestCompatibleWithJavaPaimon10WithStatsDenseStore) {
         /*creation_time=*/Timestamp(1731412938891ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::optional<std::vector<std::string>>({"f0", "f1", "f2"}),
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment2({file_meta2}, {}, {});
     expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1,
                                /*total_bucket=*/std::nullopt, data_increment2,
@@ -785,7 +842,8 @@ TEST(CommitMessageTest, TestCompatibleWithJavaPaimon10WithStatsDenseStore) {
         /*creation_time=*/Timestamp(1731412938908ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::optional<std::vector<std::string>>({"f0", "f1", "f2"}),
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment3({file_meta3}, {}, {});
     expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({20}, pool.get()), /*bucket=*/0,
                                /*total_bucket=*/std::nullopt, data_increment3,
@@ -835,7 +893,7 @@ TEST(CommitMessageTest, TestCompatibleWith09JavaPaimon1) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment1({file_meta1}, {}, {});
     expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/0,
                                /*total_bucket=*/std::nullopt, data_increment1,
@@ -854,7 +912,7 @@ TEST(CommitMessageTest, TestCompatibleWith09JavaPaimon1) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment2({file_meta2}, {}, {});
     expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1,
                                /*total_bucket=*/std::nullopt, data_increment2,
@@ -873,7 +931,7 @@ TEST(CommitMessageTest, TestCompatibleWith09JavaPaimon1) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment3({file_meta3}, {}, {});
     expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({20}, pool.get()), /*bucket=*/0,
                                /*total_bucket=*/std::nullopt, data_increment3,
@@ -923,7 +981,7 @@ TEST(CommitMessageTest, TestCompatibleWith09JavaPaimon2) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment1({file_meta1}, {}, {});
     expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1,
                                /*total_bucket=*/std::nullopt, data_increment1,
@@ -943,7 +1001,7 @@ TEST(CommitMessageTest, TestCompatibleWith09JavaPaimon2) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment2({file_meta2}, {}, {});
     expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({20}, pool.get()), /*bucket=*/0,
                                /*total_bucket=*/std::nullopt, data_increment2,
@@ -993,7 +1051,7 @@ TEST(CommitMessageTest, TestCompatibleWith09JavaPaimon3) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment1({file_meta1}, {}, {});
     expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1,
                                /*total_bucket=*/std::nullopt, data_increment1,
@@ -1046,7 +1104,7 @@ TEST(CommitMessageTest, TestPkTableCompatibleWithJavaPaimon09) {
         /*delete_row_count=*/1, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta_with_level = std::make_shared<DataFileMeta>(
         "data-2eb2a766-97e4-4fe4-88ce-eb606675c101-0.orc", /*file_size=*/789, /*row_count=*/1,
         /*min_key=*/BinaryRowGenerator::GenerateRow({std::string("Bob"), 0}, pool.get()),
@@ -1063,7 +1121,7 @@ TEST(CommitMessageTest, TestPkTableCompatibleWithJavaPaimon09) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment1({file_meta}, {}, {}, {}, {});
 
     LinkedHashMap<std::string, DeletionVectorMeta> dv_ranges;
@@ -1141,7 +1199,7 @@ TEST(CommitMessageTest, TestCompatibleWithComplexDataType) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment1({file_meta1}, {}, {});
     expected_msgs.emplace_back(BinaryRow::EmptyRow(), /*bucket=*/0, /*total_bucket=*/std::nullopt,
                                data_increment1, CompactIncrement({}, {}, {}));
diff --git a/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp
index 14fae3972..5fbd5f093 100644
--- a/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp
+++ b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp
@@ -49,7 +49,7 @@ std::shared_ptr<DataFileMeta> NewAppendFile(const std::string& file_name, int64_
         /*max_sequence_number=*/first_row_id + row_count - 1, /*schema_id=*/0, /*level=*/0,
         std::vector<std::optional<std::string>>(), Timestamp(0l, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(), std::nullopt, std::nullopt, first_row_id,
-        std::nullopt);
+        std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 }
 
 std::shared_ptr<Plan> NewDataPlan(std::vector<std::shared_ptr<DataFileMeta>> files) {
diff --git a/src/paimon/core/table/source/data_split_impl.cpp b/src/paimon/core/table/source/data_split_impl.cpp
index 5b9128a73..be0ef4030 100644
--- a/src/paimon/core/table/source/data_split_impl.cpp
+++ b/src/paimon/core/table/source/data_split_impl.cpp
@@ -223,6 +223,8 @@ DataSplitImpl::GetFileMetaSerializer(int32_t version, const std::shared_ptr<Memo
         return std::make_unique<DataFileMeta12Serializer>(pool);
     } else if (version == 7) {
         return std::make_unique<DataFileMetaFirstRowIdLegacySerializer>(pool);
+    } else if (version == 8) {
+        return std::make_unique<DataFileMetaWriteColsLegacySerializer>(pool);
     } else if (version == VERSION) {
         return std::make_unique<DataFileMetaSerializer>(pool);
     } else {
diff --git a/src/paimon/core/table/source/data_split_impl.h b/src/paimon/core/table/source/data_split_impl.h
index 01c1231c8..2d357a163 100644
--- a/src/paimon/core/table/source/data_split_impl.h
+++ b/src/paimon/core/table/source/data_split_impl.h
@@ -35,6 +35,7 @@
 #include "paimon/core/io/data_file_meta_12_serializer.h"
 #include "paimon/core/io/data_file_meta_first_row_id_legacy_serializer.h"
 #include "paimon/core/io/data_file_meta_serializer.h"
+#include "paimon/core/io/data_file_meta_write_cols_legacy_serializer.h"
 #include "paimon/core/table/source/deletion_file.h"
 #include "paimon/table/source/data_split.h"
 
@@ -44,7 +45,7 @@ namespace paimon {
 class DataSplitImpl : public DataSplit {
  public:
     static constexpr int64_t MAGIC = -2394839472490812314L;
-    static constexpr int32_t VERSION = 8;
+    static constexpr int32_t VERSION = 9;
 
     int64_t SnapshotId() const {
         return snapshot_id_;
diff --git a/src/paimon/core/table/source/data_split_test.cpp b/src/paimon/core/table/source/data_split_test.cpp
index b65988526..bb631be9a 100644
--- a/src/paimon/core/table/source/data_split_test.cpp
+++ b/src/paimon/core/table/source/data_split_test.cpp
@@ -46,6 +46,47 @@
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
+TEST(DataSplitTest, TestDeserializeVersion9GeneratedByJava) {
+    // Generated by DataSplit::serialize from Apache Paimon Java master.
+    std::string file_name = paimon::test::GetDataDir() + "/compatibility/data_split-v9";
+    auto file_system = std::make_unique<LocalFileSystem>();
+
+    ASSERT_OK_AND_ASSIGN(auto input_stream, file_system->Open(file_name));
+    std::vector<char> split_bytes(input_stream->Length().value_or(0), 0);
+    ASSERT_GT(split_bytes.size(), 0);
+    ASSERT_OK(input_stream->Read(split_bytes.data(), split_bytes.size()));
+    ASSERT_OK(input_stream->Close());
+
+    auto pool = GetDefaultPool();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Split> result,
+                         Split::Deserialize(split_bytes.data(), split_bytes.size(), pool));
+    auto result_data_split = std::dynamic_pointer_cast<DataSplitImpl>(result);
+    ASSERT_NE(result_data_split, nullptr);
+    ASSERT_EQ(result_data_split->SnapshotId(), 18);
+    ASSERT_EQ(result_data_split->Partition(),
+              BinaryRowGenerator::GenerateRow({std::string("aaaaa")}, pool.get()));
+    ASSERT_EQ(result_data_split->Bucket(), 20);
+    ASSERT_EQ(result_data_split->BucketPath(), "my path");
+    ASSERT_EQ(result_data_split->TotalBuckets(), std::optional<int32_t>(32));
+    ASSERT_EQ(result_data_split->DataFiles().size(), 1);
+    ASSERT_EQ(
+        result_data_split->DeletionFiles(),
+        (std::vector<std::optional<DeletionFile>>{DeletionFile("deletion_file", 100, 22, 33)}));
+    ASSERT_FALSE(result_data_split->IsStreaming());
+    ASSERT_FALSE(result_data_split->RawConvertible());
+
+    const std::shared_ptr<DataFileMeta>& data_file = result_data_split->DataFiles()[0];
+    ASSERT_EQ(data_file->file_name, "my_file");
+    ASSERT_TRUE(data_file->write_cols.has_value());
+    ASSERT_EQ(data_file->write_cols.value(), (std::vector<std::string>{"a", "b", "c", "f"}));
+    ASSERT_TRUE(data_file->column_max_sequence_numbers.has_value());
+    ASSERT_EQ(data_file->column_max_sequence_numbers.value(),
+              (std::vector<int64_t>{15, 100, 150, 200}));
+
+    ASSERT_OK_AND_ASSIGN(std::string serialized_bytes, Split::Serialize(result, pool));
+    ASSERT_EQ(serialized_bytes, std::string(split_bytes.data(), split_bytes.size()));
+}
+
 TEST(DataSplitTest, TestDeserializeVersion8WithWriteColsAndExternalPath) {
     std::string file_name = paimon::test::GetDataDir() +
                             "/orc/pk_dv_index_in_data_with_external.db/"
@@ -86,7 +127,8 @@ TEST(DataSplitTest, TestDeserializeVersion8WithWriteColsAndExternalPath) {
         /*value_stats_cols=*/std::nullopt,
         /*external_path=*/
         "FILE:/tmp/external/f1=10/bucket-1/data-72b62a5f-d698-4db5-b51a-04c0dc027702-0.orc",
-        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
@@ -108,7 +150,10 @@ TEST(DataSplitTest, TestDeserializeVersion8WithWriteColsAndExternalPath) {
             .value());
     ASSERT_EQ(*result_data_split, *expected_data_split) << result_data_split->ToString();
     ASSERT_OK_AND_ASSIGN(std::string serialize_bytes, Split::Serialize(result_data_split, pool));
-    ASSERT_EQ(serialize_bytes, std::string(split_bytes.data(), split_bytes.size()));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Split> roundtrip,
+                         Split::Deserialize(serialize_bytes.data(), serialize_bytes.size(), pool));
+    auto roundtrip_data_split = std::dynamic_pointer_cast<DataSplitImpl>(roundtrip);
+    ASSERT_EQ(*roundtrip_data_split, *expected_data_split) << roundtrip_data_split->ToString();
 }
 
 TEST(DataSplitTest, TestDeserializeVersion8WithWriteCols) {
@@ -149,7 +194,8 @@ TEST(DataSplitTest, TestDeserializeVersion8WithWriteCols) {
         /*creation_time=*/Timestamp(1757349273246ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
@@ -172,7 +218,10 @@ TEST(DataSplitTest, TestDeserializeVersion8WithWriteCols) {
             .value());
     ASSERT_EQ(*result_data_split, *expected_data_split) << result_data_split->ToString();
     ASSERT_OK_AND_ASSIGN(std::string serialize_bytes, Split::Serialize(result_data_split, pool));
-    ASSERT_EQ(serialize_bytes, std::string(split_bytes.data(), split_bytes.size()));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Split> roundtrip,
+                         Split::Deserialize(serialize_bytes.data(), serialize_bytes.size(), pool));
+    auto roundtrip_data_split = std::dynamic_pointer_cast<DataSplitImpl>(roundtrip);
+    ASSERT_EQ(*roundtrip_data_split, *expected_data_split) << roundtrip_data_split->ToString();
 }
 
 TEST(DataSplitTest, TestDeserializeVersion7WithFirstRowId) {
@@ -209,7 +258,8 @@ TEST(DataSplitTest, TestDeserializeVersion7WithFirstRowId) {
         /*creation_time=*/Timestamp(1754073518741ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/5, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/5, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRow::EmptyRow(),
@@ -261,7 +311,8 @@ TEST(DataSplitTest, TestDeserializeVersion7WithNullFirstRowId) {
         /*creation_time=*/Timestamp(1754068646844ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
@@ -312,7 +363,8 @@ TEST(DataSplitTest, TestDeserializeVersion6PkWithTotalBuckets) {
         /*creation_time=*/Timestamp(1743525392885ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
@@ -358,7 +410,8 @@ TEST(DataSplitTest, TestDeserializeVersion5PkWithExternalPath) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
         /*external_path=*/"file:/tmp/bucket-0/data-80110e15-97b5-4bcf-ac09-6ca2659a4950-0.orc",
-        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRow::EmptyRow(),
@@ -407,7 +460,8 @@ TEST(DataSplitTest, TestDeserializeVersion5PkWithEmptyExternalPath) {
         /*creation_time=*/Timestamp(1737052260143ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRow::EmptyRow(),
@@ -463,7 +517,8 @@ TEST(DataSplitTest, TestDeserializeVersion4PkWithSnapshot4WithDvCardinality) {
         /*creation_time=*/Timestamp(1734707235578ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::optional<std::vector<std::string>>({"f0", "f1", "f2"}),
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
         /*bucket=*/1, /*bucket_path=*/
@@ -516,7 +571,7 @@ TEST(DataSplitTest, TestDeserializeVersion3AppendWithSnapshot1) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(/*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
                                    /*bucket=*/1, /*bucket_path=*/
                                    "data/append_10.db/append_10/f1=10/bucket-1", {file_meta});
@@ -555,7 +610,8 @@ TEST(DataSplitTest, TestDeserializeVersion3AppendWithSnapshot1WithStatsDenseStor
         /*creation_time=*/Timestamp(1731412938891ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::optional<std::vector<std::string>>({"f0", "f1", "f2"}),
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
         /*bucket=*/1, /*bucket_path=*/
@@ -596,7 +652,7 @@ TEST(DataSplitTest, TestDeserializeAppendWithSnapshot1) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
         /*bucket=*/1, /*bucket_path=*/
@@ -636,7 +692,7 @@ TEST(DataSplitTest, TestDeserializeAppendWithSnapshot3) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-b913a160-a4d1-4084-af2a-18333c35668e-0.orc", /*file_size=*/506, /*row_count=*/1,
@@ -651,7 +707,7 @@ TEST(DataSplitTest, TestDeserializeAppendWithSnapshot3) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({20}, pool.get()),
@@ -709,7 +765,7 @@ TEST(DataSplitTest, TestDeserializeAppendWithSnapshot5) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
@@ -759,7 +815,7 @@ TEST(DataSplitTest, TestDeserializePkWithSnapshot2) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
         /*bucket=*/0, /*bucket_path=*/
@@ -808,7 +864,7 @@ TEST(DataSplitTest, TestDeserializePkWithSnapshot6OfSingleFile) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(/*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
                                    /*bucket=*/1, /*bucket_path=*/
                                    "data/pk_09.db/pk_09/f1=10/bucket-1", {file_meta});
@@ -866,7 +922,7 @@ TEST(DataSplitTest, TestDeserializePkWithSnapshot6OfMultiFiles) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-1c7a85f1-55bd-424f-b503-34a33be0fb96-0.orc", /*file_size=*/1148, /*row_count=*/2,
@@ -888,7 +944,7 @@ TEST(DataSplitTest, TestDeserializePkWithSnapshot6OfMultiFiles) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     auto file_meta3 = std::make_shared<DataFileMeta>(
         "data-8cdb8b8d-5830-4b3b-aa94-8a30c449277a-0.orc", /*file_size=*/810, /*row_count=*/1,
@@ -907,7 +963,7 @@ TEST(DataSplitTest, TestDeserializePkWithSnapshot6OfMultiFiles) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
@@ -967,7 +1023,7 @@ TEST(DataSplitTest, TestDeserializePkWithSnapshot8) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(/*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
                                    /*bucket=*/0, /*bucket_path=*/
                                    "data/pk_09.db/pk_09/f1=10/bucket-0", {file_meta});
@@ -1018,7 +1074,7 @@ TEST(DataSplitTest, TestDeserializePk10WithSnapshot6) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-d6d370f3-242b-45c9-8739-44bf31b2b449-0.orc", /*file_size=*/924, /*row_count=*/1,
         /*min_key=*/BinaryRowGenerator::GenerateRow({52}, pool.get()), /*max_key=*/
@@ -1035,7 +1091,7 @@ TEST(DataSplitTest, TestDeserializePk10WithSnapshot6) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({1, 1}, pool.get()), /*bucket=*/0,
@@ -1068,7 +1124,7 @@ TEST(DataSplitTest, TestPartialMergedRowCount) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(),
         DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
@@ -1078,7 +1134,7 @@ TEST(DataSplitTest, TestPartialMergedRowCount) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
         /*bucket=*/0, /*bucket_path=*/
@@ -1112,7 +1168,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithoutDeletionFiles)
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-1.orc", /*file_size=*/100, /*row_count=*/4, DataFileMeta::EmptyMinKey(),
         DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
@@ -1122,7 +1178,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithoutDeletionFiles)
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
@@ -1147,7 +1203,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithCardinality) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(),
         DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
@@ -1157,7 +1213,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithCardinality) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta3 = std::make_shared<DataFileMeta>(
         "data-2.orc", /*file_size=*/100, /*row_count=*/3, DataFileMeta::EmptyMinKey(),
         DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
@@ -1167,7 +1223,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithCardinality) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
@@ -1202,7 +1258,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountMixedCardinalityReturnsNullopt) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(),
         DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
@@ -1212,7 +1268,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountMixedCardinalityReturnsNullopt) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
@@ -1245,7 +1301,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountUnknownDeleteRowCountDoesNotBlockRa
         /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(),
         DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
@@ -1255,7 +1311,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountUnknownDeleteRowCountDoesNotBlockRa
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
@@ -1284,7 +1340,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountFallsBackToDataEvolution) {
         /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/100,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-1.orc", /*file_size=*/100, /*row_count=*/5, DataFileMeta::EmptyMinKey(),
         DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
@@ -1294,7 +1350,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountFallsBackToDataEvolution) {
         /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/100,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta3 = std::make_shared<DataFileMeta>(
         "data-2.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(),
         DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
@@ -1304,7 +1360,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountFallsBackToDataEvolution) {
         /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/200,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
@@ -1335,7 +1391,7 @@ TEST(DataSplitTest, TestDataEvolutionMergedRowCountSubtractsDeletionFileCardinal
         /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/100,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-1.orc", /*file_size=*/100, /*row_count=*/5, DataFileMeta::EmptyMinKey(),
         DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
@@ -1345,7 +1401,7 @@ TEST(DataSplitTest, TestDataEvolutionMergedRowCountSubtractsDeletionFileCardinal
         /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/100,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
@@ -1380,7 +1436,7 @@ TEST(DataSplitTest, TestSerializeDataEvolutionSplitWithDeletionFiles) {
         /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/0,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-1.orc", /*file_size=*/100, /*row_count=*/10, DataFileMeta::EmptyMinKey(),
         DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
@@ -1390,7 +1446,7 @@ TEST(DataSplitTest, TestSerializeDataEvolutionSplitWithDeletionFiles) {
         /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/0,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
@@ -1438,7 +1494,7 @@ TEST(DataSplitTest, TestDataEvolutionMergedRowCountUnavailableWithoutCardinality
         /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/100,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
@@ -1471,7 +1527,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountResolvesMissingCardinalityViaFactor
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-1.orc", /*file_size=*/100, /*row_count=*/5, DataFileMeta::EmptyMinKey(),
         DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
@@ -1481,7 +1537,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountResolvesMissingCardinalityViaFactor
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
diff --git a/src/paimon/core/table/source/fallback_data_split_test.cpp b/src/paimon/core/table/source/fallback_data_split_test.cpp
index 80d290560..5ae97e19e 100644
--- a/src/paimon/core/table/source/fallback_data_split_test.cpp
+++ b/src/paimon/core/table/source/fallback_data_split_test.cpp
@@ -115,7 +115,8 @@ TEST(FallbackDataSplitTest, TestDeserialize) {
         /*creation_time=*/Timestamp(1755880762233ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-43880011-d066-4255-ad65-891d79cde23b-0.parquet", /*file_size=*/891, /*row_count=*/1,
         /*min_key=*/BinaryRow::EmptyRow(),
@@ -129,7 +130,8 @@ TEST(FallbackDataSplitTest, TestDeserialize) {
         /*creation_time=*/Timestamp(1755884315482ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRowGenerator::GenerateRow({1}, pool.get()),
@@ -185,7 +187,8 @@ TEST(FallbackDataSplitTest, TestDeserialize2) {
         /*creation_time=*/Timestamp(1755880762585ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-625b3277-84d3-4320-80b9-89a5075bf5fd-0.parquet", /*file_size=*/891, /*row_count=*/1,
         /*min_key=*/BinaryRow::EmptyRow(),
@@ -199,7 +202,8 @@ TEST(FallbackDataSplitTest, TestDeserialize2) {
         /*creation_time=*/Timestamp(1755884315889ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     std::vector<DataSplit::SimpleDataFileMeta> file_list;
     file_list.emplace_back(
diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp
index a56469656..4126232a0 100644
--- a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp
+++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp
@@ -220,7 +220,8 @@ class PrimaryKeySortedIndexScanTest : public ::testing::Test {
             /*creation_time=*/Timestamp(1721643142456LL, 0), delete_row_count,
             /*embedded_index=*/nullptr, file_source,
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
-            /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+            /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     Result<std::shared_ptr<IndexFileMeta>> BuildPayload(std::vector<int64_t> ordinals,
diff --git a/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp
index cebc644cf..f3174fc6e 100644
--- a/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp
+++ b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp
@@ -52,7 +52,7 @@ class SnapshotReaderTest : public testing::Test {
             /*min_sequence_number=*/0, /*max_sequence_number=*/0, /*schema_id=*/0,
             DataFileMeta::DUMMY_LEVEL, std::vector<std::optional<std::string>>{}, Timestamp(0, 0),
             std::nullopt, nullptr, FileSource::Append(), std::nullopt, std::nullopt, std::nullopt,
-            std::nullopt);
+            std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     std::shared_ptr<IndexFileMeta> CreateIndexFileMeta(const std::string& index_file_name,
diff --git a/src/paimon/core/table/source/split_generator_test.cpp b/src/paimon/core/table/source/split_generator_test.cpp
index ac80eb3f1..9f1368700 100644
--- a/src/paimon/core/table/source/split_generator_test.cpp
+++ b/src/paimon/core/table/source/split_generator_test.cpp
@@ -74,7 +74,7 @@ class SplitGeneratorTest : public testing::Test {
             /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt,
             /*external_path=*/std::optional<std::string>(),
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     std::shared_ptr<DataFileMeta> CreateDataFileMeta(const std::string& file_name, int32_t level,
@@ -99,7 +99,7 @@ class SplitGeneratorTest : public testing::Test {
             /*embedded_index=*/nullptr, FileSource::Append(),
             /*external_path=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     std::shared_ptr<DataFileMeta> CreateDataFileMeta(const std::string& file_name, int32_t min_key,
@@ -117,7 +117,7 @@ class SplitGeneratorTest : public testing::Test {
             /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/0,
             /*embedded_index=*/nullptr, FileSource::Append(), /*external_path=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     std::shared_ptr<DataFileMeta> CreateDataFileMetaWithRowId(const std::string& file_name,
@@ -132,7 +132,7 @@ class SplitGeneratorTest : public testing::Test {
             /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/0,
             /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt,
             /*external_path=*/std::nullopt, first_row_id,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     }
 
     static void CheckResult(const std::vector<SplitGenerator::SplitGroup>& result_groups,
diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp
index a2a52d343..343d99605 100644
--- a/test/inte/read_inte_test.cpp
+++ b/test/inte/read_inte_test.cpp
@@ -183,7 +183,7 @@ class ReadInteTest : public testing::Test, public ::testing::WithParamInterface<
                     /*embedded_index=*/nullptr, FileSource::Append(),
                     /*value_stats_cols=*/std::nullopt,
                     /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-                    /*write_cols=*/std::nullopt));
+                    /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt));
             }
             auto bucket_str = bucket_path.substr(bucket_path.find("bucket-") + 7);
             int32_t bucket = std::stoi(bucket_str);
diff --git a/test/inte/read_inte_with_index_test.cpp b/test/inte/read_inte_with_index_test.cpp
index 7a4439734..1442f6263 100644
--- a/test/inte/read_inte_with_index_test.cpp
+++ b/test/inte/read_inte_with_index_test.cpp
@@ -945,7 +945,8 @@ TEST_P(ReadInteWithIndexTest, TestSimple) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
     ASSERT_OK_AND_ASSIGN(auto split,
@@ -1019,7 +1020,8 @@ TEST_P(ReadInteWithIndexTest, TestReadWithLimits) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
     ASSERT_OK_AND_ASSIGN(auto split,
@@ -1123,7 +1125,8 @@ TEST_P(ReadInteWithIndexTest, TestEmbeddingBitmapIndex) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/embedded_index, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
     ASSERT_OK_AND_ASSIGN(auto split,
@@ -1186,7 +1189,8 @@ TEST_P(ReadInteWithIndexTest, TestBitmapWithV1) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/embedded_index, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
     ASSERT_OK_AND_ASSIGN(auto split,
@@ -1225,7 +1229,8 @@ TEST_P(ReadInteWithIndexTest, TestNoEmbeddingBitmapIndex) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
     ASSERT_OK_AND_ASSIGN(auto split,
@@ -1264,7 +1269,8 @@ TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
     ASSERT_OK_AND_ASSIGN(auto split,
@@ -1339,7 +1345,7 @@ TEST_P(ReadInteWithIndexTest, TestNoEmbeddingBitmapIndexWithExternalPath) {
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
         /*external_path=*/external_file_path, /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
     ASSERT_OK_AND_ASSIGN(auto split,
@@ -1381,7 +1387,8 @@ TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithDv) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DeletionFile deletion_file(deletion_file_path,
                                /*offset=*/1, /*length=*/24, /*cardinality=*/2);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
@@ -1487,7 +1494,7 @@ TEST_P(ReadInteWithIndexTest, TestWithAlterTable) {
             /*embedded_index=*/embedded_index, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt,
             /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     };
 
     std::vector<uint8_t> embedded_bytes1 = {
@@ -1870,7 +1877,8 @@ TEST_P(ReadInteWithIndexTest, TestWithBsiIndex) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
     ASSERT_OK_AND_ASSIGN(auto split,
@@ -1930,7 +1938,8 @@ TEST_P(ReadInteWithIndexTest, TestWithBloomFilterIndex) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
     ASSERT_OK_AND_ASSIGN(auto split,
@@ -2136,7 +2145,8 @@ TEST_P(ReadInteWithIndexTest, TestBitmapPushDownWithMultiStripes) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
     ASSERT_OK_AND_ASSIGN(auto split,
@@ -2246,7 +2256,8 @@ TEST_P(ReadInteWithIndexTest, TestWithBitmapAndBsiAndBloomFilterIndex) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
     ASSERT_OK_AND_ASSIGN(auto split,
@@ -2328,7 +2339,8 @@ TEST_P(ReadInteWithIndexTest, TestWithIndexWithoutRegistered) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
     ASSERT_OK_AND_ASSIGN(auto split,
@@ -2465,7 +2477,8 @@ TEST_P(ReadInteWithIndexTest, TestRangeBitmapIndex) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
@@ -2510,7 +2523,8 @@ TEST_P(ReadInteWithIndexTest, TestRangeBitmapIndexMultiChunk) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
@@ -2552,7 +2566,8 @@ TEST_P(ReadInteWithIndexTest, TestWithIOException) {
         /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/path + "bucket-0/", {data_file_meta});
     ASSERT_OK_AND_ASSIGN(auto split,
diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp
index fdc38ea9f..b33fcf36b 100644
--- a/test/inte/scan_inte_test.cpp
+++ b/test/inte/scan_inte_test.cpp
@@ -148,7 +148,7 @@ class ScanInteTest : public testing::TestWithParam<ManifestCacheMode> {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     std::shared_ptr<DataFileMeta> meta_snapshot1_partition10_bucket1_ =
         std::make_shared<DataFileMeta>(
@@ -164,7 +164,7 @@ class ScanInteTest : public testing::TestWithParam<ManifestCacheMode> {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     std::shared_ptr<DataFileMeta> meta_snapshot1_partition20_bucket0_ =
         std::make_shared<DataFileMeta>(
@@ -180,7 +180,7 @@ class ScanInteTest : public testing::TestWithParam<ManifestCacheMode> {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     std::shared_ptr<DataFileMeta> meta_snapshot2_partition10_bucket1_ =
         std::make_shared<DataFileMeta>(
@@ -196,7 +196,7 @@ class ScanInteTest : public testing::TestWithParam<ManifestCacheMode> {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     std::shared_ptr<DataFileMeta> meta_snapshot2_partition20_bucket0_ =
         std::make_shared<DataFileMeta>(
@@ -212,7 +212,7 @@ class ScanInteTest : public testing::TestWithParam<ManifestCacheMode> {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     std::shared_ptr<DataFileMeta> meta_snapshot3_partition10_bucket1_ =
         std::make_shared<DataFileMeta>(
@@ -228,7 +228,7 @@ class ScanInteTest : public testing::TestWithParam<ManifestCacheMode> {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     std::shared_ptr<DataFileMeta> meta_snapshot4_partition10_bucket1_ =
         std::make_shared<DataFileMeta>(
@@ -244,7 +244,7 @@ class ScanInteTest : public testing::TestWithParam<ManifestCacheMode> {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 
     std::shared_ptr<DataFileMeta> meta_snapshot5_partition10_bucket1_ =
         std::make_shared<DataFileMeta>(
@@ -260,7 +260,7 @@ class ScanInteTest : public testing::TestWithParam<ManifestCacheMode> {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
 };
 
 TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) {
@@ -1422,7 +1422,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder1(
         BinaryRowGenerator::GenerateRow({10, 0}, pool_.get()),
         /*bucket=*/0, /*bucket_path=*/
@@ -1483,7 +1483,7 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter)
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder1(
         BinaryRowGenerator::GenerateRow({10}, pool_.get()),
         /*bucket=*/0, /*bucket_path=*/
@@ -1549,7 +1549,7 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2)
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder1(
         BinaryRowGenerator::GenerateRow({10}, pool_.get()),
         /*bucket=*/0, /*bucket_path=*/
@@ -1591,7 +1591,8 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) {
         /*creation_time=*/Timestamp(1731412938869ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::optional<std::vector<std::string>>({"f0", "f1", "f2"}),
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-c2613568-0412-4cd9-a0c4-1eae8e4ca89b-0.orc", /*file_size=*/575, /*row_count=*/3,
         /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(),
@@ -1603,7 +1604,8 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) {
         /*creation_time=*/Timestamp(1731412938891ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::optional<std::vector<std::string>>({"f0", "f1", "f2"}),
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta3 = std::make_shared<DataFileMeta>(
         "data-a6d1261a-f798-4fbd-a251-6d6c7d8060dd-0.orc", /*file_size=*/541, /*row_count=*/1,
         /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(),
@@ -1615,7 +1617,8 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) {
         /*creation_time=*/Timestamp(1731412938908ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::optional<std::vector<std::string>>({"f0", "f1", "f2"}),
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder1(
         BinaryRowGenerator::GenerateRow({10}, pool_.get()),
@@ -1689,7 +1692,8 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) {
         /*creation_time=*/Timestamp(1731412938891ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::optional<std::vector<std::string>>({"f0", "f1", "f2"}),
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(
         BinaryRowGenerator::GenerateRow({10}, pool_.get()),
@@ -1787,7 +1791,8 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) {
         /*creation_time=*/Timestamp(1732635461460ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(BinaryRowGenerator::GenerateRow({1, 1}, pool_.get()),
                                    /*bucket=*/0, /*bucket_path=*/
@@ -1833,7 +1838,8 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) {
         /*creation_time=*/Timestamp(1730458825047ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder1(BinaryRowGenerator::GenerateRow({1, 1}, pool_.get()),
                                     /*bucket=*/0, /*bucket_path=*/
@@ -1860,7 +1866,8 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) {
         /*creation_time=*/Timestamp(1730459969493ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder2(BinaryRowGenerator::GenerateRow({0, 1}, pool_.get()),
                                     /*bucket=*/0, /*bucket_path=*/
@@ -1910,7 +1917,8 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) {
         /*creation_time=*/Timestamp(1751647880163ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::optional<std::vector<std::string>>({"key0", "f0", "f1", "f2"}),
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(
         BinaryRowGenerator::GenerateRow({1}, pool_.get()),
         /*bucket=*/0, /*bucket_path=*/
@@ -1992,7 +2000,8 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) {
         /*creation_time=*/Timestamp(1745000702835ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/embedded_index, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/table_path + "bucket-0", {file_meta});
     ASSERT_OK_AND_ASSIGN(auto expected_data_split, builder.WithTotalBuckets(-1)
@@ -2057,7 +2066,8 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) {
         /*creation_time=*/Timestamp(1745235371029ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/table_path + "bucket-0", {file_meta});
     ASSERT_OK_AND_ASSIGN(auto expected_data_split, builder.WithTotalBuckets(-1)
@@ -2124,7 +2134,8 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) {
         /*creation_time=*/Timestamp(1745253323731ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/embedded_index, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/table_path + "bucket-0", {file_meta});
@@ -2198,7 +2209,8 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) {
         /*creation_time=*/Timestamp(1745251357742ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/embedded_index, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/table_path + "bucket-0", {file_meta});
@@ -2274,7 +2286,8 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) {
         /*creation_time=*/Timestamp(1745251357742ll, 0), /*delete_row_count=*/0,
         /*embedded_index=*/embedded_index, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
                                    /*bucket_path=*/table_path + "bucket-0", {file_meta});
diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp
index 451c3ce29..9035ae6e5 100644
--- a/test/inte/write_inte_test.cpp
+++ b/test/inte/write_inte_test.cpp
@@ -208,7 +208,7 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface
             file_meta->level, file_meta->extra_files, file_meta->creation_time,
             file_meta->delete_row_count, file_meta->embedded_index, file_meta->file_source,
             file_meta->value_stats_cols, file_meta->external_path, file_meta->first_row_id,
-            file_meta->write_cols);
+            file_meta->write_cols, /*column_max_sequence_numbers=*/std::nullopt);
         auto generate_null_stats = [this](const SimpleStats& stats) -> SimpleStats {
             if (stats == SimpleStats::EmptyStats()) {
                 return stats;
@@ -445,7 +445,7 @@ TEST_P(WriteInteTest, TestAppendTableBatchWrite) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta = ReconstructDataFileMeta(file_meta);
     DataIncrement data_increment({file_meta}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_1 = std::make_shared<CommitMessageImpl>(
@@ -549,7 +549,7 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithOneBucket) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta = ReconstructDataFileMeta(file_meta);
     DataIncrement data_increment({file_meta}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_1 = std::make_shared<CommitMessageImpl>(
@@ -619,7 +619,7 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithOneBucket) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_2 = ReconstructDataFileMeta(file_meta_2);
     DataIncrement data_increment_2({file_meta_2}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_2 = std::make_shared<CommitMessageImpl>(
@@ -862,7 +862,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithComplexType) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta = ReconstructDataFileMeta(file_meta);
     DataIncrement data_increment({file_meta}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_1 = std::make_shared<CommitMessageImpl>(
@@ -926,7 +926,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithComplexType) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_2 = ReconstructDataFileMeta(file_meta_2);
     DataIncrement data_increment_2({file_meta_2}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_2 = std::make_shared<CommitMessageImpl>(
@@ -1011,7 +1011,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_1 = ReconstructDataFileMeta(file_meta_1);
     DataIncrement data_increment_1({file_meta_1}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_1 = std::make_shared<CommitMessageImpl>(
@@ -1037,7 +1037,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_2 = ReconstructDataFileMeta(file_meta_2);
     DataIncrement data_increment_2({file_meta_2}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_2 = std::make_shared<CommitMessageImpl>(
@@ -1063,7 +1063,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) {
         /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_3 = ReconstructDataFileMeta(file_meta_3);
     DataIncrement data_increment_3({file_meta_3}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_3 = std::make_shared<CommitMessageImpl>(
@@ -1135,7 +1135,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_4 = ReconstructDataFileMeta(file_meta_4);
     DataIncrement data_increment_4({file_meta_4}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_4 = std::make_shared<CommitMessageImpl>(
@@ -1161,7 +1161,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_5 = ReconstructDataFileMeta(file_meta_5);
     DataIncrement data_increment_5({file_meta_5}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_5 = std::make_shared<CommitMessageImpl>(
@@ -1187,7 +1187,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_6 = ReconstructDataFileMeta(file_meta_6);
     DataIncrement data_increment_6({file_meta_6}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_6 = std::make_shared<CommitMessageImpl>(
@@ -1285,7 +1285,7 @@ TEST_P(WriteInteTest, TestPkTableBatchWrite) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_1 = ReconstructDataFileMeta(file_meta_1);
     DataIncrement data_increment_1({file_meta_1}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_1 = std::make_shared<CommitMessageImpl>(
@@ -1311,7 +1311,7 @@ TEST_P(WriteInteTest, TestPkTableBatchWrite) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_2 = ReconstructDataFileMeta(file_meta_2);
     DataIncrement data_increment_2({file_meta_2}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_2 = std::make_shared<CommitMessageImpl>(
@@ -1337,7 +1337,7 @@ TEST_P(WriteInteTest, TestPkTableBatchWrite) {
         /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_3 = ReconstructDataFileMeta(file_meta_3);
     DataIncrement data_increment_3({file_meta_3}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_3 = std::make_shared<CommitMessageImpl>(
@@ -1452,7 +1452,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_1 = ReconstructDataFileMeta(file_meta_1);
     DataIncrement data_increment_1({file_meta_1}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_1 = std::make_shared<CommitMessageImpl>(
@@ -1482,7 +1482,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) {
         /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_2 = ReconstructDataFileMeta(file_meta_2);
     DataIncrement data_increment_2({file_meta_2}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_2 = std::make_shared<CommitMessageImpl>(
@@ -1558,7 +1558,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_3 = ReconstructDataFileMeta(file_meta_3);
     DataIncrement data_increment_3({file_meta_3}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_3 = std::make_shared<CommitMessageImpl>(
@@ -1586,7 +1586,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_4 = ReconstructDataFileMeta(file_meta_4);
     DataIncrement data_increment_4({file_meta_4}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_4 = std::make_shared<CommitMessageImpl>(
@@ -1696,7 +1696,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithComplexType) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta = ReconstructDataFileMeta(file_meta);
     DataIncrement data_increment({file_meta}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_1 = std::make_shared<CommitMessageImpl>(
@@ -1769,7 +1769,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithComplexType) {
         /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_2 = ReconstructDataFileMeta(file_meta_2);
     DataIncrement data_increment_2({file_meta_2}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_2 = std::make_shared<CommitMessageImpl>(
@@ -2036,7 +2036,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         file_meta_1 = ReconstructDataFileMeta(file_meta_1);
         DataIncrement data_increment_1({file_meta_1}, {}, {});
         std::shared_ptr<CommitMessage> expected_commit_message_1 =
@@ -2065,7 +2065,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         file_meta_2 = ReconstructDataFileMeta(file_meta_2);
         DataIncrement data_increment_2({file_meta_2}, {}, {});
         std::shared_ptr<CommitMessage> expected_commit_message_2 =
@@ -2094,7 +2094,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) {
             /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         file_meta_3 = ReconstructDataFileMeta(file_meta_3);
         DataIncrement data_increment_3({file_meta_3}, {}, {});
         std::shared_ptr<CommitMessage> expected_commit_message_3 =
@@ -2126,7 +2126,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         file_meta_4 = ReconstructDataFileMeta(file_meta_4);
         DataIncrement data_increment_4({file_meta_4}, {}, {});
         std::shared_ptr<CommitMessage> expected_commit_message_4 =
@@ -2154,7 +2154,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         file_meta_5 = ReconstructDataFileMeta(file_meta_5);
         DataIncrement data_increment_5({file_meta_5}, {}, {});
         std::shared_ptr<CommitMessage> expected_commit_message_5 =
@@ -2183,7 +2183,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) {
             /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
             /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt);
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
         file_meta_6 = ReconstructDataFileMeta(file_meta_6);
         DataIncrement data_increment_6({file_meta_6}, {}, {});
         std::shared_ptr<CommitMessage> expected_commit_message_6 =
@@ -2257,7 +2257,7 @@ TEST_F(WriteInteTest, TestAppendTableWriteWithAlterTable) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment({file_meta}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message = std::make_shared<CommitMessageImpl>(
         BinaryRowGenerator::GenerateRow({1, 1}, pool_.get()), /*bucket=*/0,
@@ -2337,7 +2337,7 @@ TEST_F(WriteInteTest, TestPKTableWriteWithAlterTable) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment({file_meta}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message = std::make_shared<CommitMessageImpl>(
         BinaryRowGenerator::GenerateRow({0, 0}, pool_.get()), /*bucket=*/0,
@@ -2775,7 +2775,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithExternalPath) {
         /*level=*/0, /*extra_files=*/std::vector<std::optional<std::string>>(),
         /*creation_time=*/Timestamp(1724090888706ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt, "FILE:/tmp/xxx", std::nullopt, std::nullopt);
+        /*value_stats_cols=*/std::nullopt, "FILE:/tmp/xxx", std::nullopt, std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     file_meta = ReconstructDataFileMeta(file_meta);
     DataIncrement data_increment({file_meta}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_1 = std::make_shared<CommitMessageImpl>(
@@ -2844,7 +2845,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithExternalPath) {
         /*level=*/0, /*extra_files=*/std::vector<std::optional<std::string>>(),
         /*creation_time=*/Timestamp(1724090888706ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt, "FILE:/tmp/xxx", std::nullopt, std::nullopt);
+        /*value_stats_cols=*/std::nullopt, "FILE:/tmp/xxx", std::nullopt, std::nullopt,
+        /*column_max_sequence_numbers=*/std::nullopt);
     file_meta_2 = ReconstructDataFileMeta(file_meta_2);
     DataIncrement data_increment_2({file_meta_2}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_2 = std::make_shared<CommitMessageImpl>(
@@ -3443,7 +3445,7 @@ TEST_P(WriteInteTest, TestPkTablePostponeBucket) {
         /*delete_row_count=*/1, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment({file_meta}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message = std::make_shared<CommitMessageImpl>(
         /*partition_map=*/BinaryRow::EmptyRow(), /*bucket=*/-2,
@@ -3532,7 +3534,7 @@ TEST_F(WriteInteTest, TestBranchWrite) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment_0({file_meta_0}, {}, {});
     auto expected_commit_message_0 = std::make_shared<CommitMessageImpl>(
         BinaryRowGenerator::GenerateRow({std::string("20240726")}, pool_.get()), /*bucket=*/0,
@@ -3574,7 +3576,7 @@ TEST_F(WriteInteTest, TestBranchWrite) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     DataIncrement data_increment_1({file_meta_1}, {}, {});
     auto expected_commit_message_1 = std::make_shared<CommitMessageImpl>(
         BinaryRowGenerator::GenerateRow({std::string("20240725")}, pool_.get()), /*bucket=*/0,
@@ -3704,7 +3706,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     check_meta(commit_msgs1[0], {file_meta1});
     commit(commit_msgs1, /*latest_snapshot_id=*/1, /*next_row_id=*/2);
     check_committed_meta({{0, std::nullopt, 1, 1}});
@@ -3736,7 +3738,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/write_cols2);
+        /*write_cols=*/write_cols2, /*column_max_sequence_numbers=*/std::nullopt);
     check_meta(commit_msgs2[0], {file_meta2});
     commit(commit_msgs2, /*latest_snapshot_id=*/2, /*next_row_id=*/7);
     check_committed_meta({{0, std::nullopt, 1, 1}, {2, write_cols2, 2, 2}});
@@ -3762,7 +3764,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/write_cols3);
+        /*write_cols=*/write_cols3, /*column_max_sequence_numbers=*/std::nullopt);
     check_meta(commit_msgs3[0], {file_meta3});
     auto commit_msg_impl = std::dynamic_pointer_cast<CommitMessageImpl>(commit_msgs3[0]);
     ASSERT_TRUE(commit_msg_impl);
@@ -3834,7 +3836,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) {
         /*creation_time=*/Timestamp(1724090888706ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0,
-        /*write_cols=*/std::vector<std::string>({"f0", "f1"}));
+        /*write_cols=*/std::vector<std::string>({"f0", "f1"}),
+        /*column_max_sequence_numbers=*/std::nullopt);
     file_meta1 = ReconstructDataFileMeta(file_meta1);
     auto file_meta2 = std::make_shared<DataFileMeta>(
         "data-xxx.blob", /*file_size=*/764, /*row_count=*/3,
@@ -3845,7 +3848,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) {
         /*creation_time=*/Timestamp(1724090888706ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0,
-        /*write_cols=*/std::vector<std::string>({"blob"}));
+        /*write_cols=*/std::vector<std::string>({"blob"}),
+        /*column_max_sequence_numbers=*/std::nullopt);
     auto file_meta3 = std::make_shared<DataFileMeta>(
         "data-xxx.blob", /*file_size=*/3023, /*row_count=*/1,
         /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(),
@@ -3855,7 +3859,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) {
         /*creation_time=*/Timestamp(1724090888706ll, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/3,
-        /*write_cols=*/std::vector<std::string>({"blob"}));
+        /*write_cols=*/std::vector<std::string>({"blob"}),
+        /*column_max_sequence_numbers=*/std::nullopt);
     std::vector<std::shared_ptr<DataFileMeta>> expected_meta = {file_meta1, file_meta2, file_meta3};
 
     // NOTE: Due to the write logic of C++ Paimon and Java Paimon is different, the first_row_id in
@@ -3924,7 +3929,7 @@ TEST_P(WriteInteTest, TestAppendTableWithDateFieldAsPartitionField) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
         /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt);
+        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
     file_meta = ReconstructDataFileMeta(file_meta);
     DataIncrement data_increment({file_meta}, {}, {});
     std::shared_ptr<CommitMessage> expected_commit_message_1 = std::make_shared<CommitMessageImpl>(
@@ -4728,7 +4733,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) {
         /*creation_time=*/Timestamp(0, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0,
-        /*write_cols=*/std::vector<std::string>({"f0", "f1"}));
+        /*write_cols=*/std::vector<std::string>({"f0", "f1"}),
+        /*column_max_sequence_numbers=*/std::nullopt);
     expected_main = ReconstructDataFileMeta(expected_main);
 
     // blob1 file: 3 rows, write_cols={"blob1"}, first_row_id=0
@@ -4741,7 +4747,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) {
         /*creation_time=*/Timestamp(0, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0,
-        /*write_cols=*/std::vector<std::string>({"blob1"}));
+        /*write_cols=*/std::vector<std::string>({"blob1"}),
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     // blob2 file: 3 rows, write_cols={"blob2"}, first_row_id=0
     auto expected_blob2 = std::make_shared<DataFileMeta>(
@@ -4753,7 +4760,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) {
         /*creation_time=*/Timestamp(0, 0),
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0,
-        /*write_cols=*/std::vector<std::string>({"blob2"}));
+        /*write_cols=*/std::vector<std::string>({"blob2"}),
+        /*column_max_sequence_numbers=*/std::nullopt);
 
     ASSERT_OK_AND_ASSIGN(auto commit_msgs,
                          helper->WriteAndCommit(std::move(batch), commit_identifier++,
diff --git a/test/test_data/compatibility/commit_message-v13 b/test/test_data/compatibility/commit_message-v13
new file mode 100644
index 0000000000000000000000000000000000000000..568705a5957d5c2383fd3152e8dc3aaa1fad34d9
GIT binary patch
literal 693
zcma))Jx;?w6oe-M3Iq}qDMIIhg3baOM3oYu6x<+L@LINTNR&8`RDclX2AqQ<P;vxH
zjzP)1x4Sk12{F?6^Ul7vyZ!;t0nnmVMg%-^(2+2r$#zDnM$18(7xoF*cEPHHtnxw&
z-{>LzLEq6muCPUlWKOi^fECX!`O3Os|FnNgZuq>_)(svQsUlwAIoKgT$xTgxJ+j+&
z?2%Mx!6?5PT-qhitZH(z_@3*eoIj$4N3=ZU8BH~P|5&dscD?m=mv<2@JfUU%2~E{4
z>l{cf=e{<@+{$dD-*tSBpLKPS?{lvNu#0kP_E9MhP=)uO=C(+fE14?~QDsRwJC5Uc
uV#+Ko$MKDs+H5?Zg)`~b;U32AbARr6;hB1dy6_()F2}QH_%HnMPkaG5(?vD_

literal 0
HcmV?d00001

diff --git a/test/test_data/compatibility/data_split-v9 b/test/test_data/compatibility/data_split-v9
new file mode 100644
index 0000000000000000000000000000000000000000..2bfd4ac19a07a0b207152e9e04cd5ec10f4ec7ed
GIT binary patch
literal 738
zcma)4J5Iwu5FHy3LV!RhLP=*N3JN6iX%JOPgi>&WWWj6M!j7ZFfw+PQ#oT~%a0Db8
zYAPzk0XP69Z}wv&Xn4}|^Sqte9ed~f`E_r%^%)Y;63Ey98=%7hLokSDcx}*PT6hDI
zN4)BR3JBcD25Hd^<3!#Qt!ryu3-9>|vc7;l@B_RB+t^_Z%HTD|f)<)H+JrtKuh%*4
z+(4I&@3`uKJ6H-Ho^J?R&@bqUa*1}JO;>9h3fK}#<9_#27U)@__Qm9<uNJv~4`e+8
z`76&rpsxGJepRxrt)AO>^FY=UkoO;eKvlwb#Xk3aC6ck^)f#{E@EyPNsw7jVe+iMC
z<U_GfrP2c`?f(yBnFR2a!dH4oWfX*?W8e3AD*Qb2uf<SC*?44oQn!OMgg$3_ZuG(^
n&mJ(fd3Fv0?bCx%I~Uxy<Irj#6Pd?ZUrn3Iu{E0}-Xi)2&DByv

literal 0
HcmV?d00001


From 4027601e67f4271d5efd8ebe278efd66f2e1bee0 Mon Sep 17 00:00:00 2001
From: lxy <38709059+lxy-9602@users.noreply.github.com>
Date: Thu, 27 Aug 2026 14:55:03 +0800
Subject: [PATCH 72/93] refactor(utils): consolidate string and option handling
 (#249)

---
 .../data/variant/variant_access_utils.cpp     |   3 +-
 .../common/types/data_type_json_parser.cpp    |   6 +-
 src/paimon/common/utils/options_utils.h       |  20 +-
 .../common/utils/options_utils_test.cpp       |  12 +-
 src/paimon/common/utils/string_utils.cpp      |  57 +++-
 src/paimon/common/utils/string_utils.h        |  31 +-
 src/paimon/common/utils/string_utils_test.cpp |  32 ++
 src/paimon/core/core_options.cpp              | 281 ++++++++----------
 .../lookup_merge_tree_compact_rewriter.cpp    |  11 +-
 .../compact/merge_tree_compact_rewriter.cpp   |  11 +-
 src/paimon/core/mergetree/lookup_levels.cpp   |  11 +-
 .../append_only_file_store_write.cpp          |   8 +-
 .../commit/sequence_snapshot_properties.cpp   |  18 +-
 .../sequence_snapshot_properties_test.cpp     |   2 +-
 .../core/postpone/postpone_bucket_writer.cpp  |  18 +-
 src/paimon/core/schema/schema_validation.cpp  |   2 +-
 .../core/schema/schema_validation_test.cpp    |   6 +
 .../table/system/global_system_tables.cpp     |   7 +-
 src/paimon/format/orc/orc_format_writer.cpp   |  24 +-
 .../format/parquet/parquet_format_defs.h      |   2 +-
 src/paimon/fs/local/local_file.cpp            |   2 +-
 src/paimon/fs/local/local_file_test.cpp       |   5 +
 src/paimon/fs/s3/s3_file_system.cpp           |  12 +-
 .../global_index/lucene/jieba_analyzer.cpp    |   6 +-
 src/paimon/rest/dlf_auth.cpp                  |  25 +-
 src/paimon/rest/rest_api.cpp                  |   7 +-
 src/paimon/rest/rest_auth.cpp                 |  15 +-
 src/paimon/rest/rest_catalog.cpp              |   2 +-
 src/paimon/rest/rest_http_client.cpp          |   3 +-
 src/paimon/rest/rest_util.cpp                 |   9 +-
 30 files changed, 313 insertions(+), 335 deletions(-)

diff --git a/src/paimon/common/data/variant/variant_access_utils.cpp b/src/paimon/common/data/variant/variant_access_utils.cpp
index 78180d1ce..0238c086a 100644
--- a/src/paimon/common/data/variant/variant_access_utils.cpp
+++ b/src/paimon/common/data/variant/variant_access_utils.cpp
@@ -26,6 +26,7 @@
 #include "fmt/format.h"
 #include "paimon/common/data/variant/variant_defs.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/string_utils.h"
 
 namespace paimon {
 
@@ -61,7 +62,7 @@ std::vector<std::string> SplitDescription(const std::string& description) {
 }
 
 bool HasAccessDescription(const std::shared_ptr<arrow::Field>& field) {
-    return GetDescription(field).rfind(VariantAccessUtils::kMetadataKey, 0) == 0;
+    return StringUtils::StartsWith(GetDescription(field), VariantAccessUtils::kMetadataKey);
 }
 
 }  // namespace
diff --git a/src/paimon/common/types/data_type_json_parser.cpp b/src/paimon/common/types/data_type_json_parser.cpp
index e95582a17..d04e5ade5 100644
--- a/src/paimon/common/types/data_type_json_parser.cpp
+++ b/src/paimon/common/types/data_type_json_parser.cpp
@@ -19,7 +19,6 @@
 
 #include "paimon/common/types/data_type_json_parser.h"
 
-#include <algorithm>
 #include <cctype>
 #include <cstddef>
 #include <cstdint>
@@ -331,10 +330,7 @@ std::vector<Token> Tokenize(const std::string& chars) {
                 builder.clear();
                 cursor = ConsumeIdentifier(chars, cursor, builder);
                 auto token = builder.str();
-                auto normalized_token = token;
-                std::transform(normalized_token.begin(), normalized_token.end(),
-                               normalized_token.begin(),
-                               [](unsigned char c) { return std::toupper(c); });
+                std::string normalized_token = StringUtils::ToUpperCase(token);
                 if (Keywords().find(normalized_token) != Keywords().end()) {
                     tokens.emplace_back(TokenType::KEYWORD, cursor, normalized_token);
                 } else {
diff --git a/src/paimon/common/utils/options_utils.h b/src/paimon/common/utils/options_utils.h
index c20140071..08b09ad2d 100644
--- a/src/paimon/common/utils/options_utils.h
+++ b/src/paimon/common/utils/options_utils.h
@@ -89,13 +89,29 @@ class OptionsUtils {
         return value.status();
     }
 
+    static Result<std::string> GetNonEmptyValueFromMap(
+        const std::map<std::string, std::string>& key_value_map, const std::string& key) {
+        Result<std::string> value = GetValueFromMap<std::string>(key_value_map, key);
+        if (!value.ok()) {
+            return value.status();
+        }
+        if (value.value().empty()) {
+            return Status::Invalid(fmt::format("value for key {} must not be empty", key));
+        }
+        return value.value();
+    }
+
     /// Fetch options with specific prefix and remove prefix for key.
+    /// @param prefix Prefix used to select options and removed from the returned keys.
+    /// @param options Options to select from.
+    /// @return Options whose keys start with and are longer than `prefix`, with the prefix removed
+    /// from each key.
     static std::map<std::string, std::string> FetchOptionsWithPrefix(
         const std::string& prefix, const std::map<std::string, std::string>& options) {
         std::map<std::string, std::string> options_with_prefix;
-        int64_t prefix_len = prefix.size();
+        const std::string::size_type prefix_len = prefix.size();
         for (const auto& [key, value] : options) {
-            if (StringUtils::StartsWith(key, prefix)) {
+            if (key.size() > prefix_len && StringUtils::StartsWith(key, prefix)) {
                 options_with_prefix[key.substr(prefix_len)] = value;
             }
         }
diff --git a/src/paimon/common/utils/options_utils_test.cpp b/src/paimon/common/utils/options_utils_test.cpp
index d4641184f..61e520874 100644
--- a/src/paimon/common/utils/options_utils_test.cpp
+++ b/src/paimon/common/utils/options_utils_test.cpp
@@ -84,9 +84,19 @@ TEST(OptionsUtilsTest, TestGetOptionalValueFromMap) {
 }
 
 TEST(OptionsUtilsTest, TestFetchOptionsWithPrefix) {
-    std::map<std::string, std::string> options = {{"key1", "value1"}, {"test.key2", "value2"}};
+    std::map<std::string, std::string> options = {
+        {"key1", "value1"}, {"test.", "empty-key"}, {"test.key2", "value2"}};
     auto new_options = OptionsUtils::FetchOptionsWithPrefix("test.", options);
     std::map<std::string, std::string> expected = {{"key2", "value2"}};
     ASSERT_EQ(expected, new_options);
 }
+
+TEST(OptionsUtilsTest, TestGetNonEmptyValueFromMap) {
+    std::map<std::string, std::string> options = {{"present", "value"}, {"empty", ""}};
+    ASSERT_OK_AND_ASSIGN(std::string value,
+                         OptionsUtils::GetNonEmptyValueFromMap(options, "present"));
+    ASSERT_EQ("value", value);
+    ASSERT_TRUE(OptionsUtils::GetNonEmptyValueFromMap(options, "missing").status().IsNotExist());
+    ASSERT_TRUE(OptionsUtils::GetNonEmptyValueFromMap(options, "empty").status().IsInvalid());
+}
 }  // namespace paimon::test
diff --git a/src/paimon/common/utils/string_utils.cpp b/src/paimon/common/utils/string_utils.cpp
index 5b405895d..869e184d0 100644
--- a/src/paimon/common/utils/string_utils.cpp
+++ b/src/paimon/common/utils/string_utils.cpp
@@ -30,8 +30,28 @@
 #include "paimon/status.h"
 
 namespace paimon {
+namespace {
+
+bool IsTrimCharacter(unsigned char c) {
+    // Match the characters removed by Java String::trim for the ASCII strings handled here.
+    return c <= 0x20;
+}
+
+char ToAsciiLower(unsigned char c) {
+    return c >= 'A' && c <= 'Z' ? static_cast<char>(c + ('a' - 'A')) : static_cast<char>(c);
+}
+
+char ToAsciiUpper(unsigned char c) {
+    return c >= 'a' && c <= 'z' ? static_cast<char>(c - ('a' - 'A')) : static_cast<char>(c);
+}
+
+}  // namespace
+
 std::string StringUtils::Replace(const std::string& text, const std::string& search_string,
                                  const std::string& replacement, int32_t max) {
+    if (text.empty() || search_string.empty() || max == 0) {
+        return text;
+    }
     std::string str = text;
     size_t pos = str.find(search_string);
     int32_t count = 0;
@@ -45,6 +65,9 @@ std::string StringUtils::Replace(const std::string& text, const std::string& sea
 
 std::string StringUtils::ReplaceLast(const std::string& text, const std::string& old_str,
                                      const std::string& new_str) {
+    if (text.empty() || old_str.empty()) {
+        return text;
+    }
     std::string str = text;
     size_t pos = str.rfind(old_str);
     if (pos != std::string::npos) {
@@ -54,7 +77,8 @@ std::string StringUtils::ReplaceLast(const std::string& text, const std::string&
 }
 
 bool StringUtils::StartsWith(const std::string& str, const std::string& prefix, size_t start_pos) {
-    return (str.size() >= prefix.size()) && (str.compare(start_pos, prefix.size(), prefix) == 0);
+    return start_pos <= str.size() && prefix.size() <= str.size() - start_pos &&
+           str.compare(start_pos, prefix.size(), prefix) == 0;
 }
 bool StringUtils::EndsWith(const std::string& str, const std::string& suffix) {
     size_t s1 = str.size();
@@ -74,26 +98,45 @@ bool StringUtils::IsNullOrWhitespaceOnly(const std::string& str) {
 }
 
 void StringUtils::Trim(std::string* str) {
-    str->erase(str->find_last_not_of(' ') + 1);
-    str->erase(0, str->find_first_not_of(' '));
+    auto first = std::find_if_not(str->begin(), str->end(),
+                                  [](unsigned char c) { return IsTrimCharacter(c); });
+    auto last = std::find_if_not(str->rbegin(), str->rend(), [](unsigned char c) {
+                    return IsTrimCharacter(c);
+                }).base();
+    if (first >= last) {
+        str->clear();
+        return;
+    }
+    *str = std::string(first, last);
 }
 
 std::string StringUtils::ToLowerCase(const std::string& str) {
     std::string result;
     result.reserve(str.length());
-    std::transform(str.begin(), str.end(), std::back_inserter(result),
-                   [](unsigned char c) { return std::tolower(c); });
+    std::transform(str.begin(), str.end(), std::back_inserter(result), ToAsciiLower);
     return result;
 }
 
 std::string StringUtils::ToUpperCase(const std::string& str) {
     std::string result;
     result.reserve(str.length());
-    std::transform(str.begin(), str.end(), std::back_inserter(result),
-                   [](unsigned char c) { return std::toupper(c); });
+    std::transform(str.begin(), str.end(), std::back_inserter(result), ToAsciiUpper);
     return result;
 }
 
+bool StringUtils::EqualsIgnoreCase(const std::string& left, const std::string& right) {
+    if (left.size() != right.size()) {
+        return false;
+    }
+    for (size_t i = 0; i < left.size(); ++i) {
+        if (ToAsciiLower(static_cast<unsigned char>(left[i])) !=
+            ToAsciiLower(static_cast<unsigned char>(right[i]))) {
+            return false;
+        }
+    }
+    return true;
+}
+
 std::vector<std::string> StringUtils::Split(const std::string& text, const std::string& sep_str,
                                             bool ignore_empty) {
     std::vector<std::string> vec;
diff --git a/src/paimon/common/utils/string_utils.h b/src/paimon/common/utils/string_utils.h
index 3c0906e2e..7681a8d9e 100644
--- a/src/paimon/common/utils/string_utils.h
+++ b/src/paimon/common/utils/string_utils.h
@@ -50,24 +50,18 @@ class PAIMON_EXPORT StringUtils {
  public:
     /// Replaces all occurrences of a string within another string.
     ///
-    /// A `null` reference passed to this method is a no-op.
-    ///
     /// <pre>
-    /// StringUtils::Replace(null, *, *)        = null
     /// StringUtils::Replace("", *, *)          = ""
-    /// StringUtils::Replace("any", null, *)    = "any"
-    /// StringUtils::Replace("any", *, null)    = "any"
     /// StringUtils::Replace("any", "", *)      = "any"
-    /// StringUtils::Replace("aba", "a", null)  = "aba"
     /// StringUtils::Replace("aba", "a", "")    = "b"
     /// StringUtils::Replace("aba", "a", "z")   = "zbz"
     /// </pre>
     ///
     /// @see #replace(string text, string search_string, string replacement, int max)
-    /// @param text text to search and replace in, may be null
-    /// @param search_string the String to search for, may be null
-    /// @param replacement the String to replace it with, may be null
-    /// @return the text with any replacements processed, `null` if null string input
+    /// @param text text to search and replace in
+    /// @param search_string the String to search for
+    /// @param replacement the String to replace it with
+    /// @return the text with any replacements processed
     static std::string Replace(const std::string& text, const std::string& search_string,
                                const std::string& replacement) {
         return Replace(text, search_string, replacement, -1);
@@ -76,16 +70,10 @@ class PAIMON_EXPORT StringUtils {
     /// Replaces a String with another String inside a larger String, for the first `max` values of
     /// the search String.
     ///
-    /// A `null` reference passed to this method is a no-op.
-    ///
     /// <pre>
-    /// StringUtils::Replace(null, *, *, *)         = null
     /// StringUtils::Replace("", *, *, *)           = ""
-    /// StringUtils::Replace("any", null, *, *)     = "any"
-    /// StringUtils::Replace("any", *, null, *)     = "any"
     /// StringUtils::Replace("any", "", *, *)       = "any"
     /// StringUtils::Replace("any", *, *, 0)        = "any"
-    /// StringUtils::Replace("abaa", "a", null, -1) = "abaa"
     /// StringUtils::Replace("abaa", "a", "", -1)   = "b"
     /// StringUtils::Replace("abaa", "a", "z", 0)   = "abaa"
     /// StringUtils::Replace("abaa", "a", "z", 1)   = "zbaa"
@@ -93,11 +81,11 @@ class PAIMON_EXPORT StringUtils {
     /// StringUtils::Replace("abaa", "a", "z", -1)  = "zbzz"
     /// </pre>
     ///
-    /// @param text text to search and replace in, may be null
-    /// @param search_string the String to search for, may be null
-    /// @param replacement the String to replace it with, may be null
+    /// @param text text to search and replace in
+    /// @param search_string the String to search for
+    /// @param replacement the String to replace it with
     /// @param max maximum number of values to replace, or `-1` if no maximum
-    /// @return the text with any replacements processed, `null` if null string input
+    /// @return the text with any replacements processed
     static std::string Replace(const std::string& text, const std::string& search_string,
                                const std::string& replacement, int32_t max);
 
@@ -115,6 +103,9 @@ class PAIMON_EXPORT StringUtils {
     static std::string ToLowerCase(const std::string& str);
     static std::string ToUpperCase(const std::string& str);
 
+    /// Compares two strings using ASCII case folding.
+    static bool EqualsIgnoreCase(const std::string& left, const std::string& right);
+
     template <typename T>
     static std::string VectorToString(const std::vector<T>& vec) {
         std::vector<std::string> strs;
diff --git a/src/paimon/common/utils/string_utils_test.cpp b/src/paimon/common/utils/string_utils_test.cpp
index 11c3e0005..a4f230781 100644
--- a/src/paimon/common/utils/string_utils_test.cpp
+++ b/src/paimon/common/utils/string_utils_test.cpp
@@ -73,6 +73,8 @@ void StringUtilsTest::CheckOverFlowAndUnderFlow(const std::string& over_flow,
 }
 
 TEST_F(StringUtilsTest, TestReplaceAll) {
+    ASSERT_EQ("abc", StringUtils::Replace("abc", "", "x"));
+    ASSERT_EQ("", StringUtils::Replace("", "a", "b"));
     {
         std::string origin = "how is is you";
         std::string expect = "how are are you";
@@ -118,6 +120,8 @@ TEST_F(StringUtilsTest, TestReplaceAll) {
 }
 
 TEST_F(StringUtilsTest, TestReplaceLast) {
+    ASSERT_EQ("abc", StringUtils::ReplaceLast("abc", "", "x"));
+    ASSERT_EQ("", StringUtils::ReplaceLast("", "a", "b"));
     {
         std::string origin = "a/b/c//";
         std::string expect = "a/b/c/_";
@@ -140,6 +144,7 @@ TEST_F(StringUtilsTest, TestReplaceLast) {
 }
 
 TEST_F(StringUtilsTest, TestReplaceWithMaxCount) {
+    ASSERT_EQ("abc", StringUtils::Replace("abc", "a", "b", 0));
     {
         std::string origin = "how is is you";
         std::string expect = "how are is you";
@@ -236,6 +241,13 @@ TEST_F(StringUtilsTest, TestToUpperCase) {
     }
 }
 
+TEST_F(StringUtilsTest, TestEqualsIgnoreCase) {
+    ASSERT_TRUE(StringUtils::EqualsIgnoreCase("", ""));
+    ASSERT_TRUE(StringUtils::EqualsIgnoreCase("AbC-123", "aBc-123"));
+    ASSERT_FALSE(StringUtils::EqualsIgnoreCase("abc", "abcd"));
+    ASSERT_FALSE(StringUtils::EqualsIgnoreCase("abc", "abx"));
+}
+
 TEST_F(StringUtilsTest, TestStartsWith) {
     {
         std::string str = "abcde";
@@ -261,6 +273,26 @@ TEST_F(StringUtilsTest, TestStartsWith) {
         std::string str = "";
         ASSERT_TRUE(StringUtils::StartsWith(str, ""));
     }
+    {
+        std::string str = "abc";
+        ASSERT_TRUE(StringUtils::StartsWith(str, "", /*start_pos=*/3));
+        ASSERT_FALSE(StringUtils::StartsWith(str, "", /*start_pos=*/4));
+        ASSERT_FALSE(StringUtils::StartsWith(str, "a", /*start_pos=*/4));
+    }
+}
+
+TEST_F(StringUtilsTest, TestTrim) {
+    std::string value = " \tabc\r\n";
+    StringUtils::Trim(&value);
+    ASSERT_EQ("abc", value);
+
+    value = "\t\r\n";
+    StringUtils::Trim(&value);
+    ASSERT_TRUE(value.empty());
+
+    value.clear();
+    StringUtils::Trim(&value);
+    ASSERT_TRUE(value.empty());
 }
 TEST_F(StringUtilsTest, TestEndsWith) {
     {
diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp
index 6320ec577..a5b32b09a 100644
--- a/src/paimon/core/core_options.cpp
+++ b/src/paimon/core/core_options.cpp
@@ -19,6 +19,7 @@
 #include "paimon/core/core_options.h"
 
 #include <cstring>
+#include <initializer_list>
 #include <limits>
 #include <memory>
 #include <optional>
@@ -51,14 +52,9 @@ class ConfigParser {
     // Parse basic type configurations
     template <typename T>
     Status Parse(const std::string& key, T* value) const {
-        auto iter = config_map_.find(key);
-        if (iter != config_map_.end()) {
-            auto result = StringUtils::StringToValue<T>(iter->second);
-            if (result) {
-                *value = result.value();
-                return Status::OK();
-            }
-            return Status::Invalid(fmt::format("Invalid Config [{}: {}]", key, iter->second));
+        PAIMON_ASSIGN_OR_RAISE(std::optional<T> parsed_value, GetOptionalValue<T>(key));
+        if (parsed_value) {
+            *value = parsed_value.value();
         }
         return Status::OK();  // Return success even if the configuration does not exist
     }
@@ -66,14 +62,9 @@ class ConfigParser {
     // Parse optional basic type configurations
     template <typename T>
     Status Parse(const std::string& key, std::optional<T>* value) const {
-        auto iter = config_map_.find(key);
-        if (iter != config_map_.end()) {
-            auto result = StringUtils::StringToValue<T>(iter->second);
-            if (result) {
-                *value = result.value();
-                return Status::OK();
-            }
-            return Status::Invalid(fmt::format("Invalid Config [{}: {}]", key, iter->second));
+        PAIMON_ASSIGN_OR_RAISE(std::optional<T> parsed_value, GetOptionalValue<T>(key));
+        if (parsed_value) {
+            *value = parsed_value.value();
         }
         return Status::OK();  // Return success even if the configuration does not exist
     }
@@ -82,23 +73,26 @@ class ConfigParser {
     template <typename T>
     Status ParseList(const std::string& key, const std::string& delimiter, std::vector<T>* list,
                      bool need_trim = false) const {
-        auto iter = config_map_.find(key);
-        if (iter != config_map_.end()) {
-            auto value_str_vec = StringUtils::Split(iter->second, delimiter, /*ignore_empty=*/true);
-            for (auto& value_str : value_str_vec) {
-                if (need_trim) {
-                    StringUtils::Trim(&value_str);
-                }
-                if constexpr (std::is_same_v<T, std::string>) {
-                    list->emplace_back(value_str);
-                } else {
-                    auto value = StringUtils::StringToValue<T>(value_str);
-                    if (!value) {
-                        return Status::Invalid(
-                            fmt::format("Invalid Config [{}: {}]", key, iter->second));
-                    }
-                    list->emplace_back(value.value());
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> config_value,
+                               GetOptionalValue<std::string>(key));
+        if (!config_value) {
+            return Status::OK();
+        }
+        auto value_str_vec =
+            StringUtils::Split(config_value.value(), delimiter, /*ignore_empty=*/true);
+        for (auto& value_str : value_str_vec) {
+            if (need_trim) {
+                StringUtils::Trim(&value_str);
+            }
+            if constexpr (std::is_same_v<T, std::string>) {
+                list->emplace_back(value_str);
+            } else {
+                auto value = StringUtils::StringToValue<T>(value_str);
+                if (!value) {
+                    return Status::Invalid(
+                        fmt::format("Invalid Config [{}: {}]", key, config_value.value()));
                 }
+                list->emplace_back(value.value());
             }
         }
         return Status::OK();  // Return success even if the configuration does not exist
@@ -109,9 +103,10 @@ class ConfigParser {
     Status ParseMemorySize(const std::string& key, T* value) const {
         static_assert(std::is_same_v<T, int64_t> || std::is_same_v<T, std::optional<int64_t>>,
                       "ParseMemorySize only supports int64_t and std::optional<int64_t>");
-        auto iter = config_map_.find(key);
-        if (iter != config_map_.end()) {
-            PAIMON_ASSIGN_OR_RAISE(*value, MemorySize::ParseBytes(iter->second));
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> config_value,
+                               GetOptionalValue<std::string>(key));
+        if (config_value) {
+            PAIMON_ASSIGN_OR_RAISE(*value, MemorySize::ParseBytes(config_value.value()));
         }
         return Status::OK();
     }
@@ -121,9 +116,10 @@ class ConfigParser {
     Status ParseTimeDuration(const std::string& key, T* value) const {
         static_assert(std::is_same_v<T, int64_t> || std::is_same_v<T, std::optional<int64_t>>,
                       "ParseTimeDuration only supports int64_t and std::optional<int64_t>");
-        auto iter = config_map_.find(key);
-        if (iter != config_map_.end()) {
-            PAIMON_ASSIGN_OR_RAISE(*value, TimeDuration::Parse(iter->second));
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> config_value,
+                               GetOptionalValue<std::string>(key));
+        if (config_value) {
+            PAIMON_ASSIGN_OR_RAISE(*value, TimeDuration::Parse(config_value.value()));
         }
         return Status::OK();
     }
@@ -132,14 +128,10 @@ class ConfigParser {
     template <typename Factory, typename ObjectType>
     Status ParseObject(const std::string& key, const std::string& default_identifier,
                        std::shared_ptr<ObjectType>* value) const {
-        auto iter = config_map_.find(key);
-        if (iter != config_map_.end()) {
-            std::string normalized_value = StringUtils::ToLowerCase(iter->second);
-            PAIMON_ASSIGN_OR_RAISE(*value, Factory::Get(normalized_value, config_map_));
-        } else {
-            PAIMON_ASSIGN_OR_RAISE(
-                *value, Factory::Get(StringUtils::ToLowerCase(default_identifier), config_map_));
-        }
+        PAIMON_ASSIGN_OR_RAISE(std::string identifier, OptionsUtils::GetValueFromMap<std::string>(
+                                                           config_map_, key, default_identifier));
+        PAIMON_ASSIGN_OR_RAISE(*value,
+                               Factory::Get(StringUtils::ToLowerCase(identifier), config_map_));
         return Status::OK();
     }
 
@@ -152,11 +144,10 @@ class ConfigParser {
             *value = specified_file_system;
             return Status::OK();
         }
-        std::string default_fs_identifier = "local";
-        auto iter = config_map_.find(Options::FILE_SYSTEM);
-        if (iter != config_map_.end()) {
-            default_fs_identifier = StringUtils::ToLowerCase(iter->second);
-        }
+        PAIMON_ASSIGN_OR_RAISE(
+            std::string default_fs_identifier,
+            OptionsUtils::GetValueFromMap<std::string>(config_map_, Options::FILE_SYSTEM, "local"));
+        default_fs_identifier = StringUtils::ToLowerCase(default_fs_identifier);
         *value = std::make_shared<ResolvingFileSystem>(fs_scheme_to_identifier_map,
                                                        default_fs_identifier, config_map_);
         return Status::OK();
@@ -164,151 +155,81 @@ class ConfigParser {
 
     // Parse SortOrder
     Status ParseSortOrder(SortOrder* sort_order) const {
-        auto iter = config_map_.find(Options::SEQUENCE_FIELD_SORT_ORDER);
-        if (iter != config_map_.end()) {
-            std::string str = StringUtils::ToLowerCase(iter->second);
-            if (str == "ascending") {
-                *sort_order = SortOrder::ASCENDING;
-            } else if (str == "descending") {
-                *sort_order = SortOrder::DESCENDING;
-            } else {
-                return Status::Invalid(fmt::format("invalid sort order: {}", str));
-            }
-        }
-        return Status::OK();
+        return ParseEnum(
+            Options::SEQUENCE_FIELD_SORT_ORDER,
+            {{"ascending", SortOrder::ASCENDING}, {"descending", SortOrder::DESCENDING}},
+            "sort order", sort_order);
     }
 
     // Parse LookupCompactMode
     Status ParseLookupCompactMode(LookupCompactMode* mode) const {
-        auto iter = config_map_.find(Options::LOOKUP_COMPACT);
-        if (iter != config_map_.end()) {
-            std::string str = StringUtils::ToLowerCase(iter->second);
-            if (str == "radical") {
-                *mode = LookupCompactMode::RADICAL;
-            } else if (str == "gentle") {
-                *mode = LookupCompactMode::GENTLE;
-            } else {
-                return Status::Invalid(fmt::format("invalid lookup mode: {}", str));
-            }
-        }
-        return Status::OK();
+        return ParseEnum(
+            Options::LOOKUP_COMPACT,
+            {{"radical", LookupCompactMode::RADICAL}, {"gentle", LookupCompactMode::GENTLE}},
+            "lookup mode", mode);
     }
 
     // Parse SortEngine
     Status ParseSortEngine(SortEngine* sort_engine) const {
-        auto iter = config_map_.find(Options::SORT_ENGINE);
-        if (iter != config_map_.end()) {
-            std::string str = StringUtils::ToLowerCase(iter->second);
-            if (str == "min-heap") {
-                *sort_engine = SortEngine::MIN_HEAP;
-            } else if (str == "loser-tree") {
-                *sort_engine = SortEngine::LOSER_TREE;
-            } else {
-                return Status::Invalid(fmt::format("invalid sort engine: {}", str));
-            }
-        }
-        return Status::OK();
+        return ParseEnum(
+            Options::SORT_ENGINE,
+            {{"min-heap", SortEngine::MIN_HEAP}, {"loser-tree", SortEngine::LOSER_TREE}},
+            "sort engine", sort_engine);
     }
 
     // Parse MergeEngine
     Status ParseMergeEngine(MergeEngine* merge_engine) const {
-        auto iter = config_map_.find(Options::MERGE_ENGINE);
-        if (iter != config_map_.end()) {
-            std::string str = StringUtils::ToLowerCase(iter->second);
-            if (str == "deduplicate") {
-                *merge_engine = MergeEngine::DEDUPLICATE;
-            } else if (str == "partial-update") {
-                *merge_engine = MergeEngine::PARTIAL_UPDATE;
-            } else if (str == "aggregation") {
-                *merge_engine = MergeEngine::AGGREGATE;
-            } else if (str == "first-row") {
-                *merge_engine = MergeEngine::FIRST_ROW;
-            } else {
-                return Status::Invalid(fmt::format("invalid merge engine: {}", str));
-            }
-        }
-        return Status::OK();
+        return ParseEnum(Options::MERGE_ENGINE,
+                         {{"deduplicate", MergeEngine::DEDUPLICATE},
+                          {"partial-update", MergeEngine::PARTIAL_UPDATE},
+                          {"aggregation", MergeEngine::AGGREGATE},
+                          {"first-row", MergeEngine::FIRST_ROW}},
+                         "merge engine", merge_engine);
     }
 
     // Parse VariantShreddingInferenceMode
     Status ParseVariantShreddingInferenceMode(VariantShreddingInferenceMode* inference_mode) const {
-        auto iter = config_map_.find(Options::VARIANT_SHREDDING_INFERENCE_MODE);
-        if (iter != config_map_.end()) {
-            std::string str = StringUtils::ToLowerCase(iter->second);
-            if (str == "per-file") {
-                *inference_mode = VariantShreddingInferenceMode::PER_FILE;
-            } else if (str == "adaptive") {
-                *inference_mode = VariantShreddingInferenceMode::ADAPTIVE;
-            } else {
-                return Status::Invalid(
-                    fmt::format("invalid variant shredding inference mode: {}", str));
-            }
-        }
-        return Status::OK();
+        return ParseEnum(Options::VARIANT_SHREDDING_INFERENCE_MODE,
+                         {{"per-file", VariantShreddingInferenceMode::PER_FILE},
+                          {"adaptive", VariantShreddingInferenceMode::ADAPTIVE}},
+                         "variant shredding inference mode", inference_mode);
     }
 
     // Parse ChangelogProducer
     Status ParseChangelogProducer(ChangelogProducer* changelog_producer) const {
-        auto iter = config_map_.find(Options::CHANGELOG_PRODUCER);
-        if (iter != config_map_.end()) {
-            std::string str = StringUtils::ToLowerCase(iter->second);
-            if (str == "none") {
-                *changelog_producer = ChangelogProducer::NONE;
-            } else if (str == "input") {
-                *changelog_producer = ChangelogProducer::INPUT;
-            } else if (str == "full-compaction") {
-                *changelog_producer = ChangelogProducer::FULL_COMPACTION;
-            } else if (str == "lookup") {
-                *changelog_producer = ChangelogProducer::LOOKUP;
-            } else {
-                return Status::Invalid(fmt::format("invalid changelog producer: {}", str));
-            }
-        }
-        return Status::OK();
+        return ParseEnum(Options::CHANGELOG_PRODUCER,
+                         {{"none", ChangelogProducer::NONE},
+                          {"input", ChangelogProducer::INPUT},
+                          {"full-compaction", ChangelogProducer::FULL_COMPACTION},
+                          {"lookup", ChangelogProducer::LOOKUP}},
+                         "changelog producer", changelog_producer);
     }
 
     // Parse ExternalPathStrategy
     Status ParseExternalPathStrategy(ExternalPathStrategy* external_path_strategy) const {
-        auto iter = config_map_.find(Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY);
-        if (iter != config_map_.end()) {
-            std::string str = StringUtils::ToLowerCase(iter->second);
-            if (str == "none") {
-                *external_path_strategy = ExternalPathStrategy::NONE;
-            } else if (str == "specific-fs") {
-                *external_path_strategy = ExternalPathStrategy::SPECIFIC_FS;
-            } else if (str == "round-robin") {
-                *external_path_strategy = ExternalPathStrategy::ROUND_ROBIN;
-            } else {
-                return Status::Invalid(fmt::format("invalid external path strategy: {}", str));
-            }
-        }
-        return Status::OK();
+        return ParseEnum(Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY,
+                         {{"none", ExternalPathStrategy::NONE},
+                          {"specific-fs", ExternalPathStrategy::SPECIFIC_FS},
+                          {"round-robin", ExternalPathStrategy::ROUND_ROBIN}},
+                         "external path strategy", external_path_strategy);
     }
 
     // Parse BucketFunctionType
     Status ParseBucketFunctionType(BucketFunctionType* bucket_function_type) const {
-        auto iter = config_map_.find(Options::BUCKET_FUNCTION_TYPE);
-        if (iter != config_map_.end()) {
-            std::string str = StringUtils::ToLowerCase(iter->second);
-            if (str == "default") {
-                *bucket_function_type = BucketFunctionType::DEFAULT;
-            } else if (str == "mod") {
-                *bucket_function_type = BucketFunctionType::MOD;
-            } else if (str == "hive") {
-                *bucket_function_type = BucketFunctionType::HIVE;
-            } else {
-                return Status::Invalid(fmt::format("invalid bucket function type: {}", str));
-            }
-        }
-        return Status::OK();
+        return ParseEnum(Options::BUCKET_FUNCTION_TYPE,
+                         {{"default", BucketFunctionType::DEFAULT},
+                          {"mod", BucketFunctionType::MOD},
+                          {"hive", BucketFunctionType::HIVE}},
+                         "bucket function type", bucket_function_type);
     }
 
     // Parse StartupMode
     Status ParseStartupMode(StartupMode* startup_mode) const {
-        auto iter = config_map_.find(Options::SCAN_MODE);
-        if (iter != config_map_.end()) {
-            std::string str = StringUtils::ToLowerCase(iter->second);
-            PAIMON_ASSIGN_OR_RAISE(*startup_mode, StartupMode::FromString(str));
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> value,
+                               GetOptionalValue<std::string>(Options::SCAN_MODE));
+        if (value) {
+            PAIMON_ASSIGN_OR_RAISE(
+                *startup_mode, StartupMode::FromString(StringUtils::ToLowerCase(value.value())));
         }
         return Status::OK();
     }
@@ -372,7 +293,37 @@ class ConfigParser {
     }
 
  private:
-    const std::map<std::string, std::string> config_map_;
+    template <typename T>
+    Status ParseEnum(const std::string& key,
+                     std::initializer_list<std::pair<const char*, T>> candidates,
+                     const std::string& error_name, T* value) const {
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> config_value,
+                               GetOptionalValue<std::string>(key));
+        if (!config_value) {
+            return Status::OK();
+        }
+        std::string normalized_value = StringUtils::ToLowerCase(config_value.value());
+        for (const auto& [candidate, candidate_value] : candidates) {
+            if (normalized_value == candidate) {
+                *value = candidate_value;
+                return Status::OK();
+            }
+        }
+        return Status::Invalid(fmt::format("invalid {}: {}", error_name, normalized_value));
+    }
+
+    template <typename T>
+    Result<std::optional<T>> GetOptionalValue(const std::string& key) const {
+        Result<std::optional<T>> result =
+            OptionsUtils::GetOptionalValueFromMap<T>(config_map_, key);
+        if (!result.ok()) {
+            return Status::Invalid(
+                fmt::format("Invalid Config [{}: {}]", key, config_map_.at(key)));
+        }
+        return result.value();
+    }
+
+    const std::map<std::string, std::string>& config_map_;
 };
 
 // Impl is a private implementation of CoreOptions,
diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp
index 994b0e3fc..071456e48 100644
--- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp
+++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp
@@ -78,14 +78,9 @@ LookupMergeTreeCompactRewriter<T>::Create(
         .WithMemoryPool(pool);
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ReadContext> read_context,
                            read_context_builder.Finish());
-    // TODO(xinyu.lxy): temporarily disabled pre-buffer for parquet, which may cause high memory
-    // usage during compaction. Will fix via parquet format refactor.
-    auto new_options = options.ToMap();
-    if (new_options.find("parquet.read.enable-pre-buffer") == new_options.end()) {
-        new_options["parquet.read.enable-pre-buffer"] = "false";
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalReadContext> internal_context,
-                           InternalReadContext::Create(read_context, table_schema, new_options));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<InternalReadContext> internal_context,
+        InternalReadContext::Create(read_context, table_schema, options.ToMap()));
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr<FileStorePathFactory> path_factory,
         path_factory_cache->GetOrCreatePathFactory(options.GetFileFormat()->Identifier()));
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp
index 1b64be2da..fd7c7cbd2 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp
@@ -84,14 +84,9 @@ Result<std::unique_ptr<MergeTreeCompactRewriter>> MergeTreeCompactRewriter::Crea
         .WithMemoryPool(pool);
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ReadContext> read_context,
                            read_context_builder.Finish());
-    // TODO(xinyu.lxy): temporarily disabled pre-buffer for parquet, which may cause high memory
-    // usage during compaction. Will fix via parquet format refactor.
-    auto new_options = options.ToMap();
-    if (new_options.find("parquet.read.enable-pre-buffer") == new_options.end()) {
-        new_options["parquet.read.enable-pre-buffer"] = "false";
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalReadContext> internal_context,
-                           InternalReadContext::Create(read_context, table_schema, new_options));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<InternalReadContext> internal_context,
+        InternalReadContext::Create(read_context, table_schema, options.ToMap()));
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr<FileStorePathFactory> path_factory,
         path_factory_cache->GetOrCreatePathFactory(options.GetFileFormat()->Identifier()));
diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp
index a931fea10..512f28728 100644
--- a/src/paimon/core/mergetree/lookup_levels.cpp
+++ b/src/paimon/core/mergetree/lookup_levels.cpp
@@ -66,14 +66,9 @@ Result<std::unique_ptr<LookupLevels<T>>> LookupLevels<T>::Create(
         .WithMemoryPool(pool);
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ReadContext> read_context,
                            read_context_builder.Finish());
-    // TODO(xinyu.lxy): temporarily disabled pre-buffer for parquet, which may cause high memory
-    // usage during compaction. Will fix via parquet format refactor.
-    auto new_options = options.ToMap();
-    if (new_options.find("parquet.read.enable-pre-buffer") == new_options.end()) {
-        new_options["parquet.read.enable-pre-buffer"] = "false";
-    }
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalReadContext> internal_read_context,
-                           InternalReadContext::Create(read_context, table_schema, new_options));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<InternalReadContext> internal_read_context,
+        InternalReadContext::Create(read_context, table_schema, options.ToMap()));
     auto split_read = std::make_unique<RawFileSplitRead>(path_factory, internal_read_context, pool,
                                                          CreateDefaultExecutor());
 
diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp
index 5d6c2c930..f660093a2 100644
--- a/src/paimon/core/operation/append_only_file_store_write.cpp
+++ b/src/paimon/core/operation/append_only_file_store_write.cpp
@@ -290,14 +290,8 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyFileStoreWrite::CreateFilesReader
         .WithMemoryPool(pool_);
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
     std::map<std::string, std::string> options = options_.ToMap();
-    // TODO(xinyu.lxy): temporarily disabled pre-buffer for parquet, which may cause high
-    // memory usage during compaction. Will fix via parquet format refactor.
-    auto new_options = options;
-    if (new_options.find("parquet.read.enable-pre-buffer") == new_options.end()) {
-        new_options["parquet.read.enable-pre-buffer"] = "false";
-    }
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalReadContext> internal_read_context,
-                           InternalReadContext::Create(read_context, table_schema_, new_options));
+                           InternalReadContext::Create(read_context, table_schema_, options));
     auto read = std::make_unique<RawFileSplitRead>(file_store_path_factory_, internal_read_context,
                                                    pool_, compact_executor_);
 
diff --git a/src/paimon/core/operation/commit/sequence_snapshot_properties.cpp b/src/paimon/core/operation/commit/sequence_snapshot_properties.cpp
index 51729b991..512869f00 100644
--- a/src/paimon/core/operation/commit/sequence_snapshot_properties.cpp
+++ b/src/paimon/core/operation/commit/sequence_snapshot_properties.cpp
@@ -24,6 +24,7 @@
 #include <algorithm>
 #include <limits>
 
+#include "paimon/common/utils/string_utils.h"
 #include "paimon/core/manifest/file_kind.h"
 
 namespace paimon {
@@ -40,19 +41,12 @@ Result<std::optional<int64_t>> SequenceSnapshotProperties::MaxSequenceNumber(
         return std::optional<int64_t>();
     }
 
-    try {
-        size_t parsed = 0;
-        int64_t value = std::stoll(iter->second, &parsed);
-        if (parsed != iter->second.size()) {
-            return Status::Invalid(
-                fmt::format("Invalid {} value '{}': trailing characters are not allowed",
-                            kMaxSequenceNumberKey, iter->second));
-        }
-        return std::optional<int64_t>(value);
-    } catch (const std::exception& e) {
-        return Status::Invalid(fmt::format("Invalid {} value '{}': {}", kMaxSequenceNumberKey,
-                                           iter->second, e.what()));
+    std::optional<int64_t> value = StringUtils::StringToValue<int64_t>(iter->second);
+    if (!value) {
+        return Status::Invalid(
+            fmt::format("Invalid {} value '{}'", kMaxSequenceNumberKey, iter->second));
     }
+    return value;
 }
 
 std::optional<int64_t> SequenceSnapshotProperties::MaxSequenceNumberFromFiles(
diff --git a/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp b/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp
index 5975eb1e8..ca962e0e8 100644
--- a/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp
+++ b/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp
@@ -115,7 +115,7 @@ TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberTrailingCharacters) {
     std::map<std::string, std::string> properties{
         {SequenceSnapshotProperties::kMaxSequenceNumberKey, "123abc"}};
     ASSERT_NOK_WITH_MSG(SequenceSnapshotProperties::MaxSequenceNumber(MakeSnapshot(properties)),
-                        "trailing characters are not allowed");
+                        "Invalid sequence.generation.max-sequence-number value '123abc'");
 }
 
 TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberNotANumber) {
diff --git a/src/paimon/core/postpone/postpone_bucket_writer.cpp b/src/paimon/core/postpone/postpone_bucket_writer.cpp
index 47fd3bcb8..45bcfd364 100644
--- a/src/paimon/core/postpone/postpone_bucket_writer.cpp
+++ b/src/paimon/core/postpone/postpone_bucket_writer.cpp
@@ -34,7 +34,6 @@
 #include "paimon/common/data/shredding/shredding_write_plan_factories.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/table/special_fields.h"
-#include "paimon/common/types/data_field.h"
 #include "paimon/common/types/row_kind.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
@@ -55,27 +54,12 @@ namespace paimon {
 class InternalRow;
 class MemoryPool;
 
-namespace {
-
-std::shared_ptr<arrow::Schema> BuildPostponeBucketWriteSchema(
-    const std::shared_ptr<arrow::Schema>& value_schema) {
-    arrow::FieldVector target_fields;
-    target_fields.push_back(
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()));
-    target_fields.push_back(DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()));
-    target_fields.insert(target_fields.end(), value_schema->fields().begin(),
-                         value_schema->fields().end());
-    return arrow::schema(target_fields);
-}
-
-}  // namespace
-
 Result<std::unique_ptr<PostponeBucketWriter>> PostponeBucketWriter::Create(
     const std::vector<std::string>& trimmed_primary_keys,
     const std::shared_ptr<DataFilePathFactory>& path_factory, int64_t schema_id,
     const std::shared_ptr<arrow::Schema>& value_schema, const CoreOptions& options,
     const std::shared_ptr<MemoryPool>& pool) {
-    auto write_schema = BuildPostponeBucketWriteSchema(value_schema);
+    auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema);
     return std::unique_ptr<PostponeBucketWriter>(new PostponeBucketWriter(
         trimmed_primary_keys, path_factory, schema_id, value_schema, write_schema, options, pool));
 }
diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp
index 7342826d4..4c7dd2ce5 100644
--- a/src/paimon/core/schema/schema_validation.cpp
+++ b/src/paimon/core/schema/schema_validation.cpp
@@ -100,7 +100,7 @@ Status ValidateSharedShreddingFileFormat(const std::string& option_key,
 }
 
 Status ValidateVectorFileFormat(const std::string& option_key, const std::string& file_format) {
-    if (StringUtils::ToLowerCase(file_format) != "parquet") {
+    if (!StringUtils::EqualsIgnoreCase(file_format, "parquet")) {
         return Status::Invalid(
             fmt::format("VECTOR currently only supports parquet data files, but {} is {}.",
                         option_key, file_format));
diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp
index 47603497b..578856260 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -56,6 +56,12 @@ TEST(SchemaValidationTest, TestVectorType) {
                                              /*primary_keys=*/{}, parquet_options));
     ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
 
+    parquet_options[Options::FILE_FORMAT] = "PARQUET";
+    ASSERT_OK_AND_ASSIGN(table_schema,
+                         TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                             /*primary_keys=*/{}, parquet_options));
+    ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+
     std::map<std::string, std::string> orc_options = {{Options::BUCKET, "-1"},
                                                       {Options::FILE_FORMAT, "orc"}};
     ASSERT_OK_AND_ASSIGN(table_schema,
diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp
index dd1b4e606..6523588e8 100644
--- a/src/paimon/core/table/system/global_system_tables.cpp
+++ b/src/paimon/core/table/system/global_system_tables.cpp
@@ -108,11 +108,12 @@ VariantType OptionalStringValue(const std::map<std::string, std::string>& option
 
 Result<VariantType> OptionalLongValue(const std::map<std::string, std::string>& options,
                                       const std::string& key) {
-    if (options.find(key) == options.end()) {
+    PAIMON_ASSIGN_OR_RAISE(std::optional<int64_t> value,
+                           OptionsUtils::GetOptionalValueFromMap<int64_t>(options, key));
+    if (!value) {
         return VariantType(NullType());
     }
-    PAIMON_ASSIGN_OR_RAISE(int64_t value, OptionsUtils::GetValueFromMap<int64_t>(options, key));
-    return VariantType(value);
+    return VariantType(value.value());
 }
 
 Result<bool> IsEnabled(const GlobalSystemTableRegistryEntry& entry,
diff --git a/src/paimon/format/orc/orc_format_writer.cpp b/src/paimon/format/orc/orc_format_writer.cpp
index 1a394ca94..fc2316b21 100644
--- a/src/paimon/format/orc/orc_format_writer.cpp
+++ b/src/paimon/format/orc/orc_format_writer.cpp
@@ -40,7 +40,6 @@
 #include "orc/Writer.hh"
 #include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/metrics/metrics_impl.h"
-#include "paimon/common/options/memory_size.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/options_utils.h"
 #include "paimon/common/utils/string_utils.h"
@@ -236,18 +235,6 @@ Status OrcFormatWriter::AddMetadata(const std::map<std::string, std::string>& me
     return Status::OK();
 }
 
-namespace {
-
-Result<uint64_t> GetMemorySizeOption(const std::map<std::string, std::string>& options,
-                                     const std::string& key, uint64_t default_value) {
-    PAIMON_ASSIGN_OR_RAISE(std::string value, OptionsUtils::GetValueFromMap<std::string>(
-                                                  options, key, std::to_string(default_value)));
-    PAIMON_ASSIGN_OR_RAISE(int64_t bytes, MemorySize::ParseBytes(value));
-    return static_cast<uint64_t>(bytes);
-}
-
-}  // namespace
-
 Result<::orc::WriterOptions> OrcFormatWriter::PrepareWriterOptions(
     const std::map<std::string, std::string>& options, const std::string& file_compression,
     const std::shared_ptr<arrow::DataType>& data_type) {
@@ -261,15 +248,16 @@ Result<::orc::WriterOptions> OrcFormatWriter::PrepareWriterOptions(
         }
     }
     ::orc::WriterOptions writer_options;
-    PAIMON_ASSIGN_OR_RAISE(uint64_t stripe_size,
-                           GetMemorySizeOption(options, ORC_STRIPE_SIZE, DEFAULT_STRIPE_SIZE));
+    PAIMON_ASSIGN_OR_RAISE(
+        uint64_t stripe_size,
+        OptionsUtils::GetValueFromMap<uint64_t>(options, ORC_STRIPE_SIZE, DEFAULT_STRIPE_SIZE));
     writer_options.setStripeSize(stripe_size);
     PAIMON_ASSIGN_OR_RAISE(::orc::CompressionKind compression,
                            ToOrcCompressionKind(StringUtils::ToLowerCase(file_compression)));
     writer_options.setCompression(compression);
-    PAIMON_ASSIGN_OR_RAISE(
-        uint64_t compression_block_size,
-        GetMemorySizeOption(options, ORC_COMPRESSION_BLOCK_SIZE, DEFAULT_COMPRESSION_BLOCK_SIZE));
+    PAIMON_ASSIGN_OR_RAISE(uint64_t compression_block_size, OptionsUtils::GetValueFromMap<uint64_t>(
+                                                                options, ORC_COMPRESSION_BLOCK_SIZE,
+                                                                DEFAULT_COMPRESSION_BLOCK_SIZE));
     writer_options.setCompressionBlockSize(compression_block_size);
     PAIMON_ASSIGN_OR_RAISE(
         double dictionary_key_threshold,
diff --git a/src/paimon/format/parquet/parquet_format_defs.h b/src/paimon/format/parquet/parquet_format_defs.h
index 433103e54..8b205a09a 100644
--- a/src/paimon/format/parquet/parquet_format_defs.h
+++ b/src/paimon/format/parquet/parquet_format_defs.h
@@ -99,7 +99,7 @@ static inline const char PARQUET_READ_PREDICATE_NODE_COUNT_LIMIT[] =
 static inline const char PARQUET_READ_ENABLE_PAGE_INDEX_FILTER[] =
     "parquet.read.enable-page-index-filter";
 
-// Default is true. Compaction will set to false to reduce memory consumption.
+// Default is true.
 static inline const char PARQUET_READ_ENABLE_PRE_BUFFER[] = "parquet.read.enable-pre-buffer";
 
 static constexpr uint32_t DEFAULT_PARQUET_READ_CACHE_OPTION_PREFETCH_LIMIT = 0;
diff --git a/src/paimon/fs/local/local_file.cpp b/src/paimon/fs/local/local_file.cpp
index 645306676..a3f960af8 100644
--- a/src/paimon/fs/local/local_file.cpp
+++ b/src/paimon/fs/local/local_file.cpp
@@ -49,7 +49,7 @@ Result<std::unique_ptr<LocalFile>> LocalFile::Create(const std::string& path_str
     // local file system does not support path_string with scheme, e.g., "file:/tmp" will be
     // rewritten to "/tmp"
     PAIMON_ASSIGN_OR_RAISE(Path path, PathUtil::ToPath(path_string));
-    if (!path.scheme.empty() && StringUtils::ToLowerCase(path.scheme) != "file") {
+    if (!path.scheme.empty() && !StringUtils::EqualsIgnoreCase(path.scheme, "file")) {
         return Status::Invalid(fmt::format("invalid scheme {} for local file system", path.scheme));
     }
     if (path.path.empty() || path.path[0] != '/') {
diff --git a/src/paimon/fs/local/local_file_test.cpp b/src/paimon/fs/local/local_file_test.cpp
index f22095c90..25b9f6db9 100644
--- a/src/paimon/fs/local/local_file_test.cpp
+++ b/src/paimon/fs/local/local_file_test.cpp
@@ -27,6 +27,11 @@
 
 namespace paimon::test {
 
+TEST(LocalFileTest, TestSchemeCaseInsensitive) {
+    ASSERT_OK(LocalFile::Create("FILE:/tmp"));
+    ASSERT_NOK(LocalFile::Create("s3:/tmp"));
+}
+
 TEST(LocalFileTest, TestReadWriteEmptyContent) {
     auto test_root_dir = UniqueTestDirectory::Create();
     ASSERT_TRUE(test_root_dir);
diff --git a/src/paimon/fs/s3/s3_file_system.cpp b/src/paimon/fs/s3/s3_file_system.cpp
index 49668b70f..e31cc9ca3 100644
--- a/src/paimon/fs/s3/s3_file_system.cpp
+++ b/src/paimon/fs/s3/s3_file_system.cpp
@@ -580,22 +580,22 @@ bool IsIpAddressAuthority(const std::string& authority) {
 }
 
 const char* AwsDnsSuffixForRegion(const std::string& region) {
-    if (region.rfind("cn-", 0) == 0) {
+    if (StringUtils::StartsWith(region, "cn-")) {
         return "amazonaws.com.cn";
     }
-    if (region.rfind("eusc-de-", 0) == 0) {
+    if (StringUtils::StartsWith(region, "eusc-de-")) {
         return "amazonaws.eu";
     }
-    if (region.rfind("us-iso-", 0) == 0) {
+    if (StringUtils::StartsWith(region, "us-iso-")) {
         return "c2s.ic.gov";
     }
-    if (region.rfind("us-isob-", 0) == 0) {
+    if (StringUtils::StartsWith(region, "us-isob-")) {
         return "sc2s.sgov.gov";
     }
-    if (region.rfind("eu-isoe-", 0) == 0) {
+    if (StringUtils::StartsWith(region, "eu-isoe-")) {
         return "cloud.adc-e.uk";
     }
-    if (region.rfind("us-isof-", 0) == 0) {
+    if (StringUtils::StartsWith(region, "us-isof-")) {
         return "csp.hci.ic.gov";
     }
     return "amazonaws.com";
diff --git a/src/paimon/global_index/lucene/jieba_analyzer.cpp b/src/paimon/global_index/lucene/jieba_analyzer.cpp
index 39cecec2f..e0a71a81a 100644
--- a/src/paimon/global_index/lucene/jieba_analyzer.cpp
+++ b/src/paimon/global_index/lucene/jieba_analyzer.cpp
@@ -17,6 +17,8 @@
  */
 #include "paimon/global_index/lucene/jieba_analyzer.h"
 
+#include <cctype>
+
 #include "paimon/common/utils/string_utils.h"
 #include "paimon/global_index/lucene/lucene_utils.h"
 
@@ -94,9 +96,7 @@ void JiebaTokenizer::NormalizeCase(std::string* term) {
         }
     }
     if (is_alphanumeric && !term->empty()) {
-        std::transform(term->begin(), term->end(), term->begin(), [](char ch) {
-            return static_cast<char>(std::tolower(static_cast<unsigned char>(ch)));
-        });
+        *term = StringUtils::ToLowerCase(*term);
     }
 }
 
diff --git a/src/paimon/rest/dlf_auth.cpp b/src/paimon/rest/dlf_auth.cpp
index 4c592f31f..6a4906279 100644
--- a/src/paimon/rest/dlf_auth.cpp
+++ b/src/paimon/rest/dlf_auth.cpp
@@ -22,7 +22,6 @@
 #include <openssl/evp.h>
 
 #include <array>
-#include <cctype>
 #include <climits>
 #include <ctime>
 #include <fstream>
@@ -74,28 +73,10 @@ constexpr const char kAcsSignatureVersionHeader[] = "x-acs-signature-version";
 constexpr const char kAcsVersionHeader[] = "x-acs-version";
 constexpr const char kAcsSecurityTokenHeader[] = "x-acs-security-token";
 
-void TrimWhitespace(std::string* value) {
-    size_t begin = 0;
-    while (begin < value->size() && std::isspace(static_cast<unsigned char>((*value)[begin]))) {
-        ++begin;
-    }
-    size_t end = value->size();
-    while (end > begin && std::isspace(static_cast<unsigned char>((*value)[end - 1]))) {
-        --end;
-    }
-    *value = value->substr(begin, end - begin);
-}
-
 Result<std::string> RequiredNonEmptyOption(const std::map<std::string, std::string>& options,
                                            const std::string& key) {
-    Result<std::string> value = OptionsUtils::GetValueFromMap<std::string>(options, key);
+    Result<std::string> value = OptionsUtils::GetNonEmptyValueFromMap(options, key);
     if (!value.ok()) {
-        if (!value.status().IsNotExist()) {
-            return value.status();
-        }
-        return Status::Invalid(fmt::format("option '{}' must be configured for DLF auth", key));
-    }
-    if (value.value().empty()) {
         return Status::Invalid(fmt::format("option '{}' must be configured for DLF auth", key));
     }
     return value.value();
@@ -267,7 +248,7 @@ Result<std::string> Md5Base64(const std::string& value) {
 
 std::string Trimmed(const std::string& value) {
     std::string trimmed = value;
-    TrimWhitespace(&trimmed);
+    StringUtils::Trim(&trimmed);
     return trimmed;
 }
 
@@ -510,7 +491,7 @@ Result<DlfToken> DlfEcsTokenLoader::LoadToken() {
     }
     if (!role_name_) {
         PAIMON_ASSIGN_OR_RAISE(std::string role, Get(metadata_url_));
-        TrimWhitespace(&role);
+        StringUtils::Trim(&role);
         if (role.empty()) {
             return Status::Invalid("DLF ECS metadata service returned an empty role name");
         }
diff --git a/src/paimon/rest/rest_api.cpp b/src/paimon/rest/rest_api.cpp
index c249ede05..a28e19916 100644
--- a/src/paimon/rest/rest_api.cpp
+++ b/src/paimon/rest/rest_api.cpp
@@ -24,6 +24,7 @@
 
 #include "fmt/format.h"
 #include "paimon/catalog_options.h"
+#include "paimon/common/utils/options_utils.h"
 #include "paimon/common/utils/rapidjson_util.h"
 #include "paimon/common/utils/sensitive_config_utils.h"
 #include "paimon/logging.h"
@@ -61,13 +62,13 @@ RestApi::RestApi(std::unique_ptr<RestHttpClient> client,
 Result<std::unique_ptr<RestApi>> RestApi::Create(const std::map<std::string, std::string>& options,
                                                  const std::string& warehouse, bool config_required,
                                                  const RestHttpClient::Config& http_config) {
-    auto uri_iter = options.find(CatalogOptions::URI);
-    if (uri_iter == options.end() || uri_iter->second.empty()) {
+    Result<std::string> uri = OptionsUtils::GetNonEmptyValueFromMap(options, CatalogOptions::URI);
+    if (!uri.ok()) {
         return Status::Invalid(fmt::format("option '{}' must be configured for the rest catalog",
                                            CatalogOptions::URI));
     }
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RestHttpClient> client,
-                           RestHttpClient::Create(uri_iter->second, http_config));
+                           RestHttpClient::Create(uri.value(), http_config));
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<AuthProvider> auth_provider,
                            AuthProvider::Create(options));
 
diff --git a/src/paimon/rest/rest_auth.cpp b/src/paimon/rest/rest_auth.cpp
index 1af3b06aa..43513b701 100644
--- a/src/paimon/rest/rest_auth.cpp
+++ b/src/paimon/rest/rest_auth.cpp
@@ -20,6 +20,7 @@
 
 #include "fmt/format.h"
 #include "paimon/catalog_options.h"
+#include "paimon/common/utils/options_utils.h"
 #include "paimon/common/utils/string_utils.h"
 #include "paimon/common/utils/url_utils.h"
 #include "paimon/rest/dlf_auth.h"
@@ -50,22 +51,24 @@ Result<std::map<std::string, std::string>> BearTokenAuthProvider::MergeAuthHeade
 
 Result<std::unique_ptr<AuthProvider>> AuthProvider::Create(
     const std::map<std::string, std::string>& options) {
-    auto provider_iter = options.find(CatalogOptions::TOKEN_PROVIDER);
-    if (provider_iter == options.end() || provider_iter->second.empty()) {
+    Result<std::string> provider_value =
+        OptionsUtils::GetNonEmptyValueFromMap(options, CatalogOptions::TOKEN_PROVIDER);
+    if (!provider_value.ok()) {
         return Status::Invalid(fmt::format("option '{}' must be configured for the rest catalog",
                                            CatalogOptions::TOKEN_PROVIDER));
     }
     // Matched leniently in lower case; other clients may match provider names
     // case-sensitively, so the exact "bear" and "dlf" spellings are portable.
-    std::string provider = StringUtils::ToLowerCase(provider_iter->second);
+    std::string provider = StringUtils::ToLowerCase(provider_value.value());
     if (provider == "bear") {
-        auto token_iter = options.find(CatalogOptions::TOKEN);
-        if (token_iter == options.end() || token_iter->second.empty()) {
+        Result<std::string> token =
+            OptionsUtils::GetNonEmptyValueFromMap(options, CatalogOptions::TOKEN);
+        if (!token.ok()) {
             return Status::Invalid(
                 fmt::format("option '{}' must be configured for the bear token provider",
                             CatalogOptions::TOKEN));
         }
-        return std::make_unique<BearTokenAuthProvider>(token_iter->second);
+        return std::make_unique<BearTokenAuthProvider>(token.value());
     }
     if (provider == "dlf") {
         return DlfAuthProvider::Create(options);
diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp
index c928c6a25..eb86035c5 100644
--- a/src/paimon/rest/rest_catalog.cpp
+++ b/src/paimon/rest/rest_catalog.cpp
@@ -52,7 +52,7 @@ constexpr const char kPathOption[] = "path";
 // `BranchManager::IsMainBranch`, which names the branch directory of a table, stays
 // case-sensitive: this normalization only decides how a table is addressed on the server.
 std::optional<std::string> NormalizeBranch(std::optional<std::string> branch) {
-    if (branch && StringUtils::ToLowerCase(branch.value()) == Identifier::kDefaultMainBranch) {
+    if (branch && StringUtils::EqualsIgnoreCase(branch.value(), Identifier::kDefaultMainBranch)) {
         return std::nullopt;
     }
     return branch;
diff --git a/src/paimon/rest/rest_http_client.cpp b/src/paimon/rest/rest_http_client.cpp
index 99b233f1a..f4964bf57 100644
--- a/src/paimon/rest/rest_http_client.cpp
+++ b/src/paimon/rest/rest_http_client.cpp
@@ -228,7 +228,8 @@ std::string RestHttpClient::NormalizeUri(const std::string& uri) {
     while (!normalized.empty() && normalized.back() == '/') {
         normalized.pop_back();
     }
-    if (normalized.rfind("http://", 0) != 0 && normalized.rfind("https://", 0) != 0) {
+    if (!StringUtils::StartsWith(normalized, "http://") &&
+        !StringUtils::StartsWith(normalized, "https://")) {
         normalized = "http://" + normalized;
     }
     return normalized;
diff --git a/src/paimon/rest/rest_util.cpp b/src/paimon/rest/rest_util.cpp
index fc8e6f698..1a746987d 100644
--- a/src/paimon/rest/rest_util.cpp
+++ b/src/paimon/rest/rest_util.cpp
@@ -21,6 +21,7 @@
 #include <stdexcept>
 
 #include "fmt/format.h"
+#include "paimon/common/utils/options_utils.h"
 #include "rapidjson/error/en.h"
 #include "rapidjson/stringbuffer.h"
 #include "rapidjson/writer.h"
@@ -29,13 +30,7 @@ namespace paimon {
 
 std::map<std::string, std::string> RestUtil::ExtractPrefixMap(
     const std::map<std::string, std::string>& options, const std::string& prefix) {
-    std::map<std::string, std::string> result;
-    for (const auto& [key, value] : options) {
-        if (key.size() > prefix.size() && key.compare(0, prefix.size(), prefix) == 0) {
-            result[key.substr(prefix.size())] = value;
-        }
-    }
-    return result;
+    return OptionsUtils::FetchOptionsWithPrefix(prefix, options);
 }
 
 std::string RestUtil::ExtractRequestId(const std::map<std::string, std::string>& headers) {

From e448f7ac5324351479fba88021aacdba040d19fb Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 15:04:16 +0800
Subject: [PATCH 73/93] refactor(realtime): localize primary-key split
 validation

---
 .../core/operation/merge_file_split_read.cpp  | 55 ++++++++++---------
 .../operation/merge_file_split_read_test.cpp  | 54 ++++++++++++++++++
 2 files changed, 82 insertions(+), 27 deletions(-)

diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 6c8bbecad..3b09d8c54 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -149,28 +149,28 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
 
     Status CollectDiskReaders(const std::vector<std::shared_ptr<Split>>& disk_splits,
                               std::vector<std::unique_ptr<KeyValueRecordReader>>* readers) {
-        std::shared_ptr<DataSplitImpl> first_split =
-            std::dynamic_pointer_cast<DataSplitImpl>(disk_splits.front());
-        if (!first_split) {
-            return Status::Invalid("merge input disk split is not a data split");
+        std::vector<std::shared_ptr<DataSplitImpl>> data_splits;
+        data_splits.reserve(disk_splits.size());
+        for (const std::shared_ptr<Split>& disk_split : disk_splits) {
+            std::shared_ptr<DataSplitImpl> data_split =
+                std::dynamic_pointer_cast<DataSplitImpl>(disk_split);
+            if (!data_split) {
+                return Status::Invalid("merge input disk split is not a data split");
+            }
+            data_splits.push_back(std::move(data_split));
         }
+        const std::shared_ptr<DataSplitImpl>& first_split = data_splits.front();
         const BinaryRow& partition = first_split->Partition();
         const int32_t bucket = first_split->Bucket();
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_file_path_factory,
-                               owner_->path_factory_->CreateDataFilePathFactory(partition, bucket));
 
         std::vector<std::shared_ptr<DataFileMeta>> data_files;
         std::vector<std::optional<DeletionFile>> deletion_files;
-        for (const std::shared_ptr<Split>& disk_split : disk_splits) {
-            std::shared_ptr<DataSplitImpl> data_split =
-                std::dynamic_pointer_cast<DataSplitImpl>(disk_split);
-            if (!data_split || !(data_split->Partition() == partition) ||
-                data_split->Bucket() != bucket) {
+        for (const std::shared_ptr<DataSplitImpl>& data_split : data_splits) {
+            if (!(data_split->Partition() == partition) || data_split->Bucket() != bucket) {
                 return Status::Invalid("merge input disk splits do not share a partition-bucket");
             }
-            if (!data_split->BeforeFiles().empty() || data_split->IsStreaming() ||
-                data_split->Bucket() == BucketModeDefine::POSTPONE_BUCKET) {
-                return Status::Invalid("additional merge input requires fixed-bucket batch splits");
+            if (!data_split->BeforeFiles().empty()) {
+                return Status::Invalid("merge input disk split must not contain before files");
             }
             const std::vector<std::shared_ptr<DataFileMeta>>& split_files = data_split->DataFiles();
             const std::vector<std::optional<DeletionFile>>& split_deletion_files =
@@ -188,11 +188,16 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
                                       split_deletion_files.end());
             }
         }
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_file_path_factory,
+                               owner_->path_factory_->CreateDataFilePathFactory(partition, bucket));
 
         DeletionVector::Factory dv_factory;
         std::vector<std::vector<SortedRun>> disk_sections;
         PAIMON_RETURN_NOT_OK(
             owner_->CreateDiskSections(data_files, deletion_files, &dv_factory, &disk_sections));
+        if (disk_sections.empty()) {
+            return Status::OK();
+        }
         std::vector<std::unique_ptr<KeyValueRecordReader>> section_readers;
         ScopeGuard section_readers_guard([&section_readers]() {
             for (const std::unique_ptr<KeyValueRecordReader>& reader : section_readers) {
@@ -200,13 +205,11 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
             }
         });
         section_readers.reserve(disk_sections.size());
-        std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper;
-        if (!disk_sections.empty()) {
-            PAIMON_ASSIGN_OR_RAISE(merge_function_wrapper,
-                                   MergeFileSplitRead::CreateMergeFunctionWrapper(
-                                       owner_->options_, owner_->context_->GetTableSchema(),
-                                       owner_->value_schema_, owner_->pool_));
-        }
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper,
+            MergeFileSplitRead::CreateMergeFunctionWrapper(owner_->options_,
+                                                           owner_->context_->GetTableSchema(),
+                                                           owner_->value_schema_, owner_->pool_));
         for (const std::vector<SortedRun>& section : disk_sections) {
             PAIMON_ASSIGN_OR_RAISE(
                 std::unique_ptr<SortMergeReader> section_reader,
@@ -216,12 +219,10 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
             section_readers.push_back(
                 std::make_unique<SortMergeKeyValueRecordReader>(std::move(section_reader)));
         }
-        if (!section_readers.empty()) {
-            std::unique_ptr<KeyValueRecordReader> concat_reader =
-                std::make_unique<ConcatKeyValueRecordReader>(std::move(section_readers));
-            section_readers_guard.Release();
-            readers->push_back(std::move(concat_reader));
-        }
+        std::unique_ptr<KeyValueRecordReader> concat_reader =
+            std::make_unique<ConcatKeyValueRecordReader>(std::move(section_readers));
+        section_readers_guard.Release();
+        readers->push_back(std::move(concat_reader));
         return Status::OK();
     }
 
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index a899fd904..d1da3e4f6 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -67,6 +67,12 @@ class FileSystem;
 }  // namespace paimon
 
 namespace paimon::test {
+namespace {
+
+class TestingSplit : public Split {};
+
+}  // namespace
+
 // Parameter: min_heap/loser_tree; enable/disable IO prefetch; enable/disable multi thread row to
 // batch
 class MergeFileSplitReadTest : public ::testing::Test,
@@ -739,6 +745,54 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections)
     batch_reader->Close();
 }
 
+TEST_P(MergeFileSplitReadTest, TestRealtimeReadValidatesDiskSplits) {
+    std::string path =
+        paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
+    ReadContextBuilder context_builder(path);
+    context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"});
+    context_builder.SetOptions(
+        {{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}});
+    AddOptions(&context_builder);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
+    std::shared_ptr<InternalReadContext> internal_context = CreateInternalReadContext(read_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<MergeFileSplitRead> split_read,
+                         CreateMergeFileSplitRead(internal_context));
+
+    std::vector<std::shared_ptr<DataSplit>> prepared_splits = PrepareDataSplit();
+    std::shared_ptr<DataSplitImpl> first =
+        std::dynamic_pointer_cast<DataSplitImpl>(prepared_splits[0]);
+    ASSERT_NE(nullptr, first);
+
+    std::vector<std::shared_ptr<Split>> non_data_splits = {std::make_shared<TestingSplit>()};
+    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(non_data_splits, {}),
+                        "merge input disk split is not a data split");
+    std::vector<std::shared_ptr<Split>> mixed_partition_splits = {prepared_splits[0],
+                                                                 prepared_splits[1]};
+    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(mixed_partition_splits, {}),
+                        "merge input disk splits do not share a partition-bucket");
+
+    std::vector<std::shared_ptr<DataFileMeta>> before_data_files = first->DataFiles();
+    DataSplitImpl::Builder before_builder(first->Partition(), first->Bucket(), first->BucketPath(),
+                                          std::move(before_data_files));
+    std::vector<std::shared_ptr<DataFileMeta>> before_files = first->DataFiles();
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<DataSplitImpl> before_split,
+        before_builder.WithBeforeFiles(std::move(before_files)).RawConvertible(false).Build());
+    std::vector<std::shared_ptr<Split>> before_splits = {before_split};
+    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(before_splits, {}),
+                        "merge input disk split must not contain before files");
+
+    std::vector<std::shared_ptr<DataFileMeta>> deletion_data_files = first->DataFiles();
+    DataSplitImpl::Builder deletion_builder(first->Partition(), first->Bucket(),
+                                            first->BucketPath(), std::move(deletion_data_files));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<DataSplitImpl> deletion_split,
+        deletion_builder.WithDataDeletionFiles({std::nullopt}).RawConvertible(false).Build());
+    std::vector<std::shared_ptr<Split>> deletion_splits = {deletion_split};
+    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(deletion_splits, {}),
+                        "deletion files must be empty or match data files");
+}
+
 TEST_P(MergeFileSplitReadTest, TestLookUp) {
     std::string path =
         paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";

From 0082544d86962f45e61216b84d945a7db403a46a Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 16:38:18 +0800
Subject: [PATCH 74/93] fix(io): preserve merged reader initialization errors

---
 .../io/merged_key_value_record_reader.cpp     |  2 +-
 .../merged_key_value_record_reader_test.cpp   | 77 +++++++++++++++----
 2 files changed, 62 insertions(+), 17 deletions(-)

diff --git a/src/paimon/core/io/merged_key_value_record_reader.cpp b/src/paimon/core/io/merged_key_value_record_reader.cpp
index 8c3952874..8730086f5 100644
--- a/src/paimon/core/io/merged_key_value_record_reader.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader.cpp
@@ -123,7 +123,6 @@ Result<std::unique_ptr<KeyValueRecordReader::Iterator>> MergedKeyValueRecordRead
     if (visited_) {
         return std::unique_ptr<KeyValueRecordReader::Iterator>();
     }
-    visited_ = true;
 
     auto iterator = std::make_unique<Iterator>(this);
     Result<bool> has_next_result = iterator->HasNext();
@@ -132,6 +131,7 @@ Result<std::unique_ptr<KeyValueRecordReader::Iterator>> MergedKeyValueRecordRead
         return initialization_error_.value();
     }
     bool has_next = std::move(has_next_result).value();
+    visited_ = true;
     if (!has_next) {
         return std::unique_ptr<KeyValueRecordReader::Iterator>();
     }
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 1a94aee0e..b35e0c111 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -128,6 +128,54 @@ class MalformedBitmapBatchReader : public BatchReader {
     int32_t row_id_;
 };
 
+class ScriptedKeyValueRecordReader final : public KeyValueRecordReader {
+ public:
+    ScriptedKeyValueRecordReader(std::vector<KeyValue>&& key_values, int32_t* next_batch_count)
+        : key_values_(std::move(key_values)), next_batch_count_(next_batch_count) {}
+
+    class Iterator final : public KeyValueRecordReader::Iterator {
+     public:
+        explicit Iterator(KeyValue&& key_value) : key_value_(std::move(key_value)) {}
+
+        Result<bool> HasNext() const override {
+            return key_value_.has_value();
+        }
+
+        Result<KeyValue> Next() override {
+            KeyValue result = std::move(key_value_.value());
+            key_value_.reset();
+            return result;
+        }
+
+     private:
+        std::optional<KeyValue> key_value_;
+    };
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
+        ++(*next_batch_count_);
+        if (*next_batch_count_ == 1) {
+            return std::make_unique<Iterator>(std::move(key_values_[0]));
+        }
+        if (*next_batch_count_ == 2) {
+            return Status::IOError("scripted lookahead failure");
+        }
+        if (*next_batch_count_ == 3) {
+            return std::make_unique<Iterator>(std::move(key_values_[1]));
+        }
+        return std::unique_ptr<KeyValueRecordReader::Iterator>();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return nullptr;
+    }
+
+    void Close() override {}
+
+ private:
+    std::vector<KeyValue> key_values_;
+    int32_t* next_batch_count_;
+};
+
 }  // namespace
 
 class MergedKeyValueRecordReaderTest : public testing::Test {
@@ -459,29 +507,26 @@ TEST_F(MergedKeyValueRecordReaderTest, TestQueryReaderRequiresStoreAlignedSchema
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestMergedReaderErrorRetry) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
-    auto failing_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-    failing_reader->SetNextBatchStatus(Status::IOError("stable prepared error"));
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(failing_reader), prepared_schema,
-                                        OffsetRange(0, 1), value_schema, value_schema, pool_));
+TEST_F(MergedKeyValueRecordReaderTest, TestMergedReaderInitializationErrorIsTerminal) {
+    std::vector<DataField> key_fields = {DataField(0, arrow::field("key", arrow::int32()))};
+    std::vector<KeyValue> key_values =
+        KeyValueChecker::GenerateKeyValues({10, 11}, {{1}, {2}}, {{1}, {2}}, pool_);
+    int32_t next_batch_count = 0;
+    auto reader =
+        std::make_unique<ScriptedKeyValueRecordReader>(std::move(key_values), &next_batch_count);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<FieldsComparator> key_comparator,
-                         FieldsComparator::Create({DataField(0, key)}, true));
+                         FieldsComparator::Create(key_fields, true));
     MergedKeyValueRecordReader merged_reader(std::move(reader), key_comparator,
                                              merge_function_wrapper_);
 
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> first = merged_reader.NextBatch();
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> retry = merged_reader.NextBatch();
-    ASSERT_NOK(first);
-    ASSERT_NOK(retry);
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> second_retry =
+        merged_reader.NextBatch();
+    ASSERT_NOK_WITH_MSG(first, "scripted lookahead failure");
     ASSERT_EQ(first.status().ToString(), retry.status().ToString());
+    ASSERT_EQ(first.status().ToString(), second_retry.status().ToString());
+    ASSERT_EQ(2, next_batch_count);
 }
 
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {

From b4fb4d07d6d2cdd4844735bf076a85e95c6903a7 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 16:38:41 +0800
Subject: [PATCH 75/93] fix(realtime): validate prepared reader coverage

---
 .../merged_key_value_record_reader_test.cpp   | 316 +++++++++++++++---
 .../realtime/prepared_key_value_reader.cpp    | 184 +++++-----
 .../core/realtime/prepared_key_value_reader.h |  10 -
 test/inte/realtime_write_inte_test.cpp        |  68 +++-
 4 files changed, 403 insertions(+), 175 deletions(-)

diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index b35e0c111..8cdef7a6b 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -65,14 +65,32 @@ std::shared_ptr<arrow::Schema> MakePreparedSchema(const arrow::FieldVector& valu
     return arrow::schema(prepared_fields);
 }
 
-Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderForTest(
+Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedQueryReaderForTest(
     std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::optional<OffsetRange>& visible_offsets,
-    const std::shared_ptr<arrow::Schema>& key_schema,
+    const OffsetRange& visible_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    return PreparedKeyValueReaderFactory::Create(
-        std::move(reader), prepared_schema, visible_offsets, key_schema, value_schema, memory_pool);
+    std::vector<std::unique_ptr<BatchReader>> readers;
+    readers.push_back(std::move(reader));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
+                           PreparedKeyValueReaderFactory::CreateForQuery(
+                               std::move(readers), prepared_schema, visible_offsets, key_schema,
+                               value_schema, memory_pool));
+    return std::move(adapted_readers[0]);
+}
+
+Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedCommitReaderForTest(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const OffsetRange& sealed_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    std::vector<std::unique_ptr<BatchReader>> readers;
+    readers.push_back(std::move(reader));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
+                           PreparedKeyValueReaderFactory::CreateForCommit(
+                               std::move(readers), prepared_schema, sealed_offsets, key_schema,
+                               value_schema, memory_pool));
+    return std::move(adapted_readers[0]);
 }
 
 class TrackingBatchReader : public BatchReader {
@@ -283,7 +301,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestSkipMergedNulloptResultInHasNext) {
     }
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderOffsetFilter) {
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryAllowsCommittedPrefix) {
     std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
                                            DataField(1, arrow::field("v0", arrow::int32()))};
     std::shared_ptr<arrow::Schema> value_schema =
@@ -295,31 +313,154 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderOffsetFilter) {
         arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
         [0, 100, 0, 1, 10],
         [0, 101, 1, 2, 20],
-        [0, 102, 4, 3, 30],
-        [0, 103, 2, 4, 40],
-        [0, 104, 5, 5, 50],
-        [0, 105, 3, 6, 60]
+        [0, 102, 2, 4, 40],
+        [0, 103, 3, 6, 60]
     ])")
             .ValueOrDie());
 
-    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(2, 4),
-                                        key_schema, value_schema, pool_));
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(2, 4),
+                             key_schema, value_schema, pool_));
+    ASSERT_EQ(1, readers.size());
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> results,
-        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                    KeyValueRecordReader::Iterator>(reader.get())));
+        (ReadResultCollector::CollectKeyValueResult<
+            KeyValueRecordReader, KeyValueRecordReader::Iterator>(readers[0].get())));
 
     std::vector<RowKind*> row_kinds = {const_cast<RowKind*>(RowKind::Insert()),
                                        const_cast<RowKind*>(RowKind::Insert())};
     std::vector<int64_t> levels = {KeyValue::UNKNOWN_LEVEL, KeyValue::UNKNOWN_LEVEL};
     std::vector<KeyValue> expected = KeyValueChecker::GenerateKeyValues(
-        row_kinds, {103, 105}, levels, {{4}, {6}}, {{4, 40}, {6, 60}}, pool_);
+        row_kinds, {102, 103}, levels, {{4}, {6}}, {{4, 40}, {6, 60}}, pool_);
     KeyValueChecker::CheckResult(expected, results, 1, 2);
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsNegativeOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, -1, 1]])")
+            .ValueOrDie();
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                  /*read_batch_size=*/1),
+            prepared_schema, OffsetRange(1, 2), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "reader offset must be non-negative");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryOffsetCoverageAcrossReadersAndBatches) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> first_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 2, 1], [0, 11, 0, 2]])")
+            .ValueOrDie();
+    std::shared_ptr<arrow::Array> second_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 12, 3, 3], [0, 13, 1, 4]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
+                             value_schema, value_schema, pool_));
+    int64_t row_count = 0;
+    for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
+        ASSERT_OK_AND_ASSIGN(
+            std::vector<KeyValue> rows,
+            (ReadResultCollector::CollectKeyValueResult<
+                KeyValueRecordReader, KeyValueRecordReader::Iterator>(reader.get())));
+        row_count += static_cast<int64_t>(rows.size());
+    }
+    ASSERT_EQ(4, row_count);
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsMissingVisibleOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 0, 1], [0, 11, 2, 2]])")
+            .ValueOrDie();
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                  /*read_batch_size=*/1),
+            prepared_schema, OffsetRange(0, 3), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())),
+        "query readers did not cover the visible range");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsDuplicateVisibleOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> first_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+    std::shared_ptr<arrow::Array> second_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 11, 1, 2], [0, 12, 1, 3]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 2),
+                             value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> first_rows,
+        (ReadResultCollector::CollectKeyValueResult<
+            KeyValueRecordReader, KeyValueRecordReader::Iterator>(readers[0].get())));
+    ASSERT_EQ(1, first_rows.size());
+    ASSERT_NOK_WITH_MSG((ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                                    KeyValueRecordReader::Iterator>(
+                            readers[1].get())),
+                        "query readers did not cover the visible range");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsEmptyEofForVisibleRange) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([])").ValueOrDie();
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                  /*read_batch_size=*/1),
+            prepared_schema, OffsetRange(0, 1), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "query readers did not cover the visible range");
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsReversedVisibleOffsets) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
@@ -331,12 +472,38 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsReversedVisibleO
         std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1);
 
     Result<std::unique_ptr<KeyValueRecordReader>> result =
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(2, 1),
-                                        value_schema, value_schema, pool_);
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(2, 1), value_schema, value_schema, pool_);
     ASSERT_TRUE(result.status().IsInvalid());
     ASSERT_NOK_WITH_MSG(result, "prepared visible offset range begin exceeds end");
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsEmptyReadersForVisibleRange) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+
+    ASSERT_NOK_WITH_MSG(
+        PreparedKeyValueReaderFactory::CreateForQuery(std::move(batch_readers), prepared_schema,
+                                                      OffsetRange(0, 1), value_schema, value_schema,
+                                                      pool_),
+        "PK real-time store returned no query readers for a non-empty visible range");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestQueryAllowsEmptyReadersForEmptyVisibleRange) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(1, 1),
+                             value_schema, value_schema, pool_));
+    ASSERT_TRUE(readers.empty());
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderBitmapBounds) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
@@ -350,8 +517,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderBitmapBounds) {
 
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                        value_schema, value_schema, pool_));
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
     Result<std::vector<KeyValue>> result =
         ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
                                                    KeyValueRecordReader::Iterator>(reader.get());
@@ -359,7 +526,29 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderBitmapBounds) {
     ASSERT_NOK_WITH_MSG(result, "selected row id 1 is out of bounds for prepared batch length 1");
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedQueryReaderRejectsPartialBitmap) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 0, 1], [0, 11, 1, 2]])")
+            .ValueOrDie();
+    RoaringBitmap32 partial_bitmap;
+    partial_bitmap.Add(0);
+    auto batch_reader = std::make_unique<MockFileBatchReader>(
+        prepared_array, prepared_type, partial_bitmap, /*read_batch_size=*/2);
+    batch_reader->EnableRandomizeBatchSize(false);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 2), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "must cover every raw mutation");
+}
+
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderQueryProjection) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
@@ -373,8 +562,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
         std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> query_reader,
-        AdaptPreparedBatchReaderForTest(std::move(query_batch_reader), prepared_schema,
-                                        OffsetRange(0, 1), value_schema, value_schema, pool_));
+        CreatePreparedQueryReaderForTest(std::move(query_batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> query_results,
         (ReadResultCollector::CollectKeyValueResult<
@@ -382,11 +571,22 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderCommitSchema) {
     ASSERT_EQ(query_results.size(), 1);
     ASSERT_EQ(query_results[0].value->GetFieldCount(), 1);
     ASSERT_EQ(query_results[0].value->GetInt(0), 1);
+}
 
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsNonExactCommitSchema) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, extra});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1, 2]])")
+            .ValueOrDie();
     auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+
     ASSERT_NOK_WITH_MSG(
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, std::nullopt,
-                                        value_schema, value_schema, pool_),
+        CreatePreparedCommitReaderForTest(std::move(batch_reader), prepared_schema,
+                                          OffsetRange(0, 1), value_schema, value_schema, pool_),
         "exact");
 }
 
@@ -424,6 +624,18 @@ TEST_F(MergedKeyValueRecordReaderTest, TestCommitOffsetCoverage) {
     ASSERT_EQ(4, row_count);
 }
 
+TEST_F(MergedKeyValueRecordReaderTest, TestCommitRejectsEmptyReaders) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+
+    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForCommit(
+                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
+                            value_schema, value_schema, pool_),
+                        "PK real-time store returned no commit readers for a sealed segment");
+}
+
 TEST_F(MergedKeyValueRecordReaderTest, TestRejectsDuplicateCommitOffset) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
@@ -458,10 +670,10 @@ TEST_F(MergedKeyValueRecordReaderTest, TestBadCommitBatch) {
         arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1]])").ValueOrDie();
 
     auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, std::nullopt,
-                                        arrow::schema({key}), value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreatePreparedCommitReaderForTest(std::move(batch_reader), prepared_schema,
+                                                           OffsetRange(0, 1), arrow::schema({key}),
+                                                           value_schema, pool_));
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
@@ -478,10 +690,10 @@ TEST_F(MergedKeyValueRecordReaderTest, TestMissingCompositeKey) {
             .ValueOrDie();
 
     auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                        arrow::schema({key0, key1}), value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreatePreparedQueryReaderForTest(
+                             std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
+                             arrow::schema({key0, key1}), value_schema, pool_));
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
@@ -500,10 +712,10 @@ TEST_F(MergedKeyValueRecordReaderTest, TestQueryReaderRequiresStoreAlignedSchema
             .ValueOrDie();
 
     auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                        arrow::schema({key}), value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                                          OffsetRange(0, 1), arrow::schema({key}),
+                                                          value_schema, pool_));
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
@@ -544,8 +756,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                        value_schema, value_schema, pool_));
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
     ASSERT_NOK_WITH_MSG(
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
                                                     KeyValueRecordReader::Iterator>(reader.get())),
@@ -584,8 +796,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedValues) {
     auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        AdaptPreparedBatchReaderForTest(std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                                        key_schema, query_value_schema, pool_));
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), key_schema, query_value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> results,
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
@@ -652,8 +864,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
             &explicit_close_count);
         ASSERT_OK_AND_ASSIGN(
             std::unique_ptr<KeyValueRecordReader> reader,
-            AdaptPreparedBatchReaderForTest(std::move(tracking_reader), prepared_schema,
-                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
+            CreatePreparedQueryReaderForTest(std::move(tracking_reader), prepared_schema,
+                                             OffsetRange(0, 1), key_schema, value_schema, pool_));
         reader->Close();
     }
     ASSERT_EQ(explicit_close_count, 1);
@@ -665,8 +877,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
             &destructor_close_count);
         ASSERT_OK_AND_ASSIGN(
             std::unique_ptr<KeyValueRecordReader> reader,
-            AdaptPreparedBatchReaderForTest(std::move(tracking_reader), prepared_schema,
-                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
+            CreatePreparedQueryReaderForTest(std::move(tracking_reader), prepared_schema,
+                                             OffsetRange(0, 1), key_schema, value_schema, pool_));
     }
     ASSERT_EQ(destructor_close_count, 1);
 
@@ -676,9 +888,9 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
             std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
             &factory_failure_close_count);
         std::shared_ptr<arrow::Schema> invalid_schema = arrow::schema(value_schema->fields());
-        ASSERT_NOK(AdaptPreparedBatchReaderForTest(std::move(tracking_reader), invalid_schema,
-                                                   OffsetRange(0, 1), key_schema, value_schema,
-                                                   pool_));
+        ASSERT_NOK(CreatePreparedQueryReaderForTest(std::move(tracking_reader), invalid_schema,
+                                                    OffsetRange(0, 1), key_schema, value_schema,
+                                                    pool_));
         ASSERT_EQ(nullptr, tracking_reader);
     }
     ASSERT_EQ(factory_failure_close_count, 1);
@@ -692,8 +904,8 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
                                                                      &read_failure_close_count);
         ASSERT_OK_AND_ASSIGN(
             std::unique_ptr<KeyValueRecordReader> reader,
-            AdaptPreparedBatchReaderForTest(std::move(tracking_reader), prepared_schema,
-                                            OffsetRange(0, 1), key_schema, value_schema, pool_));
+            CreatePreparedQueryReaderForTest(std::move(tracking_reader), prepared_schema,
+                                             OffsetRange(0, 1), key_schema, value_schema, pool_));
         ASSERT_NOK_WITH_MSG(reader->NextBatch(), "prepared reader failure");
         ASSERT_EQ(read_failure_close_count, 1);
     }
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 8620ee872..71ec74062 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -60,25 +60,33 @@ void CloseReaders(const std::vector<std::unique_ptr<Reader>>& readers) {
 
 class RealtimeOffsetCoverage {
  public:
-    static Result<std::shared_ptr<RealtimeOffsetCoverage>> Create(const OffsetRange& sealed_offsets,
-                                                                  size_t reader_count) {
-        if (sealed_offsets.begin < 0 || sealed_offsets.end < sealed_offsets.begin) {
-            return Status::Invalid("PK real-time store returned an invalid sealed offset range");
+    static Result<std::shared_ptr<RealtimeOffsetCoverage>> Create(const OffsetRange& offsets,
+                                                                  size_t reader_count,
+                                                                  bool allow_committed_prefix) {
+        if (offsets.begin < 0 || offsets.end < offsets.begin) {
+            return Status::Invalid("PK real-time store returned an invalid offset range");
         }
         return std::shared_ptr<RealtimeOffsetCoverage>(
-            new RealtimeOffsetCoverage(sealed_offsets, reader_count));
+            new RealtimeOffsetCoverage(offsets, reader_count, allow_committed_prefix));
     }
 
     Status Add(const arrow::Int64Array& offsets) {
         for (int64_t row = 0; row < offsets.length(); ++row) {
             const int64_t offset = offsets.Value(row);
-            if (offset < sealed_offsets_.begin || offset >= sealed_offsets_.end) {
+            if (allow_committed_prefix_ && offset < 0) {
+                return Status::Invalid("PK real-time store reader offset must be non-negative");
+            }
+            if (allow_committed_prefix_ && offset < offsets_.begin) {
+                continue;
+            }
+            if (offset < offsets_.begin || offset >= offsets_.end) {
                 return Status::Invalid(
-                    "PK real-time store commit reader offset is outside the sealed range");
+                    allow_committed_prefix_
+                        ? "PK real-time store query reader offset is outside the visible range"
+                        : "PK real-time store commit reader offset is outside the sealed range");
             }
             if (!seen_offsets_.CheckedAdd(offset)) {
-                return Status::Invalid(
-                    "PK real-time store commit readers did not cover the sealed range");
+                return CoverageError();
             }
         }
         return Status::OK();
@@ -87,19 +95,29 @@ class RealtimeOffsetCoverage {
     Status FinishReader() {
         ++finished_reader_count_;
         if (finished_reader_count_ == reader_count_ &&
-            seen_offsets_.Cardinality() != sealed_offsets_.Count()) {
-            return Status::Invalid(
-                "PK real-time store commit readers did not cover the sealed range");
+            seen_offsets_.Cardinality() != offsets_.Count()) {
+            return CoverageError();
         }
         return Status::OK();
     }
 
  private:
-    RealtimeOffsetCoverage(const OffsetRange& sealed_offsets, size_t reader_count)
-        : sealed_offsets_(sealed_offsets), reader_count_(reader_count) {}
+    RealtimeOffsetCoverage(const OffsetRange& offsets, size_t reader_count,
+                           bool allow_committed_prefix)
+        : offsets_(offsets),
+          reader_count_(reader_count),
+          allow_committed_prefix_(allow_committed_prefix) {}
+
+    Status CoverageError() const {
+        return Status::Invalid(
+            allow_committed_prefix_
+                ? "PK real-time store query readers did not cover the visible range"
+                : "PK real-time store commit readers did not cover the sealed range");
+    }
 
-    OffsetRange sealed_offsets_;
+    OffsetRange offsets_;
     size_t reader_count_;
+    bool allow_committed_prefix_;
     RoaringBitmap64 seen_offsets_;
     size_t finished_reader_count_ = 0;
 };
@@ -151,6 +169,23 @@ Result<std::vector<int32_t>> ResolveFieldIndexes(
     return result;
 }
 
+Status ValidateReaderParameters(const std::shared_ptr<arrow::Schema>& prepared_schema,
+                                const std::shared_ptr<arrow::Schema>& key_schema,
+                                const std::shared_ptr<arrow::Schema>& value_schema,
+                                const std::shared_ptr<MemoryPool>& memory_pool) {
+    PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
+    if (!key_schema) {
+        return Status::Invalid("prepared key schema cannot be null");
+    }
+    if (!value_schema) {
+        return Status::Invalid("prepared value schema cannot be null");
+    }
+    if (!memory_pool) {
+        return Status::Invalid("prepared reader memory pool cannot be null");
+    }
+    return Status::OK();
+}
+
 Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_schema,
                                  const std::shared_ptr<arrow::Schema>& value_schema) {
     if (prepared_schema->num_fields() !=
@@ -171,22 +206,7 @@ class PreparedReaderPlan {
     static Result<std::shared_ptr<const PreparedReaderPlan>> Create(
         const std::shared_ptr<arrow::Schema>& prepared_schema,
         const std::shared_ptr<arrow::Schema>& key_schema,
-        const std::shared_ptr<arrow::Schema>& value_schema,
-        const std::shared_ptr<MemoryPool>& memory_pool, bool exact_commit_schema) {
-        PAIMON_RETURN_NOT_OK(
-            PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
-        if (!key_schema) {
-            return Status::Invalid("prepared key schema cannot be null");
-        }
-        if (!value_schema) {
-            return Status::Invalid("prepared value schema cannot be null");
-        }
-        if (!memory_pool) {
-            return Status::Invalid("prepared reader memory pool cannot be null");
-        }
-        if (exact_commit_schema) {
-            PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
-        }
+        const std::shared_ptr<arrow::Schema>& value_schema) {
         std::unordered_map<int32_t, int32_t> field_indexes;
         field_indexes.reserve(prepared_schema->num_fields() -
                               SpecialFields::kPreparedKeyValueValueStartIndex);
@@ -396,6 +416,10 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
                                 row, offsets.length()));
             }
         }
+        if (visible_offsets_.has_value() && selection.Cardinality() != offsets.length()) {
+            return Status::Invalid(
+                "PK real-time store query reader bitmap must cover every raw mutation");
+        }
         if (!visible_offsets_.has_value()) {
             selected_rows_.reserve(offsets.length());
             for (int64_t row = 0; row < offsets.length(); ++row) {
@@ -467,51 +491,17 @@ Status PreparedKeyValueReaderFactory::ValidateTransportSchema(
 
 namespace {
 
-Result<std::unique_ptr<KeyValueRecordReader>> AdaptPreparedBatchReaderImpl(
+std::unique_ptr<KeyValueRecordReader> AdaptPreparedBatchReader(
     std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<const PreparedReaderPlan>& plan,
     const std::optional<OffsetRange>& visible_offsets,
     const std::shared_ptr<MemoryPool>& memory_pool,
     const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage) {
-    std::unique_ptr<BatchReader> owned_reader = std::move(reader);
-    if (!owned_reader) {
-        return Status::Invalid("prepared batch reader cannot be null");
-    }
-    ScopeGuard close_guard([&owned_reader]() -> void { owned_reader->Close(); });
-    std::unique_ptr<KeyValueRecordReader> result = std::make_unique<PreparedKeyValueReader>(
-        std::move(owned_reader), plan, visible_offsets, memory_pool, offset_coverage);
-    close_guard.Release();
-    return result;
+    return std::make_unique<PreparedKeyValueReader>(std::move(reader), plan, visible_offsets,
+                                                    memory_pool, offset_coverage);
 }
 
 }  // namespace
 
-Result<std::unique_ptr<KeyValueRecordReader>> PreparedKeyValueReaderFactory::Create(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const std::optional<OffsetRange>& visible_offsets,
-    const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
-    std::unique_ptr<BatchReader> owned_reader = std::move(reader);
-    ScopeGuard reader_guard([&owned_reader]() {
-        if (owned_reader) {
-            owned_reader->Close();
-        }
-    });
-    if (visible_offsets.has_value() && visible_offsets->begin > visible_offsets->end) {
-        return Status::Invalid("prepared visible offset range begin exceeds end");
-    }
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<const PreparedReaderPlan> plan,
-        PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema, memory_pool,
-                                   /*exact_commit_schema=*/!visible_offsets.has_value()));
-    PAIMON_ASSIGN_OR_RAISE(
-        std::unique_ptr<KeyValueRecordReader> result,
-        AdaptPreparedBatchReaderImpl(std::move(owned_reader), plan, visible_offsets, memory_pool,
-                                     /*offset_coverage=*/nullptr));
-    reader_guard.Release();
-    return result;
-}
-
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
 PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchReader>>&& readers,
                                               const std::shared_ptr<arrow::Schema>& prepared_schema,
@@ -520,31 +510,32 @@ PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchR
                                               const std::shared_ptr<arrow::Schema>& value_schema,
                                               const std::shared_ptr<MemoryPool>& memory_pool) {
     std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
-    ScopeGuard readers_guard([&readers, &adapted_readers]() {
-        CloseReaders(readers);
-        CloseReaders(adapted_readers);
-    });
+    ScopeGuard remaining_raw_readers_guard([&readers]() { CloseReaders(readers); });
     if (visible_offsets.begin > visible_offsets.end) {
         return Status::Invalid("prepared visible offset range begin exceeds end");
     }
+    if (readers.empty() && visible_offsets.begin < visible_offsets.end) {
+        return Status::Invalid(
+            "PK real-time store returned no query readers for a non-empty visible range");
+    }
     for (const std::unique_ptr<BatchReader>& reader : readers) {
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null query reader");
         }
     }
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<const PreparedReaderPlan> plan,
-        PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema, memory_pool,
-                                   /*exact_commit_schema=*/false));
+    PAIMON_RETURN_NOT_OK(
+        ValidateReaderParameters(prepared_schema, key_schema, value_schema, memory_pool));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const PreparedReaderPlan> plan,
+                           PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
+                           RealtimeOffsetCoverage::Create(visible_offsets, readers.size(),
+                                                          /*allow_committed_prefix=*/true));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        PAIMON_ASSIGN_OR_RAISE(
-            std::unique_ptr<KeyValueRecordReader> adapted_reader,
-            AdaptPreparedBatchReaderImpl(std::move(reader), plan, visible_offsets, memory_pool,
-                                         /*offset_coverage=*/nullptr));
-        adapted_readers.push_back(std::move(adapted_reader));
+        adapted_readers.push_back(AdaptPreparedBatchReader(std::move(reader), plan, visible_offsets,
+                                                           memory_pool, offset_coverage));
     }
-    readers_guard.Release();
+    remaining_raw_readers_guard.Release();
     return adapted_readers;
 }
 
@@ -556,29 +547,30 @@ PreparedKeyValueReaderFactory::CreateForCommit(
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
-    ScopeGuard readers_guard([&readers, &adapted_readers]() {
-        CloseReaders(readers);
-        CloseReaders(adapted_readers);
-    });
+    ScopeGuard remaining_raw_readers_guard([&readers]() { CloseReaders(readers); });
+    if (readers.empty()) {
+        return Status::Invalid(
+            "PK real-time store returned no commit readers for a sealed segment");
+    }
     for (const std::unique_ptr<BatchReader>& reader : readers) {
         if (!reader) {
             return Status::Invalid("PK real-time store returned a null commit reader");
         }
     }
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<const PreparedReaderPlan> plan,
-        PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema, memory_pool,
-                                   /*exact_commit_schema=*/true));
+    PAIMON_RETURN_NOT_OK(
+        ValidateReaderParameters(prepared_schema, key_schema, value_schema, memory_pool));
+    PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const PreparedReaderPlan> plan,
+                           PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
-                           RealtimeOffsetCoverage::Create(sealed_offsets, readers.size()));
+                           RealtimeOffsetCoverage::Create(sealed_offsets, readers.size(),
+                                                          /*allow_committed_prefix=*/false));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<KeyValueRecordReader> adapted_reader,
-                               AdaptPreparedBatchReaderImpl(std::move(reader), plan, std::nullopt,
-                                                            memory_pool, offset_coverage));
-        adapted_readers.push_back(std::move(adapted_reader));
+        adapted_readers.push_back(AdaptPreparedBatchReader(std::move(reader), plan, std::nullopt,
+                                                           memory_pool, offset_coverage));
     }
-    readers_guard.Release();
+    remaining_raw_readers_guard.Release();
     return adapted_readers;
 }
 
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/prepared_key_value_reader.h
index 658ffec08..bb03e2ad4 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/prepared_key_value_reader.h
@@ -18,9 +18,7 @@
 
 #pragma once
 
-#include <cstdint>
 #include <memory>
-#include <optional>
 #include <vector>
 
 #include "arrow/type_fwd.h"
@@ -39,14 +37,6 @@ class PreparedKeyValueReaderFactory {
 
     static Status ValidateTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema);
 
-    static Result<std::unique_ptr<KeyValueRecordReader>> Create(
-        std::unique_ptr<BatchReader>&& reader,
-        const std::shared_ptr<arrow::Schema>& prepared_schema,
-        const std::optional<OffsetRange>& visible_offsets,
-        const std::shared_ptr<arrow::Schema>& key_schema,
-        const std::shared_ptr<arrow::Schema>& value_schema,
-        const std::shared_ptr<MemoryPool>& memory_pool);
-
     static Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateForQuery(
         std::vector<std::unique_ptr<BatchReader>>&& readers,
         const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& visible_offsets,
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 73ddf050b..a8a91b4ee 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -540,21 +540,20 @@ class FailAfterPhysicalFileRealtimeStore final : public DelegatingRealtimeStore
     std::shared_ptr<std::atomic<bool>> saw_artifacts_;
 };
 
-enum class CommitReaderMalformation { DROP_LAST, DUPLICATE_OFFSET, OUT_OF_RANGE_OFFSET };
+enum class ReaderMalformation { DROP_LAST, DUPLICATE_OFFSET, OUT_OF_RANGE_OFFSET };
 
 class CorruptingBatchReader final : public BatchReader {
  public:
-    CorruptingBatchReader(std::unique_ptr<BatchReader> delegate,
-                          CommitReaderMalformation malformation)
+    CorruptingBatchReader(std::unique_ptr<BatchReader> delegate, ReaderMalformation malformation)
         : delegate_(std::move(delegate)), malformation_(malformation) {}
 
     Result<ReadBatch> NextBatch() override {
         switch (malformation_) {
-            case CommitReaderMalformation::DROP_LAST:
+            case ReaderMalformation::DROP_LAST:
                 return DropLast();
-            case CommitReaderMalformation::DUPLICATE_OFFSET:
+            case ReaderMalformation::DUPLICATE_OFFSET:
                 return SubstituteOffset(/*offset=*/0);
-            case CommitReaderMalformation::OUT_OF_RANGE_OFFSET:
+            case ReaderMalformation::OUT_OF_RANGE_OFFSET:
                 return SubstituteOffset(/*offset=*/-1);
         }
         return Status::Invalid("unknown commit reader malformation");
@@ -624,14 +623,14 @@ class CorruptingBatchReader final : public BatchReader {
     }
 
     std::unique_ptr<BatchReader> delegate_;
-    CommitReaderMalformation malformation_;
+    ReaderMalformation malformation_;
     std::optional<ReadBatch> buffered_;
 };
 
 class MalformedCoverageRealtimeStore final : public DelegatingRealtimeStore {
  public:
     MalformedCoverageRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                                   CommitReaderMalformation malformation)
+                                   ReaderMalformation malformation)
         : DelegatingRealtimeStore(delegate), malformation_(malformation) {}
 
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
@@ -645,7 +644,26 @@ class MalformedCoverageRealtimeStore final : public DelegatingRealtimeStore {
     }
 
  private:
-    CommitReaderMalformation malformation_;
+    ReaderMalformation malformation_;
+};
+
+class MissingQueryOffsetRealtimeStore final : public DelegatingRealtimeStore {
+ public:
+    explicit MissingQueryOffsetRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
+        : DelegatingRealtimeStore(delegate) {}
+
+    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
+        const RealtimeQueryContext& context) override {
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+                               delegate_->CreateQueryReaders(view, offset_begin, context));
+        if (readers.empty()) {
+            return Status::Invalid("query offset drop requires a reader");
+        }
+        readers[0] = std::make_unique<CorruptingBatchReader>(std::move(readers[0]),
+                                                             ReaderMalformation::DROP_LAST);
+        return readers;
+    }
 };
 
 }  // namespace
@@ -1447,8 +1465,8 @@ class RealtimeWriteInteTest : public ::testing::Test {
         ASSERT_OK(writer->Close());
     }
 
-    void CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation malformation,
-                                                const std::string& expected_error) {
+    void CheckPkRejectsReaderMalformation(ReaderMalformation malformation,
+                                          const std::string& expected_error) {
         CreatePkTable();
         auto factory = MakeDecoratingFactory<MalformedCoverageRealtimeStore>(malformation);
         ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
@@ -2385,18 +2403,34 @@ TEST_F(RealtimeWriteInteTest, TestPkMultipleStoredBatchesMergeForQueryAndCommit)
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsMalformedCoverage) {
-    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::DROP_LAST,
-                                           "commit readers did not cover the sealed range");
+    CheckPkRejectsReaderMalformation(ReaderMalformation::DROP_LAST,
+                                     "commit readers did not cover the sealed range");
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsDuplicateOffset) {
-    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::DUPLICATE_OFFSET,
-                                           "commit readers did not cover the sealed range");
+    CheckPkRejectsReaderMalformation(ReaderMalformation::DUPLICATE_OFFSET,
+                                     "commit readers did not cover the sealed range");
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkRejectsOutOfRangeOffset) {
-    CheckPkRejectsCommitReaderMalformation(CommitReaderMalformation::OUT_OF_RANGE_OFFSET,
-                                           "offset is outside the sealed range");
+    CheckPkRejectsReaderMalformation(ReaderMalformation::OUT_OF_RANGE_OFFSET,
+                                     "offset is outside the sealed range");
+}
+
+TEST_F(RealtimeWriteInteTest, TestPkRejectsMissingQueryOffset) {
+    CreatePkTable();
+    auto factory = MakeDecoratingFactory<MissingQueryOffsetRealtimeStore>();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+                         RealtimeContext::Create(factory));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+                         CreateRealtimeWriter(realtime_context));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
+                                   /*partitioned=*/false));
+    ASSERT_OK(writer->Write(std::move(batch)));
+    ASSERT_NOK_WITH_MSG(ReadRows(realtime_context),
+                        "query readers did not cover the visible range");
+    ASSERT_OK(writer->Close());
 }
 
 TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {

From 226cd85af74b399063cf9497f7ebd7c4fef8c2c9 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 16:39:01 +0800
Subject: [PATCH 76/93] fix(mergetree): transfer sorted reader ownership safely

---
 src/paimon/core/mergetree/merge_tree_writer.cpp     |  2 +-
 .../core/mergetree/merge_tree_writer_test.cpp       | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp
index bcf98e9f7..75623fd96 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer.cpp
@@ -174,6 +174,7 @@ Status MergeTreeWriter::WriteSortedReadersToFiles(
 
     auto sort_merge_reader = std::make_unique<SortMergeReaderWithLoserTree>(
         std::move(readers), key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_);
+    raw_readers_guard.Release();
     auto create_consumer = [target_schema = write_schema_, pool = pool_]()
         -> Result<std::unique_ptr<RowToArrowArrayConverter<KeyValue, KeyValueBatch>>> {
         return KeyValueMetaProjectionConsumer::Create(target_schema, pool);
@@ -181,7 +182,6 @@ Status MergeTreeWriter::WriteSortedReadersToFiles(
     auto async_key_value_producer_consumer =
         std::make_unique<AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>>(
             std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(), 1, pool_);
-    raw_readers_guard.Release();
     ScopeGuard async_readers_guard([&]() -> void { async_key_value_producer_consumer->Close(); });
     std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>> rolling_writer;
     PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter());
diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index e93935a41..179deda44 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -642,6 +642,19 @@ TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
       [0, 0, "Alice", 10, 0, 13.1]
     ])")
             .ValueOrDie());
+    bool first_mixed_reader_closed = false;
+    bool second_mixed_reader_closed = false;
+    std::vector<std::unique_ptr<KeyValueRecordReader>> mixed_readers;
+    mixed_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
+        CreateSingleReader(sorted_reader_array), &first_mixed_reader_closed));
+    mixed_readers.push_back(nullptr);
+    mixed_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
+        CreateSingleReader(sorted_reader_array), &second_mixed_reader_closed));
+    Status mixed_status = merge_writer->WriteSortedReadersToFiles(std::move(mixed_readers));
+    ASSERT_TRUE(mixed_status.IsInvalid());
+    ASSERT_TRUE(first_mixed_reader_closed);
+    ASSERT_TRUE(second_mixed_reader_closed);
+
     Status expected_status = Status::IOError("sorted reader failure");
     bool failing_reader_closed = false;
     std::vector<std::unique_ptr<KeyValueRecordReader>> failing_readers;

From 3b65e105d8e66c22e94cef807b4063f38372c65d Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 19:40:56 +0800
Subject: [PATCH 77/93] refactor(realtime): simplify reader lifecycle

---
 .../io/merged_key_value_record_reader.cpp     |  12 +-
 .../core/io/merged_key_value_record_reader.h  |   1 -
 .../merged_key_value_record_reader_test.cpp   | 166 ++----------------
 .../core/operation/merge_file_split_read.cpp  |  23 +--
 .../operation/merge_file_split_read_test.cpp  |  15 --
 .../realtime/prepared_key_value_reader.cpp    |  20 +--
 .../core/realtime/realtime_context_test.cpp   |  11 --
 test/inte/realtime_write_inte_test.cpp        |  16 +-
 8 files changed, 22 insertions(+), 242 deletions(-)

diff --git a/src/paimon/core/io/merged_key_value_record_reader.cpp b/src/paimon/core/io/merged_key_value_record_reader.cpp
index 8730086f5..70f2bcfb9 100644
--- a/src/paimon/core/io/merged_key_value_record_reader.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader.cpp
@@ -117,21 +117,13 @@ Status MergedKeyValueRecordReader::Iterator::LoadNextRawKeyValue() const {
 }
 
 Result<std::unique_ptr<KeyValueRecordReader::Iterator>> MergedKeyValueRecordReader::NextBatch() {
-    if (initialization_error_.has_value()) {
-        return initialization_error_.value();
-    }
     if (visited_) {
         return std::unique_ptr<KeyValueRecordReader::Iterator>();
     }
+    visited_ = true;
 
     auto iterator = std::make_unique<Iterator>(this);
-    Result<bool> has_next_result = iterator->HasNext();
-    if (!has_next_result.ok()) {
-        initialization_error_ = has_next_result.status();
-        return initialization_error_.value();
-    }
-    bool has_next = std::move(has_next_result).value();
-    visited_ = true;
+    PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext());
     if (!has_next) {
         return std::unique_ptr<KeyValueRecordReader::Iterator>();
     }
diff --git a/src/paimon/core/io/merged_key_value_record_reader.h b/src/paimon/core/io/merged_key_value_record_reader.h
index 227a1593a..a1b7aa5e4 100644
--- a/src/paimon/core/io/merged_key_value_record_reader.h
+++ b/src/paimon/core/io/merged_key_value_record_reader.h
@@ -67,7 +67,6 @@ class MergedKeyValueRecordReader : public KeyValueRecordReader {
 
  private:
     bool visited_ = false;
-    std::optional<Status> initialization_error_;
     std::unique_ptr<KeyValueRecordReader> reader_;
     std::shared_ptr<FieldsComparator> key_comparator_;
     std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper_;
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index 8cdef7a6b..d61428aa4 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -146,54 +146,6 @@ class MalformedBitmapBatchReader : public BatchReader {
     int32_t row_id_;
 };
 
-class ScriptedKeyValueRecordReader final : public KeyValueRecordReader {
- public:
-    ScriptedKeyValueRecordReader(std::vector<KeyValue>&& key_values, int32_t* next_batch_count)
-        : key_values_(std::move(key_values)), next_batch_count_(next_batch_count) {}
-
-    class Iterator final : public KeyValueRecordReader::Iterator {
-     public:
-        explicit Iterator(KeyValue&& key_value) : key_value_(std::move(key_value)) {}
-
-        Result<bool> HasNext() const override {
-            return key_value_.has_value();
-        }
-
-        Result<KeyValue> Next() override {
-            KeyValue result = std::move(key_value_.value());
-            key_value_.reset();
-            return result;
-        }
-
-     private:
-        std::optional<KeyValue> key_value_;
-    };
-
-    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
-        ++(*next_batch_count_);
-        if (*next_batch_count_ == 1) {
-            return std::make_unique<Iterator>(std::move(key_values_[0]));
-        }
-        if (*next_batch_count_ == 2) {
-            return Status::IOError("scripted lookahead failure");
-        }
-        if (*next_batch_count_ == 3) {
-            return std::make_unique<Iterator>(std::move(key_values_[1]));
-        }
-        return std::unique_ptr<KeyValueRecordReader::Iterator>();
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return nullptr;
-    }
-
-    void Close() override {}
-
- private:
-    std::vector<KeyValue> key_values_;
-    int32_t* next_batch_count_;
-};
-
 }  // namespace
 
 class MergedKeyValueRecordReaderTest : public testing::Test {
@@ -461,23 +413,6 @@ TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsEmptyEofForVisibleRange)
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "query readers did not cover the visible range");
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsReversedVisibleOffsets) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
-    auto batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1);
-
-    Result<std::unique_ptr<KeyValueRecordReader>> result =
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(2, 1), value_schema, value_schema, pool_);
-    ASSERT_TRUE(result.status().IsInvalid());
-    ASSERT_NOK_WITH_MSG(result, "prepared visible offset range begin exceeds end");
-}
-
 TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsEmptyReadersForVisibleRange) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
@@ -573,23 +508,6 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderQueryProjection) {
     ASSERT_EQ(query_results[0].value->GetInt(0), 1);
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderRejectsNonExactCommitSchema) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, extra});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1, 2]])")
-            .ValueOrDie();
-    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-
-    ASSERT_NOK_WITH_MSG(
-        CreatePreparedCommitReaderForTest(std::move(batch_reader), prepared_schema,
-                                          OffsetRange(0, 1), value_schema, value_schema, pool_),
-        "exact");
-}
-
 TEST_F(MergedKeyValueRecordReaderTest, TestCommitOffsetCoverage) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
@@ -719,28 +637,6 @@ TEST_F(MergedKeyValueRecordReaderTest, TestQueryReaderRequiresStoreAlignedSchema
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestMergedReaderInitializationErrorIsTerminal) {
-    std::vector<DataField> key_fields = {DataField(0, arrow::field("key", arrow::int32()))};
-    std::vector<KeyValue> key_values =
-        KeyValueChecker::GenerateKeyValues({10, 11}, {{1}, {2}}, {{1}, {2}}, pool_);
-    int32_t next_batch_count = 0;
-    auto reader =
-        std::make_unique<ScriptedKeyValueRecordReader>(std::move(key_values), &next_batch_count);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FieldsComparator> key_comparator,
-                         FieldsComparator::Create(key_fields, true));
-    MergedKeyValueRecordReader merged_reader(std::move(reader), key_comparator,
-                                             merge_function_wrapper_);
-
-    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> first = merged_reader.NextBatch();
-    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> retry = merged_reader.NextBatch();
-    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> second_retry =
-        merged_reader.NextBatch();
-    ASSERT_NOK_WITH_MSG(first, "scripted lookahead failure");
-    ASSERT_EQ(first.status().ToString(), retry.status().ToString());
-    ASSERT_EQ(first.status().ToString(), second_retry.status().ToString());
-    ASSERT_EQ(2, next_batch_count);
-}
-
 TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
@@ -843,7 +739,7 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedValues) {
     ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(1), 23);
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
+TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderFactoryFailureClosesReaders) {
     std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
                                            DataField(1, arrow::field("v0", arrow::int32()))};
     std::shared_ptr<arrow::Schema> value_schema =
@@ -857,59 +753,17 @@ TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderLifecycle) {
     ])")
             .ValueOrDie());
 
-    int32_t explicit_close_count = 0;
-    {
-        auto tracking_reader = std::make_unique<TrackingBatchReader>(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
-            &explicit_close_count);
-        ASSERT_OK_AND_ASSIGN(
-            std::unique_ptr<KeyValueRecordReader> reader,
-            CreatePreparedQueryReaderForTest(std::move(tracking_reader), prepared_schema,
-                                             OffsetRange(0, 1), key_schema, value_schema, pool_));
-        reader->Close();
-    }
-    ASSERT_EQ(explicit_close_count, 1);
-
-    int32_t destructor_close_count = 0;
-    {
-        auto tracking_reader = std::make_unique<TrackingBatchReader>(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
-            &destructor_close_count);
-        ASSERT_OK_AND_ASSIGN(
-            std::unique_ptr<KeyValueRecordReader> reader,
-            CreatePreparedQueryReaderForTest(std::move(tracking_reader), prepared_schema,
-                                             OffsetRange(0, 1), key_schema, value_schema, pool_));
-    }
-    ASSERT_EQ(destructor_close_count, 1);
-
     int32_t factory_failure_close_count = 0;
-    {
-        std::unique_ptr<BatchReader> tracking_reader = std::make_unique<TrackingBatchReader>(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
-            &factory_failure_close_count);
-        std::shared_ptr<arrow::Schema> invalid_schema = arrow::schema(value_schema->fields());
-        ASSERT_NOK(CreatePreparedQueryReaderForTest(std::move(tracking_reader), invalid_schema,
-                                                    OffsetRange(0, 1), key_schema, value_schema,
-                                                    pool_));
-        ASSERT_EQ(nullptr, tracking_reader);
-    }
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(std::make_unique<TrackingBatchReader>(
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
+        &factory_failure_close_count));
+    batch_readers.push_back(nullptr);
+    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForQuery(
+                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
+                            key_schema, value_schema, pool_),
+                        "PK real-time store returned a null query reader");
     ASSERT_EQ(factory_failure_close_count, 1);
-
-    int32_t read_failure_close_count = 0;
-    {
-        auto failing_reader =
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-        failing_reader->SetNextBatchStatus(Status::IOError("prepared reader failure"));
-        auto tracking_reader = std::make_unique<TrackingBatchReader>(std::move(failing_reader),
-                                                                     &read_failure_close_count);
-        ASSERT_OK_AND_ASSIGN(
-            std::unique_ptr<KeyValueRecordReader> reader,
-            CreatePreparedQueryReaderForTest(std::move(tracking_reader), prepared_schema,
-                                             OffsetRange(0, 1), key_schema, value_schema, pool_));
-        ASSERT_NOK_WITH_MSG(reader->NextBatch(), "prepared reader failure");
-        ASSERT_EQ(read_failure_close_count, 1);
-    }
-    ASSERT_EQ(read_failure_close_count, 1);
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 3b09d8c54..835ed0932 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -149,28 +149,17 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
 
     Status CollectDiskReaders(const std::vector<std::shared_ptr<Split>>& disk_splits,
                               std::vector<std::unique_ptr<KeyValueRecordReader>>* readers) {
-        std::vector<std::shared_ptr<DataSplitImpl>> data_splits;
-        data_splits.reserve(disk_splits.size());
+        std::shared_ptr<DataSplitImpl> first_split;
+        std::vector<std::shared_ptr<DataFileMeta>> data_files;
+        std::vector<std::optional<DeletionFile>> deletion_files;
         for (const std::shared_ptr<Split>& disk_split : disk_splits) {
             std::shared_ptr<DataSplitImpl> data_split =
                 std::dynamic_pointer_cast<DataSplitImpl>(disk_split);
             if (!data_split) {
                 return Status::Invalid("merge input disk split is not a data split");
             }
-            data_splits.push_back(std::move(data_split));
-        }
-        const std::shared_ptr<DataSplitImpl>& first_split = data_splits.front();
-        const BinaryRow& partition = first_split->Partition();
-        const int32_t bucket = first_split->Bucket();
-
-        std::vector<std::shared_ptr<DataFileMeta>> data_files;
-        std::vector<std::optional<DeletionFile>> deletion_files;
-        for (const std::shared_ptr<DataSplitImpl>& data_split : data_splits) {
-            if (!(data_split->Partition() == partition) || data_split->Bucket() != bucket) {
-                return Status::Invalid("merge input disk splits do not share a partition-bucket");
-            }
-            if (!data_split->BeforeFiles().empty()) {
-                return Status::Invalid("merge input disk split must not contain before files");
+            if (!first_split) {
+                first_split = data_split;
             }
             const std::vector<std::shared_ptr<DataFileMeta>>& split_files = data_split->DataFiles();
             const std::vector<std::optional<DeletionFile>>& split_deletion_files =
@@ -188,6 +177,8 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
                                       split_deletion_files.end());
             }
         }
+        const BinaryRow& partition = first_split->Partition();
+        const int32_t bucket = first_split->Bucket();
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_file_path_factory,
                                owner_->path_factory_->CreateDataFilePathFactory(partition, bucket));
 
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index d1da3e4f6..7a859cae5 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -766,21 +766,6 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadValidatesDiskSplits) {
     std::vector<std::shared_ptr<Split>> non_data_splits = {std::make_shared<TestingSplit>()};
     ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(non_data_splits, {}),
                         "merge input disk split is not a data split");
-    std::vector<std::shared_ptr<Split>> mixed_partition_splits = {prepared_splits[0],
-                                                                 prepared_splits[1]};
-    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(mixed_partition_splits, {}),
-                        "merge input disk splits do not share a partition-bucket");
-
-    std::vector<std::shared_ptr<DataFileMeta>> before_data_files = first->DataFiles();
-    DataSplitImpl::Builder before_builder(first->Partition(), first->Bucket(), first->BucketPath(),
-                                          std::move(before_data_files));
-    std::vector<std::shared_ptr<DataFileMeta>> before_files = first->DataFiles();
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<DataSplitImpl> before_split,
-        before_builder.WithBeforeFiles(std::move(before_files)).RawConvertible(false).Build());
-    std::vector<std::shared_ptr<Split>> before_splits = {before_split};
-    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(before_splits, {}),
-                        "merge input disk split must not contain before files");
 
     std::vector<std::shared_ptr<DataFileMeta>> deletion_data_files = first->DataFiles();
     DataSplitImpl::Builder deletion_builder(first->Partition(), first->Bucket(),
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index 71ec74062..c2d729900 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -264,10 +264,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
           pool_(pool),
           offset_coverage_(offset_coverage) {}
 
-    ~PreparedKeyValueReader() override {
-        Close();
-    }
-
     class Iterator final : public KeyValueRecordReader::Iterator {
      public:
         explicit Iterator(PreparedKeyValueReader* reader) : reader_(reader) {}
@@ -298,15 +294,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     };
 
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
-        if (first_error_.has_value()) {
-            return first_error_.value();
-        }
-        Result<std::unique_ptr<KeyValueRecordReader::Iterator>> result = NextBatchImpl();
-        if (!result.ok()) {
-            first_error_ = result.status();
-            Close();
-        }
-        return result;
+        return NextBatchImpl();
     }
 
     std::shared_ptr<Metrics> GetReaderMetrics() const override {
@@ -314,10 +302,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     }
 
     void Close() override {
-        if (closed_) {
-            return;
-        }
-        closed_ = true;
         ResetBatchState();
         reader_->Close();
     }
@@ -454,8 +438,6 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     }
 
  private:
-    bool closed_ = false;
-    std::optional<Status> first_error_;
     std::unique_ptr<BatchReader> reader_;
     std::shared_ptr<const PreparedReaderPlan> plan_;
     std::optional<OffsetRange> visible_offsets_;
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 6dd1d7577..82f831f6a 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -234,17 +234,6 @@ TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
     ASSERT_EQ(10, fourth);
 }
 
-TEST(RealtimeContextTest, TestMaterializedSequenceRejectsMissingStore) {
-    auto factory = std::make_shared<TestingRealtimeStoreFactory>();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
-
-    Result<int64_t> result = context->AdvanceMaterializedMaxSequenceNumber(
-        RealtimePartitionBucket({{"dt", "missing"}}, /*bucket=*/3),
-        /*max_sequence_number=*/4);
-    ASSERT_TRUE(result.status().IsKeyError());
-    ASSERT_NOK_WITH_MSG(result, "real-time store not found for partition {dt=missing}, bucket 3");
-}
-
 TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
     auto factory = std::make_shared<TestingRealtimeStoreFactory>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index a8a91b4ee..73391b1a6 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -2445,21 +2445,9 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
                          MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
     ASSERT_OK(writer->Write(std::move(batch)));
 
-    auto release_reader = [&](bool explicit_close) -> Status {
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
-                               CreateQueryReader(realtime_context));
-        if (explicit_close) {
-            reader->Close();
-        }
-        return Status::OK();
-    };
-
-    ASSERT_OK(release_reader(/*explicit_close=*/true));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader, CreateQueryReader(realtime_context));
+    reader->Close();
     ASSERT_EQ(1, state->query_close_count->load(std::memory_order_acquire));
-
-    ASSERT_OK(release_reader(/*explicit_close=*/false));
-    ASSERT_EQ(2, state->query_close_count->load(std::memory_order_acquire));
-
     ASSERT_OK(writer->Close());
 }
 

From e8a5b2913e9518b83207064135bc21e5fe5982a1 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 21:27:58 +0800
Subject: [PATCH 78/93] refactor(realtime): simplify prepared reader validation

---
 src/paimon/CMakeLists.txt                     |   1 +
 .../merged_key_value_record_reader_test.cpp   | 616 -----------------
 .../realtime/prepared_key_value_reader.cpp    |  63 +-
 .../prepared_key_value_reader_test.cpp        | 621 ++++++++++++++++++
 4 files changed, 628 insertions(+), 673 deletions(-)
 create mode 100644 src/paimon/core/realtime/prepared_key_value_reader_test.cpp

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 0a78b0902..41fe5f6ba 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -791,6 +791,7 @@ if(PAIMON_BUILD_TESTS)
                     core/memory/writer_memory_manager_test.cpp
                     core/realtime/arrow_realtime_store_test.cpp
                     core/realtime/primary_key_realtime_store_test.cpp
+                    core/realtime/prepared_key_value_reader_test.cpp
                     core/realtime/realtime_context_test.cpp
                     core/realtime/realtime_reader_test.cpp
                     core/mergetree/levels_test.cpp
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index d61428aa4..a9395c9ab 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -45,109 +45,6 @@
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
-
-namespace {
-
-std::shared_ptr<arrow::Field> MakeField(const std::string& name,
-                                        const std::shared_ptr<arrow::DataType>& type,
-                                        int32_t field_id, bool nullable = true) {
-    return DataField::ConvertDataFieldToArrowField(
-        DataField(field_id, arrow::field(name, type, nullable)));
-}
-
-std::shared_ptr<arrow::Schema> MakePreparedSchema(const arrow::FieldVector& value_fields) {
-    arrow::FieldVector prepared_fields = {
-        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-            ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
-    prepared_fields.insert(prepared_fields.end(), value_fields.begin(), value_fields.end());
-    return arrow::schema(prepared_fields);
-}
-
-Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedQueryReaderForTest(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const OffsetRange& visible_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
-    std::vector<std::unique_ptr<BatchReader>> readers;
-    readers.push_back(std::move(reader));
-    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
-                           PreparedKeyValueReaderFactory::CreateForQuery(
-                               std::move(readers), prepared_schema, visible_offsets, key_schema,
-                               value_schema, memory_pool));
-    return std::move(adapted_readers[0]);
-}
-
-Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedCommitReaderForTest(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
-    const OffsetRange& sealed_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
-    const std::shared_ptr<arrow::Schema>& value_schema,
-    const std::shared_ptr<MemoryPool>& memory_pool) {
-    std::vector<std::unique_ptr<BatchReader>> readers;
-    readers.push_back(std::move(reader));
-    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
-                           PreparedKeyValueReaderFactory::CreateForCommit(
-                               std::move(readers), prepared_schema, sealed_offsets, key_schema,
-                               value_schema, memory_pool));
-    return std::move(adapted_readers[0]);
-}
-
-class TrackingBatchReader : public BatchReader {
- public:
-    TrackingBatchReader(std::unique_ptr<BatchReader>&& delegate, int32_t* close_count)
-        : delegate_(std::move(delegate)), close_count_(close_count) {}
-
-    Result<ReadBatch> NextBatch() override {
-        return delegate_->NextBatch();
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        ++(*close_count_);
-        delegate_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> delegate_;
-    int32_t* close_count_;
-};
-
-class MalformedBitmapBatchReader : public BatchReader {
- public:
-    MalformedBitmapBatchReader(std::unique_ptr<BatchReader>&& delegate, int32_t row_id)
-        : delegate_(std::move(delegate)), row_id_(row_id) {}
-
-    Result<ReadBatch> NextBatch() override {
-        return delegate_->NextBatch();
-    }
-
-    Result<ReadBatchWithBitmap> NextBatchWithBitmap() override {
-        PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, delegate_->NextBatchWithBitmap());
-        if (!IsEofBatch(batch)) {
-            batch.second.Add(row_id_);
-        }
-        return batch;
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        delegate_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> delegate_;
-    int32_t row_id_;
-};
-
-}  // namespace
-
 class MergedKeyValueRecordReaderTest : public testing::Test {
  public:
     void SetUp() override {
@@ -253,517 +150,4 @@ TEST_F(MergedKeyValueRecordReaderTest, TestSkipMergedNulloptResultInHasNext) {
     }
 }
 
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryAllowsCommittedPrefix) {
-    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
-                                           DataField(1, arrow::field("v0", arrow::int32()))};
-    std::shared_ptr<arrow::Schema> value_schema =
-        DataField::ConvertDataFieldsToArrowSchema(value_fields);
-    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
-        [0, 100, 0, 1, 10],
-        [0, 101, 1, 2, 20],
-        [0, 102, 2, 4, 40],
-        [0, 103, 3, 6, 60]
-    ])")
-            .ValueOrDie());
-
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-    batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(2, 4),
-                             key_schema, value_schema, pool_));
-    ASSERT_EQ(1, readers.size());
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<KeyValue> results,
-        (ReadResultCollector::CollectKeyValueResult<
-            KeyValueRecordReader, KeyValueRecordReader::Iterator>(readers[0].get())));
-
-    std::vector<RowKind*> row_kinds = {const_cast<RowKind*>(RowKind::Insert()),
-                                       const_cast<RowKind*>(RowKind::Insert())};
-    std::vector<int64_t> levels = {KeyValue::UNKNOWN_LEVEL, KeyValue::UNKNOWN_LEVEL};
-    std::vector<KeyValue> expected = KeyValueChecker::GenerateKeyValues(
-        row_kinds, {102, 103}, levels, {{4}, {6}}, {{4, 40}, {6, 60}}, pool_);
-    KeyValueChecker::CheckResult(expected, results, 1, 2);
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsNegativeOffset) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, -1, 1]])")
-            .ValueOrDie();
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
-                                                  /*read_batch_size=*/1),
-            prepared_schema, OffsetRange(1, 2), value_schema, value_schema, pool_));
-
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "reader offset must be non-negative");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryOffsetCoverageAcrossReadersAndBatches) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> first_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
-                                                  R"([[0, 10, 2, 1], [0, 11, 0, 2]])")
-            .ValueOrDie();
-    std::shared_ptr<arrow::Array> second_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
-                                                  R"([[0, 12, 3, 3], [0, 13, 1, 4]])")
-            .ValueOrDie();
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-    batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
-    batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
-
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
-                             value_schema, value_schema, pool_));
-    int64_t row_count = 0;
-    for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
-        ASSERT_OK_AND_ASSIGN(
-            std::vector<KeyValue> rows,
-            (ReadResultCollector::CollectKeyValueResult<
-                KeyValueRecordReader, KeyValueRecordReader::Iterator>(reader.get())));
-        row_count += static_cast<int64_t>(rows.size());
-    }
-    ASSERT_EQ(4, row_count);
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsMissingVisibleOffset) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
-                                                  R"([[0, 10, 0, 1], [0, 11, 2, 2]])")
-            .ValueOrDie();
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
-                                                  /*read_batch_size=*/1),
-            prepared_schema, OffsetRange(0, 3), value_schema, value_schema, pool_));
-
-    ASSERT_NOK_WITH_MSG(
-        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                    KeyValueRecordReader::Iterator>(reader.get())),
-        "query readers did not cover the visible range");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsDuplicateVisibleOffset) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> first_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
-    std::shared_ptr<arrow::Array> second_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
-                                                  R"([[0, 11, 1, 2], [0, 12, 1, 3]])")
-            .ValueOrDie();
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-    batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
-    batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 2),
-                             value_schema, value_schema, pool_));
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<KeyValue> first_rows,
-        (ReadResultCollector::CollectKeyValueResult<
-            KeyValueRecordReader, KeyValueRecordReader::Iterator>(readers[0].get())));
-    ASSERT_EQ(1, first_rows.size());
-    ASSERT_NOK_WITH_MSG((ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                                    KeyValueRecordReader::Iterator>(
-                            readers[1].get())),
-                        "query readers did not cover the visible range");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsEmptyEofForVisibleRange) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([])").ValueOrDie();
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
-                                                  /*read_batch_size=*/1),
-            prepared_schema, OffsetRange(0, 1), value_schema, value_schema, pool_));
-
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "query readers did not cover the visible range");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryRejectsEmptyReadersForVisibleRange) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-
-    ASSERT_NOK_WITH_MSG(
-        PreparedKeyValueReaderFactory::CreateForQuery(std::move(batch_readers), prepared_schema,
-                                                      OffsetRange(0, 1), value_schema, value_schema,
-                                                      pool_),
-        "PK real-time store returned no query readers for a non-empty visible range");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryAllowsEmptyReadersForEmptyVisibleRange) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(1, 1),
-                             value_schema, value_schema, pool_));
-    ASSERT_TRUE(readers.empty());
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderBitmapBounds) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
-    auto batch_reader = std::make_unique<MalformedBitmapBatchReader>(
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1),
-        /*row_id=*/1);
-
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
-    Result<std::vector<KeyValue>> result =
-        ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                   KeyValueRecordReader::Iterator>(reader.get());
-    ASSERT_TRUE(result.status().IsInvalid());
-    ASSERT_NOK_WITH_MSG(result, "selected row id 1 is out of bounds for prepared batch length 1");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedQueryReaderRejectsPartialBitmap) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
-                                                  R"([[0, 10, 0, 1], [0, 11, 1, 2]])")
-            .ValueOrDie();
-    RoaringBitmap32 partial_bitmap;
-    partial_bitmap.Add(0);
-    auto batch_reader = std::make_unique<MockFileBatchReader>(
-        prepared_array, prepared_type, partial_bitmap, /*read_batch_size=*/2);
-    batch_reader->EnableRandomizeBatchSize(false);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 2), value_schema, value_schema, pool_));
-
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "must cover every raw mutation");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderQueryProjection) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, extra});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1, 2]])")
-            .ValueOrDie());
-
-    auto query_batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> query_reader,
-        CreatePreparedQueryReaderForTest(std::move(query_batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<KeyValue> query_results,
-        (ReadResultCollector::CollectKeyValueResult<
-            KeyValueRecordReader, KeyValueRecordReader::Iterator>(query_reader.get())));
-    ASSERT_EQ(query_results.size(), 1);
-    ASSERT_EQ(query_results[0].value->GetFieldCount(), 1);
-    ASSERT_EQ(query_results[0].value->GetInt(0), 1);
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestCommitOffsetCoverage) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> first_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
-                                                  R"([[0, 10, 2, 1], [0, 11, 0, 3]])")
-            .ValueOrDie();
-    std::shared_ptr<arrow::Array> second_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
-                                                  R"([[0, 12, 1, 2], [0, 13, 3, 4]])")
-            .ValueOrDie();
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-    batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
-    batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
-
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForCommit(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
-                             value_schema, value_schema, pool_));
-    int64_t row_count = 0;
-    for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
-        ASSERT_OK_AND_ASSIGN(
-            std::vector<KeyValue> rows,
-            (ReadResultCollector::CollectKeyValueResult<
-                KeyValueRecordReader, KeyValueRecordReader::Iterator>(reader.get())));
-        row_count += static_cast<int64_t>(rows.size());
-    }
-    ASSERT_EQ(4, row_count);
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestCommitRejectsEmptyReaders) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-
-    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForCommit(
-                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
-                            value_schema, value_schema, pool_),
-                        "PK real-time store returned no commit readers for a sealed segment");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestRejectsDuplicateCommitOffset) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(
-            prepared_type, R"([[0, 10, 0, 1], [0, 11, 0, 2], [0, 12, 2, 3]])")
-            .ValueOrDie();
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-    batch_readers.push_back(std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
-                                                                  /*read_batch_size=*/1));
-
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForCommit(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 3),
-                             value_schema, value_schema, pool_));
-    ASSERT_NOK_WITH_MSG((ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                                    KeyValueRecordReader::Iterator>(
-                            readers[0].get())),
-                        "did not cover the sealed range");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestBadCommitBatch) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Field> value = MakeField("value", arrow::int32(), 1);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key, value});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, value});
-    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
-    std::shared_ptr<arrow::Array> actual =
-        arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1]])").ValueOrDie();
-
-    auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
-                         CreatePreparedCommitReaderForTest(std::move(batch_reader), prepared_schema,
-                                                           OffsetRange(0, 1), arrow::schema({key}),
-                                                           value_schema, pool_));
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestMissingCompositeKey) {
-    std::shared_ptr<arrow::Field> key0 = MakeField("key0", arrow::int32(), 0);
-    std::shared_ptr<arrow::Field> key1 = MakeField("key1", arrow::int32(), 1);
-    std::shared_ptr<arrow::Field> value = MakeField("value", arrow::int32(), 2);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key0, key1, value});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key0, key1, value});
-    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key0, value});
-    std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
-    std::shared_ptr<arrow::Array> actual =
-        arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1, 20]])")
-            .ValueOrDie();
-
-    auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
-                         CreatePreparedQueryReaderForTest(
-                             std::move(batch_reader), prepared_schema, OffsetRange(0, 1),
-                             arrow::schema({key0, key1}), value_schema, pool_));
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestQueryReaderRequiresStoreAlignedSchema) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Field> old_value = MakeField("old_value", arrow::int32(), 1);
-    std::shared_ptr<arrow::Field> renamed_value = MakeField("renamed_value", arrow::int32(), 1);
-    std::shared_ptr<arrow::Field> added = MakeField("added", arrow::int32(), 2);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key, renamed_value, added});
-    std::shared_ptr<arrow::Schema> prepared_schema =
-        MakePreparedSchema({key, renamed_value, added});
-    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key, old_value});
-    std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
-    std::shared_ptr<arrow::Array> actual =
-        arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1, 20]])")
-            .ValueOrDie();
-
-    auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
-                         CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                                          OffsetRange(0, 1), arrow::schema({key}),
-                                                          value_schema, pool_));
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderSafeDecode) {
-    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-
-    arrow::FieldVector invalid_fields = prepared_schema->fields();
-    invalid_fields[0] = invalid_fields[0]->WithName("wrong_value_kind");
-    invalid_fields[3] = MakeField("wrong_key", arrow::int32(), 99);
-    std::shared_ptr<arrow::DataType> invalid_type = arrow::struct_(invalid_fields);
-    auto invalid_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(invalid_type, R"([[0, 10, 0, 1]])").ValueOrDie());
-
-    auto batch_reader = std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
-    ASSERT_NOK_WITH_MSG(
-        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                    KeyValueRecordReader::Iterator>(reader.get())),
-        "prepared batch field");
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderNestedValues) {
-    std::shared_ptr<arrow::Field> id = MakeField("id", arrow::int32(), 0);
-    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({id});
-    std::shared_ptr<arrow::Field> query_item_b = MakeField("renamed_b", arrow::int32(), 11);
-    std::shared_ptr<arrow::Field> query_item_a = MakeField("renamed_a", arrow::int32(), 10);
-    std::shared_ptr<arrow::Field> query_items = MakeField(
-        "items_renamed",
-        arrow::list(arrow::field("element", arrow::struct_({query_item_b, query_item_a}))), 2);
-    std::shared_ptr<arrow::Field> query_attr_y = MakeField("renamed_y", arrow::int32(), 21);
-    std::shared_ptr<arrow::Field> query_attr_x = MakeField("renamed_x", arrow::int32(), 20);
-    std::shared_ptr<arrow::Field> query_attrs =
-        MakeField("attrs_renamed",
-                  arrow::map(arrow::utf8(), arrow::struct_({query_attr_y, query_attr_x})), 3);
-    std::shared_ptr<arrow::Field> query_key_right = MakeField("renamed_right", arrow::int32(), 31);
-    std::shared_ptr<arrow::Field> query_key_left = MakeField("renamed_left", arrow::int32(), 30);
-    std::shared_ptr<arrow::Field> query_keyed_values =
-        MakeField("keyed_values_renamed",
-                  arrow::map(arrow::struct_({query_key_right, query_key_left}), arrow::int32()), 4);
-    std::shared_ptr<arrow::Schema> query_value_schema =
-        arrow::schema({id, query_items, query_attrs, query_keyed_values});
-    std::shared_ptr<arrow::Schema> prepared_schema =
-        MakePreparedSchema(query_value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(
-            prepared_type,
-            R"([[0, 10, 0, 1, [[200, 100], [400, 300]], [["k1", [8, 7]], ["k2", [10, 9]]], [[[12, 11], 13], [[22, 21], 23]]]])")
-            .ValueOrDie();
-
-    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), key_schema, query_value_schema, pool_));
-    ASSERT_OK_AND_ASSIGN(
-        std::vector<KeyValue> results,
-        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
-                                                    KeyValueRecordReader::Iterator>(reader.get())));
-
-    ASSERT_EQ(results.size(), 1);
-    ASSERT_EQ(results[0].key->GetInt(0), 1);
-    ASSERT_EQ(results[0].value->GetFieldCount(), 4);
-    ASSERT_EQ(results[0].value->GetInt(0), 1);
-
-    std::shared_ptr<InternalArray> item_array = results[0].value->GetArray(1);
-    ASSERT_EQ(item_array->Size(), 2);
-    std::shared_ptr<InternalRow> first_item = item_array->GetRow(0, 2);
-    ASSERT_EQ(first_item->GetInt(0), 200);
-    ASSERT_EQ(first_item->GetInt(1), 100);
-    std::shared_ptr<InternalRow> second_item = item_array->GetRow(1, 2);
-    ASSERT_EQ(second_item->GetInt(0), 400);
-    ASSERT_EQ(second_item->GetInt(1), 300);
-
-    std::shared_ptr<InternalMap> attr_map = results[0].value->GetMap(2);
-    ASSERT_EQ(attr_map->Size(), 2);
-    std::shared_ptr<InternalArray> key_array = attr_map->KeyArray();
-    ASSERT_EQ(std::string(key_array->GetStringView(0)), "k1");
-    ASSERT_EQ(std::string(key_array->GetStringView(1)), "k2");
-    std::shared_ptr<InternalArray> value_array = attr_map->ValueArray();
-    std::shared_ptr<InternalRow> first_attr = value_array->GetRow(0, 2);
-    ASSERT_EQ(first_attr->GetInt(0), 8);
-    ASSERT_EQ(first_attr->GetInt(1), 7);
-    std::shared_ptr<InternalRow> second_attr = value_array->GetRow(1, 2);
-    ASSERT_EQ(second_attr->GetInt(0), 10);
-    ASSERT_EQ(second_attr->GetInt(1), 9);
-
-    std::shared_ptr<InternalMap> keyed_value_map = results[0].value->GetMap(3);
-    ASSERT_EQ(keyed_value_map->Size(), 2);
-    std::shared_ptr<InternalArray> struct_keys = keyed_value_map->KeyArray();
-    std::shared_ptr<InternalRow> first_key = struct_keys->GetRow(0, 2);
-    ASSERT_EQ(first_key->GetInt(0), 12);
-    ASSERT_EQ(first_key->GetInt(1), 11);
-    std::shared_ptr<InternalRow> second_key = struct_keys->GetRow(1, 2);
-    ASSERT_EQ(second_key->GetInt(0), 22);
-    ASSERT_EQ(second_key->GetInt(1), 21);
-    ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(0), 13);
-    ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(1), 23);
-}
-
-TEST_F(MergedKeyValueRecordReaderTest, TestPreparedReaderFactoryFailureClosesReaders) {
-    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
-                                           DataField(1, arrow::field("v0", arrow::int32()))};
-    std::shared_ptr<arrow::Schema> value_schema =
-        DataField::ConvertDataFieldsToArrowSchema(value_fields);
-    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
-        [0, 10, 0, 1, 100]
-    ])")
-            .ValueOrDie());
-
-    int32_t factory_failure_close_count = 0;
-    std::vector<std::unique_ptr<BatchReader>> batch_readers;
-    batch_readers.push_back(std::make_unique<TrackingBatchReader>(
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
-        &factory_failure_close_count));
-    batch_readers.push_back(nullptr);
-    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForQuery(
-                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
-                            key_schema, value_schema, pool_),
-                        "PK real-time store returned a null query reader");
-    ASSERT_EQ(factory_failure_close_count, 1);
-}
-
 }  // namespace paimon::test
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/prepared_key_value_reader.cpp
index c2d729900..d5e29da52 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/prepared_key_value_reader.cpp
@@ -169,38 +169,6 @@ Result<std::vector<int32_t>> ResolveFieldIndexes(
     return result;
 }
 
-Status ValidateReaderParameters(const std::shared_ptr<arrow::Schema>& prepared_schema,
-                                const std::shared_ptr<arrow::Schema>& key_schema,
-                                const std::shared_ptr<arrow::Schema>& value_schema,
-                                const std::shared_ptr<MemoryPool>& memory_pool) {
-    PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
-    if (!key_schema) {
-        return Status::Invalid("prepared key schema cannot be null");
-    }
-    if (!value_schema) {
-        return Status::Invalid("prepared value schema cannot be null");
-    }
-    if (!memory_pool) {
-        return Status::Invalid("prepared reader memory pool cannot be null");
-    }
-    return Status::OK();
-}
-
-Status ValidateExactCommitSchema(const std::shared_ptr<arrow::Schema>& prepared_schema,
-                                 const std::shared_ptr<arrow::Schema>& value_schema) {
-    if (prepared_schema->num_fields() !=
-        value_schema->num_fields() + SpecialFields::kPreparedKeyValueValueStartIndex) {
-        return Status::Invalid("commit requires the exact prepared writer schema");
-    }
-    for (int32_t i = 0; i < value_schema->num_fields(); ++i) {
-        if (!prepared_schema->field(i + SpecialFields::kPreparedKeyValueValueStartIndex)
-                 ->Equals(value_schema->field(i), true)) {
-            return Status::Invalid("commit requires the exact prepared writer schema");
-        }
-    }
-    return Status::OK();
-}
-
 class PreparedReaderPlan {
  public:
     static Result<std::shared_ptr<const PreparedReaderPlan>> Create(
@@ -471,19 +439,6 @@ Status PreparedKeyValueReaderFactory::ValidateTransportSchema(
     return Status::OK();
 }
 
-namespace {
-
-std::unique_ptr<KeyValueRecordReader> AdaptPreparedBatchReader(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<const PreparedReaderPlan>& plan,
-    const std::optional<OffsetRange>& visible_offsets,
-    const std::shared_ptr<MemoryPool>& memory_pool,
-    const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage) {
-    return std::make_unique<PreparedKeyValueReader>(std::move(reader), plan, visible_offsets,
-                                                    memory_pool, offset_coverage);
-}
-
-}  // namespace
-
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
 PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchReader>>&& readers,
                                               const std::shared_ptr<arrow::Schema>& prepared_schema,
@@ -493,9 +448,6 @@ PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchR
                                               const std::shared_ptr<MemoryPool>& memory_pool) {
     std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
     ScopeGuard remaining_raw_readers_guard([&readers]() { CloseReaders(readers); });
-    if (visible_offsets.begin > visible_offsets.end) {
-        return Status::Invalid("prepared visible offset range begin exceeds end");
-    }
     if (readers.empty() && visible_offsets.begin < visible_offsets.end) {
         return Status::Invalid(
             "PK real-time store returned no query readers for a non-empty visible range");
@@ -505,8 +457,7 @@ PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchR
             return Status::Invalid("PK real-time store returned a null query reader");
         }
     }
-    PAIMON_RETURN_NOT_OK(
-        ValidateReaderParameters(prepared_schema, key_schema, value_schema, memory_pool));
+    PAIMON_RETURN_NOT_OK(ValidateTransportSchema(prepared_schema));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const PreparedReaderPlan> plan,
                            PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
@@ -514,8 +465,8 @@ PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchR
                                                           /*allow_committed_prefix=*/true));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        adapted_readers.push_back(AdaptPreparedBatchReader(std::move(reader), plan, visible_offsets,
-                                                           memory_pool, offset_coverage));
+        adapted_readers.push_back(std::make_unique<PreparedKeyValueReader>(
+            std::move(reader), plan, visible_offsets, memory_pool, offset_coverage));
     }
     remaining_raw_readers_guard.Release();
     return adapted_readers;
@@ -539,9 +490,7 @@ PreparedKeyValueReaderFactory::CreateForCommit(
             return Status::Invalid("PK real-time store returned a null commit reader");
         }
     }
-    PAIMON_RETURN_NOT_OK(
-        ValidateReaderParameters(prepared_schema, key_schema, value_schema, memory_pool));
-    PAIMON_RETURN_NOT_OK(ValidateExactCommitSchema(prepared_schema, value_schema));
+    PAIMON_RETURN_NOT_OK(ValidateTransportSchema(prepared_schema));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const PreparedReaderPlan> plan,
                            PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
@@ -549,8 +498,8 @@ PreparedKeyValueReaderFactory::CreateForCommit(
                                                           /*allow_committed_prefix=*/false));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        adapted_readers.push_back(AdaptPreparedBatchReader(std::move(reader), plan, std::nullopt,
-                                                           memory_pool, offset_coverage));
+        adapted_readers.push_back(std::make_unique<PreparedKeyValueReader>(
+            std::move(reader), plan, std::nullopt, memory_pool, offset_coverage));
     }
     remaining_raw_readers_guard.Release();
     return adapted_readers;
diff --git a/src/paimon/core/realtime/prepared_key_value_reader_test.cpp b/src/paimon/core/realtime/prepared_key_value_reader_test.cpp
new file mode 100644
index 000000000..c0515df72
--- /dev/null
+++ b/src/paimon/core/realtime/prepared_key_value_reader_test.cpp
@@ -0,0 +1,621 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/realtime/prepared_key_value_reader.h"
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/array/array_nested.h"
+#include "arrow/ipc/json_simple.h"
+#include "gtest/gtest.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/realtime/offset_range.h"
+#include "paimon/testing/mock/mock_file_batch_reader.h"
+#include "paimon/testing/utils/key_value_checker.h"
+#include "paimon/testing/utils/read_result_collector.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+namespace {
+
+std::shared_ptr<arrow::Field> MakeField(const std::string& name,
+                                        const std::shared_ptr<arrow::DataType>& type,
+                                        int32_t field_id, bool nullable = true) {
+    return DataField::ConvertDataFieldToArrowField(
+        DataField(field_id, arrow::field(name, type, nullable)));
+}
+
+std::shared_ptr<arrow::Schema> MakePreparedSchema(const arrow::FieldVector& value_fields) {
+    arrow::FieldVector prepared_fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+            ->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
+    prepared_fields.insert(prepared_fields.end(), value_fields.begin(), value_fields.end());
+    return arrow::schema(prepared_fields);
+}
+
+Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedQueryReaderForTest(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const OffsetRange& visible_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    std::vector<std::unique_ptr<BatchReader>> readers;
+    readers.push_back(std::move(reader));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
+                           PreparedKeyValueReaderFactory::CreateForQuery(
+                               std::move(readers), prepared_schema, visible_offsets, key_schema,
+                               value_schema, memory_pool));
+    return std::move(adapted_readers[0]);
+}
+
+Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedCommitReaderForTest(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const OffsetRange& sealed_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
+    std::vector<std::unique_ptr<BatchReader>> readers;
+    readers.push_back(std::move(reader));
+    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
+                           PreparedKeyValueReaderFactory::CreateForCommit(
+                               std::move(readers), prepared_schema, sealed_offsets, key_schema,
+                               value_schema, memory_pool));
+    return std::move(adapted_readers[0]);
+}
+
+class TrackingBatchReader : public BatchReader {
+ public:
+    TrackingBatchReader(std::unique_ptr<BatchReader>&& delegate, int32_t* close_count)
+        : delegate_(std::move(delegate)), close_count_(close_count) {}
+
+    Result<ReadBatch> NextBatch() override {
+        return delegate_->NextBatch();
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        ++(*close_count_);
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    int32_t* close_count_;
+};
+
+class MalformedBitmapBatchReader : public BatchReader {
+ public:
+    MalformedBitmapBatchReader(std::unique_ptr<BatchReader>&& delegate, int32_t row_id)
+        : delegate_(std::move(delegate)), row_id_(row_id) {}
+
+    Result<ReadBatch> NextBatch() override {
+        return delegate_->NextBatch();
+    }
+
+    Result<ReadBatchWithBitmap> NextBatchWithBitmap() override {
+        PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, delegate_->NextBatchWithBitmap());
+        if (!IsEofBatch(batch)) {
+            batch.second.Add(row_id_);
+        }
+        return batch;
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return delegate_->GetReaderMetrics();
+    }
+
+    void Close() override {
+        delegate_->Close();
+    }
+
+ private:
+    std::unique_ptr<BatchReader> delegate_;
+    int32_t row_id_;
+};
+
+}  // namespace
+
+class PreparedKeyValueReaderTest : public testing::Test {
+ protected:
+    std::shared_ptr<MemoryPool> pool_ = GetDefaultPool();
+};
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryAllowsCommittedPrefix) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
+                                           DataField(1, arrow::field("v0", arrow::int32()))};
+    std::shared_ptr<arrow::Schema> value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+        [0, 100, 0, 1, 10],
+        [0, 101, 1, 2, 20],
+        [0, 102, 2, 4, 40],
+        [0, 103, 3, 6, 60]
+    ])")
+            .ValueOrDie());
+
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(2, 4),
+                             key_schema, value_schema, pool_));
+    ASSERT_EQ(1, readers.size());
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> results,
+        (ReadResultCollector::CollectKeyValueResult<
+            KeyValueRecordReader, KeyValueRecordReader::Iterator>(readers[0].get())));
+
+    std::vector<RowKind*> row_kinds = {const_cast<RowKind*>(RowKind::Insert()),
+                                       const_cast<RowKind*>(RowKind::Insert())};
+    std::vector<int64_t> levels = {KeyValue::UNKNOWN_LEVEL, KeyValue::UNKNOWN_LEVEL};
+    std::vector<KeyValue> expected = KeyValueChecker::GenerateKeyValues(
+        row_kinds, {102, 103}, levels, {{4}, {6}}, {{4, 40}, {6, 60}}, pool_);
+    KeyValueChecker::CheckResult(expected, results, 1, 2);
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsNegativeOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, -1, 1]])")
+            .ValueOrDie();
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                  /*read_batch_size=*/1),
+            prepared_schema, OffsetRange(1, 2), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "reader offset must be non-negative");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryOffsetCoverageAcrossReadersAndBatches) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> first_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 2, 1], [0, 11, 0, 2]])")
+            .ValueOrDie();
+    std::shared_ptr<arrow::Array> second_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 12, 3, 3], [0, 13, 1, 4]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
+                             value_schema, value_schema, pool_));
+    int64_t row_count = 0;
+    for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
+        ASSERT_OK_AND_ASSIGN(
+            std::vector<KeyValue> rows,
+            (ReadResultCollector::CollectKeyValueResult<
+                KeyValueRecordReader, KeyValueRecordReader::Iterator>(reader.get())));
+        row_count += static_cast<int64_t>(rows.size());
+    }
+    ASSERT_EQ(4, row_count);
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsMissingVisibleOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 0, 1], [0, 11, 2, 2]])")
+            .ValueOrDie();
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                  /*read_batch_size=*/1),
+            prepared_schema, OffsetRange(0, 3), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())),
+        "query readers did not cover the visible range");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsDuplicateVisibleOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> first_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+    std::shared_ptr<arrow::Array> second_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 11, 1, 2], [0, 12, 1, 3]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 2),
+                             value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> first_rows,
+        (ReadResultCollector::CollectKeyValueResult<
+            KeyValueRecordReader, KeyValueRecordReader::Iterator>(readers[0].get())));
+    ASSERT_EQ(1, first_rows.size());
+    ASSERT_NOK_WITH_MSG((ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                                    KeyValueRecordReader::Iterator>(
+                            readers[1].get())),
+                        "query readers did not cover the visible range");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsEmptyEofForVisibleRange) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([])").ValueOrDie();
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                  /*read_batch_size=*/1),
+            prepared_schema, OffsetRange(0, 1), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "query readers did not cover the visible range");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsEmptyReadersForVisibleRange) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+
+    ASSERT_NOK_WITH_MSG(
+        PreparedKeyValueReaderFactory::CreateForQuery(std::move(batch_readers), prepared_schema,
+                                                      OffsetRange(0, 1), value_schema, value_schema,
+                                                      pool_),
+        "PK real-time store returned no query readers for a non-empty visible range");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestQueryAllowsEmptyReadersForEmptyVisibleRange) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForQuery(
+                             std::move(batch_readers), prepared_schema, OffsetRange(1, 1),
+                             value_schema, value_schema, pool_));
+    ASSERT_TRUE(readers.empty());
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderBitmapBounds) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+    auto batch_reader = std::make_unique<MalformedBitmapBatchReader>(
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1),
+        /*row_id=*/1);
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
+    Result<std::vector<KeyValue>> result =
+        ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                   KeyValueRecordReader::Iterator>(reader.get());
+    ASSERT_TRUE(result.status().IsInvalid());
+    ASSERT_NOK_WITH_MSG(result, "selected row id 1 is out of bounds for prepared batch length 1");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestPreparedQueryReaderRejectsPartialBitmap) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 0, 1], [0, 11, 1, 2]])")
+            .ValueOrDie();
+    RoaringBitmap32 partial_bitmap;
+    partial_bitmap.Add(0);
+    auto batch_reader = std::make_unique<MockFileBatchReader>(
+        prepared_array, prepared_type, partial_bitmap, /*read_batch_size=*/2);
+    batch_reader->EnableRandomizeBatchSize(false);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 2), value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "must cover every raw mutation");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderQueryProjection) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, extra});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1, 2]])")
+            .ValueOrDie());
+
+    auto query_batch_reader =
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> query_reader,
+        CreatePreparedQueryReaderForTest(std::move(query_batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> query_results,
+        (ReadResultCollector::CollectKeyValueResult<
+            KeyValueRecordReader, KeyValueRecordReader::Iterator>(query_reader.get())));
+    ASSERT_EQ(query_results.size(), 1);
+    ASSERT_EQ(query_results[0].value->GetFieldCount(), 1);
+    ASSERT_EQ(query_results[0].value->GetInt(0), 1);
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestCommitOffsetCoverage) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> first_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 10, 2, 1], [0, 11, 0, 3]])")
+            .ValueOrDie();
+    std::shared_ptr<arrow::Array> second_array =
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+                                                  R"([[0, 12, 1, 2], [0, 13, 3, 4]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+    batch_readers.push_back(
+        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForCommit(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
+                             value_schema, value_schema, pool_));
+    int64_t row_count = 0;
+    for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
+        ASSERT_OK_AND_ASSIGN(
+            std::vector<KeyValue> rows,
+            (ReadResultCollector::CollectKeyValueResult<
+                KeyValueRecordReader, KeyValueRecordReader::Iterator>(reader.get())));
+        row_count += static_cast<int64_t>(rows.size());
+    }
+    ASSERT_EQ(4, row_count);
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestCommitRejectsEmptyReaders) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+
+    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForCommit(
+                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
+                            value_schema, value_schema, pool_),
+                        "PK real-time store returned no commit readers for a sealed segment");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestRejectsDuplicateCommitOffset) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(
+            prepared_type, R"([[0, 10, 0, 1], [0, 11, 0, 2], [0, 12, 2, 3]])")
+            .ValueOrDie();
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+                                                                  /*read_batch_size=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
+                         PreparedKeyValueReaderFactory::CreateForCommit(
+                             std::move(batch_readers), prepared_schema, OffsetRange(0, 3),
+                             value_schema, value_schema, pool_));
+    ASSERT_NOK_WITH_MSG((ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                                    KeyValueRecordReader::Iterator>(
+                            readers[0].get())),
+                        "did not cover the sealed range");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestBadCommitBatch) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Field> value = MakeField("value", arrow::int32(), 1);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key, value});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, value});
+    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
+    std::shared_ptr<arrow::Array> actual =
+        arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+
+    auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreatePreparedCommitReaderForTest(std::move(batch_reader), prepared_schema,
+                                                           OffsetRange(0, 1), arrow::schema({key}),
+                                                           value_schema, pool_));
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderSafeDecode) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+
+    arrow::FieldVector invalid_fields = prepared_schema->fields();
+    invalid_fields[0] = invalid_fields[0]->WithName("wrong_value_kind");
+    invalid_fields[3] = MakeField("wrong_key", arrow::int32(), 99);
+    std::shared_ptr<arrow::DataType> invalid_type = arrow::struct_(invalid_fields);
+    auto invalid_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(invalid_type, R"([[0, 10, 0, 1]])").ValueOrDie());
+
+    auto batch_reader = std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
+    ASSERT_NOK_WITH_MSG(
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())),
+        "prepared batch field");
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderNestedValues) {
+    std::shared_ptr<arrow::Field> id = MakeField("id", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({id});
+    std::shared_ptr<arrow::Field> query_item_b = MakeField("renamed_b", arrow::int32(), 11);
+    std::shared_ptr<arrow::Field> query_item_a = MakeField("renamed_a", arrow::int32(), 10);
+    std::shared_ptr<arrow::Field> query_items = MakeField(
+        "items_renamed",
+        arrow::list(arrow::field("element", arrow::struct_({query_item_b, query_item_a}))), 2);
+    std::shared_ptr<arrow::Field> query_attr_y = MakeField("renamed_y", arrow::int32(), 21);
+    std::shared_ptr<arrow::Field> query_attr_x = MakeField("renamed_x", arrow::int32(), 20);
+    std::shared_ptr<arrow::Field> query_attrs =
+        MakeField("attrs_renamed",
+                  arrow::map(arrow::utf8(), arrow::struct_({query_attr_y, query_attr_x})), 3);
+    std::shared_ptr<arrow::Field> query_key_right = MakeField("renamed_right", arrow::int32(), 31);
+    std::shared_ptr<arrow::Field> query_key_left = MakeField("renamed_left", arrow::int32(), 30);
+    std::shared_ptr<arrow::Field> query_keyed_values =
+        MakeField("keyed_values_renamed",
+                  arrow::map(arrow::struct_({query_key_right, query_key_left}), arrow::int32()), 4);
+    std::shared_ptr<arrow::Schema> query_value_schema =
+        arrow::schema({id, query_items, query_attrs, query_keyed_values});
+    std::shared_ptr<arrow::Schema> prepared_schema =
+        MakePreparedSchema(query_value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Array> prepared_array =
+        arrow::ipc::internal::json::ArrayFromJSON(
+            prepared_type,
+            R"([[0, 10, 0, 1, [[200, 100], [400, 300]], [["k1", [8, 7]], ["k2", [10, 9]]], [[[12, 11], 13], [[22, 21], 23]]]])")
+            .ValueOrDie();
+
+    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<KeyValueRecordReader> reader,
+        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
+                                         OffsetRange(0, 1), key_schema, query_value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(
+        std::vector<KeyValue> results,
+        (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
+                                                    KeyValueRecordReader::Iterator>(reader.get())));
+
+    ASSERT_EQ(results.size(), 1);
+    ASSERT_EQ(results[0].key->GetInt(0), 1);
+    ASSERT_EQ(results[0].value->GetFieldCount(), 4);
+    ASSERT_EQ(results[0].value->GetInt(0), 1);
+
+    std::shared_ptr<InternalArray> item_array = results[0].value->GetArray(1);
+    ASSERT_EQ(item_array->Size(), 2);
+    std::shared_ptr<InternalRow> first_item = item_array->GetRow(0, 2);
+    ASSERT_EQ(first_item->GetInt(0), 200);
+    ASSERT_EQ(first_item->GetInt(1), 100);
+    std::shared_ptr<InternalRow> second_item = item_array->GetRow(1, 2);
+    ASSERT_EQ(second_item->GetInt(0), 400);
+    ASSERT_EQ(second_item->GetInt(1), 300);
+
+    std::shared_ptr<InternalMap> attr_map = results[0].value->GetMap(2);
+    ASSERT_EQ(attr_map->Size(), 2);
+    std::shared_ptr<InternalArray> key_array = attr_map->KeyArray();
+    ASSERT_EQ(std::string(key_array->GetStringView(0)), "k1");
+    ASSERT_EQ(std::string(key_array->GetStringView(1)), "k2");
+    std::shared_ptr<InternalArray> value_array = attr_map->ValueArray();
+    std::shared_ptr<InternalRow> first_attr = value_array->GetRow(0, 2);
+    ASSERT_EQ(first_attr->GetInt(0), 8);
+    ASSERT_EQ(first_attr->GetInt(1), 7);
+    std::shared_ptr<InternalRow> second_attr = value_array->GetRow(1, 2);
+    ASSERT_EQ(second_attr->GetInt(0), 10);
+    ASSERT_EQ(second_attr->GetInt(1), 9);
+
+    std::shared_ptr<InternalMap> keyed_value_map = results[0].value->GetMap(3);
+    ASSERT_EQ(keyed_value_map->Size(), 2);
+    std::shared_ptr<InternalArray> struct_keys = keyed_value_map->KeyArray();
+    std::shared_ptr<InternalRow> first_key = struct_keys->GetRow(0, 2);
+    ASSERT_EQ(first_key->GetInt(0), 12);
+    ASSERT_EQ(first_key->GetInt(1), 11);
+    std::shared_ptr<InternalRow> second_key = struct_keys->GetRow(1, 2);
+    ASSERT_EQ(second_key->GetInt(0), 22);
+    ASSERT_EQ(second_key->GetInt(1), 21);
+    ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(0), 13);
+    ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(1), 23);
+}
+
+TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderFactoryFailureClosesReaders) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
+                                           DataField(1, arrow::field("v0", arrow::int32()))};
+    std::shared_ptr<arrow::Schema> value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
+    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+        [0, 10, 0, 1, 100]
+    ])")
+            .ValueOrDie());
+
+    int32_t factory_failure_close_count = 0;
+    std::vector<std::unique_ptr<BatchReader>> batch_readers;
+    batch_readers.push_back(std::make_unique<TrackingBatchReader>(
+        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
+        &factory_failure_close_count));
+    batch_readers.push_back(nullptr);
+    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForQuery(
+                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
+                            key_schema, value_schema, pool_),
+                        "PK real-time store returned a null query reader");
+    ASSERT_EQ(factory_failure_close_count, 1);
+}
+
+}  // namespace paimon::test

From 87796a390455ba5eff66b23e2ba22fdc270d6b54 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 21:28:15 +0800
Subject: [PATCH 79/93] fix(read): close realtime readers on setup failure

---
 .../core/mergetree/merge_tree_writer_test.cpp | 28 ------------
 .../core/operation/merge_file_split_read.cpp  | 40 +++++++++++++++--
 .../operation/merge_file_split_read_test.cpp  | 45 +++++++++++++++++--
 3 files changed, 78 insertions(+), 35 deletions(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 179deda44..84ebc0b84 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -588,34 +588,6 @@ TEST_P(MergeTreeWriterTest, TestMergeSortedReaders) {
     CheckFileContent(path_factory->ToPath(new_file), expected_array);
 }
 
-TEST_P(MergeTreeWriterTest, TestSortedReaderOwnership) {
-    ASSERT_OK_AND_ASSIGN(CoreOptions options,
-                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
-
-    auto dir = UniqueTestDirectory::Create();
-    ASSERT_TRUE(dir);
-    auto path_factory = std::make_shared<DataFilePathFactory>();
-    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
-
-    ASSERT_OK_AND_ASSIGN(auto merge_writer,
-                         CreateMergeWriter(-1, dir->Str(), path_factory, 0, options));
-
-    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
-      [0, 0, "Alice", 10, 0, 13.1]
-    ])")
-            .ValueOrDie());
-
-    bool closed = false;
-    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
-    sorted_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(
-        CreateSingleReader(sorted_reader_array), &closed));
-
-    ASSERT_OK(merge_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
-    ASSERT_TRUE(closed);
-    ASSERT_OK(merge_writer->Close());
-}
-
 TEST_P(MergeTreeWriterTest, TestSortedReaderFailure) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
                          CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 835ed0932..3b64ed20e 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -132,6 +132,13 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
         const std::vector<std::shared_ptr<Split>>& disk_splits,
         std::vector<std::unique_ptr<KeyValueRecordReader>>&& additional_readers,
         MergeFileSplitRead* owner) {
+        ScopeGuard additional_readers_guard([&additional_readers]() {
+            for (const std::unique_ptr<KeyValueRecordReader>& reader : additional_readers) {
+                if (reader) {
+                    reader->Close();
+                }
+            }
+        });
         RealtimeReaderBuilder builder(owner);
         std::vector<std::unique_ptr<KeyValueRecordReader>> readers;
         if (!disk_splits.empty()) {
@@ -141,6 +148,7 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
         for (std::unique_ptr<KeyValueRecordReader>& additional_reader : additional_readers) {
             readers.push_back(std::move(additional_reader));
         }
+        additional_readers_guard.Release();
         return builder.CreateMergedReader(std::move(readers));
     }
 
@@ -219,15 +227,32 @@ class MergeFileSplitRead::RealtimeReaderBuilder {
 
     Result<std::unique_ptr<BatchReader>> CreateMergedReader(
         std::vector<std::unique_ptr<KeyValueRecordReader>>&& record_readers) {
+        ScopeGuard record_readers_guard([&record_readers]() {
+            for (const std::unique_ptr<KeyValueRecordReader>& reader : record_readers) {
+                if (reader) {
+                    reader->Close();
+                }
+            }
+        });
         if (record_readers.empty()) {
+            record_readers_guard.Release();
             return std::make_unique<ConcatBatchReader>(std::vector<std::unique_ptr<BatchReader>>{},
                                                        owner_->pool_);
         }
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> sort_merge_reader,
                                owner_->CreateSortMergeReader(std::move(record_readers)));
-        return owner_->CreateProjectedReader(std::move(sort_merge_reader),
-                                             owner_->context_->GetPredicate(),
-                                             /*complete_row_kind=*/true);
+        record_readers_guard.Release();
+        ScopeGuard sort_merge_reader_guard([&sort_merge_reader]() {
+            if (sort_merge_reader) {
+                sort_merge_reader->Close();
+            }
+        });
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> result,
+                               owner_->CreateProjectedReader(std::move(sort_merge_reader),
+                                                             owner_->context_->GetPredicate(),
+                                                             /*complete_row_kind=*/true));
+        sort_merge_reader_guard.Release();
+        return result;
     }
 
     MergeFileSplitRead* owner_;
@@ -665,8 +690,15 @@ Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateProjectedReader(
             std::move(sort_merge_reader), raw_read_schema_, projection_,
             options_.GetReadBatchSize(), thread_number, pool_);
     }
-    PAIMON_ASSIGN_OR_RAISE(projection_reader,
+    ScopeGuard projection_reader_guard([&projection_reader]() {
+        if (projection_reader) {
+            projection_reader->Close();
+        }
+    });
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> filtered_reader,
                            ApplyPredicateFilterIfNeeded(std::move(projection_reader), predicate));
+    projection_reader_guard.Release();
+    projection_reader = std::move(filtered_reader);
     if (complete_row_kind) {
         return std::make_unique<CompleteRowKindBatchReader>(std::move(projection_reader), pool_);
     }
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index 7a859cae5..b79baf5d6 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -71,6 +71,26 @@ namespace {
 
 class TestingSplit : public Split {};
 
+class TrackingKeyValueRecordReader : public KeyValueRecordReader {
+ public:
+    explicit TrackingKeyValueRecordReader(int32_t* close_count) : close_count_(close_count) {}
+
+    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
+        return std::unique_ptr<KeyValueRecordReader::Iterator>();
+    }
+
+    void Close() override {
+        ++(*close_count_);
+    }
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override {
+        return nullptr;
+    }
+
+ private:
+    int32_t* close_count_;
+};
+
 }  // namespace
 
 // Parameter: min_heap/loser_tree; enable/disable IO prefetch; enable/disable multi thread row to
@@ -693,7 +713,6 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections)
     context_builder.SetOptions(
         {{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}});
     AddOptions(&context_builder);
-    context_builder.EnableMultiThreadRowToBatch(false);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
     std::shared_ptr<InternalReadContext> internal_context = CreateInternalReadContext(read_context);
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<MergeFileSplitRead> split_read,
@@ -745,14 +764,13 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections)
     batch_reader->Close();
 }
 
-TEST_P(MergeFileSplitReadTest, TestRealtimeReadValidatesDiskSplits) {
+TEST_F(MergeFileSplitReadTest, TestRealtimeReadValidatesDiskSplits) {
     std::string path =
         paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
     ReadContextBuilder context_builder(path);
     context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"});
     context_builder.SetOptions(
         {{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}});
-    AddOptions(&context_builder);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
     std::shared_ptr<InternalReadContext> internal_context = CreateInternalReadContext(read_context);
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<MergeFileSplitRead> split_read,
@@ -778,6 +796,27 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadValidatesDiskSplits) {
                         "deletion files must be empty or match data files");
 }
 
+TEST_F(MergeFileSplitReadTest, TestRealtimeReaderFailureClosesPluginReader) {
+    std::string path =
+        paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
+    ReadContextBuilder context_builder(path);
+    context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"});
+    context_builder.SetOptions(
+        {{Options::MERGE_ENGINE, "aggregation"}, {"fields.v0.aggregate-function", "unsupported"}});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
+    std::shared_ptr<InternalReadContext> internal_context = CreateInternalReadContext(read_context);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<MergeFileSplitRead> split_read,
+                         CreateMergeFileSplitRead(internal_context));
+
+    int32_t close_count = 0;
+    std::vector<std::unique_ptr<KeyValueRecordReader>> plugin_readers;
+    plugin_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(&close_count));
+
+    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader({}, std::move(plugin_readers)),
+                        "unsupported");
+    ASSERT_EQ(1, close_count);
+}
+
 TEST_P(MergeFileSplitReadTest, TestLookUp) {
     std::string path =
         paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";

From 982eb59256ebf14d943e202950b35ad30b8fc671 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Thu, 27 Aug 2026 23:15:44 +0800
Subject: [PATCH 80/93] refactor(realtime): clarify primary-key reader
 contracts

---
 include/paimon/realtime/realtime_store.h      |  23 +-
 src/paimon/CMakeLists.txt                     |   4 +-
 src/paimon/common/table/special_fields.h      |  14 -
 .../common/table/special_fields_test.cpp      |  21 -
 .../merged_key_value_record_reader_test.cpp   |   2 +-
 .../operation/key_value_file_store_write.cpp  |  14 +-
 .../key_value_file_store_write_test.cpp       |  52 ++-
 .../core/operation/merge_file_split_read.cpp  |   1 +
 .../realtime/primary_key_realtime_store.cpp   |  23 +-
 .../realtime/primary_key_realtime_store.h     |   2 +-
 .../primary_key_realtime_store_test.cpp       |  52 +--
 ...er.cpp => realtime_primary_key_reader.cpp} | 234 +++++------
 ...reader.h => realtime_primary_key_reader.h} |  28 +-
 ...p => realtime_primary_key_reader_test.cpp} | 394 ++++++++++--------
 .../realtime/realtime_primary_key_writer.cpp  |  74 ++--
 .../realtime/realtime_primary_key_writer.h    |  13 +-
 .../table/source/key_value_table_read.cpp     |  65 +--
 .../core/table/source/key_value_table_read.h  |   4 +-
 18 files changed, 535 insertions(+), 485 deletions(-)
 rename src/paimon/core/realtime/{prepared_key_value_reader.cpp => realtime_primary_key_reader.cpp} (65%)
 rename src/paimon/core/realtime/{prepared_key_value_reader.h => realtime_primary_key_reader.h} (63%)
 rename src/paimon/core/realtime/{prepared_key_value_reader_test.cpp => realtime_primary_key_reader_test.cpp} (56%)

diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index cbfe96595..81e64b9fd 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -50,7 +50,7 @@ enum class PAIMON_EXPORT RealtimeStoreMode {
 /// Parameters used by a `RealtimeStoreFactory` to create a store.
 struct PAIMON_EXPORT RealtimeStoreCreateRequest {
     /// Schema whose ownership is transferred to the factory. Append mode receives the complete
-    /// table write schema. Primary-key mode receives the prepared transport schema:
+    /// table write schema. Primary-key mode receives the realtime primary-key transport schema:
     /// [_VALUE_KIND, _SEQUENCE_NUMBER, _REALTIME_OFFSET, table write fields].
     std::unique_ptr<::ArrowSchema> write_schema;
     /// Table options available to the store implementation.
@@ -66,8 +66,8 @@ struct PAIMON_EXPORT RealtimeStoreCreateRequest {
 /// A record batch and its framework-assigned contiguous offset range.
 ///
 /// Append-mode batches contain table write fields, and row `i` has offset
-/// `offset_range.begin + i`. Primary-key batches use the prepared transport schema, are sorted
-/// by full primary key then sequence number, and retain the original offset in
+/// `offset_range.begin + i`. Primary-key batches use the realtime primary-key transport schema,
+/// are sorted by full primary key then sequence number, and retain the original offset in
 /// `_REALTIME_OFFSET`.
 struct PAIMON_EXPORT RealtimeWriteBatch {
     /// Input batch whose ownership is transferred to `RealtimeStore::Write`.
@@ -104,7 +104,8 @@ class PAIMON_EXPORT RealtimeReadView {
 /// Parameters used by a `RealtimeStore` to create readers for a query.
 struct PAIMON_EXPORT RealtimeQueryContext {
     /// Append mode receives the requested output fields before the mandatory leading
-    /// `_VALUE_KIND` field is added. Primary-key mode receives the requested prepared schema.
+    /// `_VALUE_KIND` field is added. Primary-key mode receives the requested realtime primary-key
+    /// transport schema.
     /// This schema is borrowed and remains valid only during `CreateQueryReaders`; plugins must
     /// import or copy it synchronously.
     ::ArrowSchema* read_schema;
@@ -145,8 +146,8 @@ class PAIMON_EXPORT RealtimeStore {
     ///
     /// The returned readers collectively expose every sealed row exactly once. Append-mode readers
     /// preserve write order and contain `_VALUE_KIND` followed by table write fields. Primary-key
-    /// readers use the prepared transport schema; each reader's complete stream is sorted by full
-    /// primary key then sequence number.
+    /// readers use the realtime primary-key transport schema; each reader's complete stream is
+    /// sorted by full primary key then sequence number.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
         const std::shared_ptr<RealtimeSegmentHandle>& segment) = 0;
 
@@ -161,11 +162,11 @@ class PAIMON_EXPORT RealtimeStore {
     ///
     /// Append-mode batches contain `_VALUE_KIND` followed by the requested fields except a
     /// duplicate `_VALUE_KIND`, and collectively expose every matching row exactly once.
-    /// Primary-key batches use the requested prepared transport schema, including nested field-ID
-    /// alignment, and may contain multiple mutations per key; each reader's complete stream is
-    /// sorted by full primary key then sequence number, and the readers collectively expose every
-    /// raw mutation exactly once. Paimon retains `view` for the lifetime of the resulting framework
-    /// reader.
+    /// Primary-key batches use the requested realtime primary-key transport schema, including
+    /// nested field-ID alignment, and may contain multiple mutations per key; each reader's
+    /// complete stream is sorted by full primary key then sequence number, and the readers
+    /// collectively expose every raw mutation exactly once. Paimon retains `view` for the lifetime
+    /// of the resulting framework reader.
     virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) = 0;
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 41fe5f6ba..7e4986ab8 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -382,7 +382,7 @@ set(PAIMON_CORE_SRCS
     core/operation/write_restore.cpp
     core/realtime/arrow_realtime_store.cpp
     core/realtime/arrow_realtime_store_factory.cpp
-    core/realtime/prepared_key_value_reader.cpp
+    core/realtime/realtime_primary_key_reader.cpp
     core/realtime/primary_key_realtime_store.cpp
     core/realtime/realtime_append_only_writer.cpp
     core/realtime/realtime_context.cpp
@@ -791,7 +791,7 @@ if(PAIMON_BUILD_TESTS)
                     core/memory/writer_memory_manager_test.cpp
                     core/realtime/arrow_realtime_store_test.cpp
                     core/realtime/primary_key_realtime_store_test.cpp
-                    core/realtime/prepared_key_value_reader_test.cpp
+                    core/realtime/realtime_primary_key_reader_test.cpp
                     core/realtime/realtime_context_test.cpp
                     core/realtime/realtime_reader_test.cpp
                     core/mergetree/levels_test.cpp
diff --git a/src/paimon/common/table/special_fields.h b/src/paimon/common/table/special_fields.h
index 0e07882a8..8908f9f0c 100644
--- a/src/paimon/common/table/special_fields.h
+++ b/src/paimon/common/table/special_fields.h
@@ -36,10 +36,6 @@ struct SpecialFields {
 
     static constexpr char KEY_FIELD_PREFIX[] = "_KEY_";
     static constexpr int32_t KEY_VALUE_SPECIAL_FIELD_COUNT = 2;
-    static constexpr int32_t kPreparedKeyValueValueKindIndex = 0;
-    static constexpr int32_t kPreparedKeyValueSequenceNumberIndex = 1;
-    static constexpr int32_t kPreparedKeyValueRealtimeOffsetIndex = 2;
-    static constexpr int32_t kPreparedKeyValueValueStartIndex = 3;
 
     static const DataField& SequenceNumber() {
         static const DataField data_field = DataField(
@@ -97,16 +93,6 @@ struct SpecialFields {
         target_fields.insert(target_fields.end(), schema->fields().begin(), schema->fields().end());
         return arrow::schema(target_fields);
     }
-
-    static std::shared_ptr<arrow::Schema> PreparedKeyValueSchema(
-        const arrow::FieldVector& value_fields) {
-        arrow::FieldVector fields = {
-            DataField::ConvertDataFieldToArrowField(ValueKind())->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(SequenceNumber())->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(RealtimeOffset())};
-        fields.insert(fields.end(), value_fields.begin(), value_fields.end());
-        return arrow::schema(std::move(fields));
-    }
 };
 
 }  // namespace paimon
diff --git a/src/paimon/common/table/special_fields_test.cpp b/src/paimon/common/table/special_fields_test.cpp
index a0a0980a5..58a025ba2 100644
--- a/src/paimon/common/table/special_fields_test.cpp
+++ b/src/paimon/common/table/special_fields_test.cpp
@@ -66,27 +66,6 @@ TEST(SpecialFieldsTest, TestKeyValueSpecialFieldCount) {
     ASSERT_EQ(SpecialFields::KEY_VALUE_SPECIAL_FIELD_COUNT, 2);
 }
 
-TEST(SpecialFieldsTest, TestPreparedKeyValueSchema) {
-    arrow::FieldVector value_fields = {arrow::field("key", arrow::int64(), false),
-                                       arrow::field("value", arrow::utf8())};
-    std::shared_ptr<arrow::Schema> schema = SpecialFields::PreparedKeyValueSchema(value_fields);
-
-    ASSERT_EQ(SpecialFields::kPreparedKeyValueValueKindIndex, 0);
-    ASSERT_EQ(SpecialFields::kPreparedKeyValueSequenceNumberIndex, 1);
-    ASSERT_EQ(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex, 2);
-    ASSERT_EQ(SpecialFields::kPreparedKeyValueValueStartIndex, 3);
-    ASSERT_EQ(schema->field(0)->name(), "_VALUE_KIND");
-    ASSERT_EQ(schema->field(1)->name(), "_SEQUENCE_NUMBER");
-    ASSERT_EQ(schema->field(2)->name(), "_REALTIME_OFFSET");
-    ASSERT_EQ(schema->field(3)->name(), "key");
-    ASSERT_EQ(schema->field(4)->name(), "value");
-    ASSERT_FALSE(schema->field(0)->nullable());
-    ASSERT_FALSE(schema->field(1)->nullable());
-    ASSERT_EQ(schema->field(2)->nullable(), SpecialFields::RealtimeOffset().Nullable());
-    ASSERT_FALSE(schema->field(3)->nullable());
-    ASSERT_TRUE(schema->field(4)->nullable());
-}
-
 TEST(SpecialFieldsTest, TestIsSystemField) {
     ASSERT_TRUE(SpecialFields::IsSystemField("_SEQUENCE_NUMBER"));
     ASSERT_TRUE(SpecialFields::IsSystemField("_VALUE_KIND"));
diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index a9395c9ab..d484b28ee 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -35,7 +35,7 @@
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
-#include "paimon/core/realtime/prepared_key_value_reader.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/realtime/offset_range.h"
 #include "paimon/testing/mock/mock_file_batch_reader.h"
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 86f432998..152a4ed01 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -35,6 +35,7 @@
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/key_value_file_store_scan.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/core/realtime/realtime_primary_key_writer.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/utils/file_store_path_factory.h"
@@ -124,7 +125,7 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
     std::shared_ptr<CompactManager> compact_manager;
     std::shared_ptr<RealtimeContextImpl> realtime_context_impl;
     std::optional<RealtimeStoreState> realtime_store_state;
-    std::shared_ptr<arrow::Schema> prepared_schema;
+    std::shared_ptr<arrow::Schema> transport_schema;
     if (realtime_context_) {
         std::vector<std::pair<std::string, std::string>> partition_values;
         PAIMON_ASSIGN_OR_RAISE(partition_values,
@@ -132,10 +133,10 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
         partition_map =
             std::map<std::string, std::string>(partition_values.begin(), partition_values.end());
         PAIMON_ASSIGN_OR_RAISE(realtime_context_impl, RealtimeContextImpl::Cast(realtime_context_));
-        prepared_schema = SpecialFields::PreparedKeyValueSchema(schema_->fields());
+        transport_schema = RealtimePrimaryKeyLayout::CreateSchema(schema_->fields());
         auto c_write_schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(
-            arrow::ExportSchema(*prepared_schema, c_write_schema.get()));
+            arrow::ExportSchema(*transport_schema, c_write_schema.get()));
         PAIMON_ASSIGN_OR_RAISE(
             RealtimeStoreState store_state,
             realtime_context_impl->GetOrCreateRealtimeStore(
@@ -164,9 +165,10 @@ Result<std::shared_ptr<BatchWriter>> KeyValueFileStoreWrite::CreateWriter(
     if (!realtime_context_) {
         return std::shared_ptr<BatchWriter>(std::move(writer));
     }
-    return RealtimePrimaryKeyWriter::Create(
-        partition_map, bucket, schema_, prepared_schema, trimmed_primary_keys, key_comparator_,
-        realtime_context_impl, realtime_store_state.value(), restore_max_seq_number, writer, pool_);
+    return RealtimePrimaryKeyWriter::Create(partition_map, bucket, schema_, transport_schema,
+                                            trimmed_primary_keys, key_comparator_, options_,
+                                            realtime_context_impl, realtime_store_state.value(),
+                                            restore_max_seq_number, writer, pool_);
 }
 
 Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index fc6bfff11..01ba8459d 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -248,7 +248,8 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
     }
 
     Result<std::vector<std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>>>
-    ReadPreparedRows(const std::shared_ptr<RealtimeContext>& realtime_context) const {
+    ReadRealtimePrimaryKeyTransportRows(
+        const std::shared_ptr<RealtimeContext>& realtime_context) const {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> context,
                                RealtimeContextImpl::Cast(realtime_context));
         PAIMON_ASSIGN_OR_RAISE(std::vector<RealtimePartitionBucketView> views,
@@ -256,7 +257,7 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
         if (views.size() != 1) {
             return Status::Invalid("expected exactly one real-time store");
         }
-        std::shared_ptr<arrow::Schema> prepared_schema = arrow::schema({
+        std::shared_ptr<arrow::Schema> transport_schema = arrow::schema({
             DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
                 ->WithNullable(false),
             DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
@@ -268,7 +269,7 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
                 DataField(1, arrow::field("value", arrow::utf8()))),
         });
         auto c_schema = std::make_unique<ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*transport_schema, c_schema.get()));
         RealtimeQueryContext query_context{c_schema.get(), nullptr, false};
         PAIMON_ASSIGN_OR_RAISE(
             std::vector<std::unique_ptr<BatchReader>> readers,
@@ -286,7 +287,7 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
                 std::shared_ptr<arrow::StructArray> values =
                     std::dynamic_pointer_cast<arrow::StructArray>(array);
                 if (!values || values->num_fields() != 5) {
-                    return Status::Invalid("unexpected prepared real-time batch");
+                    return Status::Invalid("unexpected realtime primary-key transport batch");
                 }
                 std::shared_ptr<arrow::Int8Array> row_kinds =
                     std::dynamic_pointer_cast<arrow::Int8Array>(values->field(0));
@@ -299,7 +300,7 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
                 std::shared_ptr<arrow::StringArray> payloads =
                     std::dynamic_pointer_cast<arrow::StringArray>(values->field(4));
                 if (!row_kinds || !sequences || !offsets || !ids || !payloads) {
-                    return Status::Invalid("unexpected prepared real-time column type");
+                    return Status::Invalid("unexpected realtime primary-key transport column type");
                 }
                 for (int64_t row = 0; row < values->length(); ++row) {
                     rows.emplace_back(row_kinds->Value(row), ids->Value(row),
@@ -456,12 +457,13 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
                   {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE,
                    RecordBatch::RowKind::UPDATE_AFTER});
     ASSERT_OK(writer->Write(std::move(batch)));
-    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
-    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> prepared_rows,
-                         ReadPreparedRows(realtime_context));
-    ASSERT_EQ(
-        (std::vector<PreparedRow>{{0, 1, "old", 0, 0}, {2, 1, "new", 2, 2}, {3, 2, "two", 1, 1}}),
-        prepared_rows);
+    using RealtimePrimaryKeyTransportRow =
+        std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimePrimaryKeyTransportRow> transport_rows,
+                         ReadRealtimePrimaryKeyTransportRows(realtime_context));
+    ASSERT_EQ((std::vector<RealtimePrimaryKeyTransportRow>{
+                  {0, 1, "old", 0, 0}, {2, 1, "new", 2, 2}, {3, 2, "two", 1, 1}}),
+              transport_rows);
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progresses,
                          writer->PrepareCommitWithProgress(0));
     ASSERT_EQ(1, progresses.size());
@@ -508,10 +510,11 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
     ASSERT_GT(pool->allocation_count, allocations_before_write);
     ASSERT_OK(writer->Close());
     writer.reset();
-    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
-    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> retained_rows,
-                         ReadPreparedRows(realtime_context));
-    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "one", 0, 0}}), retained_rows);
+    using RealtimePrimaryKeyTransportRow =
+        std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimePrimaryKeyTransportRow> retained_rows,
+                         ReadRealtimePrimaryKeyTransportRows(realtime_context));
+    ASSERT_EQ((std::vector<RealtimePrimaryKeyTransportRow>{{0, 1, "one", 0, 0}}), retained_rows);
 
     std::shared_ptr<TestingMemoryPool> rejecting_pool = std::make_shared<TestingMemoryPool>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> rejecting_context,
@@ -531,8 +534,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
     ASSERT_NOK_WITH_MSG(rejecting_writer->Write(MakeBatch(schema, R"([[2, "two"]])")),
                         "Out of memory");
     ASSERT_GT(rejecting_pool->allocation_count, rejecting_allocations_before_write);
-    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> rejected_rows,
-                         ReadPreparedRows(rejecting_context));
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimePrimaryKeyTransportRow> rejected_rows,
+                         ReadRealtimePrimaryKeyTransportRows(rejecting_context));
     ASSERT_TRUE(rejected_rows.empty());
     ASSERT_OK(rejecting_writer->Close());
 }
@@ -587,15 +590,18 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) {
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                          FileStoreWrite::Create(std::move(write_context)));
     ASSERT_OK(writer->Write(MakeBatch(schema, R"([[1, "legal"]])")));
-    using PreparedRow = std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
-    ASSERT_OK_AND_ASSIGN(std::vector<PreparedRow> prepared_rows,
-                         ReadPreparedRows(realtime_context));
-    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "legal", max - 1, max - 1}}), prepared_rows);
+    using RealtimePrimaryKeyTransportRow =
+        std::tuple<int8_t, int64_t, std::string, int64_t, int64_t>;
+    ASSERT_OK_AND_ASSIGN(std::vector<RealtimePrimaryKeyTransportRow> transport_rows,
+                         ReadRealtimePrimaryKeyTransportRows(realtime_context));
+    ASSERT_EQ((std::vector<RealtimePrimaryKeyTransportRow>{{0, 1, "legal", max - 1, max - 1}}),
+              transport_rows);
 
     ASSERT_NOK_WITH_MSG(writer->Write(MakeBatch(schema, R"([[2, "overflow"]])")),
                         "real-time offset range exceeds INT64_MAX");
-    ASSERT_OK_AND_ASSIGN(prepared_rows, ReadPreparedRows(realtime_context));
-    ASSERT_EQ((std::vector<PreparedRow>{{0, 1, "legal", max - 1, max - 1}}), prepared_rows);
+    ASSERT_OK_AND_ASSIGN(transport_rows, ReadRealtimePrimaryKeyTransportRows(realtime_context));
+    ASSERT_EQ((std::vector<RealtimePrimaryKeyTransportRow>{{0, 1, "legal", max - 1, max - 1}}),
+              transport_rows);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context_impl,
                          RealtimeContextImpl::Cast(realtime_context));
     ASSERT_OK_AND_ASSIGN(std::vector<RealtimePartitionBucketView> views,
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 3b64ed20e..3bcc705aa 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -677,6 +677,7 @@ Result<std::unique_ptr<BatchReader>> MergeFileSplitRead::CreateProjectedReader(
     if (!force_keep_delete_) {
         sort_merge_reader = std::make_unique<DropDeleteReader>(std::move(sort_merge_reader));
     }
+    // KeyValueProjectionReader converts KeyValue objects to arrow array according to projection
     std::unique_ptr<BatchReader> projection_reader;
     if (!context_->EnableMultiThreadRowToBatch()) {
         PAIMON_ASSIGN_OR_RAISE(
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.cpp b/src/paimon/core/realtime/primary_key_realtime_store.cpp
index a22ba8fed..421eb0c8d 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store.cpp
@@ -33,7 +33,6 @@
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/scope_guard.h"
-#include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/macros.h"
 #include "paimon/memory/memory_pool.h"
@@ -138,9 +137,9 @@ class StoredBatchReader final : public BatchReader {
 
 class PrimaryKeyRealtimeStore::Impl {
  public:
-    Impl(std::shared_ptr<arrow::Schema> prepared_schema,
+    Impl(std::shared_ptr<arrow::Schema> transport_schema,
          std::shared_ptr<arrow::MemoryPool> arrow_pool)
-        : prepared_schema_(std::move(prepared_schema)), arrow_pool_(std::move(arrow_pool)) {}
+        : transport_schema_(std::move(transport_schema)), arrow_pool_(std::move(arrow_pool)) {}
 
     Status Write(RealtimeWriteBatch&& write_batch) {
         if (!write_batch.batch || !write_batch.batch->GetData()) {
@@ -154,15 +153,15 @@ class PrimaryKeyRealtimeStore::Impl {
         PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
             std::shared_ptr<arrow::Array> array,
             arrow::ImportArray(write_batch.batch->GetData(),
-                               arrow::struct_(prepared_schema_->fields())));
+                               arrow::struct_(transport_schema_->fields())));
         if (!array || array->type_id() != arrow::Type::STRUCT) {
-            return Status::Invalid("PK real-time prepared batch is not a StructArray");
+            return Status::Invalid("PK real-time transport batch is not a StructArray");
         }
-        std::shared_ptr<arrow::StructArray> prepared =
+        std::shared_ptr<arrow::StructArray> transport =
             checked_pointer_cast<arrow::StructArray>(array);
         std::lock_guard<std::mutex> lock(mutex_);
-        building_.push_back(StoredBatch{prepared, write_batch.offset_range,
-                                        ArrowUtils::GetArrayMemoryUsage(prepared->data())});
+        building_.push_back(StoredBatch{transport, write_batch.offset_range,
+                                        ArrowUtils::GetArrayMemoryUsage(transport->data())});
         building_memory_usage_ += building_.back().memory_usage;
         return Status::OK();
     }
@@ -218,7 +217,6 @@ class PrimaryKeyRealtimeStore::Impl {
         }
         PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> read_schema,
                                           arrow::ImportSchema(context.read_schema));
-        PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(read_schema));
         std::vector<std::unique_ptr<BatchReader>> readers;
         for (const std::shared_ptr<Segment>& segment : typed->Segments()) {
             for (const StoredBatch& batch : segment->Batches()) {
@@ -260,7 +258,7 @@ class PrimaryKeyRealtimeStore::Impl {
     }
 
  private:
-    std::shared_ptr<arrow::Schema> prepared_schema_;
+    std::shared_ptr<arrow::Schema> transport_schema_;
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     mutable std::mutex mutex_;
     std::vector<StoredBatch> building_;
@@ -273,15 +271,14 @@ PrimaryKeyRealtimeStore::PrimaryKeyRealtimeStore(std::unique_ptr<Impl>&& impl)
 PrimaryKeyRealtimeStore::~PrimaryKeyRealtimeStore() = default;
 
 Result<std::shared_ptr<PrimaryKeyRealtimeStore>> PrimaryKeyRealtimeStore::Create(
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& transport_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
-    PAIMON_RETURN_NOT_OK(PreparedKeyValueReaderFactory::ValidateTransportSchema(prepared_schema));
     if (!memory_pool) {
         return Status::Invalid("PK real-time store memory pool is null");
     }
     std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(memory_pool);
     return std::shared_ptr<PrimaryKeyRealtimeStore>(new PrimaryKeyRealtimeStore(
-        std::make_unique<Impl>(prepared_schema, std::move(arrow_pool))));
+        std::make_unique<Impl>(transport_schema, std::move(arrow_pool))));
 }
 Status PrimaryKeyRealtimeStore::Write(RealtimeWriteBatch&& batch) {
     return impl_->Write(std::move(batch));
diff --git a/src/paimon/core/realtime/primary_key_realtime_store.h b/src/paimon/core/realtime/primary_key_realtime_store.h
index 01e1926ab..46c0fe8f7 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store.h
+++ b/src/paimon/core/realtime/primary_key_realtime_store.h
@@ -35,7 +35,7 @@ class MemoryPool;
 class PrimaryKeyRealtimeStore final : public RealtimeStore {
  public:
     static Result<std::shared_ptr<PrimaryKeyRealtimeStore>> Create(
-        const std::shared_ptr<arrow::Schema>& prepared_schema,
+        const std::shared_ptr<arrow::Schema>& transport_schema,
         const std::shared_ptr<MemoryPool>& memory_pool);
 
     ~PrimaryKeyRealtimeStore() override;
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 46d9a8e7d..301cc92a9 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -49,7 +49,7 @@ std::shared_ptr<arrow::Field> FieldWithId(const std::string& name,
         ->WithNullable(nullable);
 }
 
-std::shared_ptr<arrow::Schema> PreparedSchema() {
+std::shared_ptr<arrow::Schema> TransportSchema() {
     return arrow::schema(
         {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
          DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
@@ -60,7 +60,7 @@ std::shared_ptr<arrow::Schema> PreparedSchema() {
              DataField(1, arrow::field("value", arrow::utf8())))});
 }
 
-std::shared_ptr<arrow::Schema> NestedPreparedSchema() {
+std::shared_ptr<arrow::Schema> NestedTransportSchema() {
     return arrow::schema(
         {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
          DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
@@ -76,7 +76,7 @@ std::shared_ptr<arrow::Schema> NestedPreparedSchema() {
 
 std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) {
     std::shared_ptr<arrow::Array> array =
-        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(PreparedSchema()->fields()), json)
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(TransportSchema()->fields()), json)
             .ValueOrDie();
     auto c_array = std::make_unique<ArrowArray>();
     EXPECT_TRUE(arrow::ExportArray(*array, c_array.get()).ok());
@@ -157,7 +157,7 @@ class TestingMemoryPool final : public MemoryPool {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+                         PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
                          store->SealForCommit());
     ASSERT_FALSE(segment.has_value());
@@ -180,37 +180,9 @@ TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
         RealtimeWriteBatch{MakeBatch(R"([[0, 4, 3, 4, "four"]])"), OffsetRange(3, 4)}));
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestBadTransportPrefix) {
-    const std::shared_ptr<arrow::Schema> valid = PreparedSchema();
-    std::vector<arrow::FieldVector> invalid_fields;
-
-    arrow::FieldVector wrong_type = valid->fields();
-    wrong_type[0] = DataField::ConvertDataFieldToArrowField(
-                        DataField(SpecialFields::ValueKind().Id(),
-                                  arrow::field("_VALUE_KIND", arrow::int32(), false)))
-                        ->WithNullable(false);
-    invalid_fields.push_back(std::move(wrong_type));
-
-    arrow::FieldVector nullable_sequence = valid->fields();
-    nullable_sequence[1] = nullable_sequence[1]->WithNullable(true);
-    invalid_fields.push_back(std::move(nullable_sequence));
-
-    arrow::FieldVector wrong_offset_id = valid->fields();
-    wrong_offset_id[2] = DataField::ConvertDataFieldToArrowField(
-                             DataField(99, arrow::field("_REALTIME_OFFSET", arrow::int64(), false)))
-                             ->WithNullable(false);
-    invalid_fields.push_back(std::move(wrong_offset_id));
-
-    for (const arrow::FieldVector& fields : invalid_fields) {
-        ASSERT_NOK_WITH_MSG(
-            PrimaryKeyRealtimeStore::Create(arrow::schema(fields), GetDefaultPool()),
-            "prepared schema field");
-    }
-}
-
 TEST(PrimaryKeyRealtimeStoreTest, TestCommitReaderPerStoredBatch) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+                         PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
         MakeBatch(R"([[1, 6, 1, 1, "before"], [0, 5, 0, 3, "three"]])"), OffsetRange(0, 2)}));
     ASSERT_OK(store->Write(
@@ -264,7 +236,7 @@ void AssertSlicedBatch(BatchReader* reader) {
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestSlicedReadersExportZeroOffsets) {
-    std::shared_ptr<arrow::Schema> schema = NestedPreparedSchema();
+    std::shared_ptr<arrow::Schema> schema = NestedTransportSchema();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
                          PrimaryKeyRealtimeStore::Create(schema, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
@@ -293,7 +265,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestSlicedReadersExportZeroOffsets) {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadBatchReaders) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+                         PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 10, 0, 1, "a"]])"), OffsetRange(0, 1)}));
     ASSERT_OK(
@@ -313,7 +285,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadBatchReaders) {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+                         PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 0, 4, 1, "one"]])"), OffsetRange(4, 5)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -349,7 +321,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_FALSE(current_view->GetOffsetRange().has_value());
 
     auto c_schema = std::make_unique<ArrowSchema>();
-    ASSERT_TRUE(arrow::ExportSchema(*PreparedSchema(), c_schema.get()).ok());
+    ASSERT_TRUE(arrow::ExportSchema(*TransportSchema(), c_schema.get()).ok());
     RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
@@ -362,7 +334,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(PreparedSchema(), GetDefaultPool()));
+                         PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
     ASSERT_OK(
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 2, "two"]])"), OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
@@ -372,7 +344,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
         store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 2, 1, 1, "one"]])"), OffsetRange(1, 2)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
     auto c_schema = std::make_unique<ArrowSchema>();
-    ASSERT_TRUE(arrow::ExportSchema(*PreparedSchema(), c_schema.get()).ok());
+    ASSERT_TRUE(arrow::ExportSchema(*TransportSchema(), c_schema.get()).ok());
     RealtimeQueryContext context{/*read_schema=*/c_schema.get(), /*predicate=*/nullptr,
                                  /*enable_predicate_pushdown=*/false};
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
@@ -384,7 +356,7 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
 }
 
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryPoolOutlivesStoreReaderAndExport) {
-    const std::shared_ptr<arrow::Schema> stored_schema = PreparedSchema();
+    const std::shared_ptr<arrow::Schema> stored_schema = TransportSchema();
     std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
     std::weak_ptr<TestingMemoryPool> pool_lifetime = pool;
     auto write_schema = std::make_unique<ArrowSchema>();
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.cpp b/src/paimon/core/realtime/realtime_primary_key_reader.cpp
similarity index 65%
rename from src/paimon/core/realtime/prepared_key_value_reader.cpp
rename to src/paimon/core/realtime/realtime_primary_key_reader.cpp
index d5e29da52..19ba0a985 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_reader.cpp
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-#include "paimon/core/realtime/prepared_key_value_reader.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 
 #include <cstdint>
 #include <memory>
@@ -122,19 +122,20 @@ class RealtimeOffsetCoverage {
     size_t finished_reader_count_ = 0;
 };
 
-Status CheckPreparedField(const std::shared_ptr<arrow::Schema>& schema, int32_t field_idx,
-                          const DataField& expected_field) {
+Status CheckTransportField(const std::shared_ptr<arrow::Schema>& schema, int32_t field_idx,
+                           const DataField& expected_field) {
     if (schema->num_fields() <= field_idx) {
-        return Status::Invalid(fmt::format("prepared schema missing transport field {} at index {}",
-                                           expected_field.Name(), field_idx));
+        return Status::Invalid(
+            fmt::format("realtime primary-key transport schema is missing field {} at index {}",
+                        expected_field.Name(), field_idx));
     }
     const std::shared_ptr<arrow::Field>& field = schema->field(field_idx);
     PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(field));
     if (field->name() != expected_field.Name() || !field->type()->Equals(*expected_field.Type()) ||
         field->nullable() || field_id != expected_field.Id()) {
         return Status::Invalid(fmt::format(
-            "prepared schema field {} must be non-null {}:{} with field id {}, got {}:{} "
-            "nullable={} field id {}",
+            "realtime primary-key transport schema field {} must be non-null {}:{} with field id "
+            "{}, got {}:{} nullable={} field id {}",
             field_idx, expected_field.Name(), expected_field.Type()->ToString(),
             expected_field.Id(), field->name(), field->type()->ToString(), field->nullable(),
             field_id));
@@ -143,7 +144,7 @@ Status CheckPreparedField(const std::shared_ptr<arrow::Schema>& schema, int32_t
 }
 
 Result<std::vector<int32_t>> ResolveFieldIndexes(
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& transport_schema,
     const std::unordered_map<int32_t, int32_t>& field_indexes,
     const std::shared_ptr<arrow::Schema>& row_schema) {
     std::vector<int32_t> result;
@@ -153,50 +154,49 @@ Result<std::vector<int32_t>> ResolveFieldIndexes(
                                NestedProjectionUtils::GetPaimonFieldId(row_field));
         auto field_index = field_indexes.find(field_id);
         if (field_index == field_indexes.end()) {
-            return Status::Invalid(
-                fmt::format("cannot find field id {} in prepared schema", field_id));
+            return Status::Invalid(fmt::format(
+                "cannot find field id {} in realtime primary-key transport schema", field_id));
         }
-        const std::shared_ptr<arrow::Field>& prepared_field =
-            prepared_schema->field(field_index->second);
-        if (!prepared_field->type()->Equals(row_field->type())) {
+        const std::shared_ptr<arrow::Field>& transport_field =
+            transport_schema->field(field_index->second);
+        if (!transport_field->type()->Equals(row_field->type())) {
             return Status::Invalid(fmt::format(
-                "prepared field id {} type {} does not match row "
-                "type {}",
-                field_id, prepared_field->type()->ToString(), row_field->type()->ToString()));
+                "realtime primary-key transport field id {} type {} does not match row type {}",
+                field_id, transport_field->type()->ToString(), row_field->type()->ToString()));
         }
         result.push_back(field_index->second);
     }
     return result;
 }
 
-class PreparedReaderPlan {
+class RealtimePrimaryKeyReaderPlan {
  public:
-    static Result<std::shared_ptr<const PreparedReaderPlan>> Create(
-        const std::shared_ptr<arrow::Schema>& prepared_schema,
+    static Result<std::shared_ptr<const RealtimePrimaryKeyReaderPlan>> Create(
+        const std::shared_ptr<arrow::Schema>& transport_schema,
         const std::shared_ptr<arrow::Schema>& key_schema,
         const std::shared_ptr<arrow::Schema>& value_schema) {
         std::unordered_map<int32_t, int32_t> field_indexes;
-        field_indexes.reserve(prepared_schema->num_fields() -
-                              SpecialFields::kPreparedKeyValueValueStartIndex);
-        for (int32_t i = SpecialFields::kPreparedKeyValueValueStartIndex;
-             i < prepared_schema->num_fields(); ++i) {
+        field_indexes.reserve(transport_schema->num_fields() -
+                              RealtimePrimaryKeyLayout::kValueStartIndex);
+        for (int32_t i = RealtimePrimaryKeyLayout::kValueStartIndex;
+             i < transport_schema->num_fields(); ++i) {
             PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(
-                                                         prepared_schema->field(i)));
+                                                         transport_schema->field(i)));
             if (!field_indexes.emplace(field_id, i).second) {
-                return Status::Invalid(
-                    fmt::format("duplicate field id {} in prepared schema", field_id));
+                return Status::Invalid(fmt::format(
+                    "duplicate field id {} in realtime primary-key transport schema", field_id));
             }
         }
         PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> key_field_indexes,
-                               ResolveFieldIndexes(prepared_schema, field_indexes, key_schema));
+                               ResolveFieldIndexes(transport_schema, field_indexes, key_schema));
         PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> value_field_indexes,
-                               ResolveFieldIndexes(prepared_schema, field_indexes, value_schema));
-        return std::shared_ptr<const PreparedReaderPlan>(new PreparedReaderPlan(
-            prepared_schema, std::move(key_field_indexes), std::move(value_field_indexes)));
+                               ResolveFieldIndexes(transport_schema, field_indexes, value_schema));
+        return std::shared_ptr<const RealtimePrimaryKeyReaderPlan>(new RealtimePrimaryKeyReaderPlan(
+            transport_schema, std::move(key_field_indexes), std::move(value_field_indexes)));
     }
 
-    const std::shared_ptr<arrow::Schema>& PreparedSchema() const {
-        return prepared_schema_;
+    const std::shared_ptr<arrow::Schema>& TransportSchema() const {
+        return transport_schema_;
     }
 
     const std::vector<int32_t>& KeyFieldIndexes() const {
@@ -208,24 +208,25 @@ class PreparedReaderPlan {
     }
 
  private:
-    PreparedReaderPlan(const std::shared_ptr<arrow::Schema>& schema,
-                       std::vector<int32_t>&& key_indexes, std::vector<int32_t>&& value_indexes)
-        : prepared_schema_(schema),
+    RealtimePrimaryKeyReaderPlan(const std::shared_ptr<arrow::Schema>& schema,
+                                 std::vector<int32_t>&& key_indexes,
+                                 std::vector<int32_t>&& value_indexes)
+        : transport_schema_(schema),
           key_field_indexes_(std::move(key_indexes)),
           value_field_indexes_(std::move(value_indexes)) {}
 
-    const std::shared_ptr<arrow::Schema> prepared_schema_;
+    const std::shared_ptr<arrow::Schema> transport_schema_;
     const std::vector<int32_t> key_field_indexes_;
     const std::vector<int32_t> value_field_indexes_;
 };
 
-class PreparedKeyValueReader final : public KeyValueRecordReader {
+class RealtimePrimaryKeyReader final : public KeyValueRecordReader {
  public:
-    PreparedKeyValueReader(std::unique_ptr<BatchReader>&& reader,
-                           const std::shared_ptr<const PreparedReaderPlan>& plan,
-                           const std::optional<OffsetRange>& visible_offsets,
-                           const std::shared_ptr<MemoryPool>& pool,
-                           const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage)
+    RealtimePrimaryKeyReader(std::unique_ptr<BatchReader>&& reader,
+                             const std::shared_ptr<const RealtimePrimaryKeyReaderPlan>& plan,
+                             const std::optional<OffsetRange>& visible_offsets,
+                             const std::shared_ptr<MemoryPool>& pool,
+                             const std::shared_ptr<RealtimeOffsetCoverage>& offset_coverage)
         : reader_(std::move(reader)),
           plan_(plan),
           visible_offsets_(visible_offsets),
@@ -234,7 +235,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
     class Iterator final : public KeyValueRecordReader::Iterator {
      public:
-        explicit Iterator(PreparedKeyValueReader* reader) : reader_(reader) {}
+        explicit Iterator(RealtimePrimaryKeyReader* reader) : reader_(reader) {}
 
         Result<bool> HasNext() const override {
             return cursor_ < reader_->RowCount();
@@ -242,7 +243,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
         Result<KeyValue> Next() override {
             if (cursor_ >= reader_->RowCount()) {
-                return Status::Invalid("No more prepared key values in current iterator");
+                return Status::Invalid("No more realtime primary-key values in current iterator");
             }
             const int64_t row = reader_->RowAt(cursor_);
             std::shared_ptr<InternalRow> key =
@@ -257,7 +258,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         }
 
      private:
-        PreparedKeyValueReader* reader_;
+        RealtimePrimaryKeyReader* reader_;
         int64_t cursor_ = 0;
     };
 
@@ -278,13 +279,8 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
     Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatchImpl() {
         while (true) {
             ResetBatchState();
-            BatchReader::ReadBatchWithBitmap batch_with_bitmap;
-            if (visible_offsets_.has_value()) {
-                PAIMON_ASSIGN_OR_RAISE(batch_with_bitmap, reader_->NextBatchWithBitmap());
-            } else {
-                PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
-                batch_with_bitmap.first = std::move(batch);
-            }
+            PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap,
+                                   reader_->NextBatchWithBitmap());
             if (BatchReader::IsEofBatch(batch_with_bitmap)) {
                 if (offset_coverage_ && !offset_coverage_finished_) {
                     offset_coverage_finished_ = true;
@@ -297,23 +293,24 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
             PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> arrow_array,
                                               arrow::ImportArray(c_array.get(), c_schema.get()));
             if (!arrow_array || arrow_array->type_id() != arrow::Type::STRUCT) {
-                return Status::Invalid("cannot cast prepared batch to StructArray");
+                return Status::Invalid(
+                    "cannot cast realtime primary-key transport batch to StructArray");
             }
             std::shared_ptr<arrow::StructArray> data_batch =
                 checked_pointer_cast<arrow::StructArray>(arrow_array);
-            PAIMON_RETURN_NOT_OK(ValidatePreparedBatch(data_batch));
+            PAIMON_RETURN_NOT_OK(ValidateTransportBatch(data_batch));
 
             std::shared_ptr<arrow::NumericArray<arrow::Int64Type>> offset_array =
                 checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
-                    data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex));
+                    data_batch->field(RealtimePrimaryKeyLayout::kRealtimeOffsetIndex));
             if (offset_coverage_) {
                 PAIMON_RETURN_NOT_OK(offset_coverage_->Add(*offset_array));
             }
 
             row_kind_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int8Type>>(
-                data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex));
+                data_batch->field(RealtimePrimaryKeyLayout::kValueKindIndex));
             sequence_number_array_ = checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(
-                data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex));
+                data_batch->field(RealtimePrimaryKeyLayout::kSequenceNumberIndex));
             arrow::ArrayVector key_fields;
             key_fields.reserve(plan_->KeyFieldIndexes().size());
             for (int32_t index : plan_->KeyFieldIndexes()) {
@@ -336,53 +333,48 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
         }
     }
 
-    Status ValidatePreparedBatch(const std::shared_ptr<arrow::StructArray>& data_batch) const {
-        if (data_batch->num_fields() != plan_->PreparedSchema()->num_fields()) {
+    Status ValidateTransportBatch(const std::shared_ptr<arrow::StructArray>& data_batch) const {
+        if (data_batch->num_fields() != plan_->TransportSchema()->num_fields()) {
             return Status::Invalid(fmt::format(
-                "prepared batch field count {} does not match prepared schema field count {}",
-                data_batch->num_fields(), plan_->PreparedSchema()->num_fields()));
+                "realtime primary-key transport batch field count {} does not match schema field "
+                "count {}",
+                data_batch->num_fields(), plan_->TransportSchema()->num_fields()));
         }
         const arrow::FieldVector& batch_fields = data_batch->type()->fields();
         for (int32_t i = 0; i < data_batch->num_fields(); ++i) {
-            if (!batch_fields[i]->Equals(plan_->PreparedSchema()->field(i), true)) {
+            if (!batch_fields[i]->Equals(plan_->TransportSchema()->field(i), true)) {
                 return Status::Invalid(fmt::format(
-                    "prepared batch field {} does not match declared prepared schema", i));
+                    "realtime primary-key transport batch field {} does not match declared schema",
+                    i));
             }
         }
-        if (data_batch->field(SpecialFields::kPreparedKeyValueValueKindIndex)->null_count() != 0 ||
-            data_batch->field(SpecialFields::kPreparedKeyValueSequenceNumberIndex)->null_count() !=
-                0 ||
-            data_batch->field(SpecialFields::kPreparedKeyValueRealtimeOffsetIndex)->null_count() !=
-                0) {
-            return Status::Invalid("prepared transport columns must not contain nulls");
+        if (data_batch->field(RealtimePrimaryKeyLayout::kValueKindIndex)->null_count() != 0 ||
+            data_batch->field(RealtimePrimaryKeyLayout::kSequenceNumberIndex)->null_count() != 0 ||
+            data_batch->field(RealtimePrimaryKeyLayout::kRealtimeOffsetIndex)->null_count() != 0) {
+            return Status::Invalid("realtime primary-key transport columns must not contain nulls");
         }
         return Status::OK();
     }
 
     Result<bool> SelectRows(const arrow::Int64Array& offsets, RoaringBitmap32&& selection) {
         for (auto iter = selection.Begin(); iter != selection.End(); ++iter) {
-            const int32_t row = *iter;
-            if (row < 0 || row >= offsets.length()) {
+            const uint32_t row = *iter;
+            if (static_cast<int64_t>(row) >= offsets.length()) {
                 return Status::Invalid(
-                    fmt::format("selected row id {} is out of bounds for prepared batch length {}",
+                    fmt::format("selected row id {} is out of bounds for realtime primary-key "
+                                "transport batch length {}",
                                 row, offsets.length()));
             }
         }
-        if (visible_offsets_.has_value() && selection.Cardinality() != offsets.length()) {
+        if (selection.Cardinality() != offsets.length()) {
             return Status::Invalid(
-                "PK real-time store query reader bitmap must cover every raw mutation");
-        }
-        if (!visible_offsets_.has_value()) {
-            selected_rows_.reserve(offsets.length());
-            for (int64_t row = 0; row < offsets.length(); ++row) {
-                selected_rows_.push_back(row);
-            }
-            return true;
+                "PK real-time store reader bitmap must cover every raw "
+                "transport row");
         }
-        for (auto iter = selection.Begin(); iter != selection.End(); ++iter) {
-            const int32_t row = *iter;
-            const int64_t offset = offsets.Value(row);
-            if (offset >= visible_offsets_->begin && offset < visible_offsets_->end) {
+        selected_rows_.reserve(offsets.length());
+        for (int64_t row = 0; row < offsets.length(); ++row) {
+            if (!visible_offsets_.has_value() || (offsets.Value(row) >= visible_offsets_->begin &&
+                                                  offsets.Value(row) < visible_offsets_->end)) {
                 selected_rows_.push_back(row);
             }
         }
@@ -407,7 +399,7 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
  private:
     std::unique_ptr<BatchReader> reader_;
-    std::shared_ptr<const PreparedReaderPlan> plan_;
+    std::shared_ptr<const RealtimePrimaryKeyReaderPlan> plan_;
     std::optional<OffsetRange> visible_offsets_;
     std::shared_ptr<MemoryPool> pool_;
     std::shared_ptr<RealtimeOffsetCoverage> offset_coverage_;
@@ -421,31 +413,39 @@ class PreparedKeyValueReader final : public KeyValueRecordReader {
 
 }  // namespace
 
-Status PreparedKeyValueReaderFactory::ValidateTransportSchema(
-    const std::shared_ptr<arrow::Schema>& prepared_schema) {
-    if (!prepared_schema ||
-        prepared_schema->num_fields() < SpecialFields::kPreparedKeyValueValueStartIndex) {
-        return Status::Invalid("prepared schema must contain realtime transport fields");
+std::shared_ptr<arrow::Schema> RealtimePrimaryKeyLayout::CreateSchema(
+    const std::vector<std::shared_ptr<arrow::Field>>& value_fields) {
+    arrow::FieldVector fields = {
+        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
+            ->WithNullable(false),
+        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
+    fields.insert(fields.end(), value_fields.begin(), value_fields.end());
+    return arrow::schema(std::move(fields));
+}
+
+Status RealtimePrimaryKeyLayout::ValidateSchema(
+    const std::shared_ptr<arrow::Schema>& transport_schema) {
+    if (!transport_schema || transport_schema->num_fields() < kValueStartIndex) {
+        return Status::Invalid(
+            "realtime primary-key transport schema must contain transport fields");
     }
-    PAIMON_RETURN_NOT_OK(CheckPreparedField(prepared_schema,
-                                            SpecialFields::kPreparedKeyValueValueKindIndex,
-                                            SpecialFields::ValueKind()));
-    PAIMON_RETURN_NOT_OK(CheckPreparedField(prepared_schema,
-                                            SpecialFields::kPreparedKeyValueSequenceNumberIndex,
-                                            SpecialFields::SequenceNumber()));
-    PAIMON_RETURN_NOT_OK(CheckPreparedField(prepared_schema,
-                                            SpecialFields::kPreparedKeyValueRealtimeOffsetIndex,
-                                            SpecialFields::RealtimeOffset()));
+    PAIMON_RETURN_NOT_OK(
+        CheckTransportField(transport_schema, kValueKindIndex, SpecialFields::ValueKind()));
+    PAIMON_RETURN_NOT_OK(CheckTransportField(transport_schema, kSequenceNumberIndex,
+                                             SpecialFields::SequenceNumber()));
+    PAIMON_RETURN_NOT_OK(CheckTransportField(transport_schema, kRealtimeOffsetIndex,
+                                             SpecialFields::RealtimeOffset()));
     return Status::OK();
 }
 
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
-PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchReader>>&& readers,
-                                              const std::shared_ptr<arrow::Schema>& prepared_schema,
-                                              const OffsetRange& visible_offsets,
-                                              const std::shared_ptr<arrow::Schema>& key_schema,
-                                              const std::shared_ptr<arrow::Schema>& value_schema,
-                                              const std::shared_ptr<MemoryPool>& memory_pool) {
+RealtimePrimaryKeyReaderFactory::CreateForQuery(
+    std::vector<std::unique_ptr<BatchReader>>&& readers,
+    const std::shared_ptr<arrow::Schema>& transport_schema, const OffsetRange& visible_offsets,
+    const std::shared_ptr<arrow::Schema>& key_schema,
+    const std::shared_ptr<arrow::Schema>& value_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool) {
     std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers;
     ScopeGuard remaining_raw_readers_guard([&readers]() { CloseReaders(readers); });
     if (readers.empty() && visible_offsets.begin < visible_offsets.end) {
@@ -457,15 +457,16 @@ PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchR
             return Status::Invalid("PK real-time store returned a null query reader");
         }
     }
-    PAIMON_RETURN_NOT_OK(ValidateTransportSchema(prepared_schema));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const PreparedReaderPlan> plan,
-                           PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema));
+    PAIMON_RETURN_NOT_OK(RealtimePrimaryKeyLayout::ValidateSchema(transport_schema));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<const RealtimePrimaryKeyReaderPlan> plan,
+        RealtimePrimaryKeyReaderPlan::Create(transport_schema, key_schema, value_schema));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
                            RealtimeOffsetCoverage::Create(visible_offsets, readers.size(),
                                                           /*allow_committed_prefix=*/true));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        adapted_readers.push_back(std::make_unique<PreparedKeyValueReader>(
+        adapted_readers.push_back(std::make_unique<RealtimePrimaryKeyReader>(
             std::move(reader), plan, visible_offsets, memory_pool, offset_coverage));
     }
     remaining_raw_readers_guard.Release();
@@ -473,9 +474,9 @@ PreparedKeyValueReaderFactory::CreateForQuery(std::vector<std::unique_ptr<BatchR
 }
 
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>>
-PreparedKeyValueReaderFactory::CreateForCommit(
+RealtimePrimaryKeyReaderFactory::CreateForCommit(
     std::vector<std::unique_ptr<BatchReader>>&& readers,
-    const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
+    const std::shared_ptr<arrow::Schema>& transport_schema, const OffsetRange& sealed_offsets,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
@@ -490,15 +491,16 @@ PreparedKeyValueReaderFactory::CreateForCommit(
             return Status::Invalid("PK real-time store returned a null commit reader");
         }
     }
-    PAIMON_RETURN_NOT_OK(ValidateTransportSchema(prepared_schema));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const PreparedReaderPlan> plan,
-                           PreparedReaderPlan::Create(prepared_schema, key_schema, value_schema));
+    PAIMON_RETURN_NOT_OK(RealtimePrimaryKeyLayout::ValidateSchema(transport_schema));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<const RealtimePrimaryKeyReaderPlan> plan,
+        RealtimePrimaryKeyReaderPlan::Create(transport_schema, key_schema, value_schema));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetCoverage> offset_coverage,
                            RealtimeOffsetCoverage::Create(sealed_offsets, readers.size(),
                                                           /*allow_committed_prefix=*/false));
     adapted_readers.reserve(readers.size());
     for (std::unique_ptr<BatchReader>& reader : readers) {
-        adapted_readers.push_back(std::make_unique<PreparedKeyValueReader>(
+        adapted_readers.push_back(std::make_unique<RealtimePrimaryKeyReader>(
             std::move(reader), plan, std::nullopt, memory_pool, offset_coverage));
     }
     remaining_raw_readers_guard.Release();
diff --git a/src/paimon/core/realtime/prepared_key_value_reader.h b/src/paimon/core/realtime/realtime_primary_key_reader.h
similarity index 63%
rename from src/paimon/core/realtime/prepared_key_value_reader.h
rename to src/paimon/core/realtime/realtime_primary_key_reader.h
index bb03e2ad4..d175c3b63 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader.h
+++ b/src/paimon/core/realtime/realtime_primary_key_reader.h
@@ -18,6 +18,7 @@
 
 #pragma once
 
+#include <cstdint>
 #include <memory>
 #include <vector>
 
@@ -30,23 +31,38 @@ namespace paimon {
 class BatchReader;
 class MemoryPool;
 
-class PreparedKeyValueReaderFactory {
+/// Defines the Arrow field layout for PK realtime transport batches.
+class RealtimePrimaryKeyLayout {
  public:
-    PreparedKeyValueReaderFactory() = delete;
-    ~PreparedKeyValueReaderFactory() = delete;
+    RealtimePrimaryKeyLayout() = delete;
+    ~RealtimePrimaryKeyLayout() = delete;
 
-    static Status ValidateTransportSchema(const std::shared_ptr<arrow::Schema>& prepared_schema);
+    static constexpr int32_t kValueKindIndex = 0;
+    static constexpr int32_t kSequenceNumberIndex = 1;
+    static constexpr int32_t kRealtimeOffsetIndex = 2;
+    static constexpr int32_t kValueStartIndex = 3;
+
+    static std::shared_ptr<arrow::Schema> CreateSchema(
+        const std::vector<std::shared_ptr<arrow::Field>>& value_fields);
+
+    static Status ValidateSchema(const std::shared_ptr<arrow::Schema>& transport_schema);
+};
+
+class RealtimePrimaryKeyReaderFactory {
+ public:
+    RealtimePrimaryKeyReaderFactory() = delete;
+    ~RealtimePrimaryKeyReaderFactory() = delete;
 
     static Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateForQuery(
         std::vector<std::unique_ptr<BatchReader>>&& readers,
-        const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& visible_offsets,
+        const std::shared_ptr<arrow::Schema>& transport_schema, const OffsetRange& visible_offsets,
         const std::shared_ptr<arrow::Schema>& key_schema,
         const std::shared_ptr<arrow::Schema>& value_schema,
         const std::shared_ptr<MemoryPool>& memory_pool);
 
     static Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateForCommit(
         std::vector<std::unique_ptr<BatchReader>>&& readers,
-        const std::shared_ptr<arrow::Schema>& prepared_schema, const OffsetRange& sealed_offsets,
+        const std::shared_ptr<arrow::Schema>& transport_schema, const OffsetRange& sealed_offsets,
         const std::shared_ptr<arrow::Schema>& key_schema,
         const std::shared_ptr<arrow::Schema>& value_schema,
         const std::shared_ptr<MemoryPool>& memory_pool);
diff --git a/src/paimon/core/realtime/prepared_key_value_reader_test.cpp b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
similarity index 56%
rename from src/paimon/core/realtime/prepared_key_value_reader_test.cpp
rename to src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
index c0515df72..beb1a6435 100644
--- a/src/paimon/core/realtime/prepared_key_value_reader_test.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-#include "paimon/core/realtime/prepared_key_value_reader.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 
 #include <cstdint>
 #include <memory>
@@ -49,40 +49,34 @@ std::shared_ptr<arrow::Field> MakeField(const std::string& name,
         DataField(field_id, arrow::field(name, type, nullable)));
 }
 
-std::shared_ptr<arrow::Schema> MakePreparedSchema(const arrow::FieldVector& value_fields) {
-    arrow::FieldVector prepared_fields = {
-        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-            ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
-    prepared_fields.insert(prepared_fields.end(), value_fields.begin(), value_fields.end());
-    return arrow::schema(prepared_fields);
+std::shared_ptr<arrow::Schema> MakeTransportSchema(const arrow::FieldVector& value_fields) {
+    return RealtimePrimaryKeyLayout::CreateSchema(value_fields);
 }
 
-Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedQueryReaderForTest(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+Result<std::unique_ptr<KeyValueRecordReader>> CreateRealtimePrimaryKeyQueryReaderForTest(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& transport_schema,
     const OffsetRange& visible_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     std::vector<std::unique_ptr<BatchReader>> readers;
     readers.push_back(std::move(reader));
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
-                           PreparedKeyValueReaderFactory::CreateForQuery(
-                               std::move(readers), prepared_schema, visible_offsets, key_schema,
+                           RealtimePrimaryKeyReaderFactory::CreateForQuery(
+                               std::move(readers), transport_schema, visible_offsets, key_schema,
                                value_schema, memory_pool));
     return std::move(adapted_readers[0]);
 }
 
-Result<std::unique_ptr<KeyValueRecordReader>> CreatePreparedCommitReaderForTest(
-    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& prepared_schema,
+Result<std::unique_ptr<KeyValueRecordReader>> CreateRealtimePrimaryKeyCommitReaderForTest(
+    std::unique_ptr<BatchReader>&& reader, const std::shared_ptr<arrow::Schema>& transport_schema,
     const OffsetRange& sealed_offsets, const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     std::vector<std::unique_ptr<BatchReader>> readers;
     readers.push_back(std::move(reader));
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> adapted_readers,
-                           PreparedKeyValueReaderFactory::CreateForCommit(
-                               std::move(readers), prepared_schema, sealed_offsets, key_schema,
+                           RealtimePrimaryKeyReaderFactory::CreateForCommit(
+                               std::move(readers), transport_schema, sealed_offsets, key_schema,
                                value_schema, memory_pool));
     return std::move(adapted_readers[0]);
 }
@@ -142,21 +136,69 @@ class MalformedBitmapBatchReader : public BatchReader {
 
 }  // namespace
 
-class PreparedKeyValueReaderTest : public testing::Test {
+class RealtimePrimaryKeyReaderTest : public testing::Test {
  protected:
     std::shared_ptr<MemoryPool> pool_ = GetDefaultPool();
 };
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryAllowsCommittedPrefix) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestTransportSchemaLayout) {
+    arrow::FieldVector value_fields = {arrow::field("key", arrow::int64(), false),
+                                       arrow::field("value", arrow::utf8())};
+    std::shared_ptr<arrow::Schema> schema = MakeTransportSchema(value_fields);
+
+    ASSERT_EQ(RealtimePrimaryKeyLayout::kValueKindIndex, 0);
+    ASSERT_EQ(RealtimePrimaryKeyLayout::kSequenceNumberIndex, 1);
+    ASSERT_EQ(RealtimePrimaryKeyLayout::kRealtimeOffsetIndex, 2);
+    ASSERT_EQ(RealtimePrimaryKeyLayout::kValueStartIndex, 3);
+    ASSERT_EQ(schema->field(0)->name(), "_VALUE_KIND");
+    ASSERT_EQ(schema->field(1)->name(), "_SEQUENCE_NUMBER");
+    ASSERT_EQ(schema->field(2)->name(), "_REALTIME_OFFSET");
+    ASSERT_EQ(schema->field(3)->name(), "key");
+    ASSERT_EQ(schema->field(4)->name(), "value");
+    ASSERT_FALSE(schema->field(0)->nullable());
+    ASSERT_FALSE(schema->field(1)->nullable());
+    ASSERT_EQ(schema->field(2)->nullable(), SpecialFields::RealtimeOffset().Nullable());
+    ASSERT_FALSE(schema->field(3)->nullable());
+    ASSERT_TRUE(schema->field(4)->nullable());
+}
+
+TEST_F(RealtimePrimaryKeyReaderTest, TestTransportSchemaValidation) {
+    const std::shared_ptr<arrow::Schema> valid = MakeTransportSchema({});
+    std::vector<arrow::FieldVector> invalid_fields;
+
+    arrow::FieldVector wrong_type = valid->fields();
+    wrong_type[0] = DataField::ConvertDataFieldToArrowField(
+                        DataField(SpecialFields::ValueKind().Id(),
+                                  arrow::field("_VALUE_KIND", arrow::int32(), false)))
+                        ->WithNullable(false);
+    invalid_fields.push_back(std::move(wrong_type));
+
+    arrow::FieldVector nullable_sequence = valid->fields();
+    nullable_sequence[1] = nullable_sequence[1]->WithNullable(true);
+    invalid_fields.push_back(std::move(nullable_sequence));
+
+    arrow::FieldVector wrong_offset_id = valid->fields();
+    wrong_offset_id[2] = DataField::ConvertDataFieldToArrowField(
+                             DataField(99, arrow::field("_REALTIME_OFFSET", arrow::int64(), false)))
+                             ->WithNullable(false);
+    invalid_fields.push_back(std::move(wrong_offset_id));
+
+    for (const arrow::FieldVector& fields : invalid_fields) {
+        ASSERT_NOK_WITH_MSG(RealtimePrimaryKeyLayout::ValidateSchema(arrow::schema(fields)),
+                            "transport schema field");
+    }
+}
+
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryAllowsCommittedPrefix) {
     std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
                                            DataField(1, arrow::field("v0", arrow::int32()))};
     std::shared_ptr<arrow::Schema> value_schema =
         DataField::ConvertDataFieldsToArrowSchema(value_fields);
     std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    auto transport_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([
         [0, 100, 0, 1, 10],
         [0, 101, 1, 2, 20],
         [0, 102, 2, 4, 40],
@@ -166,10 +208,10 @@ TEST_F(PreparedKeyValueReaderTest, TestQueryAllowsCommittedPrefix) {
 
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
     batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 2));
+        std::make_unique<MockFileBatchReader>(transport_array, transport_type, 2));
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(2, 4),
+                         RealtimePrimaryKeyReaderFactory::CreateForQuery(
+                             std::move(batch_readers), transport_schema, OffsetRange(2, 4),
                              key_schema, value_schema, pool_));
     ASSERT_EQ(1, readers.size());
     ASSERT_OK_AND_ASSIGN(
@@ -185,46 +227,46 @@ TEST_F(PreparedKeyValueReaderTest, TestQueryAllowsCommittedPrefix) {
     KeyValueChecker::CheckResult(expected, results, 1, 2);
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsNegativeOffset) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryRejectsNegativeOffset) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, -1, 1]])")
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([[0, 10, -1, 1]])")
             .ValueOrDie();
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+        CreateRealtimePrimaryKeyQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(transport_array, transport_type,
                                                   /*read_batch_size=*/1),
-            prepared_schema, OffsetRange(1, 2), value_schema, value_schema, pool_));
+            transport_schema, OffsetRange(1, 2), value_schema, value_schema, pool_));
 
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "reader offset must be non-negative");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryOffsetCoverageAcrossReadersAndBatches) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryOffsetCoverageAcrossReadersAndBatches) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
     std::shared_ptr<arrow::Array> first_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
                                                   R"([[0, 10, 2, 1], [0, 11, 0, 2]])")
             .ValueOrDie();
     std::shared_ptr<arrow::Array> second_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
                                                   R"([[0, 12, 3, 3], [0, 13, 1, 4]])")
             .ValueOrDie();
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
     batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+        std::make_unique<MockFileBatchReader>(first_array, transport_type, /*read_batch_size=*/1));
     batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+        std::make_unique<MockFileBatchReader>(second_array, transport_type, /*read_batch_size=*/1));
 
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
+                         RealtimePrimaryKeyReaderFactory::CreateForQuery(
+                             std::move(batch_readers), transport_schema, OffsetRange(0, 4),
                              value_schema, value_schema, pool_));
     int64_t row_count = 0;
     for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
@@ -237,21 +279,21 @@ TEST_F(PreparedKeyValueReaderTest, TestQueryOffsetCoverageAcrossReadersAndBatche
     ASSERT_EQ(4, row_count);
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsMissingVisibleOffset) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryRejectsMissingVisibleOffset) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
                                                   R"([[0, 10, 0, 1], [0, 11, 2, 2]])")
             .ValueOrDie();
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+        CreateRealtimePrimaryKeyQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(transport_array, transport_type,
                                                   /*read_batch_size=*/1),
-            prepared_schema, OffsetRange(0, 3), value_schema, value_schema, pool_));
+            transport_schema, OffsetRange(0, 3), value_schema, value_schema, pool_));
 
     ASSERT_NOK_WITH_MSG(
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
@@ -259,25 +301,26 @@ TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsMissingVisibleOffset) {
         "query readers did not cover the visible range");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsDuplicateVisibleOffset) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryRejectsDuplicateVisibleOffset) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
     std::shared_ptr<arrow::Array> first_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([[0, 10, 0, 1]])")
+            .ValueOrDie();
     std::shared_ptr<arrow::Array> second_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
                                                   R"([[0, 11, 1, 2], [0, 12, 1, 3]])")
             .ValueOrDie();
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
     batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+        std::make_unique<MockFileBatchReader>(first_array, transport_type, /*read_batch_size=*/1));
     batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+        std::make_unique<MockFileBatchReader>(second_array, transport_type, /*read_batch_size=*/1));
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 2),
+                         RealtimePrimaryKeyReaderFactory::CreateForQuery(
+                             std::move(batch_readers), transport_schema, OffsetRange(0, 2),
                              value_schema, value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> first_rows,
@@ -290,109 +333,134 @@ TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsDuplicateVisibleOffset) {
                         "query readers did not cover the visible range");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsEmptyEofForVisibleRange) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryRejectsEmptyEofForVisibleRange) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([])").ValueOrDie();
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([])").ValueOrDie();
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(
-            std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+        CreateRealtimePrimaryKeyQueryReaderForTest(
+            std::make_unique<MockFileBatchReader>(transport_array, transport_type,
                                                   /*read_batch_size=*/1),
-            prepared_schema, OffsetRange(0, 1), value_schema, value_schema, pool_));
+            transport_schema, OffsetRange(0, 1), value_schema, value_schema, pool_));
 
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "query readers did not cover the visible range");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryRejectsEmptyReadersForVisibleRange) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryRejectsEmptyReadersForVisibleRange) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
 
     ASSERT_NOK_WITH_MSG(
-        PreparedKeyValueReaderFactory::CreateForQuery(std::move(batch_readers), prepared_schema,
-                                                      OffsetRange(0, 1), value_schema, value_schema,
-                                                      pool_),
+        RealtimePrimaryKeyReaderFactory::CreateForQuery(std::move(batch_readers), transport_schema,
+                                                        OffsetRange(0, 1), value_schema,
+                                                        value_schema, pool_),
         "PK real-time store returned no query readers for a non-empty visible range");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestQueryAllowsEmptyReadersForEmptyVisibleRange) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryAllowsEmptyReadersForEmptyVisibleRange) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
 
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForQuery(
-                             std::move(batch_readers), prepared_schema, OffsetRange(1, 1),
+                         RealtimePrimaryKeyReaderFactory::CreateForQuery(
+                             std::move(batch_readers), transport_schema, OffsetRange(1, 1),
                              value_schema, value_schema, pool_));
     ASSERT_TRUE(readers.empty());
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderBitmapBounds) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryBitmapBounds) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1]])").ValueOrDie();
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([[0, 10, 0, 1]])")
+            .ValueOrDie();
     auto batch_reader = std::make_unique<MalformedBitmapBatchReader>(
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, /*batch_size=*/1),
+        std::make_unique<MockFileBatchReader>(transport_array, transport_type, /*batch_size=*/1),
         /*row_id=*/1);
 
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreateRealtimePrimaryKeyQueryReaderForTest(
+                             std::move(batch_reader), transport_schema, OffsetRange(0, 1),
+                             value_schema, value_schema, pool_));
     Result<std::vector<KeyValue>> result =
         ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
                                                    KeyValueRecordReader::Iterator>(reader.get());
     ASSERT_TRUE(result.status().IsInvalid());
-    ASSERT_NOK_WITH_MSG(result, "selected row id 1 is out of bounds for prepared batch length 1");
+    ASSERT_NOK_WITH_MSG(result,
+                        "selected row id 1 is out of bounds for realtime primary-key transport "
+                        "batch length 1");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestPreparedQueryReaderRejectsPartialBitmap) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryRejectsPartialBitmap) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
                                                   R"([[0, 10, 0, 1], [0, 11, 1, 2]])")
             .ValueOrDie();
     RoaringBitmap32 partial_bitmap;
     partial_bitmap.Add(0);
     auto batch_reader = std::make_unique<MockFileBatchReader>(
-        prepared_array, prepared_type, partial_bitmap, /*read_batch_size=*/2);
+        transport_array, transport_type, partial_bitmap, /*read_batch_size=*/2);
     batch_reader->EnableRandomizeBatchSize(false);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 2), value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreateRealtimePrimaryKeyQueryReaderForTest(
+                             std::move(batch_reader), transport_schema, OffsetRange(0, 2),
+                             value_schema, value_schema, pool_));
 
-    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "must cover every raw mutation");
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "must cover every raw transport row");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderQueryProjection) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestCommitRejectsPartialBitmap) {
+    std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
+    std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
+                                                  R"([[0, 10, 0, 1], [0, 11, 1, 2]])")
+            .ValueOrDie();
+    RoaringBitmap32 partial_bitmap;
+    partial_bitmap.Add(0);
+    auto batch_reader = std::make_unique<MockFileBatchReader>(
+        transport_array, transport_type, partial_bitmap, /*read_batch_size=*/2);
+    batch_reader->EnableRandomizeBatchSize(false);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreateRealtimePrimaryKeyCommitReaderForTest(
+                             std::move(batch_reader), transport_schema, OffsetRange(0, 2),
+                             value_schema, value_schema, pool_));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "must cover every raw transport row");
+}
+
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryProjection) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Field> extra = MakeField("extra", arrow::int32(), 1);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, extra});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([[0, 10, 0, 1, 2]])")
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key, extra});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    auto transport_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([[0, 10, 0, 1, 2]])")
             .ValueOrDie());
 
     auto query_batch_reader =
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> query_reader,
-        CreatePreparedQueryReaderForTest(std::move(query_batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
+        std::make_unique<MockFileBatchReader>(transport_array, transport_type, 1);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> query_reader,
+                         CreateRealtimePrimaryKeyQueryReaderForTest(
+                             std::move(query_batch_reader), transport_schema, OffsetRange(0, 1),
+                             value_schema, value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> query_results,
         (ReadResultCollector::CollectKeyValueResult<
@@ -402,28 +470,28 @@ TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderQueryProjection) {
     ASSERT_EQ(query_results[0].value->GetInt(0), 1);
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestCommitOffsetCoverage) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestCommitOffsetCoverage) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
     std::shared_ptr<arrow::Array> first_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
                                                   R"([[0, 10, 2, 1], [0, 11, 0, 3]])")
             .ValueOrDie();
     std::shared_ptr<arrow::Array> second_array =
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type,
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type,
                                                   R"([[0, 12, 1, 2], [0, 13, 3, 4]])")
             .ValueOrDie();
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
     batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(first_array, prepared_type, /*read_batch_size=*/1));
+        std::make_unique<MockFileBatchReader>(first_array, transport_type, /*read_batch_size=*/1));
     batch_readers.push_back(
-        std::make_unique<MockFileBatchReader>(second_array, prepared_type, /*read_batch_size=*/1));
+        std::make_unique<MockFileBatchReader>(second_array, transport_type, /*read_batch_size=*/1));
 
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForCommit(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 4),
+                         RealtimePrimaryKeyReaderFactory::CreateForCommit(
+                             std::move(batch_readers), transport_schema, OffsetRange(0, 4),
                              value_schema, value_schema, pool_));
     int64_t row_count = 0;
     for (const std::unique_ptr<KeyValueRecordReader>& reader : readers) {
@@ -436,34 +504,34 @@ TEST_F(PreparedKeyValueReaderTest, TestCommitOffsetCoverage) {
     ASSERT_EQ(4, row_count);
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestCommitRejectsEmptyReaders) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestCommitRejectsEmptyReaders) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
 
-    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForCommit(
-                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
+    ASSERT_NOK_WITH_MSG(RealtimePrimaryKeyReaderFactory::CreateForCommit(
+                            std::move(batch_readers), transport_schema, OffsetRange(0, 1),
                             value_schema, value_schema, pool_),
                         "PK real-time store returned no commit readers for a sealed segment");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestRejectsDuplicateCommitOffset) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestRejectsDuplicateCommitOffset) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
         arrow::ipc::internal::json::ArrayFromJSON(
-            prepared_type, R"([[0, 10, 0, 1], [0, 11, 0, 2], [0, 12, 2, 3]])")
+            transport_type, R"([[0, 10, 0, 1], [0, 11, 0, 2], [0, 12, 2, 3]])")
             .ValueOrDie();
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
-    batch_readers.push_back(std::make_unique<MockFileBatchReader>(prepared_array, prepared_type,
+    batch_readers.push_back(std::make_unique<MockFileBatchReader>(transport_array, transport_type,
                                                                   /*read_batch_size=*/1));
 
     ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
-                         PreparedKeyValueReaderFactory::CreateForCommit(
-                             std::move(batch_readers), prepared_schema, OffsetRange(0, 3),
+                         RealtimePrimaryKeyReaderFactory::CreateForCommit(
+                             std::move(batch_readers), transport_schema, OffsetRange(0, 3),
                              value_schema, value_schema, pool_));
     ASSERT_NOK_WITH_MSG((ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
                                                                     KeyValueRecordReader::Iterator>(
@@ -471,30 +539,30 @@ TEST_F(PreparedKeyValueReaderTest, TestRejectsDuplicateCommitOffset) {
                         "did not cover the sealed range");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestBadCommitBatch) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestBadCommitBatch) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Field> value = MakeField("value", arrow::int32(), 1);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key, value});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key, value});
-    std::shared_ptr<arrow::Schema> actual_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key, value});
+    std::shared_ptr<arrow::Schema> actual_schema = MakeTransportSchema({key});
     std::shared_ptr<arrow::DataType> actual_type = arrow::struct_(actual_schema->fields());
     std::shared_ptr<arrow::Array> actual =
         arrow::ipc::internal::json::ArrayFromJSON(actual_type, R"([[0, 10, 0, 1]])").ValueOrDie();
 
     auto batch_reader = std::make_unique<MockFileBatchReader>(actual, actual_type, 1);
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
-                         CreatePreparedCommitReaderForTest(std::move(batch_reader), prepared_schema,
-                                                           OffsetRange(0, 1), arrow::schema({key}),
-                                                           value_schema, pool_));
+                         CreateRealtimePrimaryKeyCommitReaderForTest(
+                             std::move(batch_reader), transport_schema, OffsetRange(0, 1),
+                             arrow::schema({key}), value_schema, pool_));
     ASSERT_NOK_WITH_MSG(reader->NextBatch(), "field count");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderSafeDecode) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestSafeDecode) {
     std::shared_ptr<arrow::Field> key = MakeField("key", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> value_schema = arrow::schema({key});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema({key});
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema({key});
 
-    arrow::FieldVector invalid_fields = prepared_schema->fields();
+    arrow::FieldVector invalid_fields = transport_schema->fields();
     invalid_fields[0] = invalid_fields[0]->WithName("wrong_value_kind");
     invalid_fields[3] = MakeField("wrong_key", arrow::int32(), 99);
     std::shared_ptr<arrow::DataType> invalid_type = arrow::struct_(invalid_fields);
@@ -502,17 +570,17 @@ TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderSafeDecode) {
         arrow::ipc::internal::json::ArrayFromJSON(invalid_type, R"([[0, 10, 0, 1]])").ValueOrDie());
 
     auto batch_reader = std::make_unique<MockFileBatchReader>(invalid_array, invalid_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), value_schema, value_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreateRealtimePrimaryKeyQueryReaderForTest(
+                             std::move(batch_reader), transport_schema, OffsetRange(0, 1),
+                             value_schema, value_schema, pool_));
     ASSERT_NOK_WITH_MSG(
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
                                                     KeyValueRecordReader::Iterator>(reader.get())),
-        "prepared batch field");
+        "transport batch field");
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderNestedValues) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestNestedValues) {
     std::shared_ptr<arrow::Field> id = MakeField("id", arrow::int32(), 0);
     std::shared_ptr<arrow::Schema> key_schema = arrow::schema({id});
     std::shared_ptr<arrow::Field> query_item_b = MakeField("renamed_b", arrow::int32(), 11);
@@ -532,20 +600,20 @@ TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderNestedValues) {
                   arrow::map(arrow::struct_({query_key_right, query_key_left}), arrow::int32()), 4);
     std::shared_ptr<arrow::Schema> query_value_schema =
         arrow::schema({id, query_items, query_attrs, query_keyed_values});
-    std::shared_ptr<arrow::Schema> prepared_schema =
-        MakePreparedSchema(query_value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    std::shared_ptr<arrow::Array> prepared_array =
+    std::shared_ptr<arrow::Schema> transport_schema =
+        MakeTransportSchema(query_value_schema->fields());
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
         arrow::ipc::internal::json::ArrayFromJSON(
-            prepared_type,
+            transport_type,
             R"([[0, 10, 0, 1, [[200, 100], [400, 300]], [["k1", [8, 7]], ["k2", [10, 9]]], [[[12, 11], 13], [[22, 21], 23]]]])")
             .ValueOrDie();
 
-    auto batch_reader = std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1);
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<KeyValueRecordReader> reader,
-        CreatePreparedQueryReaderForTest(std::move(batch_reader), prepared_schema,
-                                         OffsetRange(0, 1), key_schema, query_value_schema, pool_));
+    auto batch_reader = std::make_unique<MockFileBatchReader>(transport_array, transport_type, 1);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreateRealtimePrimaryKeyQueryReaderForTest(
+                             std::move(batch_reader), transport_schema, OffsetRange(0, 1),
+                             key_schema, query_value_schema, pool_));
     ASSERT_OK_AND_ASSIGN(
         std::vector<KeyValue> results,
         (ReadResultCollector::CollectKeyValueResult<KeyValueRecordReader,
@@ -591,16 +659,16 @@ TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderNestedValues) {
     ASSERT_EQ(keyed_value_map->ValueArray()->GetInt(1), 23);
 }
 
-TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderFactoryFailureClosesReaders) {
+TEST_F(RealtimePrimaryKeyReaderTest, TestFactoryFailureClosesReaders) {
     std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
                                            DataField(1, arrow::field("v0", arrow::int32()))};
     std::shared_ptr<arrow::Schema> value_schema =
         DataField::ConvertDataFieldsToArrowSchema(value_fields);
     std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
-    std::shared_ptr<arrow::Schema> prepared_schema = MakePreparedSchema(value_schema->fields());
-    std::shared_ptr<arrow::DataType> prepared_type = arrow::struct_(prepared_schema->fields());
-    auto prepared_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(prepared_type, R"([
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    auto transport_array = std::dynamic_pointer_cast<arrow::StructArray>(
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([
         [0, 10, 0, 1, 100]
     ])")
             .ValueOrDie());
@@ -608,11 +676,11 @@ TEST_F(PreparedKeyValueReaderTest, TestPreparedReaderFactoryFailureClosesReaders
     int32_t factory_failure_close_count = 0;
     std::vector<std::unique_ptr<BatchReader>> batch_readers;
     batch_readers.push_back(std::make_unique<TrackingBatchReader>(
-        std::make_unique<MockFileBatchReader>(prepared_array, prepared_type, 1),
+        std::make_unique<MockFileBatchReader>(transport_array, transport_type, 1),
         &factory_failure_close_count));
     batch_readers.push_back(nullptr);
-    ASSERT_NOK_WITH_MSG(PreparedKeyValueReaderFactory::CreateForQuery(
-                            std::move(batch_readers), prepared_schema, OffsetRange(0, 1),
+    ASSERT_NOK_WITH_MSG(RealtimePrimaryKeyReaderFactory::CreateForQuery(
+                            std::move(batch_readers), transport_schema, OffsetRange(0, 1),
                             key_schema, value_schema, pool_),
                         "PK real-time store returned a null query reader");
     ASSERT_EQ(factory_failure_close_count, 1);
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.cpp b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
index 58103c599..61cd14009 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.cpp
@@ -31,22 +31,24 @@
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/core_options.h"
 #include "paimon/core/io/merged_key_value_record_reader.h"
-#include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
+#include "paimon/core/mergetree/compact/merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/mergetree/merge_tree_writer.h"
-#include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/core/utils/commit_increment.h"
+#include "paimon/core/utils/primary_key_table_utils.h"
 #include "paimon/macros.h"
 
 namespace paimon {
 
 namespace {
 
-Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
+Result<std::shared_ptr<arrow::StructArray>> CreateRealtimePrimaryKeyTransportBatch(
     std::unique_ptr<RecordBatch>&& batch, const std::shared_ptr<arrow::Schema>& write_schema,
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& transport_schema,
     const std::vector<std::string>& trimmed_primary_keys, int64_t first_sequence_number,
     int64_t first_offset, arrow::MemoryPool* arrow_pool) {
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
@@ -81,8 +83,8 @@ Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
                                   std::move(offset_array)};
     columns.insert(columns.end(), values->fields().begin(), values->fields().end());
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-        std::shared_ptr<arrow::StructArray> prepared,
-        arrow::StructArray::Make(std::move(columns), prepared_schema->fields()));
+        std::shared_ptr<arrow::StructArray> transport,
+        arrow::StructArray::Make(std::move(columns), transport_schema->fields()));
 
     std::vector<arrow::compute::SortKey> sort_keys;
     sort_keys.reserve(trimmed_primary_keys.size() + 1);
@@ -95,10 +97,10 @@ Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
     arrow::compute::SortOptions options(sort_keys, arrow::compute::NullPlacement::AtStart);
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
         arrow::Datum indices,
-        arrow::compute::SortIndices(arrow::Datum(prepared), options, &context));
+        arrow::compute::SortIndices(arrow::Datum(transport), options, &context));
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
         arrow::Datum sorted,
-        arrow::compute::Take(arrow::Datum(prepared), indices,
+        arrow::compute::Take(arrow::Datum(transport), indices,
                              arrow::compute::TakeOptions::NoBoundsCheck(), &context));
     return checked_pointer_cast<arrow::StructArray>(sorted.make_array());
 }
@@ -108,9 +110,9 @@ Result<std::shared_ptr<arrow::StructArray>> PrepareBatch(
 Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Create(
     const std::map<std::string, std::string>& partition, int32_t bucket,
     const std::shared_ptr<arrow::Schema>& write_schema,
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& transport_schema,
     const std::vector<std::string>& trimmed_primary_keys,
-    const std::shared_ptr<FieldsComparator>& key_comparator,
+    const std::shared_ptr<FieldsComparator>& key_comparator, const CoreOptions& options,
     const std::shared_ptr<RealtimeContextImpl>& realtime_context,
     const RealtimeStoreState& store_state, int64_t restored_max_sequence_number,
     const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
@@ -119,6 +121,10 @@ Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Crea
         restored_max_sequence_number == std::numeric_limits<int64_t>::max()) {
         return Status::Invalid("PK restored sequence number is invalid");
     }
+    if (options.GetMergeEngine() != MergeEngine::DEDUPLICATE) {
+        return Status::NotImplemented("PK realtime supports only the DEDUPLICATE merge engine");
+    }
+    PAIMON_RETURN_NOT_OK(RealtimePrimaryKeyLayout::ValidateSchema(transport_schema));
     arrow::FieldVector key_fields;
     key_fields.reserve(trimmed_primary_keys.size());
     for (const std::string& key : trimmed_primary_keys) {
@@ -134,8 +140,9 @@ Result<std::shared_ptr<RealtimePrimaryKeyWriter>> RealtimePrimaryKeyWriter::Crea
                                partition_bucket, restored_max_sequence_number));
     return std::shared_ptr<RealtimePrimaryKeyWriter>(new RealtimePrimaryKeyWriter(
         store_state.store, merge_tree_writer, realtime_context, partition_bucket, write_schema,
-        prepared_schema, arrow::schema(std::move(key_fields)), trimmed_primary_keys, key_comparator,
-        store_state.initial_offset, initial_max_sequence_number, memory_pool));
+        transport_schema, arrow::schema(std::move(key_fields)), trimmed_primary_keys,
+        key_comparator, options, store_state.initial_offset, initial_max_sequence_number,
+        memory_pool));
 }
 
 RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
@@ -144,11 +151,12 @@ RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
     const std::shared_ptr<RealtimeContextImpl>& realtime_context,
     const RealtimePartitionBucket& partition_bucket,
     const std::shared_ptr<arrow::Schema>& write_schema,
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& transport_schema,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::vector<std::string>& trimmed_primary_keys,
-    const std::shared_ptr<FieldsComparator>& key_comparator, int64_t next_offset,
-    int64_t last_sequence_number, const std::shared_ptr<MemoryPool>& memory_pool)
+    const std::shared_ptr<FieldsComparator>& key_comparator, const CoreOptions& options,
+    int64_t next_offset, int64_t last_sequence_number,
+    const std::shared_ptr<MemoryPool>& memory_pool)
     : memory_pool_(memory_pool),
       arrow_pool_(GetArrowPool(memory_pool)),
       realtime_store_(realtime_store),
@@ -156,10 +164,11 @@ RealtimePrimaryKeyWriter::RealtimePrimaryKeyWriter(
       realtime_context_(realtime_context),
       partition_bucket_(partition_bucket),
       write_schema_(write_schema),
-      prepared_schema_(prepared_schema),
+      transport_schema_(transport_schema),
       key_schema_(key_schema),
       trimmed_primary_keys_(trimmed_primary_keys),
       key_comparator_(key_comparator),
+      options_(options),
       next_offset_(next_offset),
       last_sequence_number_(last_sequence_number) {}
 
@@ -190,16 +199,17 @@ Status RealtimePrimaryKeyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
     }
     const int64_t first_sequence = last_sequence_number_ + 1;
     PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<arrow::StructArray> prepared,
-        PrepareBatch(std::move(batch), write_schema_, prepared_schema_, trimmed_primary_keys_,
-                     first_sequence, next_offset_, arrow_pool_.get()));
+        std::shared_ptr<arrow::StructArray> transport,
+        CreateRealtimePrimaryKeyTransportBatch(std::move(batch), write_schema_, transport_schema_,
+                                               trimmed_primary_keys_, first_sequence, next_offset_,
+                                               arrow_pool_.get()));
     auto output = std::make_unique<ArrowArray>();
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*prepared, output.get()));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*transport, output.get()));
     PAIMON_RETURN_NOT_OK(RetainArrowArrayMemoryPool(output.get(), arrow_pool_));
     RecordBatchBuilder builder(output.get());
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> prepared_batch, builder.Finish());
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> transport_batch, builder.Finish());
     PAIMON_RETURN_NOT_OK(realtime_store_->Write(RealtimeWriteBatch{
-        std::move(prepared_batch), OffsetRange(next_offset_, next_offset_ + count)}));
+        std::move(transport_batch), OffsetRange(next_offset_, next_offset_ + count)}));
     next_offset_ += count;
     last_sequence_number_ += count;
     PAIMON_RETURN_NOT_OK(realtime_context_->AdvanceMaterializedMaxSequenceNumber(
@@ -239,16 +249,20 @@ Status RealtimePrimaryKeyWriter::FlushSegment(const std::shared_ptr<RealtimeSegm
                                               const OffsetRange& sealed_offsets) {
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
                            realtime_store_->CreateCommitReaders(segment));
-    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
-                           PreparedKeyValueReaderFactory::CreateForCommit(
-                               std::move(readers), prepared_schema_, sealed_offsets, key_schema_,
-                               write_schema_, memory_pool_));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::vector<std::unique_ptr<KeyValueRecordReader>> realtime_primary_key_readers,
+        RealtimePrimaryKeyReaderFactory::CreateForCommit(std::move(readers), transport_schema_,
+                                                         sealed_offsets, key_schema_, write_schema_,
+                                                         memory_pool_));
     std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
-    sorted_readers.reserve(prepared_readers.size());
-    for (std::unique_ptr<KeyValueRecordReader>& prepared_reader : prepared_readers) {
-        auto merge_function = std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/false);
+    sorted_readers.reserve(realtime_primary_key_readers.size());
+    for (std::unique_ptr<KeyValueRecordReader>& realtime_primary_key_reader :
+         realtime_primary_key_readers) {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<MergeFunction> merge_function,
+                               PrimaryKeyTableUtils::CreateMergeFunction(
+                                   write_schema_, trimmed_primary_keys_, options_, memory_pool_));
         sorted_readers.push_back(std::make_unique<MergedKeyValueRecordReader>(
-            std::move(prepared_reader), key_comparator_,
+            std::move(realtime_primary_key_reader), key_comparator_,
             std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function))));
     }
     return merge_tree_writer_->WriteSortedReadersToFiles(std::move(sorted_readers));
diff --git a/src/paimon/core/realtime/realtime_primary_key_writer.h b/src/paimon/core/realtime/realtime_primary_key_writer.h
index 4a26f930d..cdd3d889f 100644
--- a/src/paimon/core/realtime/realtime_primary_key_writer.h
+++ b/src/paimon/core/realtime/realtime_primary_key_writer.h
@@ -26,6 +26,7 @@
 #include <string>
 #include <vector>
 
+#include "paimon/core/core_options.h"
 #include "paimon/core/utils/batch_writer.h"
 #include "paimon/realtime/realtime_context.h"
 #include "paimon/realtime/realtime_store.h"
@@ -48,9 +49,9 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
     static Result<std::shared_ptr<RealtimePrimaryKeyWriter>> Create(
         const std::map<std::string, std::string>& partition, int32_t bucket,
         const std::shared_ptr<arrow::Schema>& write_schema,
-        const std::shared_ptr<arrow::Schema>& prepared_schema,
+        const std::shared_ptr<arrow::Schema>& transport_schema,
         const std::vector<std::string>& trimmed_primary_keys,
-        const std::shared_ptr<FieldsComparator>& key_comparator,
+        const std::shared_ptr<FieldsComparator>& key_comparator, const CoreOptions& options,
         const std::shared_ptr<RealtimeContextImpl>& realtime_context,
         const RealtimeStoreState& store_state, int64_t restored_max_sequence_number,
         const std::shared_ptr<MergeTreeWriter>& merge_tree_writer,
@@ -72,11 +73,12 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
                              const std::shared_ptr<RealtimeContextImpl>& realtime_context,
                              const RealtimePartitionBucket& partition_bucket,
                              const std::shared_ptr<arrow::Schema>& write_schema,
-                             const std::shared_ptr<arrow::Schema>& prepared_schema,
+                             const std::shared_ptr<arrow::Schema>& transport_schema,
                              const std::shared_ptr<arrow::Schema>& key_schema,
                              const std::vector<std::string>& trimmed_primary_keys,
                              const std::shared_ptr<FieldsComparator>& key_comparator,
-                             int64_t next_offset, int64_t last_sequence_number,
+                             const CoreOptions& options, int64_t next_offset,
+                             int64_t last_sequence_number,
                              const std::shared_ptr<MemoryPool>& memory_pool);
 
     Status FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment,
@@ -89,10 +91,11 @@ class RealtimePrimaryKeyWriter final : public BatchWriter {
     std::shared_ptr<RealtimeContextImpl> realtime_context_;
     RealtimePartitionBucket partition_bucket_;
     std::shared_ptr<arrow::Schema> write_schema_;
-    std::shared_ptr<arrow::Schema> prepared_schema_;
+    std::shared_ptr<arrow::Schema> transport_schema_;
     std::shared_ptr<arrow::Schema> key_schema_;
     std::vector<std::string> trimmed_primary_keys_;
     std::shared_ptr<FieldsComparator> key_comparator_;
+    CoreOptions options_;
     int64_t next_offset_;
     int64_t last_sequence_number_;
     std::mutex realtime_store_mutex_;
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 110994331..afb852a6b 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -36,8 +36,8 @@
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/operation/merge_file_split_read.h"
 #include "paimon/core/operation/raw_file_split_read.h"
-#include "paimon/core/realtime/prepared_key_value_reader.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/core/realtime/realtime_reader.h"
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/table/source/pk_count_reader.h"
@@ -56,55 +56,57 @@ struct ColumnarBatchContext;
 
 namespace {
 
-Result<std::shared_ptr<arrow::Schema>> CreatePreparedQuerySchema(
+Result<std::shared_ptr<arrow::Schema>> CreateRealtimePrimaryKeyQueryTransportSchema(
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema) {
-    arrow::FieldVector prepared_value_fields;
-    prepared_value_fields.reserve(key_schema->num_fields() + value_schema->num_fields());
+    arrow::FieldVector transport_value_fields;
+    transport_value_fields.reserve(key_schema->num_fields() + value_schema->num_fields());
     std::unordered_set<int32_t> field_ids;
     for (const std::shared_ptr<arrow::Field>& field : key_schema->fields()) {
         PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(field));
         if (field_ids.insert(field_id).second) {
-            prepared_value_fields.push_back(field);
+            transport_value_fields.push_back(field);
         }
     }
     for (const std::shared_ptr<arrow::Field>& field : value_schema->fields()) {
         PAIMON_ASSIGN_OR_RAISE(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(field));
         if (field_ids.insert(field_id).second) {
-            prepared_value_fields.push_back(field);
+            transport_value_fields.push_back(field);
         }
     }
-    return SpecialFields::PreparedKeyValueSchema(prepared_value_fields);
+    return RealtimePrimaryKeyLayout::CreateSchema(transport_value_fields);
 }
 
 Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
     const std::shared_ptr<RealtimeSplit>& split, const RealtimePartitionBucketView& memory,
-    const std::shared_ptr<arrow::Schema>& prepared_schema,
+    const std::shared_ptr<arrow::Schema>& transport_schema,
     const std::shared_ptr<arrow::Schema>& key_schema,
     const std::shared_ptr<arrow::Schema>& value_schema,
     const std::shared_ptr<FieldsComparator>& key_comparator,
     const std::shared_ptr<InternalReadContext>& context,
     const std::shared_ptr<MemoryPool>& memory_pool) {
     auto c_schema = std::make_unique<ArrowSchema>();
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*prepared_schema, c_schema.get()));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*transport_schema, c_schema.get()));
     ScopeGuard schema_guard([schema = c_schema.get()]() { ArrowSchemaRelease(schema); });
     RealtimeQueryContext query_context{c_schema.get(), nullptr, false};
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> batch_readers,
                            memory.store->CreateQueryReaders(memory.read_view, 0, query_context));
-    PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> prepared_readers,
-                           PreparedKeyValueReaderFactory::CreateForQuery(
-                               std::move(batch_readers), prepared_schema,
-                               OffsetRange(split->CommittedEndOffset(), split->MemoryEndOffset()),
-                               key_schema, value_schema, memory_pool));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::vector<std::unique_ptr<KeyValueRecordReader>> realtime_primary_key_readers,
+        RealtimePrimaryKeyReaderFactory::CreateForQuery(
+            std::move(batch_readers), transport_schema,
+            OffsetRange(split->CommittedEndOffset(), split->MemoryEndOffset()), key_schema,
+            value_schema, memory_pool));
     std::vector<std::unique_ptr<KeyValueRecordReader>> result;
-    result.reserve(prepared_readers.size());
-    for (std::unique_ptr<KeyValueRecordReader>& prepared_reader : prepared_readers) {
+    result.reserve(realtime_primary_key_readers.size());
+    for (std::unique_ptr<KeyValueRecordReader>& realtime_primary_key_reader :
+         realtime_primary_key_readers) {
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<MergeFunction> merge,
                                PrimaryKeyTableUtils::CreateMergeFunction(
                                    value_schema, context->GetTableSchema()->PrimaryKeys(),
                                    context->GetCoreOptions(), memory_pool));
         result.push_back(std::make_unique<MergedKeyValueRecordReader>(
-            std::move(prepared_reader), key_comparator,
+            std::move(realtime_primary_key_reader), key_comparator,
             std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge))));
     }
     return result;
@@ -112,17 +114,17 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateMemoryReaders(
 
 }  // namespace
 
-KeyValueTableRead::KeyValueTableRead(std::vector<std::unique_ptr<SplitRead>>&& split_reads,
-                                     const std::shared_ptr<FileStorePathFactory>& path_factory,
-                                     const std::shared_ptr<InternalReadContext>& context,
-                                     const std::shared_ptr<arrow::Schema>& prepared_query_schema,
-                                     const std::shared_ptr<MemoryPool>& memory_pool,
-                                     const std::shared_ptr<Executor>& executor)
+KeyValueTableRead::KeyValueTableRead(
+    std::vector<std::unique_ptr<SplitRead>>&& split_reads,
+    const std::shared_ptr<FileStorePathFactory>& path_factory,
+    const std::shared_ptr<InternalReadContext>& context,
+    const std::shared_ptr<arrow::Schema>& realtime_primary_key_transport_schema,
+    const std::shared_ptr<MemoryPool>& memory_pool, const std::shared_ptr<Executor>& executor)
     : TableRead(memory_pool),
       split_reads_(std::move(split_reads)),
       path_factory_(path_factory),
       context_(context),
-      prepared_query_schema_(prepared_query_schema),
+      realtime_primary_key_transport_schema_(realtime_primary_key_transport_schema),
       executor_(executor) {}
 
 Result<std::unique_ptr<TableRead>> KeyValueTableRead::Create(
@@ -136,17 +138,18 @@ Result<std::unique_ptr<TableRead>> KeyValueTableRead::Create(
     PAIMON_ASSIGN_OR_RAISE(
         std::unique_ptr<MergeFileSplitRead> merge_file_split_read,
         MergeFileSplitRead::Create(path_factory, context, memory_pool, executor));
-    std::shared_ptr<arrow::Schema> prepared_query_schema;
+    std::shared_ptr<arrow::Schema> realtime_primary_key_transport_schema;
     if (context->GetRealtimeContext()) {
-        PAIMON_ASSIGN_OR_RAISE(prepared_query_schema,
-                               CreatePreparedQuerySchema(merge_file_split_read->GetKeySchema(),
+        PAIMON_ASSIGN_OR_RAISE(
+            realtime_primary_key_transport_schema,
+            CreateRealtimePrimaryKeyQueryTransportSchema(merge_file_split_read->GetKeySchema(),
                                                          merge_file_split_read->GetValueSchema()));
     }
     split_reads.emplace_back(std::move(merge_file_split_read));
 
-    return std::unique_ptr<TableRead>(new KeyValueTableRead(std::move(split_reads), path_factory,
-                                                            context, prepared_query_schema,
-                                                            memory_pool, executor));
+    return std::unique_ptr<TableRead>(
+        new KeyValueTableRead(std::move(split_reads), path_factory, context,
+                              realtime_primary_key_transport_schema, memory_pool, executor));
 }
 
 void KeyValueTableRead::ForceKeepDelete(bool force_keep_delete) {
@@ -289,7 +292,7 @@ Result<std::unique_ptr<BatchReader>> KeyValueTableRead::CreateRealtimeReader(
         if (merge_read) {
             PAIMON_ASSIGN_OR_RAISE(
                 std::vector<std::unique_ptr<KeyValueRecordReader>> memory_readers,
-                CreateMemoryReaders(realtime_split, memory, prepared_query_schema_,
+                CreateMemoryReaders(realtime_split, memory, realtime_primary_key_transport_schema_,
                                     merge_read->GetKeySchema(), merge_read->GetValueSchema(),
                                     merge_read->GetKeyComparator(), context_, GetMemoryPool()));
             PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
diff --git a/src/paimon/core/table/source/key_value_table_read.h b/src/paimon/core/table/source/key_value_table_read.h
index 54f802cf6..1dd59b016 100644
--- a/src/paimon/core/table/source/key_value_table_read.h
+++ b/src/paimon/core/table/source/key_value_table_read.h
@@ -59,7 +59,7 @@ class KeyValueTableRead : public TableRead {
     KeyValueTableRead(std::vector<std::unique_ptr<SplitRead>>&& split_reads,
                       const std::shared_ptr<FileStorePathFactory>& path_factory,
                       const std::shared_ptr<InternalReadContext>& context,
-                      const std::shared_ptr<arrow::Schema>& prepared_query_schema,
+                      const std::shared_ptr<arrow::Schema>& realtime_primary_key_transport_schema,
                       const std::shared_ptr<MemoryPool>& memory_pool,
                       const std::shared_ptr<Executor>& executor);
 
@@ -69,7 +69,7 @@ class KeyValueTableRead : public TableRead {
     std::vector<std::unique_ptr<SplitRead>> split_reads_;
     std::shared_ptr<FileStorePathFactory> path_factory_;
     std::shared_ptr<InternalReadContext> context_;
-    std::shared_ptr<arrow::Schema> prepared_query_schema_;
+    std::shared_ptr<arrow::Schema> realtime_primary_key_transport_schema_;
     std::shared_ptr<Executor> executor_;
     bool force_keep_delete_ = false;
 };

From d4ebe09a12e101172fec84a65c32bf90354ac3e7 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 28 Aug 2026 12:21:40 +0800
Subject: [PATCH 81/93] fix(realtime): resolve CI failures

---
 src/paimon/core/mergetree/merge_tree_writer.h |  3 +--
 .../realtime/arrow_realtime_store_test.cpp    |  8 --------
 .../primary_key_realtime_store_test.cpp       |  8 ++++----
 test/inte/realtime_write_inte_test.cpp        | 20 +++++++++++++------
 4 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h
index 17a9dc51c..c2a9131e9 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.h
+++ b/src/paimon/core/mergetree/merge_tree_writer.h
@@ -72,8 +72,7 @@ class MergeTreeWriter : public BatchWriter {
 
     /// Consumes readers whose complete streams are individually sorted by primary key and sequence
     /// number. Readers are closed on success or failure.
-    Status WriteSortedReadersToFiles(
-        std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers);
+    Status WriteSortedReadersToFiles(std::vector<std::unique_ptr<KeyValueRecordReader>>&& readers);
 
     Status Compact(bool full_compaction) override;
 
diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
index d4bda2000..864b1f810 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
@@ -271,14 +271,6 @@ TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) {
     ASSERT_EQ(std::vector<int64_t>({0, 1}), ReadIds(unfiltered_batch));
 }
 
-TEST_F(ArrowRealtimeStoreTest, TestFactoryRejectsInvalidMode) {
-    ArrowRealtimeStoreFactory factory;
-    std::unique_ptr<ArrowSchema> write_schema = MakeReadSchema(schema_);
-    RealtimeStoreCreateRequest request{std::move(write_schema),
-                                       /*options=*/{}, pool_, static_cast<RealtimeStoreMode>(-1)};
-    ASSERT_NOK_WITH_MSG(factory.Create(std::move(request)), "invalid real-time store mode: -1");
-}
-
 TEST_F(ArrowRealtimeStoreTest, TestMissingStatisticsRetainsNonMatchingBatch) {
     ASSERT_OK(
         store_->Write(RealtimeWriteBatch{MakeBatch(R"([[0, "a"], [1, "b"]])"), OffsetRange(0, 2)}));
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 301cc92a9..d3d5dff5e 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -361,11 +361,11 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryPoolOutlivesStoreReaderAndExport) {
     std::weak_ptr<TestingMemoryPool> pool_lifetime = pool;
     auto write_schema = std::make_unique<ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*stored_schema, write_schema.get()).ok());
-    RealtimeStoreCreateRequest request{std::move(write_schema),
-                                       /*options=*/{}, pool, RealtimeStoreMode::PRIMARY_KEY};
     ArrowRealtimeStoreFactory factory;
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> store, factory.Create(std::move(request)));
-    request.memory_pool.reset();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeStore> store,
+                         factory.Create(RealtimeStoreCreateRequest{
+                             std::move(write_schema),
+                             /*options=*/{}, pool, RealtimeStoreMode::PRIMARY_KEY}));
     ASSERT_OK(store->Write(
         RealtimeWriteBatch{MakeBatch(R"([[0, 1, 0, 7, "seven"]])"), OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 73391b1a6..6b611300e 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -45,6 +45,7 @@
 #include "paimon/catalog/identifier.h"
 #include "paimon/commit_context.h"
 #include "paimon/common/factories/io_hook.h"
+#include "paimon/common/reader/reader_utils.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/path_util.h"
@@ -564,11 +565,18 @@ class CorruptingBatchReader final : public BatchReader {
     }
 
     void Close() override {
-        buffered_.reset();
+        ReleaseBuffered();
         delegate_->Close();
     }
 
  private:
+    void ReleaseBuffered() {
+        if (buffered_.has_value()) {
+            ReaderUtils::ReleaseReadBatch(std::move(buffered_.value()));
+            buffered_.reset();
+        }
+    }
+
     Result<ReadBatch> DropLast() {
         if (!buffered_.has_value()) {
             PAIMON_ASSIGN_OR_RAISE(ReadBatch first, delegate_->NextBatch());
@@ -579,7 +587,7 @@ class CorruptingBatchReader final : public BatchReader {
         }
         PAIMON_ASSIGN_OR_RAISE(ReadBatch next, delegate_->NextBatch());
         if (BatchReader::IsEofBatch(next)) {
-            buffered_.reset();
+            ReleaseBuffered();
             return MakeEofBatch();
         }
         ReadBatch result = std::move(buffered_.value());
@@ -2096,8 +2104,8 @@ TEST_F(RealtimeWriteInteTest, TestPkPartitionBucketRecovery) {
     }
     ASSERT_EQ(OffsetRange(0, 2), first_ranges.at(p0b0));
     ASSERT_EQ(OffsetRange(0, 3), first_ranges.at(p1b1));
-    ASSERT_EQ((std::make_pair<int64_t, int64_t>(0, 1)), first_sequences.at(p0b0));
-    ASSERT_EQ((std::make_pair<int64_t, int64_t>(0, 2)), first_sequences.at(p1b1));
+    ASSERT_EQ((std::pair<int64_t, int64_t>(0, 1)), first_sequences.at(p0b0));
+    ASSERT_EQ((std::pair<int64_t, int64_t>(0, 2)), first_sequences.at(p1b1));
     ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id,
                          Commit(first_progress, /*commit_identifier=*/0));
     ASSERT_OK(first_writer->RefreshCommittedSnapshot(first_snapshot_id));
@@ -2139,8 +2147,8 @@ TEST_F(RealtimeWriteInteTest, TestPkPartitionBucketRecovery) {
     }
     ASSERT_EQ(OffsetRange(2, 4), second_ranges.at(p0b0));
     ASSERT_EQ(OffsetRange(3, 5), second_ranges.at(p1b1));
-    ASSERT_EQ((std::make_pair<int64_t, int64_t>(2, 3)), second_sequences.at(p0b0));
-    ASSERT_EQ((std::make_pair<int64_t, int64_t>(3, 4)), second_sequences.at(p1b1));
+    ASSERT_EQ((std::pair<int64_t, int64_t>(2, 3)), second_sequences.at(p0b0));
+    ASSERT_EQ((std::pair<int64_t, int64_t>(3, 4)), second_sequences.at(p1b1));
     ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
                          Commit(second_progress, /*commit_identifier=*/1));
     ASSERT_OK(second_writer->RefreshCommittedSnapshot(second_snapshot_id));

From 49f0ea7bf3ab6a0f9b0d89839fcc8a898bc4011d Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 28 Aug 2026 14:02:37 +0800
Subject: [PATCH 82/93] test(realtime): remove redundant reader test code

---
 .../merged_key_value_record_reader_test.cpp   |  6 ----
 .../operation/merge_file_split_read_test.cpp  | 34 -------------------
 2 files changed, 40 deletions(-)

diff --git a/src/paimon/core/io/merged_key_value_record_reader_test.cpp b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
index d484b28ee..858e302e8 100644
--- a/src/paimon/core/io/merged_key_value_record_reader_test.cpp
+++ b/src/paimon/core/io/merged_key_value_record_reader_test.cpp
@@ -18,10 +18,7 @@
 
 #include "paimon/core/io/merged_key_value_record_reader.h"
 
-#include <cstdint>
 #include <memory>
-#include <optional>
-#include <string>
 #include <utility>
 #include <vector>
 
@@ -31,13 +28,10 @@
 #include "gtest/gtest.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
-#include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
-#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/memory/memory_pool.h"
-#include "paimon/realtime/offset_range.h"
 #include "paimon/testing/mock/mock_file_batch_reader.h"
 #include "paimon/testing/mock/mock_key_value_data_file_record_reader.h"
 #include "paimon/testing/utils/key_value_checker.h"
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index ad7c21b22..df9f5a02f 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -69,8 +69,6 @@ class FileSystem;
 namespace paimon::test {
 namespace {
 
-class TestingSplit : public Split {};
-
 class TrackingKeyValueRecordReader : public KeyValueRecordReader {
  public:
     explicit TrackingKeyValueRecordReader(int32_t* close_count) : close_count_(close_count) {}
@@ -764,38 +762,6 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections)
     batch_reader->Close();
 }
 
-TEST_F(MergeFileSplitReadTest, TestRealtimeReadValidatesDiskSplits) {
-    std::string path =
-        paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
-    ReadContextBuilder context_builder(path);
-    context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"});
-    context_builder.SetOptions(
-        {{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}});
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
-    std::shared_ptr<InternalReadContext> internal_context = CreateInternalReadContext(read_context);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<MergeFileSplitRead> split_read,
-                         CreateMergeFileSplitRead(internal_context));
-
-    std::vector<std::shared_ptr<DataSplit>> prepared_splits = PrepareDataSplit();
-    std::shared_ptr<DataSplitImpl> first =
-        std::dynamic_pointer_cast<DataSplitImpl>(prepared_splits[0]);
-    ASSERT_NE(nullptr, first);
-
-    std::vector<std::shared_ptr<Split>> non_data_splits = {std::make_shared<TestingSplit>()};
-    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(non_data_splits, {}),
-                        "merge input disk split is not a data split");
-
-    std::vector<std::shared_ptr<DataFileMeta>> deletion_data_files = first->DataFiles();
-    DataSplitImpl::Builder deletion_builder(first->Partition(), first->Bucket(),
-                                            first->BucketPath(), std::move(deletion_data_files));
-    ASSERT_OK_AND_ASSIGN(
-        std::shared_ptr<DataSplitImpl> deletion_split,
-        deletion_builder.WithDataDeletionFiles({std::nullopt}).RawConvertible(false).Build());
-    std::vector<std::shared_ptr<Split>> deletion_splits = {deletion_split};
-    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader(deletion_splits, {}),
-                        "deletion files must be empty or match data files");
-}
-
 TEST_F(MergeFileSplitReadTest, TestRealtimeReaderFailureClosesPluginReader) {
     std::string path =
         paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";

From 34d0cb240e3c24a7e677f5c182127bb5d8248f4c Mon Sep 17 00:00:00 2001
From: lxy <38709059+lxy-9602@users.noreply.github.com>
Date: Fri, 28 Aug 2026 14:48:06 +0800
Subject: [PATCH 83/93] feat(mosaic): support Mosaic file format (#238)

---
 .github/workflows/build_and_test.yaml         |   5 +-
 .github/workflows/gcc8_test.yaml              |   3 +
 .github/workflows/release_candidate.yaml      |   2 +-
 CMakeLists.txt                                |  20 +
 ci/scripts/build_paimon.sh                    |   1 +
 ci/scripts/setup_rust.sh                      |   7 +-
 cmake_modules/ThirdpartyToolchain.cmake       |  59 ++
 include/paimon/defs.h                         |   4 +
 src/paimon/common/data/binary_array.h         |   2 +-
 src/paimon/common/data/binary_row.h           |   2 +-
 src/paimon/common/defs.cpp                    |   1 +
 .../reader/blob_fallback_batch_reader.cpp     |  15 +-
 .../prefetch_file_batch_reader_impl.cpp       |  19 +-
 .../reader/prefetch_file_batch_reader_impl.h  |   9 +-
 src/paimon/common/utils/arrow/arrow_utils.cpp |  17 +-
 src/paimon/common/utils/arrow/arrow_utils.h   |   2 +
 .../common/utils/arrow/arrow_utils_test.cpp   |  28 +
 .../core/operation/abstract_split_read.cpp    |   3 +-
 src/paimon/core/schema/schema_validation.cpp  |  71 +++
 src/paimon/core/schema/schema_validation.h    |   4 +
 .../core/schema/schema_validation_test.cpp    |  70 +++
 .../core/stats/simple_stats_converter.h       |   3 +-
 src/paimon/format/mosaic/CMakeLists.txt       |  60 ++
 src/paimon/format/mosaic/mosaic_ffi.h         |  24 +
 .../mosaic/mosaic_file_batch_reader.cpp       | 286 +++++++++
 .../format/mosaic/mosaic_file_batch_reader.h  |  95 +++
 .../format/mosaic/mosaic_file_format.cpp      |  57 ++
 src/paimon/format/mosaic/mosaic_file_format.h |  49 ++
 .../mosaic/mosaic_file_format_factory.cpp     |  36 ++
 .../mosaic/mosaic_file_format_factory.h       |  41 ++
 .../format/mosaic/mosaic_file_format_test.cpp | 554 ++++++++++++++++++
 src/paimon/format/mosaic/mosaic_format_defs.h |  40 ++
 .../format/mosaic/mosaic_format_writer.cpp    | 131 +++++
 .../format/mosaic/mosaic_format_writer.h      |  70 +++
 .../format/mosaic/mosaic_reader_builder.h     |  53 ++
 src/paimon/format/mosaic/mosaic_stats.cpp     | 311 ++++++++++
 src/paimon/format/mosaic/mosaic_stats.h       |  62 ++
 .../format/mosaic/mosaic_stats_extractor.cpp  |  84 +++
 .../format/mosaic/mosaic_stats_extractor.h    |  55 ++
 src/paimon/format/mosaic/mosaic_stream.cpp    | 155 +++++
 src/paimon/format/mosaic/mosaic_stream.h      |  76 +++
 .../format/mosaic/mosaic_writer_builder.cpp   | 104 ++++
 .../format/mosaic/mosaic_writer_builder.h     |  56 ++
 src/paimon/format/orc/orc_format_writer.cpp   |  13 +-
 .../format/orc/orc_format_writer_test.cpp     |   7 +
 .../format/parquet/parquet_writer_builder.cpp |  13 +-
 .../parquet/parquet_writer_builder_test.cpp   |  15 +
 .../testing/utils/read_result_collector.h     |  24 +-
 test/inte/append_compaction_inte_test.cpp     |   5 +-
 test/inte/blob_table_inte_test.cpp            |  68 ++-
 test/inte/data_evolution_table_test.cpp       |  97 ++-
 test/inte/pk_compaction_inte_test.cpp         |   9 +-
 test/inte/scan_and_read_inte_test.cpp         |  91 +++
 test/inte/write_and_read_inte_test.cpp        |  68 ++-
 test/inte/write_inte_test.cpp                 |  23 +-
 .../append_java_compat/README.md              |  60 ++
 ...a05a4-4c30-4777-81fa-43a43f7c260d-0.mosaic | Bin 0 -> 2007 bytes
 ...est-9d1301ba-d0b3-4d85-ae44-e93f6194876f-0 | Bin 0 -> 2291 bytes
 ...ist-1572ca97-622c-452d-8a80-a992a0684230-0 | Bin 0 -> 1006 bytes
 ...ist-1572ca97-622c-452d-8a80-a992a0684230-1 | Bin 0 -> 1113 bytes
 .../append_java_compat/schema/schema-0        | 162 +++++
 .../append_java_compat/snapshot/EARLIEST      |   1 +
 .../append_java_compat/snapshot/LATEST        |   1 +
 .../append_java_compat/snapshot/snapshot-1    |  18 +
 .../append_python_compat/README.md            |  62 ++
 ...c1f82-5909-45cd-a6a8-ff0314c8e365-0.mosaic | Bin 0 -> 2004 bytes
 ...est-82c6d797-6335-462b-b86e-a076ae20578a-0 | Bin 0 -> 2209 bytes
 ...ist-16d5961d-78cd-498a-9cff-80ef60258224-0 | Bin 0 -> 799 bytes
 ...ist-16d5961d-78cd-498a-9cff-80ef60258224-1 | Bin 0 -> 897 bytes
 .../append_python_compat/schema/schema-0      | 201 +++++++
 .../append_python_compat/snapshot/LATEST      |   1 +
 .../append_python_compat/snapshot/snapshot-1  |  15 +
 third_party/versions.txt                      |   5 +
 73 files changed, 3595 insertions(+), 110 deletions(-)
 create mode 100644 src/paimon/format/mosaic/CMakeLists.txt
 create mode 100644 src/paimon/format/mosaic/mosaic_ffi.h
 create mode 100644 src/paimon/format/mosaic/mosaic_file_batch_reader.cpp
 create mode 100644 src/paimon/format/mosaic/mosaic_file_batch_reader.h
 create mode 100644 src/paimon/format/mosaic/mosaic_file_format.cpp
 create mode 100644 src/paimon/format/mosaic/mosaic_file_format.h
 create mode 100644 src/paimon/format/mosaic/mosaic_file_format_factory.cpp
 create mode 100644 src/paimon/format/mosaic/mosaic_file_format_factory.h
 create mode 100644 src/paimon/format/mosaic/mosaic_file_format_test.cpp
 create mode 100644 src/paimon/format/mosaic/mosaic_format_defs.h
 create mode 100644 src/paimon/format/mosaic/mosaic_format_writer.cpp
 create mode 100644 src/paimon/format/mosaic/mosaic_format_writer.h
 create mode 100644 src/paimon/format/mosaic/mosaic_reader_builder.h
 create mode 100644 src/paimon/format/mosaic/mosaic_stats.cpp
 create mode 100644 src/paimon/format/mosaic/mosaic_stats.h
 create mode 100644 src/paimon/format/mosaic/mosaic_stats_extractor.cpp
 create mode 100644 src/paimon/format/mosaic/mosaic_stats_extractor.h
 create mode 100644 src/paimon/format/mosaic/mosaic_stream.cpp
 create mode 100644 src/paimon/format/mosaic/mosaic_stream.h
 create mode 100644 src/paimon/format/mosaic/mosaic_writer_builder.cpp
 create mode 100644 src/paimon/format/mosaic/mosaic_writer_builder.h
 create mode 100644 test/test_data/mosaic/append_java_compat.db/append_java_compat/README.md
 create mode 100644 test/test_data/mosaic/append_java_compat.db/append_java_compat/bucket-0/data-2c5a05a4-4c30-4777-81fa-43a43f7c260d-0.mosaic
 create mode 100644 test/test_data/mosaic/append_java_compat.db/append_java_compat/manifest/manifest-9d1301ba-d0b3-4d85-ae44-e93f6194876f-0
 create mode 100644 test/test_data/mosaic/append_java_compat.db/append_java_compat/manifest/manifest-list-1572ca97-622c-452d-8a80-a992a0684230-0
 create mode 100644 test/test_data/mosaic/append_java_compat.db/append_java_compat/manifest/manifest-list-1572ca97-622c-452d-8a80-a992a0684230-1
 create mode 100644 test/test_data/mosaic/append_java_compat.db/append_java_compat/schema/schema-0
 create mode 100644 test/test_data/mosaic/append_java_compat.db/append_java_compat/snapshot/EARLIEST
 create mode 100644 test/test_data/mosaic/append_java_compat.db/append_java_compat/snapshot/LATEST
 create mode 100644 test/test_data/mosaic/append_java_compat.db/append_java_compat/snapshot/snapshot-1
 create mode 100644 test/test_data/mosaic/append_python_compat.db/append_python_compat/README.md
 create mode 100644 test/test_data/mosaic/append_python_compat.db/append_python_compat/bucket-0/data-087c1f82-5909-45cd-a6a8-ff0314c8e365-0.mosaic
 create mode 100644 test/test_data/mosaic/append_python_compat.db/append_python_compat/manifest/manifest-82c6d797-6335-462b-b86e-a076ae20578a-0
 create mode 100644 test/test_data/mosaic/append_python_compat.db/append_python_compat/manifest/manifest-list-16d5961d-78cd-498a-9cff-80ef60258224-0
 create mode 100644 test/test_data/mosaic/append_python_compat.db/append_python_compat/manifest/manifest-list-16d5961d-78cd-498a-9cff-80ef60258224-1
 create mode 100644 test/test_data/mosaic/append_python_compat.db/append_python_compat/schema/schema-0
 create mode 100644 test/test_data/mosaic/append_python_compat.db/append_python_compat/snapshot/LATEST
 create mode 100644 test/test_data/mosaic/append_python_compat.db/append_python_compat/snapshot/snapshot-1

diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml
index b19e49069..0c8baf82a 100644
--- a/.github/workflows/build_and_test.yaml
+++ b/.github/workflows/build_and_test.yaml
@@ -75,7 +75,6 @@ jobs:
           - name: asan-ubsan-x86_64
             build_args: --enable_asan --enable_ubsan
           - name: tsan-x86_64
-            skip_rust: true
             build_args: --enable_tsan
           - name: gcc-debug-aarch64
             runner: ubuntu-24.04-arm
@@ -96,7 +95,6 @@ jobs:
             build_args: --build_type Release
           - name: tsan-aarch64
             runner: ubuntu-24.04-arm
-            skip_rust: true
             build_args: --enable_tsan
     steps:
       - name: Checkout paimon-cpp
@@ -108,8 +106,7 @@ jobs:
         uses: ./.github/actions/setup-ccache
         with:
           cache-key-prefix: ccache-${{ matrix.name }}
-      - name: Install Rust toolchain (tantivy-fts)
-        if: ${{ !matrix.skip_rust }}
+      - name: Install Rust toolchain (Mosaic and tantivy-fts)
         shell: bash
         run: ci/scripts/setup_rust.sh
       - name: Install HTTP and TLS development dependencies
diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml
index aae2ce98b..c0b4c1593 100644
--- a/.github/workflows/gcc8_test.yaml
+++ b/.github/workflows/gcc8_test.yaml
@@ -60,6 +60,9 @@ jobs:
           ls -la
       - name: Checkout paimon-cpp
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - name: Install Rust toolchain (Mosaic)
+        shell: bash
+        run: ci/scripts/setup_rust.sh
       - name: Setup ccache
         uses: ./.github/actions/setup-ccache
         with:
diff --git a/.github/workflows/release_candidate.yaml b/.github/workflows/release_candidate.yaml
index 6b5b1ebda..fdf9f5391 100644
--- a/.github/workflows/release_candidate.yaml
+++ b/.github/workflows/release_candidate.yaml
@@ -131,7 +131,7 @@ jobs:
           name: source-archive
           path: release/ci
 
-      - name: Install Rust toolchain (tantivy-fts)
+      - name: Install Rust toolchain (Mosaic and tantivy-fts)
         shell: bash
         run: ci/scripts/setup_rust.sh
 
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 084e6bf03..46a55426a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -60,6 +60,7 @@ option(PAIMON_USE_UBSAN "Use Undefined Behavior Sanitizer" OFF)
 option(PAIMON_USE_CXX11_ABI "Use C++11 ABI" ON)
 option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
 option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
+option(PAIMON_ENABLE_MOSAIC "Whether to enable mosaic file format (Rust FFI)" OFF)
 option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
 option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF)
 option(PAIMON_ENABLE_NETWORK_TESTS
@@ -86,6 +87,9 @@ endif()
 if(PAIMON_ENABLE_AVRO)
     add_definitions(-DPAIMON_ENABLE_AVRO)
 endif()
+if(PAIMON_ENABLE_MOSAIC)
+    add_definitions(-DPAIMON_ENABLE_MOSAIC)
+endif()
 if(PAIMON_ENABLE_JINDO)
     add_definitions(-DPAIMON_ENABLE_JINDO)
 endif()
@@ -388,6 +392,11 @@ if(PAIMON_BUILD_TESTS OR PAIMON_BUILD_BENCHMARKS)
     paimon_link_libraries_whole_archive(PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS
                                         paimon_parquet_file_format_static)
 
+    if(PAIMON_ENABLE_MOSAIC)
+        paimon_link_libraries_whole_archive(PAIMON_MOSAIC_FILE_FORMAT_STATIC_LINK_LIBS
+                                            paimon_mosaic_file_format_static)
+    endif()
+
     if(PAIMON_ENABLE_ORC)
         paimon_link_libraries_whole_archive(PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS
                                             paimon_orc_file_format_static)
@@ -439,6 +448,14 @@ if(PAIMON_BUILD_TESTS)
     paimon_link_libraries_whole_archive(PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS
                                         paimon_parquet_file_format_static)
 
+    if(PAIMON_ENABLE_MOSAIC)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_mosaic_file_format_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
+        paimon_link_libraries_whole_archive(PAIMON_MOSAIC_FILE_FORMAT_STATIC_LINK_LIBS
+                                            paimon_mosaic_file_format_static)
+    endif()
+
     if(PAIMON_ENABLE_ORC)
         paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
                                            paimon_orc_file_format_shared)
@@ -520,6 +537,9 @@ add_subdirectory(src/paimon/format/blob)
 add_subdirectory(src/paimon/format/orc)
 add_subdirectory(src/paimon/format/parquet)
 add_subdirectory(src/paimon/format/avro)
+if(PAIMON_ENABLE_MOSAIC)
+    add_subdirectory(src/paimon/format/mosaic)
+endif()
 if(PAIMON_ENABLE_LUMINA)
     add_subdirectory(src/paimon/global_index/lumina)
 endif()
diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh
index 50983d4da..58ed9b0a0 100755
--- a/ci/scripts/build_paimon.sh
+++ b/ci/scripts/build_paimon.sh
@@ -147,6 +147,7 @@ CMAKE_ARGS=(
     "-G Ninja"
     "-DCMAKE_BUILD_TYPE=${build_type}"
     "-DPAIMON_BUILD_TESTS=ON"
+    "-DPAIMON_ENABLE_MOSAIC=ON"
     "-DPAIMON_ENABLE_JINDO=ON"
     "-DPAIMON_ENABLE_S3=ON"
     "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}"
diff --git a/ci/scripts/setup_rust.sh b/ci/scripts/setup_rust.sh
index bdb8f622e..8c1e58392 100755
--- a/ci/scripts/setup_rust.sh
+++ b/ci/scripts/setup_rust.sh
@@ -15,12 +15,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Install the Rust toolchain + cbindgen required to build the
-# tantivy-fts FFI crate (crates/tantivy_ffi) from CI.
+# Install the Rust toolchain used by the Mosaic and tantivy-fts FFI builds, plus cbindgen required
+# by tantivy-fts.
 #
 # The dev container (see .devcontainer/) already has these preinstalled;
-# this script is for the GitHub Actions runners. Called by
-# .github/workflows/build_and_test.yaml before ci/scripts/build_paimon.sh.
+# this script is for the GitHub Actions runners and is called before ci/scripts/build_paimon.sh.
 #
 # Idempotent: a second invocation is a no-op when the tools already exist.
 
diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
index 776519104..a35ecd497 100644
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
@@ -91,6 +91,18 @@ else()
     endif()
 endif()
 
+if(DEFINED ENV{PAIMON_MOSAIC_URL})
+    set(MOSAIC_SOURCE_URL "$ENV{PAIMON_MOSAIC_URL}")
+else()
+    if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_MOSAIC_PKG_NAME}")
+        set_urls(MOSAIC_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_MOSAIC_PKG_NAME}")
+    else()
+        set_urls(MOSAIC_SOURCE_URL
+                 "https://downloads.apache.org/paimon/paimon-mosaic-${PAIMON_MOSAIC_BUILD_VERSION}/${PAIMON_MOSAIC_PKG_NAME}"
+        )
+    endif()
+endif()
+
 if(DEFINED ENV{PAIMON_RAPIDJSON_URL})
     set(RAPIDJSON_SOURCE_URL "$ENV{PAIMON_RAPIDJSON_URL}")
 else()
@@ -1314,6 +1326,50 @@ macro(build_lumina)
     install(FILES "${LUMINA_DYNAMIC_LIB}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
 endmacro()
 
+macro(build_mosaic)
+    message(STATUS "Building Apache Paimon Mosaic Rust FFI from source")
+    find_program(PAIMON_CARGO_EXECUTABLE cargo REQUIRED)
+
+    set(MOSAIC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/mosaic_ep-install")
+    set(MOSAIC_INCLUDE_DIR "${MOSAIC_PREFIX}/include")
+    set(MOSAIC_LIB_DIR "${MOSAIC_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+    set(MOSAIC_DYNAMIC_LIB
+        "${MOSAIC_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_mosaic_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}"
+    )
+    set(MOSAIC_CARGO_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}/mosaic_ep-cargo")
+    set(MOSAIC_CARGO_DYNAMIC_LIB
+        "${MOSAIC_CARGO_TARGET_DIR}/release/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_mosaic_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}"
+    )
+
+    file(MAKE_DIRECTORY "${MOSAIC_INCLUDE_DIR}")
+    file(MAKE_DIRECTORY "${MOSAIC_LIB_DIR}")
+
+    externalproject_add(mosaic_ep
+                        URL ${MOSAIC_SOURCE_URL}
+                        URL_HASH "SHA256=${PAIMON_MOSAIC_BUILD_SHA256_CHECKSUM}"
+                        ${THIRDPARTY_LOG_OPTIONS}
+                        CONFIGURE_COMMAND ""
+                        BUILD_COMMAND ${CMAKE_COMMAND} -E env
+                                      CARGO_TARGET_DIR=${MOSAIC_CARGO_TARGET_DIR}
+                                      ${PAIMON_CARGO_EXECUTABLE} build --release
+                                      --manifest-path <SOURCE_DIR>/ffi/Cargo.toml
+                        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+                                ${MOSAIC_CARGO_DYNAMIC_LIB} ${MOSAIC_DYNAMIC_LIB}
+                        INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory
+                                        <SOURCE_DIR>/include ${MOSAIC_INCLUDE_DIR}
+                        BUILD_BYPRODUCTS "${MOSAIC_DYNAMIC_LIB}")
+
+    add_library(paimon_mosaic_ffi SHARED IMPORTED GLOBAL)
+    set_target_properties(paimon_mosaic_ffi
+                          PROPERTIES IMPORTED_LOCATION "${MOSAIC_DYNAMIC_LIB}"
+                                     IMPORTED_NO_SONAME TRUE
+                                     INTERFACE_INCLUDE_DIRECTORIES
+                                     "${MOSAIC_INCLUDE_DIR}")
+    add_dependencies(paimon_mosaic_ffi mosaic_ep)
+
+    install(FILES "${MOSAIC_DYNAMIC_LIB}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+endmacro()
+
 macro(build_jindosdk_nextarch)
     message(STATUS "Building jindosdk-nextarch from local source")
 
@@ -1953,6 +2009,9 @@ paimon_warn_if_mixed_arrow_dependencies()
 resolve_dependency(TBB)
 resolve_dependency(glog)
 
+if(PAIMON_ENABLE_MOSAIC)
+    build_mosaic()
+endif()
 if(PAIMON_ENABLE_AVRO)
     resolve_dependency(Avro)
 endif()
diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index 8062d3d2d..c96cf5d4f 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -156,6 +156,10 @@ struct PAIMON_EXPORT Options {
     /// 9, but the read and write speed will significantly decrease. Default value is 1.
     static const char FILE_COMPRESSION_ZSTD_LEVEL[];
 
+    /// "file.block-size" - File block size of format, default value of orc stripe is 64 MB,
+    /// parquet row group is 128 MB, and Mosaic row group is 256 MB.
+    static const char FILE_BLOCK_SIZE[];
+
     /// "manifest.target-file-size" - Suggested file size of a manifest file.
     /// Default value is 8MB.
     static const char MANIFEST_TARGET_FILE_SIZE[];
diff --git a/src/paimon/common/data/binary_array.h b/src/paimon/common/data/binary_array.h
index c505ce7fa..8ab07e5f5 100644
--- a/src/paimon/common/data/binary_array.h
+++ b/src/paimon/common/data/binary_array.h
@@ -52,7 +52,7 @@ class MemorySegment;
 /// [size(int)] + [null bits(4-byte word boundaries)] + [values or offset&length] + [variable length
 /// part].
 
-class BinaryArray final : public BinarySection, public InternalArray {
+class PAIMON_EXPORT BinaryArray final : public BinarySection, public InternalArray {
  public:
     BinaryArray() = default;
 
diff --git a/src/paimon/common/data/binary_row.h b/src/paimon/common/data/binary_row.h
index e3dfe8b07..7dda5492a 100644
--- a/src/paimon/common/data/binary_row.h
+++ b/src/paimon/common/data/binary_row.h
@@ -59,7 +59,7 @@ class MemoryPool;
 /// @note: Unlike the Java implementation where variable-length data may span multiple
 /// MemorySegments, in this C++ implementation both the fixed-length part and the
 /// variable-length part reside within a single MemorySegment.
-class BinaryRow final : public BinarySection, public InternalRow, public DataSetters {
+class PAIMON_EXPORT BinaryRow final : public BinarySection, public InternalRow, public DataSetters {
  public:
     BinaryRow() : BinaryRow(0) {}
     explicit BinaryRow(int32_t arity);
diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp
index 36b9c1c99..454961461 100644
--- a/src/paimon/common/defs.cpp
+++ b/src/paimon/common/defs.cpp
@@ -46,6 +46,7 @@ const char Options::PAGE_SIZE[] = "page-size";
 const char Options::PARTITION_DEFAULT_NAME[] = "partition.default-name";
 const char Options::FILE_COMPRESSION[] = "file.compression";
 const char Options::FILE_COMPRESSION_ZSTD_LEVEL[] = "file.compression.zstd-level";
+const char Options::FILE_BLOCK_SIZE[] = "file.block-size";
 const char Options::MANIFEST_TARGET_FILE_SIZE[] = "manifest.target-file-size";
 const char Options::MANIFEST_FORMAT[] = "manifest.format";
 const char Options::MANIFEST_COMPRESSION[] = "manifest.compression";
diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.cpp b/src/paimon/common/reader/blob_fallback_batch_reader.cpp
index 6ca3106b3..21db0ade6 100644
--- a/src/paimon/common/reader/blob_fallback_batch_reader.cpp
+++ b/src/paimon/common/reader/blob_fallback_batch_reader.cpp
@@ -31,6 +31,7 @@
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/reader/reader_utils.h"
 #include "paimon/common/table/special_fields.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
@@ -238,11 +239,9 @@ Result<std::shared_ptr<arrow::Array>> BlobFallbackBatchReader::AssembleRowIdRun(
             break;
         }
     }
-    if (pieces.size() == 1 && pieces[0]->offset() == 0) {
+    if (pieces.size() == 1) {
         return pieces[0];
     }
-    // Concatenate flattens non-zero offsets left by Slice, so the exported batch honors the
-    // zero-offset BatchReader contract.
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> concat_array,
                                       arrow::Concatenate(pieces, arrow_pool_.get()));
     return concat_array;
@@ -303,11 +302,9 @@ Result<std::shared_ptr<arrow::Array>> BlobFallbackBatchReader::AssembleColumn(
         }
         run_start = run_end;
     }
-    if (pieces.size() == 1 && pieces[0]->offset() == 0) {
+    if (pieces.size() == 1) {
         return pieces[0];
     }
-    // Concatenate flattens non-zero offsets left by Slice, so the exported batch honors the
-    // zero-offset BatchReader contract.
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> concat_array,
                                       arrow::Concatenate(pieces, arrow_pool_.get()));
     return concat_array;
@@ -360,12 +357,14 @@ Result<BatchReader::ReadBatch> BlobFallbackBatchReader::NextBatch() {
                                AssembleColumn(field_idx, group_choice, group_chunks));
         columns.push_back(std::move(column));
     }
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> target_array,
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> array,
                                       arrow::StructArray::Make(columns, read_schema_->fields()));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> normalized_array,
+                           ArrowUtils::NormalizeArrayOffsets(array, arrow_pool_.get()));
     std::unique_ptr<ArrowArray> c_array = std::make_unique<ArrowArray>();
     std::unique_ptr<ArrowSchema> c_schema = std::make_unique<ArrowSchema>();
     PAIMON_RETURN_NOT_OK_FROM_ARROW(
-        arrow::ExportArray(*target_array, c_array.get(), c_schema.get()));
+        arrow::ExportArray(*normalized_array, c_array.get(), c_schema.get()));
     return std::make_pair(std::move(c_array), std::move(c_schema));
 }
 
diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp
index 369bb58fe..4b6411b7b 100644
--- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp
+++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp
@@ -30,6 +30,8 @@
 #include "paimon/common/io/cache_input_stream.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/reader/reader_utils.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/read_ahead_cache.h"
 #include "paimon/common/utils/scope_guard.h"
@@ -121,7 +123,7 @@ Result<std::unique_ptr<PrefetchFileBatchReaderImpl>> PrefetchFileBatchReaderImpl
 
     auto reader = std::unique_ptr<PrefetchFileBatchReaderImpl>(
         new PrefetchFileBatchReaderImpl(readers, batch_size, prefetch_queue_capacity,
-                                        enable_adaptive_prefetch_strategy, executor, cache));
+                                        enable_adaptive_prefetch_strategy, executor, cache, pool));
     if (initialize_read_ranges) {
         // normally initialize read ranges should be false, as set read schema will refresh read
         // ranges, and set read schema will always be called before read.
@@ -133,11 +135,13 @@ Result<std::unique_ptr<PrefetchFileBatchReaderImpl>> PrefetchFileBatchReaderImpl
 PrefetchFileBatchReaderImpl::PrefetchFileBatchReaderImpl(
     const std::vector<std::shared_ptr<PrefetchFileBatchReader>>& readers, int32_t batch_size,
     uint32_t prefetch_queue_capacity, bool enable_adaptive_prefetch_strategy,
-    const std::shared_ptr<Executor>& executor, const std::shared_ptr<ReadAheadCache>& cache)
+    const std::shared_ptr<Executor>& executor, const std::shared_ptr<ReadAheadCache>& cache,
+    const std::shared_ptr<MemoryPool>& pool)
     : readers_(std::move(readers)),
       batch_size_(batch_size),
       executor_(executor),
       cache_(cache),
+      arrow_pool_(GetArrowPool(pool)),
       prefetch_queue_capacity_(prefetch_queue_capacity),
       enable_adaptive_prefetch_strategy_(enable_adaptive_prefetch_strategy) {
     for (size_t i = 0; i < readers_.size(); i++) {
@@ -470,12 +474,15 @@ Status PrefetchFileBatchReaderImpl::HandleReadResult(
         } else if (slice_end < c_array->length) {
             // partially out of range, data before read_range.second has been effectively consumed
             readers_pos_[reader_idx]->store(read_range.second);
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> src_array,
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
                                               arrow::ImportArray(c_array.get(), c_schema.get()));
-            auto array = src_array->Slice(0, slice_end);
+            std::shared_ptr<arrow::Array> sliced_array = array->Slice(/*offset=*/0, slice_end);
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr<arrow::Array> normalized_array,
+                ArrowUtils::NormalizeArrayOffsets(sliced_array, arrow_pool_.get()));
             PAIMON_RETURN_NOT_OK_FROM_ARROW(
-                arrow::ExportArray(*array, c_array.get(), c_schema.get()));
-            bitmap.RemoveRange(slice_end, src_array->length());
+                arrow::ExportArray(*normalized_array, c_array.get(), c_schema.get()));
+            bitmap.RemoveRange(slice_end, array->length());
             global_row_ids =
                 std::vector<uint64_t>(global_row_ids.begin(), global_row_ids.begin() + slice_end);
         } else {
diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h
index 4750501e2..08d3bd370 100644
--- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h
+++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h
@@ -45,8 +45,13 @@
 
 struct ArrowSchema;
 
+namespace arrow {
+class MemoryPool;
+}  // namespace arrow
+
 namespace paimon {
 
+class MemoryPool;
 class ReaderBuilder;
 class FileSystem;
 class Executor;
@@ -113,7 +118,8 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader {
     PrefetchFileBatchReaderImpl(
         const std::vector<std::shared_ptr<PrefetchFileBatchReader>>& readers, int32_t batch_size,
         uint32_t prefetch_queue_capacity, bool enable_adaptive_prefetch_strategy,
-        const std::shared_ptr<Executor>& executor, const std::shared_ptr<ReadAheadCache>& cache);
+        const std::shared_ptr<Executor>& executor, const std::shared_ptr<ReadAheadCache>& cache,
+        const std::shared_ptr<MemoryPool>& pool);
 
     Status CleanUp();
     void Workloop();
@@ -160,6 +166,7 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader {
     std::condition_variable cv_;
     std::shared_ptr<Executor> executor_;
     std::shared_ptr<ReadAheadCache> cache_;
+    std::unique_ptr<arrow::MemoryPool> arrow_pool_;
 
     mutable std::shared_mutex rw_mutex_;
     std::unique_ptr<std::thread> background_thread_;
diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp b/src/paimon/common/utils/arrow/arrow_utils.cpp
index c6a07b2bf..97bb77813 100644
--- a/src/paimon/common/utils/arrow/arrow_utils.cpp
+++ b/src/paimon/common/utils/arrow/arrow_utils.cpp
@@ -38,12 +38,21 @@ namespace paimon {
 
 namespace {
 
-bool HasNonZeroOffset(const std::shared_ptr<arrow::ArrayData>& data) {
+bool NeedsNormalization(const std::shared_ptr<arrow::ArrayData>& data) {
     if (data->offset != 0) {
         return true;
     }
+    if (data->type->id() == arrow::Type::STRUCT) {
+        for (const auto& child : data->child_data) {
+            // StructArray::Slice(0, length) shortens only the parent ArrayData. Its children may
+            // still describe the full unsliced arrays, which cannot be imported as a RecordBatch.
+            if (child->length != data->length) {
+                return true;
+            }
+        }
+    }
     for (const auto& child : data->child_data) {
-        if (HasNonZeroOffset(child)) {
+        if (NeedsNormalization(child)) {
             return true;
         }
     }
@@ -234,7 +243,7 @@ Result<std::shared_ptr<arrow::ArrayData>> RebaseFixedWidth(
 /// proportional to the number of rows instead of the number of value bytes.
 Result<std::shared_ptr<arrow::ArrayData>> RebaseToZeroOffset(
     const std::shared_ptr<arrow::ArrayData>& data, arrow::MemoryPool* pool) {
-    if (!HasNonZeroOffset(data)) {
+    if (!NeedsNormalization(data)) {
         return data;
     }
     // An empty array may not carry the buffers the layouts below slice.
@@ -443,7 +452,7 @@ Result<std::shared_ptr<arrow::RecordBatch>> ArrowUtils::NormalizeRecordBatchOffs
     arrow::ArrayVector normalized_columns;
     for (int32_t i = 0; i < record_batch->num_columns(); ++i) {
         const std::shared_ptr<arrow::Array>& column = record_batch->column(i);
-        if (!HasNonZeroOffset(column->data())) {
+        if (!NeedsNormalization(column->data())) {
             continue;
         }
         if (normalized_columns.empty()) {
diff --git a/src/paimon/common/utils/arrow/arrow_utils.h b/src/paimon/common/utils/arrow/arrow_utils.h
index 13bd81549..d82d84d88 100644
--- a/src/paimon/common/utils/arrow/arrow_utils.h
+++ b/src/paimon/common/utils/arrow/arrow_utils.h
@@ -57,6 +57,8 @@ class PAIMON_EXPORT ArrowUtils {
     static Result<std::shared_ptr<arrow::RecordBatch>> NormalizeRecordBatchOffsets(
         const std::shared_ptr<arrow::RecordBatch>& record_batch, arrow::MemoryPool* pool);
 
+    /// Returns an Array with zero offsets. Struct children are also sliced to the parent's visible
+    /// range so the result can be exported and imported as a RecordBatch.
     static Result<std::shared_ptr<arrow::Array>> NormalizeArrayOffsets(
         const std::shared_ptr<arrow::Array>& array, arrow::MemoryPool* pool);
 
diff --git a/src/paimon/common/utils/arrow/arrow_utils_test.cpp b/src/paimon/common/utils/arrow/arrow_utils_test.cpp
index 4e1fdaa0c..2aad598b5 100644
--- a/src/paimon/common/utils/arrow/arrow_utils_test.cpp
+++ b/src/paimon/common/utils/arrow/arrow_utils_test.cpp
@@ -20,6 +20,7 @@
 #include "paimon/common/utils/arrow/arrow_utils.h"
 
 #include "arrow/api.h"
+#include "arrow/c/bridge.h"
 #include "arrow/ipc/api.h"
 #include "gtest/gtest.h"
 #include "paimon/common/types/data_field.h"
@@ -501,6 +502,33 @@ TEST(ArrowUtilsTest, TestNormalizeRecordBatchOffsets) {
     ASSERT_EQ(unchanged_batch.get(), normalized_batch.get());
 }
 
+TEST(ArrowUtilsTest, TestNormalizeArrayOffsetsSlicesZeroOffsetStructChildren) {
+    std::shared_ptr<arrow::Array> ints =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[0, 1, 2, 3]").ValueOrDie();
+    std::shared_ptr<arrow::Array> texts =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["a", "b", "c", "d"])")
+            .ValueOrDie();
+    std::shared_ptr<arrow::Array> array =
+        arrow::StructArray::Make({ints, texts}, std::vector<std::string>{"i", "s"}).ValueOrDie();
+    std::shared_ptr<arrow::Array> sliced = array->Slice(/*offset=*/0, /*length=*/2);
+    ASSERT_EQ(0, sliced->offset());
+    ASSERT_EQ(4, sliced->data()->child_data[0]->length);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> normalized,
+                         ArrowUtils::NormalizeArrayOffsets(sliced, arrow::default_memory_pool()));
+    ASSERT_TRUE(normalized->Equals(sliced));
+    ASSERT_EQ(0, normalized->offset());
+    ASSERT_EQ(2, normalized->data()->child_data[0]->length);
+    ASSERT_EQ(2, normalized->data()->child_data[1]->length);
+
+    ::ArrowArray c_array = {};
+    ::ArrowSchema c_schema = {};
+    ASSERT_TRUE(arrow::ExportArray(*normalized, &c_array, &c_schema).ok());
+    std::shared_ptr<arrow::RecordBatch> batch =
+        arrow::ImportRecordBatch(&c_array, &c_schema).ValueOrDie();
+    ASSERT_EQ(2, batch->num_rows());
+}
+
 namespace {
 
 /// A buffer that rebasing must expose as a view into the source.
diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp
index 89f5071e9..91157f71a 100644
--- a/src/paimon/core/operation/abstract_split_read.cpp
+++ b/src/paimon/core/operation/abstract_split_read.cpp
@@ -157,8 +157,9 @@ Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::CreateFileBatchReade
         reader_builder =
             std::make_unique<LateMaterializingReaderBuilder>(std::move(reader_builder), pool_);
     }
+    // TODO(xinyu.lxy): test format table for mosaic format
     if (context_->EnablePrefetch() && file_format_identifier != "blob" &&
-        file_format_identifier != "avro") {
+        file_format_identifier != "avro" && file_format_identifier != "mosaic") {
         PAIMON_ASSIGN_OR_RAISE(
             std::unique_ptr<PrefetchFileBatchReaderImpl> prefetch_reader,
             PrefetchFileBatchReaderImpl::Create(
diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp
index 4c7dd2ce5..a6844d9e6 100644
--- a/src/paimon/core/schema/schema_validation.cpp
+++ b/src/paimon/core/schema/schema_validation.cpp
@@ -197,6 +197,7 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) {
 
     PAIMON_RETURN_NOT_OK(ValidateRowTracking(schema, options));
     PAIMON_RETURN_NOT_OK(ValidateBlobFields(schema, options));
+    PAIMON_RETURN_NOT_OK(ValidateMosaicDataFields(schema, options));
     PAIMON_RETURN_NOT_OK(ValidateMapStorageLayout(schema, options));
     PAIMON_RETURN_NOT_OK(ValidateVectorFields(schema, options));
     return Status::OK();
@@ -561,6 +562,76 @@ Status SchemaValidation::ValidateBlobFields(const TableSchema& schema, const Cor
     return Status::OK();
 }
 
+Status SchemaValidation::ValidateMosaicDataField(const std::shared_ptr<arrow::Field>& field) {
+    if (VariantTypeUtils::IsVariantField(field)) {
+        return Status::Invalid("Mosaic file format does not support type VARIANT");
+    }
+    if (BlobUtils::IsBlobField(field)) {
+        return Status::Invalid("Mosaic file format does not support type BLOB");
+    }
+
+    const std::shared_ptr<arrow::DataType>& type = field->type();
+    switch (type->id()) {
+        case arrow::Type::BOOL:
+        case arrow::Type::INT8:
+        case arrow::Type::INT16:
+        case arrow::Type::INT32:
+        case arrow::Type::INT64:
+        case arrow::Type::FLOAT:
+        case arrow::Type::DOUBLE:
+        case arrow::Type::DATE32:
+        case arrow::Type::STRING:
+        case arrow::Type::BINARY:
+        case arrow::Type::TIME32:
+        case arrow::Type::DECIMAL128:
+            return Status::OK();
+        case arrow::Type::TIMESTAMP: {
+            const auto& timestamp_type = checked_cast<const arrow::TimestampType&>(*type);
+            if (timestamp_type.unit() == arrow::TimeUnit::SECOND) {
+                return Status::Invalid("Mosaic file format does not support TIMESTAMP(0)");
+            }
+            return Status::OK();
+        }
+        case arrow::Type::LIST:
+            return ValidateMosaicDataField(type->field(0));
+        case arrow::Type::MAP: {
+            const auto& map_type = checked_cast<const arrow::MapType&>(*type);
+            PAIMON_RETURN_NOT_OK(ValidateMosaicDataField(map_type.key_field()));
+            return ValidateMosaicDataField(map_type.item_field());
+        }
+        case arrow::Type::FIXED_SIZE_LIST:
+            return Status::Invalid("Mosaic file format does not support type VECTOR");
+        case arrow::Type::STRUCT:
+            return Status::Invalid("Mosaic file format does not support type ROW");
+        default:
+            break;
+    }
+    return Status::Invalid(
+        fmt::format("Mosaic file format does not support type {}", type->ToString()));
+}
+
+Status SchemaValidation::ValidateMosaicDataFields(const TableSchema& schema,
+                                                  const CoreOptions& options) {
+    if (StringUtils::ToLowerCase(options.GetFileFormat()->Identifier()) != "mosaic") {
+        return Status::OK();
+    }
+
+    const std::vector<std::string> inline_blob_fields = options.GetBlobInlineFields();
+    const std::set<std::string> inline_blob_field_set(inline_blob_fields.begin(),
+                                                      inline_blob_fields.end());
+    // Match Java SchemaValidation by validating only fields stored in the normal data file. C++
+    // permits BLOB only as a top-level field; descriptor and view fields are inline, so Mosaic
+    // must reject them here.
+    for (const DataField& field : schema.Fields()) {
+        if (BlobUtils::IsBlobField(field.ArrowField()) &&
+            inline_blob_field_set.count(field.Name()) == 0) {
+            continue;
+        }
+        PAIMON_RETURN_NOT_OK(ValidateMosaicDataField(field.ArrowField()));
+    }
+    return Status::OK();
+}
+
 Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema,
                                                   const CoreOptions& options) {
     // Extract all field names that have map.storage-layout configured from options
diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h
index abf4d5b02..d331bf10f 100644
--- a/src/paimon/core/schema/schema_validation.h
+++ b/src/paimon/core/schema/schema_validation.h
@@ -73,6 +73,10 @@ class SchemaValidation {
 
     static Status ValidateBlobFields(const TableSchema& schema, const CoreOptions& options);
 
+    static Status ValidateMosaicDataFields(const TableSchema& schema, const CoreOptions& options);
+
+    static Status ValidateMosaicDataField(const std::shared_ptr<arrow::Field>& field);
+
     static Status ValidateMapStorageLayout(const TableSchema& schema, const CoreOptions& options);
 
     static Status ValidateVectorFields(const TableSchema& schema, const CoreOptions& options);
diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp
index 578856260..65ab7f7d1 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -24,6 +24,7 @@
 #include "arrow/api.h"
 #include "gtest/gtest.h"
 #include "paimon/common/data/blob_utils.h"
+#include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/defs.h"
 #include "paimon/testing/utils/testharness.h"
@@ -115,6 +116,75 @@ TEST(SchemaValidationTest, TestVectorType) {
                         "VECTOR fields in data-evolution tables are not implemented yet.");
 }
 
+#ifdef PAIMON_ENABLE_MOSAIC
+TEST(SchemaValidationTest, TestMosaicDataTypes) {
+    std::map<std::string, std::string> options = {{Options::BUCKET, "-1"},
+                                                  {Options::FILE_FORMAT, "mosaic"}};
+    arrow::FieldVector supported_fields = {
+        arrow::field("f0", arrow::boolean()),
+        arrow::field("f1", arrow::int8()),
+        arrow::field("f2", arrow::int16()),
+        arrow::field("f3", arrow::int32()),
+        arrow::field("f4", arrow::int64()),
+        arrow::field("f5", arrow::float32()),
+        arrow::field("f6", arrow::float64()),
+        arrow::field("f7", arrow::utf8()),
+        arrow::field("f8", arrow::binary()),
+        arrow::field("f9", arrow::date32()),
+        arrow::field("f10", arrow::timestamp(arrow::TimeUnit::NANO)),
+        arrow::field("f11", arrow::decimal128(38, 2)),
+        arrow::field("f12", arrow::list(arrow::float32())),
+        arrow::field("f13", arrow::map(arrow::int8(), arrow::int16())),
+    };
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
+                         TableSchema::Create(/*schema_id=*/0, arrow::schema(supported_fields),
+                                             /*partition_keys=*/{}, /*primary_keys=*/{}, options));
+    ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+
+    arrow::FieldVector unsupported_fields = {
+        arrow::field("row", arrow::struct_({arrow::field("value", arrow::int32())})),
+        VariantTypeUtils::ToArrowField("variant"),
+        arrow::field("vector", arrow::fixed_size_list(arrow::float32(), 3)),
+        arrow::field("timestamp", arrow::timestamp(arrow::TimeUnit::SECOND)),
+        arrow::field("nested_row",
+                     arrow::list(arrow::struct_({arrow::field("value", arrow::int32())}))),
+    };
+    std::vector<std::string> expected_errors = {"type ROW", "type VARIANT", "type VECTOR",
+                                                "TIMESTAMP(0)", "type ROW"};
+    for (size_t i = 0; i < unsupported_fields.size(); ++i) {
+        SCOPED_TRACE("field=" + unsupported_fields[i]->name());
+        ASSERT_OK_AND_ASSIGN(
+            table_schema,
+            TableSchema::Create(/*schema_id=*/0, arrow::schema({unsupported_fields[i]}),
+                                /*partition_keys=*/{}, /*primary_keys=*/{}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            expected_errors[i]);
+    }
+
+    std::shared_ptr<arrow::Field> blob_field = BlobUtils::ToArrowField("blob", false);
+    std::map<std::string, std::string> blob_options = {
+        {Options::BUCKET, "-1"},
+        {Options::FILE_FORMAT, "mosaic"},
+        {Options::ROW_TRACKING_ENABLED, "true"},
+        {Options::DATA_EVOLUTION_ENABLED, "true"},
+    };
+    ASSERT_OK_AND_ASSIGN(
+        table_schema,
+        TableSchema::Create(/*schema_id=*/0,
+                            arrow::schema({arrow::field("id", arrow::int32()), blob_field}),
+                            /*partition_keys=*/{}, /*primary_keys=*/{}, blob_options));
+    ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+
+    blob_options[Options::BLOB_DESCRIPTOR_FIELD] = "blob";
+    ASSERT_OK_AND_ASSIGN(
+        table_schema,
+        TableSchema::Create(/*schema_id=*/0,
+                            arrow::schema({arrow::field("id", arrow::int32()), blob_field}),
+                            /*partition_keys=*/{}, /*primary_keys=*/{}, blob_options));
+    ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), "type BLOB");
+}
+#endif
+
 TEST(SchemaValidationTest, TestRowTracking) {
     auto f0 = arrow::field("f0", arrow::utf8());
     auto f1 = arrow::field("f1", arrow::int32());
diff --git a/src/paimon/core/stats/simple_stats_converter.h b/src/paimon/core/stats/simple_stats_converter.h
index 903cf6e08..91eb04937 100644
--- a/src/paimon/core/stats/simple_stats_converter.h
+++ b/src/paimon/core/stats/simple_stats_converter.h
@@ -24,6 +24,7 @@
 
 #include "paimon/format/column_stats.h"
 #include "paimon/result.h"
+#include "paimon/visibility.h"
 
 namespace paimon {
 
@@ -31,7 +32,7 @@ class SimpleStats;
 class ColumnStats;
 class MemoryPool;
 
-class SimpleStatsConverter {
+class PAIMON_EXPORT SimpleStatsConverter {
  public:
     static Result<SimpleStats> ToBinary(const std::vector<std::shared_ptr<ColumnStats>>& stats,
                                         MemoryPool* pool);
diff --git a/src/paimon/format/mosaic/CMakeLists.txt b/src/paimon/format/mosaic/CMakeLists.txt
new file mode 100644
index 000000000..ca1612f97
--- /dev/null
+++ b/src/paimon/format/mosaic/CMakeLists.txt
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if(PAIMON_ENABLE_MOSAIC)
+    set(PAIMON_MOSAIC_FILE_FORMAT
+        mosaic_file_batch_reader.cpp
+        mosaic_file_format.cpp
+        mosaic_file_format_factory.cpp
+        mosaic_format_writer.cpp
+        mosaic_stats.cpp
+        mosaic_stats_extractor.cpp
+        mosaic_stream.cpp
+        mosaic_writer_builder.cpp)
+
+    add_paimon_lib(paimon_mosaic_file_format
+                   SOURCES
+                   ${PAIMON_MOSAIC_FILE_FORMAT}
+                   DEPENDENCIES
+                   paimon_shared
+                   paimon_mosaic_ffi
+                   STATIC_LINK_LIBS
+                   arrow
+                   fmt
+                   paimon_mosaic_ffi
+                   Threads::Threads
+                   SHARED_LINK_LIBS
+                   paimon_shared
+                   paimon_mosaic_ffi
+                   SHARED_LINK_FLAGS
+                   ${PAIMON_VERSION_SCRIPT_FLAGS})
+
+    target_link_libraries(paimon_mosaic_file_format_objlib PUBLIC paimon_mosaic_ffi)
+
+    if(PAIMON_BUILD_TESTS)
+        add_paimon_test(mosaic_format_test
+                        SOURCES
+                        mosaic_file_format_test.cpp
+                        STATIC_LINK_LIBS
+                        paimon_shared
+                        test_utils_static
+                        ${PAIMON_MOSAIC_FILE_FORMAT_STATIC_LINK_LIBS}
+                        ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS}
+                        paimon_mosaic_ffi
+                        ${GTEST_LINK_TOOLCHAIN})
+    endif()
+endif()
diff --git a/src/paimon/format/mosaic/mosaic_ffi.h b/src/paimon/format/mosaic/mosaic_ffi.h
new file mode 100644
index 000000000..5b7193eec
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_ffi.h
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+extern "C" {
+#include "mosaic.h"  // NOLINT(build/include_subdir)
+}
diff --git a/src/paimon/format/mosaic/mosaic_file_batch_reader.cpp b/src/paimon/format/mosaic/mosaic_file_batch_reader.cpp
new file mode 100644
index 000000000..b065c3632
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_file_batch_reader.cpp
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/format/mosaic/mosaic_file_batch_reader.h"
+
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "fmt/format.h"
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/predicate/predicate_filter.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/math.h"
+#include "paimon/core/stats/simple_stats.h"
+#include "paimon/core/stats/simple_stats_converter.h"
+#include "paimon/format/mosaic/mosaic_stats.h"
+#include "paimon/fs/file_system.h"
+
+namespace paimon::mosaic {
+
+MosaicFileBatchReader::MosaicFileBatchReader(
+    const std::shared_ptr<InputStream>& input, int32_t batch_size,
+    std::unique_ptr<MosaicInputContext> input_context, MosaicReaderHandle* reader,
+    const std::shared_ptr<arrow::Schema>& file_schema, uint32_t num_row_groups, uint64_t total_rows,
+    const std::shared_ptr<MemoryPool>& pool, const std::shared_ptr<arrow::MemoryPool>& arrow_pool)
+    : input_(input),
+      batch_size_(batch_size),
+      input_context_(std::move(input_context)),
+      reader_(reader),
+      file_schema_(file_schema),
+      num_row_groups_(num_row_groups),
+      total_rows_(total_rows),
+      pool_(pool),
+      arrow_pool_(arrow_pool),
+      metrics_(std::make_shared<MetricsImpl>()) {}
+
+Result<std::unique_ptr<MosaicFileBatchReader>> MosaicFileBatchReader::Create(
+    const std::shared_ptr<InputStream>& input, int32_t batch_size,
+    const std::shared_ptr<MemoryPool>& pool) {
+    if (input == nullptr || pool == nullptr || batch_size <= 0) {
+        return Status::Invalid(
+            "Mosaic reader requires non-null input and memory pool, and positive batch size");
+    }
+    std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(pool);
+    PAIMON_ASSIGN_OR_RAISE(int64_t signed_length, input->Length());
+    PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(signed_length, "Mosaic input length"));
+    auto length = static_cast<uint64_t>(signed_length);
+    auto input_context = std::make_unique<MosaicInputContext>(input, length);
+    MosaicInputFile input_file = {};
+    input_file.ctx = input_context.get();
+    input_file.read_at_fn = MosaicInputContext::ReadAt;
+    input_file.length_fn = MosaicInputContext::Length;
+    std::unique_ptr<MosaicReaderHandle, decltype(&mosaic_reader_free)> reader(
+        mosaic_reader_open(input_file), mosaic_reader_free);
+    if (reader == nullptr) {
+        return MosaicFfiError("open Mosaic reader", input_context->GetCallbackStatus());
+    }
+
+    ::ArrowSchema ffi_schema = {};
+    if (mosaic_reader_export_schema(reader.get(), &ffi_schema) != 0) {
+        return MosaicFfiError("read Mosaic schema", input_context->GetCallbackStatus());
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> file_schema,
+                                      arrow::ImportSchema(&ffi_schema));
+
+    uint32_t num_row_groups = 0;
+    if (mosaic_reader_num_row_groups(reader.get(), &num_row_groups) != 0) {
+        return MosaicFfiError("read Mosaic row group count", input_context->GetCallbackStatus());
+    }
+    uint64_t total_rows = 0;
+    for (uint32_t row_group = 0; row_group < num_row_groups; ++row_group) {
+        uint32_t row_count = 0;
+        if (mosaic_reader_row_group_num_rows(reader.get(), row_group, &row_count) != 0) {
+            return MosaicFfiError("read Mosaic row count", input_context->GetCallbackStatus());
+        }
+        total_rows += row_count;
+    }
+    return std::unique_ptr<MosaicFileBatchReader>(
+        new MosaicFileBatchReader(input, batch_size, std::move(input_context), reader.release(),
+                                  file_schema, num_row_groups, total_rows, pool, arrow_pool));
+}
+
+MosaicFileBatchReader::~MosaicFileBatchReader() {
+    CloseInternal();
+}
+
+Result<std::shared_ptr<arrow::Array>> MosaicFileBatchReader::ReadNextRowGroup() {
+    while (next_row_group_ < num_row_groups_) {
+        uint32_t row_group = next_row_group_++;
+        uint32_t row_count = 0;
+        if (mosaic_reader_row_group_num_rows(reader_, row_group, &row_count) != 0) {
+            return MosaicFfiError("read Mosaic row count", input_context_->GetCallbackStatus());
+        }
+        current_row_group_first_row_ = next_row_group_first_row_;
+        next_row_group_first_row_ += row_count;
+        PAIMON_ASSIGN_OR_RAISE(bool matches, MatchesRowGroup(row_group, row_count));
+        if (!matches) {
+            continue;
+        }
+
+        MosaicRowGroupReaderHandle* row_group_reader =
+            mosaic_reader_open_row_group(reader_, row_group);
+        if (row_group_reader == nullptr) {
+            return MosaicFfiError("open Mosaic row group", input_context_->GetCallbackStatus());
+        }
+        MosaicRecordBatchHandle* record_batch =
+            mosaic_row_group_reader_read_columns(row_group_reader);
+        mosaic_row_group_reader_free(row_group_reader);
+        if (record_batch == nullptr) {
+            return MosaicFfiError("read Mosaic row group", input_context_->GetCallbackStatus());
+        }
+        ::ArrowArray ffi_array = {};
+        ::ArrowSchema ffi_schema = {};
+        int32_t export_result = mosaic_record_batch_export(record_batch, &ffi_array, &ffi_schema);
+        mosaic_record_batch_free(record_batch);
+        if (export_result != 0) {
+            return MosaicFfiError("export Mosaic row group", input_context_->GetCallbackStatus());
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> batch,
+                                          arrow::ImportArray(&ffi_array, &ffi_schema));
+        if (batch->length() != row_count) {
+            return Status::Invalid("Mosaic row group row count mismatch");
+        }
+        if (batch->length() != 0) {
+            return batch;
+        }
+    }
+    return std::shared_ptr<arrow::Array>();
+}
+
+Result<bool> MosaicFileBatchReader::MatchesRowGroup(uint32_t row_group, uint32_t row_count) {
+    if (predicate_filter_ == nullptr) {
+        return true;
+    }
+    PAIMON_ASSIGN_OR_RAISE(
+        MosaicStatsUtils::RowGroupStatistics stats,
+        MosaicStatsUtils::ReadRowGroupStatistics(row_group, input_context_.get(), reader_));
+    // This matches the Java Mosaic reader: a file without row-group statistics is always kept.
+    if (stats.empty()) {
+        return true;
+    }
+    std::vector<MosaicStatsUtils::RowGroupStatistics> row_group_stats;
+    row_group_stats.push_back(std::move(stats));
+    PAIMON_ASSIGN_OR_RAISE(
+        ColumnStatsVector column_stats,
+        MosaicStatsUtils::ConvertColumnStatistics(file_schema_, row_group_stats,
+                                                  /*missing_null_count_is_zero=*/false));
+    PAIMON_ASSIGN_OR_RAISE(SimpleStats simple_stats,
+                           SimpleStatsConverter::ToBinary(column_stats, pool_.get()));
+    return predicate_filter_->Test(file_schema_, row_count, simple_stats.MinValues(),
+                                   simple_stats.MaxValues(), simple_stats.NullCounts());
+}
+
+Result<BatchReader::ReadBatch> MosaicFileBatchReader::NextBatch() {
+    if (closed_) {
+        return Status::Invalid("Mosaic reader is closed");
+    }
+    if (current_batch_ == nullptr || current_batch_offset_ == current_batch_->length()) {
+        PAIMON_ASSIGN_OR_RAISE(current_batch_, ReadNextRowGroup());
+        current_batch_offset_ = 0;
+    }
+    if (current_batch_ == nullptr) {
+        previous_batch_row_count_ = 0;
+        return BatchReader::MakeEofBatch();
+    }
+
+    int64_t row_count =
+        std::min<int64_t>(batch_size_, current_batch_->length() - current_batch_offset_);
+    std::shared_ptr<arrow::Array> sliced_array =
+        current_batch_->Slice(current_batch_offset_, row_count);
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> normalized_array,
+                           ArrowUtils::NormalizeArrayOffsets(sliced_array, arrow_pool_.get()));
+
+    previous_first_row_ = current_row_group_first_row_ + current_batch_offset_;
+    previous_batch_row_count_ = row_count;
+    current_batch_offset_ += row_count;
+    auto ffi_array = std::make_unique<::ArrowArray>();
+    auto ffi_schema = std::make_unique<::ArrowSchema>();
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(
+        arrow::ExportArray(*normalized_array, ffi_array.get(), ffi_schema.get()));
+    return std::make_pair(std::move(ffi_array), std::move(ffi_schema));
+}
+
+Result<std::unique_ptr<::ArrowSchema>> MosaicFileBatchReader::GetFileSchema() const {
+    auto schema = std::make_unique<::ArrowSchema>();
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema_, schema.get()));
+    return schema;
+}
+
+Status MosaicFileBatchReader::SetReadSchema(
+    ::ArrowSchema* read_schema, const std::shared_ptr<Predicate>& predicate,
+    const std::optional<RoaringBitmap32>& selection_bitmap) {
+    if (read_schema == nullptr) {
+        return Status::Invalid("Mosaic read schema is nullptr");
+    }
+    (void)selection_bitmap;
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> schema,
+                                      arrow::ImportSchema(read_schema));
+    std::vector<std::string> names;
+    std::vector<const char*> name_pointers;
+    names.reserve(schema->num_fields());
+    name_pointers.reserve(schema->num_fields());
+    for (const std::shared_ptr<arrow::Field>& field : schema->fields()) {
+        names.push_back(field->name());
+    }
+    for (const std::string& name : names) {
+        name_pointers.push_back(name.c_str());
+    }
+    if (mosaic_reader_set_projection(reader_, name_pointers.data(), name_pointers.size()) != 0) {
+        return MosaicFfiError("set Mosaic projection", input_context_->GetCallbackStatus());
+    }
+    predicate_filter_ = std::dynamic_pointer_cast<PredicateFilter>(predicate);
+    next_row_group_ = 0;
+    next_row_group_first_row_ = 0;
+    current_row_group_first_row_ = 0;
+    current_batch_.reset();
+    current_batch_offset_ = 0;
+    previous_first_row_ = std::numeric_limits<uint64_t>::max();
+    previous_batch_row_count_ = 0;
+    return Status::OK();
+}
+
+Result<uint64_t> MosaicFileBatchReader::GetPreviousBatchFileRowId(uint64_t batch_row_id) const {
+    if (previous_batch_row_count_ == 0) {
+        return Status::Invalid(previous_first_row_ == std::numeric_limits<uint64_t>::max()
+                                   ? "no Mosaic batch has been read yet"
+                                   : "last Mosaic batch was EOF");
+    }
+    if (batch_row_id >= previous_batch_row_count_) {
+        return Status::Invalid(fmt::format("batch row id {} is out of range {}", batch_row_id,
+                                           previous_batch_row_count_));
+    }
+    return previous_first_row_ + batch_row_id;
+}
+
+Result<uint64_t> MosaicFileBatchReader::GetNumberOfRows() const {
+    return total_rows_;
+}
+
+std::shared_ptr<Metrics> MosaicFileBatchReader::GetReaderMetrics() const {
+    return metrics_;
+}
+
+void MosaicFileBatchReader::Close() {
+    CloseInternal();
+}
+
+void MosaicFileBatchReader::CloseInternal() {
+    if (!closed_) {
+        if (reader_ != nullptr) {
+            mosaic_reader_free(reader_);
+            reader_ = nullptr;
+        }
+        if (input_ != nullptr) {
+            (void)input_->Close();
+        }
+        input_context_.reset();
+        input_.reset();
+        current_batch_.reset();
+        closed_ = true;
+    }
+}
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_file_batch_reader.h b/src/paimon/format/mosaic/mosaic_file_batch_reader.h
new file mode 100644
index 000000000..030e26c44
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_file_batch_reader.h
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <limits>
+#include <memory>
+
+#include "arrow/type_fwd.h"
+#include "paimon/format/mosaic/mosaic_ffi.h"
+#include "paimon/format/mosaic/mosaic_stream.h"
+#include "paimon/reader/file_batch_reader.h"
+#include "paimon/result.h"
+
+namespace paimon {
+class InputStream;
+class MemoryPool;
+class Metrics;
+class PredicateFilter;
+}  // namespace paimon
+
+namespace paimon::mosaic {
+
+class MosaicFileBatchReader : public FileBatchReader {
+ public:
+    static Result<std::unique_ptr<MosaicFileBatchReader>> Create(
+        const std::shared_ptr<InputStream>& input, int32_t batch_size,
+        const std::shared_ptr<MemoryPool>& pool);
+
+    ~MosaicFileBatchReader() override;
+
+    Result<ReadBatch> NextBatch() override;
+    Result<std::unique_ptr<::ArrowSchema>> GetFileSchema() const override;
+    Status SetReadSchema(::ArrowSchema* read_schema, const std::shared_ptr<Predicate>& predicate,
+                         const std::optional<RoaringBitmap32>& selection_bitmap) override;
+    Result<uint64_t> GetPreviousBatchFileRowId(uint64_t batch_row_id) const override;
+    Result<uint64_t> GetNumberOfRows() const override;
+    std::shared_ptr<Metrics> GetReaderMetrics() const override;
+    void Close() override;
+    bool SupportPreciseBitmapSelection() const override {
+        return false;
+    }
+
+ private:
+    MosaicFileBatchReader(const std::shared_ptr<InputStream>& input, int32_t batch_size,
+                          std::unique_ptr<MosaicInputContext> input_context,
+                          MosaicReaderHandle* reader,
+                          const std::shared_ptr<arrow::Schema>& file_schema,
+                          uint32_t num_row_groups, uint64_t total_rows,
+                          const std::shared_ptr<MemoryPool>& pool,
+                          const std::shared_ptr<arrow::MemoryPool>& arrow_pool);
+
+    Result<std::shared_ptr<arrow::Array>> ReadNextRowGroup();
+    Result<bool> MatchesRowGroup(uint32_t row_group, uint32_t row_count);
+    void CloseInternal();
+
+    std::shared_ptr<InputStream> input_;
+    int32_t batch_size_;
+    std::unique_ptr<MosaicInputContext> input_context_;
+    MosaicReaderHandle* reader_;
+    std::shared_ptr<arrow::Schema> file_schema_;
+    uint32_t num_row_groups_;
+    uint64_t total_rows_;
+    uint32_t next_row_group_ = 0;
+    uint64_t next_row_group_first_row_ = 0;
+    uint64_t current_row_group_first_row_ = 0;
+    std::shared_ptr<arrow::Array> current_batch_;
+    int64_t current_batch_offset_ = 0;
+    uint64_t previous_first_row_ = std::numeric_limits<uint64_t>::max();
+    uint64_t previous_batch_row_count_ = 0;
+    std::shared_ptr<MemoryPool> pool_;
+    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+    std::shared_ptr<PredicateFilter> predicate_filter_;
+    std::shared_ptr<Metrics> metrics_;
+    bool closed_ = false;
+};
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_file_format.cpp b/src/paimon/format/mosaic/mosaic_file_format.cpp
new file mode 100644
index 000000000..937a72624
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_file_format.cpp
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/format/mosaic/mosaic_file_format.h"
+
+#include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/format/mosaic/mosaic_reader_builder.h"
+#include "paimon/format/mosaic/mosaic_stats_extractor.h"
+#include "paimon/format/mosaic/mosaic_writer_builder.h"
+
+namespace paimon::mosaic {
+
+Result<std::unique_ptr<ReaderBuilder>> MosaicFileFormat::CreateReaderBuilder(
+    int32_t batch_size) const {
+    return std::make_unique<MosaicReaderBuilder>(batch_size);
+}
+
+Result<std::unique_ptr<WriterBuilder>> MosaicFileFormat::CreateWriterBuilder(
+    ::ArrowSchema* schema, int32_t batch_size) const {
+    (void)batch_size;
+    if (schema == nullptr) {
+        return Status::Invalid("Mosaic writer schema is nullptr");
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> typed_schema,
+                                      arrow::ImportSchema(schema));
+    return std::make_unique<MosaicWriterBuilder>(typed_schema, options_);
+}
+
+Result<std::unique_ptr<FormatStatsExtractor>> MosaicFileFormat::CreateStatsExtractor(
+    ::ArrowSchema* schema) const {
+    if (schema == nullptr) {
+        return Status::Invalid("Mosaic stats schema is nullptr");
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> typed_schema,
+                                      arrow::ImportSchema(schema));
+    return std::make_unique<MosaicStatsExtractor>(typed_schema);
+}
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_file_format.h b/src/paimon/format/mosaic/mosaic_file_format.h
new file mode 100644
index 000000000..90fb9fc9d
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_file_format.h
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include "paimon/format/file_format.h"
+
+namespace paimon::mosaic {
+
+class MosaicFileFormat : public FileFormat {
+ public:
+    explicit MosaicFileFormat(const std::map<std::string, std::string>& options)
+        : identifier_("mosaic"), options_(options) {}
+
+    const std::string& Identifier() const override {
+        return identifier_;
+    }
+    Result<std::unique_ptr<ReaderBuilder>> CreateReaderBuilder(int32_t batch_size) const override;
+    Result<std::unique_ptr<WriterBuilder>> CreateWriterBuilder(::ArrowSchema* schema,
+                                                               int32_t batch_size) const override;
+    Result<std::unique_ptr<FormatStatsExtractor>> CreateStatsExtractor(
+        ::ArrowSchema* schema) const override;
+
+ private:
+    std::string identifier_;
+    std::map<std::string, std::string> options_;
+};
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_file_format_factory.cpp b/src/paimon/format/mosaic/mosaic_file_format_factory.cpp
new file mode 100644
index 000000000..0b6b6b397
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_file_format_factory.cpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/format/mosaic/mosaic_file_format_factory.h"
+
+#include "paimon/factories/factory.h"
+#include "paimon/format/mosaic/mosaic_file_format.h"
+
+namespace paimon::mosaic {
+
+const char MosaicFileFormatFactory::IDENTIFIER[] = "mosaic";
+
+Result<std::unique_ptr<FileFormat>> MosaicFileFormatFactory::Create(
+    const std::map<std::string, std::string>& options) const {
+    return std::make_unique<MosaicFileFormat>(options);
+}
+
+REGISTER_PAIMON_FACTORY(MosaicFileFormatFactory);
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_file_format_factory.h b/src/paimon/format/mosaic/mosaic_file_format_factory.h
new file mode 100644
index 000000000..e0326765d
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_file_format_factory.h
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include "paimon/format/file_format_factory.h"
+
+namespace paimon::mosaic {
+
+class MosaicFileFormatFactory : public FileFormatFactory {
+ public:
+    static const char IDENTIFIER[];
+
+    const char* Identifier() const override {
+        return IDENTIFIER;
+    }
+    Result<std::unique_ptr<FileFormat>> Create(
+        const std::map<std::string, std::string>& options) const override;
+};
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_file_format_test.cpp b/src/paimon/format/mosaic/mosaic_file_format_test.cpp
new file mode 100644
index 000000000..5dc37b5af
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_file_format_test.cpp
@@ -0,0 +1,554 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <array>
+#include <cstdint>
+#include <initializer_list>
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/array/concatenate.h"
+#include "arrow/c/bridge.h"
+#include "arrow/ipc/json_simple.h"
+#include "gtest/gtest.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/format/column_stats.h"
+#include "paimon/format/file_format.h"
+#include "paimon/format/file_format_factory.h"
+#include "paimon/format/format_writer.h"
+#include "paimon/format/mosaic/mosaic_format_defs.h"
+#include "paimon/format/reader_builder.h"
+#include "paimon/format/writer_builder.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/fs/local/local_file_system.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/reader/batch_reader.h"
+#include "paimon/testing/utils/read_result_collector.h"
+#include "paimon/testing/utils/testharness.h"
+#include "paimon/testing/utils/timezone_guard.h"
+
+namespace paimon::mosaic::test {
+
+class MosaicFileFormatTest : public ::testing::Test {
+ public:
+    // The footer layout stores the number of buckets followed by the number of row groups.
+    using FooterLayout = std::pair<uint32_t, uint32_t>;
+
+    void SetUp() override {
+        ASSERT_OK_AND_ASSIGN(format_,
+                             FileFormatFactory::Get("mosaic", {{"file.format", "mosaic"}}));
+        file_system_ = std::make_shared<LocalFileSystem>();
+        directory_ = paimon::test::UniqueTestDirectory::Create();
+        ASSERT_NE(directory_, nullptr);
+        pool_ = GetDefaultPool();
+        arrow_pool_ = GetArrowPool(pool_);
+    }
+
+    Status WriteFile(const std::string& path, const std::shared_ptr<arrow::Schema>& schema,
+                     const std::shared_ptr<arrow::Array>& array, int32_t batch_size,
+                     const std::shared_ptr<FileFormat>& format = nullptr) const {
+        ::ArrowSchema ffi_schema = {};
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &ffi_schema));
+        const std::shared_ptr<FileFormat>& writer_format = format == nullptr ? format_ : format;
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<WriterBuilder> writer_builder,
+                               writer_format->CreateWriterBuilder(&ffi_schema, batch_size));
+        writer_builder->WithMemoryPool(pool_);
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OutputStream> output,
+                               file_system_->Create(path, /*overwrite=*/false));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatWriter> writer,
+                               writer_builder->Build(output, "zstd"));
+        for (int64_t offset = 0; offset < array->length(); offset += batch_size) {
+            std::shared_ptr<arrow::Array> slice = array->Slice(offset, batch_size);
+            ::ArrowArray ffi_array = {};
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*slice, &ffi_array));
+            PAIMON_RETURN_NOT_OK(writer->AddBatch(&ffi_array));
+        }
+        PAIMON_RETURN_NOT_OK(writer->Finish());
+        return output->Close();
+    }
+
+    Result<FooterLayout> ReadFooterLayout(const std::string& path) const {
+        constexpr int64_t kFooterSize = 32;
+        constexpr int64_t kLayoutSize = 8;
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> input, file_system_->Open(path));
+        PAIMON_ASSIGN_OR_RAISE(int64_t file_size, input->Length());
+        if (file_size < kFooterSize) {
+            return Status::Invalid("Mosaic file is shorter than its footer");
+        }
+        std::array<uint8_t, kLayoutSize> layout = {};
+        PAIMON_ASSIGN_OR_RAISE(int64_t bytes_read,
+                               input->Read(reinterpret_cast<char*>(layout.data()), kLayoutSize,
+                                           file_size - kFooterSize + /*layout offset=*/16));
+        if (bytes_read != kLayoutSize) {
+            return Status::IOError("short read while reading Mosaic footer");
+        }
+        auto decode_uint32 = [&layout](size_t offset) -> uint32_t {
+            return (uint32_t{layout[offset]} << 24) | (uint32_t{layout[offset + 1]} << 16) |
+                   (uint32_t{layout[offset + 2]} << 8) | uint32_t{layout[offset + 3]};
+        };
+        return std::make_pair(decode_uint32(0), decode_uint32(4));
+    }
+
+    Result<std::shared_ptr<arrow::Array>> ReadFile(const std::string& path,
+                                                   const std::shared_ptr<arrow::Schema>& schema,
+                                                   int32_t batch_size,
+                                                   uint64_t expected_row_count) const {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReaderBuilder> reader_builder,
+                               format_->CreateReaderBuilder(batch_size));
+        reader_builder->WithMemoryPool(pool_);
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> input, file_system_->Open(path));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileBatchReader> reader,
+                               reader_builder->Build(input));
+        ::ArrowSchema ffi_schema = {};
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &ffi_schema));
+        PAIMON_RETURN_NOT_OK(reader->SetReadSchema(&ffi_schema, /*predicate=*/nullptr,
+                                                   /*selection_bitmap=*/std::nullopt));
+        PAIMON_ASSIGN_OR_RAISE(uint64_t total_rows, reader->GetNumberOfRows());
+        if (total_rows != expected_row_count) {
+            return Status::Invalid("unexpected Mosaic row count");
+        }
+
+        std::vector<std::shared_ptr<arrow::Array>> batches;
+        uint64_t expected_first_row = 0;
+        while (true) {
+            PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
+            if (BatchReader::IsEofBatch(batch)) {
+                break;
+            }
+            PAIMON_RETURN_NOT_OK(paimon::test::ReadResultCollector::CheckBatchOffset(batch));
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr<arrow::Array> arrow_batch,
+                arrow::ImportArray(batch.first.get(), batch.second.get()));
+            if (arrow_batch->length() > batch_size) {
+                return Status::Invalid("Mosaic read batch exceeds configured batch size");
+            }
+            PAIMON_ASSIGN_OR_RAISE(uint64_t first_row,
+                                   reader->GetPreviousBatchFileRowId(/*batch_row_id=*/0));
+            if (first_row != expected_first_row) {
+                return Status::Invalid("unexpected Mosaic batch first row");
+            }
+            expected_first_row += arrow_batch->length();
+            batches.push_back(std::move(arrow_batch));
+        }
+        if (expected_first_row != expected_row_count) {
+            return Status::Invalid("Mosaic batches do not contain all rows");
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> result,
+                                          arrow::Concatenate(batches, arrow_pool_.get()));
+        return result;
+    }
+
+    void AssertReadWithBatchSizes(const std::string& path,
+                                  const std::shared_ptr<arrow::Schema>& schema,
+                                  const std::shared_ptr<arrow::Array>& expected,
+                                  std::initializer_list<int32_t> batch_sizes) const {
+        for (int32_t batch_size : batch_sizes) {
+            SCOPED_TRACE("batch_size=" + std::to_string(batch_size));
+            ASSERT_OK_AND_ASSIGN(
+                std::shared_ptr<arrow::Array> actual,
+                ReadFile(path, schema, batch_size,
+                         /*expected_row_count=*/static_cast<uint64_t>(expected->length())));
+            ASSERT_TRUE(actual->Equals(expected)) << actual->ToString() << "\nvs\n"
+                                                  << expected->ToString();
+        }
+    }
+
+ protected:
+    std::shared_ptr<FileFormat> format_;
+    std::shared_ptr<LocalFileSystem> file_system_;
+    std::unique_ptr<paimon::test::UniqueTestDirectory> directory_;
+    std::shared_ptr<MemoryPool> pool_;
+    std::unique_ptr<arrow::MemoryPool> arrow_pool_;
+};
+
+TEST_F(MosaicFileFormatTest, WriteThenRead) {
+    std::string path = PathUtil::JoinPath(directory_->Str(), "data.mosaic");
+    arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false),
+                                 arrow::field("name", arrow::utf8())};
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
+    std::shared_ptr<arrow::DataType> data_type = arrow::struct_(fields);
+    std::shared_ptr<arrow::Array> expected =
+        arrow::ipc::internal::json::ArrayFromJSON(
+            data_type, R"([[1,"one"],[2,null],[3,"three"],[4,"four"],[5,"five"]])")
+            .ValueOrDie();
+
+    ASSERT_OK(WriteFile(path, schema, expected, /*batch_size=*/2));
+    AssertReadWithBatchSizes(path, schema, expected, {1, 2, 3, 5, 8});
+}
+
+TEST_F(MosaicFileFormatTest, EmptyProjectionPreservesRowCount) {
+    std::string path = PathUtil::JoinPath(directory_->Str(), "empty-projection.mosaic");
+    arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false),
+                                 arrow::field("name", arrow::utf8())};
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
+    std::shared_ptr<arrow::Array> data =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields),
+                                                  R"([[1,"one"],[2,"two"],[3,"three"]])")
+            .ValueOrDie();
+    ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/2));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReaderBuilder> reader_builder,
+                         format_->CreateReaderBuilder(/*batch_size=*/2));
+    reader_builder->WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input, file_system_->Open(path));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileBatchReader> reader, reader_builder->Build(input));
+    ::ArrowSchema ffi_schema = {};
+    ASSERT_TRUE(arrow::ExportSchema(*arrow::schema({}), &ffi_schema).ok());
+    ASSERT_OK(reader->SetReadSchema(&ffi_schema, /*predicate=*/nullptr,
+                                    /*selection_bitmap=*/std::nullopt));
+
+    for (int64_t expected_rows : {2, 1}) {
+        ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
+        ASSERT_FALSE(BatchReader::IsEofBatch(batch));
+        std::shared_ptr<arrow::RecordBatch> record_batch =
+            arrow::ImportRecordBatch(batch.first.get(), batch.second.get()).ValueOrDie();
+        ASSERT_EQ(record_batch->num_columns(), 0);
+        ASSERT_EQ(record_batch->num_rows(), expected_rows);
+    }
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof_batch, reader->NextBatch());
+    ASSERT_TRUE(BatchReader::IsEofBatch(eof_batch));
+}
+
+TEST_F(MosaicFileFormatTest, SetReadSchemaResetsReaderToFirstRow) {
+    std::string path = PathUtil::JoinPath(directory_->Str(), "reset-reader.mosaic");
+    arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false),
+                                 arrow::field("name", arrow::utf8())};
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
+    std::shared_ptr<arrow::Array> data =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields),
+                                                  R"([[1,"one"],[2,"two"],[3,"three"]])")
+            .ValueOrDie();
+    ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/2));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReaderBuilder> reader_builder,
+                         format_->CreateReaderBuilder(/*batch_size=*/2));
+    reader_builder->WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input, file_system_->Open(path));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileBatchReader> reader, reader_builder->Build(input));
+
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch first_batch, reader->NextBatch());
+    ASSERT_OK_AND_ASSIGN(uint64_t first_row, reader->GetPreviousBatchFileRowId(/*batch_row_id=*/0));
+    ASSERT_EQ(first_row, 0);
+    std::shared_ptr<arrow::Array> first_array =
+        arrow::ImportArray(first_batch.first.get(), first_batch.second.get()).ValueOrDie();
+    ASSERT_TRUE(first_array->Equals(data->Slice(/*offset=*/0, /*length=*/2)));
+
+    std::shared_ptr<arrow::Schema> projected_schema = arrow::schema({fields[1]});
+    ::ArrowSchema ffi_schema = {};
+    ASSERT_TRUE(arrow::ExportSchema(*projected_schema, &ffi_schema).ok());
+    ASSERT_OK(reader->SetReadSchema(&ffi_schema, /*predicate=*/nullptr,
+                                    /*selection_bitmap=*/std::nullopt));
+    ASSERT_NOK(reader->GetPreviousBatchFileRowId(/*batch_row_id=*/0));
+
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch projected_batch, reader->NextBatch());
+    ASSERT_OK_AND_ASSIGN(first_row, reader->GetPreviousBatchFileRowId(/*batch_row_id=*/0));
+    ASSERT_EQ(first_row, 0);
+    std::shared_ptr<arrow::Array> projected_array =
+        arrow::ImportArray(projected_batch.first.get(), projected_batch.second.get()).ValueOrDie();
+    std::shared_ptr<arrow::Array> expected =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[1]}),
+                                                  R"([["one"],["two"]])")
+            .ValueOrDie();
+    ASSERT_TRUE(projected_array->Equals(expected)) << projected_array->ToString();
+}
+
+TEST_F(MosaicFileFormatTest, WriterOptions) {
+    std::map<std::string, std::string> options = {
+        {"file.format", "mosaic"},      {Options::FILE_BLOCK_SIZE, "1 B"},
+        {MOSAIC_NUM_BUCKETS, "2"},      {MOSAIC_MAX_DICT_TOTAL_BYTES, "1 KB"},
+        {MOSAIC_MAX_DICT_ENTRIES, "2"}, {MOSAIC_PAGE_SIZE_THRESHOLD, "1 B"},
+    };
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileFormat> configured_format,
+                         FileFormatFactory::Get("mosaic", options));
+    std::string path = PathUtil::JoinPath(directory_->Str(), "writer-options.mosaic");
+    arrow::FieldVector fields = {
+        arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::int32()),
+        arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::int32()),
+        arrow::field("f4", arrow::int32()), arrow::field("f5", arrow::int32()),
+    };
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
+    std::shared_ptr<arrow::Array> expected =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields),
+                                                  R"([[1,2,3,4,5,6],
+                                                     [7,8,9,10,11,12],
+                                                     [13,14,15,16,17,18],
+                                                     [19,20,21,22,23,24],
+                                                     [25,26,27,28,29,30]])")
+            .ValueOrDie();
+
+    ASSERT_OK(WriteFile(path, schema, expected, /*batch_size=*/2, configured_format));
+    ASSERT_OK_AND_ASSIGN(FooterLayout footer_layout, ReadFooterLayout(path));
+    ASSERT_EQ(footer_layout.first, 2);
+    ASSERT_EQ(footer_layout.second, 3);
+    AssertReadWithBatchSizes(path, schema, expected, {10});
+}
+
+TEST_F(MosaicFileFormatTest, ExtractStatistics) {
+    std::map<std::string, std::string> options = {
+        {"file.format", "mosaic"},
+        {Options::FILE_BLOCK_SIZE, "1 B"},
+        {MOSAIC_STATS_COLUMNS, " id, name, ts, amount, all_null "},
+    };
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileFormat> configured_format,
+                         FileFormatFactory::Get("mosaic", options));
+    std::string path = PathUtil::JoinPath(directory_->Str(), "statistics.mosaic");
+    arrow::FieldVector fields = {
+        arrow::field("id", arrow::int32()),
+        arrow::field("name", arrow::utf8()),
+        arrow::field("ts", arrow::timestamp(arrow::TimeUnit::NANO)),
+        arrow::field("amount", arrow::decimal128(10, 2)),
+        arrow::field("untracked", arrow::int64()),
+        arrow::field("all_null", arrow::int32()),
+    };
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
+    std::shared_ptr<arrow::Array> data =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+          [3,"three","1970-01-01 00:00:00.000000003","3.30",30,null],
+          [1,"one","1970-01-01 00:00:00.000000001","1.10",10,null],
+          [null,null,null,null,null,null],
+          [5,"five","1970-01-01 00:00:00.000000005","5.50",50,null],
+          [2,"","1970-01-01 00:00:00.000000002","2.20",20,null]
+        ])")
+            .ValueOrDie();
+    ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/2, configured_format));
+
+    ::ArrowSchema ffi_schema = {};
+    ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatStatsExtractor> extractor,
+                         configured_format->CreateStatsExtractor(&ffi_schema));
+    ASSERT_OK_AND_ASSIGN(auto result, extractor->ExtractWithFileInfo(file_system_, path, pool_));
+    ASSERT_EQ(result.second.GetRowCount(), 5);
+    std::vector<std::string> expected_stats = {
+        "min 1, max 5, null count 1",
+        "min , max three, null count 1",
+        "min 1970-01-01 00:00:00.000000001, max 1970-01-01 00:00:00.000000005, null count 1",
+        "min 1.10, max 5.50, null count 1",
+        "min null, max null, null count null",
+        "min null, max null, null count 5",
+    };
+    ASSERT_EQ(result.first.size(), expected_stats.size());
+    for (size_t i = 0; i < expected_stats.size(); ++i) {
+        ASSERT_EQ(result.first[i]->ToString(), expected_stats[i]);
+    }
+}
+
+TEST_F(MosaicFileFormatTest, RowGroupPredicateFiltering) {
+    std::map<std::string, std::string> options = {
+        {"file.format", "mosaic"},
+        {Options::FILE_BLOCK_SIZE, "1 B"},
+        {MOSAIC_STATS_COLUMNS, "id"},
+    };
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileFormat> configured_format,
+                         FileFormatFactory::Get("mosaic", options));
+    std::string path = PathUtil::JoinPath(directory_->Str(), "predicate.mosaic");
+    arrow::FieldVector fields = {arrow::field("id", arrow::int32()),
+                                 arrow::field("untracked", arrow::int32())};
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
+    std::shared_ptr<arrow::Array> data =
+        arrow::ipc::internal::json::ArrayFromJSON(
+            arrow::struct_(fields), R"([[1,null],[2,2],[10,10],[11,11],[20,20],[21,21]])")
+            .ValueOrDie();
+    ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/2, configured_format));
+    ASSERT_OK_AND_ASSIGN(FooterLayout footer_layout, ReadFooterLayout(path));
+    ASSERT_EQ(footer_layout.second, 3);
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReaderBuilder> reader_builder,
+                         configured_format->CreateReaderBuilder(/*batch_size=*/10));
+    reader_builder->WithMemoryPool(pool_);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input, file_system_->Open(path));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileBatchReader> reader, reader_builder->Build(input));
+
+    ::ArrowSchema ffi_schema = {};
+    ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok());
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::GreaterThan(
+        /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(15));
+    ASSERT_OK(reader->SetReadSchema(&ffi_schema, predicate, /*selection_bitmap=*/std::nullopt));
+    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
+    std::shared_ptr<arrow::Array> actual =
+        arrow::ImportArray(batch.first.get(), batch.second.get()).ValueOrDie();
+    ASSERT_TRUE(actual->Equals(data->Slice(/*offset=*/4, /*length=*/2))) << actual->ToString();
+    ASSERT_OK_AND_ASSIGN(uint64_t first_row, reader->GetPreviousBatchFileRowId(/*batch_row_id=*/0));
+    ASSERT_EQ(first_row, 4);
+    ASSERT_OK_AND_ASSIGN(batch, reader->NextBatch());
+    ASSERT_TRUE(BatchReader::IsEofBatch(batch));
+
+    ffi_schema = {};
+    ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok());
+    predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id",
+                                              FieldType::INT, Literal(100));
+    ASSERT_OK(reader->SetReadSchema(&ffi_schema, predicate, /*selection_bitmap=*/std::nullopt));
+    ASSERT_OK_AND_ASSIGN(batch, reader->NextBatch());
+    ASSERT_TRUE(BatchReader::IsEofBatch(batch));
+
+    ffi_schema = {};
+    ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok());
+    predicate = PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"untracked",
+                                              FieldType::INT, Literal(100));
+    ASSERT_OK(reader->SetReadSchema(&ffi_schema, predicate, /*selection_bitmap=*/std::nullopt));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> actual_without_stats,
+                         paimon::test::ReadResultCollector::CollectResult(reader.get()));
+    ASSERT_TRUE(actual_without_stats->Equals(arrow::ChunkedArray(data)))
+        << actual_without_stats->ToString();
+
+    ffi_schema = {};
+    ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok());
+    predicate =
+        PredicateBuilder::IsNull(/*field_index=*/1, /*field_name=*/"untracked", FieldType::INT);
+    ASSERT_OK(reader->SetReadSchema(&ffi_schema, predicate, /*selection_bitmap=*/std::nullopt));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> actual_is_null_without_stats,
+                         paimon::test::ReadResultCollector::CollectResult(reader.get()));
+    ASSERT_TRUE(actual_is_null_without_stats->Equals(arrow::ChunkedArray(data)))
+        << actual_is_null_without_stats->ToString();
+}
+
+TEST_F(MosaicFileFormatTest, WriteThenReadSupportedTypes) {
+    std::string path = PathUtil::JoinPath(directory_->Str(), "supported-types.mosaic");
+    arrow::FieldVector fields = {
+        arrow::field("f0", arrow::boolean()),
+        arrow::field("f1", arrow::int8()),
+        arrow::field("f2", arrow::int16()),
+        arrow::field("f3", arrow::int32()),
+        arrow::field("f4", arrow::int64()),
+        arrow::field("f5", arrow::float32()),
+        arrow::field("f6", arrow::float64()),
+        arrow::field("f7", arrow::utf8()),
+        arrow::field("f8", arrow::binary()),
+        arrow::field("f9", arrow::map(arrow::int8(), arrow::int16())),
+        arrow::field("f10", arrow::list(arrow::float32())),
+        arrow::field("f12", arrow::timestamp(arrow::TimeUnit::NANO)),
+        arrow::field("f13", arrow::date32()),
+        arrow::field("f14", arrow::decimal128(2, 2)),
+        arrow::field("f15", arrow::decimal128(30, 2)),
+    };
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
+    std::shared_ptr<arrow::Array> expected =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+          [true,-128,-32768,-2147483648,-4294967298,0.5,1.141592659,"mosaic","binary",
+           [[-1,-2],[1,2]],[1.5,null],"1970-01-01 00:00:00.000000001",-1,"-0.99","-123456789987654321.45"],
+          [false,127,32767,2147483647,4294967296,2.0,3.141592657,"", "",
+           [],[],"2030-12-31 23:59:59.999999999",12345,"0.78","123456789987654321.45"],
+          [null,null,null,null,null,null,null,null,null,null,null,null,null,null,null]
+        ])")
+            .ValueOrDie();
+
+    ASSERT_OK(WriteFile(path, schema, expected, /*batch_size=*/2));
+    AssertReadWithBatchSizes(path, schema, expected, {1, 2, 3, 5});
+}
+
+TEST_F(MosaicFileFormatTest, WriteThenReadNestedTypes) {
+    std::string path = PathUtil::JoinPath(directory_->Str(), "nested.mosaic");
+    std::shared_ptr<arrow::DataType> list_type = arrow::list(arrow::int32());
+    std::shared_ptr<arrow::DataType> map_type = arrow::map(arrow::utf8(), arrow::int64());
+    std::shared_ptr<arrow::DataType> nested_list_type = arrow::list(list_type);
+    std::shared_ptr<arrow::DataType> list_of_maps_type =
+        arrow::list(arrow::map(arrow::utf8(), arrow::int32()));
+    std::shared_ptr<arrow::DataType> map_of_lists_type =
+        arrow::map(arrow::utf8(), arrow::list(arrow::int32()));
+    arrow::FieldVector fields = {
+        arrow::field("id", arrow::int32(), false),
+        arrow::field("list_col", list_type),
+        arrow::field("map_col", map_type),
+        arrow::field("nested_list_col", nested_list_type),
+        arrow::field("list_of_maps_col", list_of_maps_type),
+        arrow::field("map_of_lists_col", map_of_lists_type),
+    };
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
+    std::shared_ptr<arrow::Array> expected =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields),
+                                                  R"([
+              [1,[1,null,3],[["a",10],["b",null]],[[1,2],[3]],
+                 [[["a",1]],[["b",2],["c",3]]],[["x",[1,2]],["y",[]]]],
+              [2,[],[],[[4]],[],[]],
+              [3,null,null,null,null,null],
+              [4,[4,5],[["c",30]],[[],[5,null]],[[]],[["z",[null,5]]]],
+              [5,[6],[["d",40],["e",50]],[],[[["d",4]]],[["w",[]]]]
+            ])")
+            .ValueOrDie();
+
+    ASSERT_OK(WriteFile(path, schema, expected, /*batch_size=*/2));
+    AssertReadWithBatchSizes(path, schema, expected, {1, 2, 3, 5, 8});
+}
+
+TEST_F(MosaicFileFormatTest, WriteThenReadTimestampTypes) {
+    const std::string timezone = "Asia/Tokyo";
+    paimon::test::TimezoneGuard timezone_guard(timezone);
+    std::string path = PathUtil::JoinPath(directory_->Str(), "timestamp-types.mosaic");
+    arrow::FieldVector fields = {
+        arrow::field("ts_milli", arrow::timestamp(arrow::TimeUnit::MILLI)),
+        arrow::field("ts_micro", arrow::timestamp(arrow::TimeUnit::MICRO)),
+        arrow::field("ts_nano", arrow::timestamp(arrow::TimeUnit::NANO)),
+        arrow::field("ts_tz_milli", arrow::timestamp(arrow::TimeUnit::MILLI, timezone)),
+        arrow::field("ts_tz_micro", arrow::timestamp(arrow::TimeUnit::MICRO, timezone)),
+        arrow::field("ts_tz_nano", arrow::timestamp(arrow::TimeUnit::NANO, timezone)),
+    };
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
+    std::shared_ptr<arrow::Array> expected =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+          ["1970-01-01 00:00:00.001","1970-01-01 00:00:00.000001","1970-01-01 00:00:00.000000001",
+           "1970-01-01 00:00:00.002","1970-01-01 00:00:00.000002","1970-01-01 00:00:00.000000002"],
+          ["2030-12-31 23:59:59.999","2030-12-31 23:59:59.999999","2030-12-31 23:59:59.999999999",
+           "2031-01-01 00:00:00.001","2031-01-01 00:00:00.000001","2031-01-01 00:00:00.000000001"],
+          [null,null,null,null,null,null]
+        ])")
+            .ValueOrDie();
+
+    ASSERT_OK(WriteFile(path, schema, expected, /*batch_size=*/2));
+    AssertReadWithBatchSizes(path, schema, expected, {1, 2, 3, 5});
+}
+
+TEST_F(MosaicFileFormatTest, RejectUnsupportedTimestampSecond) {
+    std::string path = PathUtil::JoinPath(directory_->Str(), "timestamp-second.mosaic");
+    arrow::FieldVector fields = {
+        arrow::field("ts_second", arrow::timestamp(arrow::TimeUnit::SECOND)),
+    };
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
+    std::shared_ptr<arrow::Array> array =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields),
+                                                  R"([["1970-01-01 00:00:01"]])")
+            .ValueOrDie();
+
+    ASSERT_NOK_WITH_MSG(WriteFile(path, schema, array, /*batch_size=*/1),
+                        "unsupported Timestamp unit: Second");
+}
+
+TEST_F(MosaicFileFormatTest, RejectUnsupportedStructType) {
+    std::string path = PathUtil::JoinPath(directory_->Str(), "struct.mosaic");
+    std::shared_ptr<arrow::DataType> struct_type = arrow::struct_(
+        {arrow::field("value", arrow::int32()), arrow::field("label", arrow::utf8())});
+    arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false),
+                                 arrow::field("struct_col", struct_type)};
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
+    std::shared_ptr<arrow::Array> array =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields),
+                                                  R"([[1,[10,"ten"]],[2,null]])")
+            .ValueOrDie();
+
+    ASSERT_NOK_WITH_MSG(WriteFile(path, schema, array, /*batch_size=*/2),
+                        "unsupported DataType: Struct");
+}
+
+}  // namespace paimon::mosaic::test
diff --git a/src/paimon/format/mosaic/mosaic_format_defs.h b/src/paimon/format/mosaic/mosaic_format_defs.h
new file mode 100644
index 000000000..4b5265edf
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_format_defs.h
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+namespace paimon::mosaic {
+
+/// Number of column buckets for parallel IO.
+static inline const char MOSAIC_NUM_BUCKETS[] = "mosaic.num-buckets";
+
+/// Max dict size per column.
+static inline const char MOSAIC_MAX_DICT_TOTAL_BYTES[] = "mosaic.max-dict-total-bytes";
+
+/// Max dict entries per column.
+static inline const char MOSAIC_MAX_DICT_ENTRIES[] = "mosaic.max-dict-entries";
+
+/// Min avg column page size to enable paged mode.
+static inline const char MOSAIC_PAGE_SIZE_THRESHOLD[] = "mosaic.page-size-threshold";
+
+/// Comma-separated list of column names to collect statistics for. Empty means no statistics
+/// collection.
+static inline const char MOSAIC_STATS_COLUMNS[] = "mosaic.stats-columns";
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_format_writer.cpp b/src/paimon/format/mosaic/mosaic_format_writer.cpp
new file mode 100644
index 000000000..828ec565b
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_format_writer.cpp
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/format/mosaic/mosaic_format_writer.h"
+
+#include <utility>
+
+#include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/fs/file_system.h"
+
+namespace paimon::mosaic {
+
+MosaicFormatWriter::MosaicFormatWriter(const std::shared_ptr<OutputStream>& output,
+                                       const std::shared_ptr<arrow::Schema>& schema,
+                                       std::unique_ptr<MosaicOutputContext> output_context,
+                                       MosaicWriterHandle* writer)
+    : output_(output),
+      schema_(schema),
+      output_context_(std::move(output_context)),
+      writer_(writer),
+      metrics_(std::make_shared<MetricsImpl>()) {}
+
+Result<std::unique_ptr<MosaicFormatWriter>> MosaicFormatWriter::Create(
+    const std::shared_ptr<OutputStream>& output, const std::shared_ptr<arrow::Schema>& schema,
+    const MosaicWriterOptions& options) {
+    if (output == nullptr || schema == nullptr) {
+        return Status::Invalid("Mosaic writer requires non-null output and schema");
+    }
+    auto output_context = std::make_unique<MosaicOutputContext>(output);
+    MosaicOutputFile output_file = {};
+    output_file.ctx = output_context.get();
+    output_file.write_fn = MosaicOutputContext::Write;
+    output_file.flush_fn = MosaicOutputContext::Flush;
+    output_file.get_pos_fn = MosaicOutputContext::GetPos;
+
+    ::ArrowSchema ffi_schema = {};
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &ffi_schema));
+    MosaicWriterHandle* writer = mosaic_writer_open(output_file, &ffi_schema, options);
+    if (writer == nullptr) {
+        Status status = MosaicFfiError("open Mosaic writer", output_context->GetCallbackStatus());
+        ArrowSchemaRelease(&ffi_schema);
+        return status;
+    }
+    return std::unique_ptr<MosaicFormatWriter>(
+        new MosaicFormatWriter(output, schema, std::move(output_context), writer));
+}
+
+MosaicFormatWriter::~MosaicFormatWriter() {
+    if (writer_ != nullptr) {
+        if (!finished_) {
+            mosaic_writer_close(writer_);
+        }
+        mosaic_writer_free(writer_);
+    }
+}
+
+Status MosaicFormatWriter::AddBatch(::ArrowArray* batch) {
+    if (batch == nullptr) {
+        return Status::Invalid("Mosaic writer batch is nullptr");
+    }
+    if (finished_) {
+        return Status::Invalid("cannot add a batch after Mosaic writer is finished");
+    }
+    ::ArrowSchema ffi_schema = {};
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, &ffi_schema));
+    if (mosaic_writer_write_batch(writer_, batch, &ffi_schema) != 0) {
+        Status status = MosaicFfiError("write Mosaic batch", output_context_->GetCallbackStatus());
+        ArrowSchemaRelease(&ffi_schema);
+        return status;
+    }
+    return Status::OK();
+}
+
+Status MosaicFormatWriter::Flush() {
+    if (finished_) {
+        return Status::OK();
+    }
+    return output_->Flush();
+}
+
+Status MosaicFormatWriter::Finish() {
+    if (finished_) {
+        return Status::OK();
+    }
+    if (mosaic_writer_close(writer_) != 0) {
+        return MosaicFfiError("finish Mosaic writer", output_context_->GetCallbackStatus());
+    }
+    finished_ = true;
+    return Status::OK();
+}
+
+Result<bool> MosaicFormatWriter::ReachTargetSize(bool suggested_check, int64_t target_size) const {
+    if (!suggested_check) {
+        return false;
+    }
+    int64_t estimated_size = 0;
+    if (mosaic_writer_estimated_file_size(writer_, &estimated_size) != 0) {
+        return MosaicFfiError("estimate Mosaic file size", output_context_->GetCallbackStatus());
+    }
+    return estimated_size >= target_size;
+}
+
+std::shared_ptr<Metrics> MosaicFormatWriter::GetWriterMetrics() const {
+    return metrics_;
+}
+
+Status MosaicFormatWriter::AddMetadata(const std::map<std::string, std::string>& metadata) {
+    (void)metadata;
+    return Status::NotImplemented("Mosaic writer metadata is not supported");
+}
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_format_writer.h b/src/paimon/format/mosaic/mosaic_format_writer.h
new file mode 100644
index 000000000..b96aba887
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_format_writer.h
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include "paimon/format/format_writer.h"
+#include "paimon/format/mosaic/mosaic_ffi.h"
+#include "paimon/format/mosaic/mosaic_stream.h"
+#include "paimon/result.h"
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+namespace paimon {
+class Metrics;
+class OutputStream;
+}  // namespace paimon
+
+namespace paimon::mosaic {
+
+class MosaicFormatWriter : public FormatWriter {
+ public:
+    static Result<std::unique_ptr<MosaicFormatWriter>> Create(
+        const std::shared_ptr<OutputStream>& output, const std::shared_ptr<arrow::Schema>& schema,
+        const MosaicWriterOptions& options);
+
+    ~MosaicFormatWriter() override;
+
+    Status AddBatch(::ArrowArray* batch) override;
+    Status Flush() override;
+    Status Finish() override;
+    Result<bool> ReachTargetSize(bool suggested_check, int64_t target_size) const override;
+    std::shared_ptr<Metrics> GetWriterMetrics() const override;
+    Status AddMetadata(const std::map<std::string, std::string>& metadata) override;
+
+ private:
+    MosaicFormatWriter(const std::shared_ptr<OutputStream>& output,
+                       const std::shared_ptr<arrow::Schema>& schema,
+                       std::unique_ptr<MosaicOutputContext> output_context,
+                       MosaicWriterHandle* writer);
+
+    std::shared_ptr<OutputStream> output_;
+    std::shared_ptr<arrow::Schema> schema_;
+    std::unique_ptr<MosaicOutputContext> output_context_;
+    MosaicWriterHandle* writer_;
+    std::shared_ptr<Metrics> metrics_;
+    bool finished_ = false;
+};
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_reader_builder.h b/src/paimon/format/mosaic/mosaic_reader_builder.h
new file mode 100644
index 000000000..ac1afba92
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_reader_builder.h
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "paimon/format/mosaic/mosaic_file_batch_reader.h"
+#include "paimon/format/reader_builder.h"
+#include "paimon/memory/memory_pool.h"
+
+namespace paimon::mosaic {
+
+class MosaicReaderBuilder : public ReaderBuilder {
+ public:
+    explicit MosaicReaderBuilder(int32_t batch_size)
+        : batch_size_(batch_size), pool_(GetDefaultPool()) {}
+
+    ReaderBuilder* WithMemoryPool(const std::shared_ptr<MemoryPool>& pool) override {
+        pool_ = pool;
+        return this;
+    }
+
+    Result<std::unique_ptr<FileBatchReader>> Build(
+        const std::shared_ptr<InputStream>& input) const override {
+        if (pool_ == nullptr) {
+            return Status::Invalid("Mosaic reader memory pool is nullptr");
+        }
+        return MosaicFileBatchReader::Create(input, batch_size_, pool_);
+    }
+
+ private:
+    int32_t batch_size_;
+    std::shared_ptr<MemoryPool> pool_;
+};
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_stats.cpp b/src/paimon/format/mosaic/mosaic_stats.cpp
new file mode 100644
index 000000000..17b788200
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_stats.cpp
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/format/mosaic/mosaic_stats.h"
+
+#include <cstring>
+#include <functional>
+#include <limits>
+#include <utility>
+
+#include "arrow/api.h"
+#include "fmt/format.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/date_time_utils.h"
+#include "paimon/common/utils/fields_comparator.h"
+#include "paimon/common/utils/math.h"
+#include "paimon/data/decimal.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/defs.h"
+#include "paimon/format/column_stats.h"
+#include "paimon/format/mosaic/mosaic_stream.h"
+#include "paimon/io/byte_order.h"
+
+namespace paimon::mosaic {
+
+namespace {
+
+template <typename T>
+using OptionalMinMax = std::pair<std::optional<T>, std::optional<T>>;
+
+template <typename T>
+Result<T> DecodeBigEndian(const std::vector<uint8_t>& bytes) {
+    if (bytes.size() != sizeof(T)) {
+        return Status::Invalid(
+            fmt::format("invalid Mosaic statistic size {}, expected {}", bytes.size(), sizeof(T)));
+    }
+    T value;
+    std::memcpy(&value, bytes.data(), sizeof(T));
+    if constexpr (SystemByteOrder() == ByteOrder::PAIMON_LITTLE_ENDIAN) {
+        value = EndianSwapValue(value);
+    }
+    return value;
+}
+
+template <typename T, typename Decoder, typename Less = std::less<T>>
+Result<std::pair<std::optional<T>, std::optional<T>>> CollectMinMax(
+    const std::vector<const MosaicStatsUtils::ColumnStatistics*>& stats, Decoder decoder,
+    Less less = Less()) {
+    std::optional<T> min;
+    std::optional<T> max;
+    for (const MosaicStatsUtils::ColumnStatistics* stat : stats) {
+        if (stat->min.has_value() != stat->max.has_value()) {
+            return Status::Invalid("Mosaic statistics contain incomplete min/max values");
+        }
+        if (!stat->min.has_value()) {
+            continue;
+        }
+        PAIMON_ASSIGN_OR_RAISE(T row_group_min, decoder(stat->min.value()));
+        PAIMON_ASSIGN_OR_RAISE(T row_group_max, decoder(stat->max.value()));
+        if (!min.has_value() || less(row_group_min, min.value())) {
+            min = std::move(row_group_min);
+        }
+        if (!max.has_value() || less(max.value(), row_group_max)) {
+            max = std::move(row_group_max);
+        }
+    }
+    return std::make_pair(std::move(min), std::move(max));
+}
+
+Result<std::optional<int64_t>> CollectNullCount(
+    const std::vector<const MosaicStatsUtils::ColumnStatistics*>& stats,
+    bool missing_null_count_is_zero) {
+    if (stats.empty()) {
+        return missing_null_count_is_zero ? std::optional<int64_t>(0) : std::nullopt;
+    }
+    uint64_t result = 0;
+    for (const MosaicStatsUtils::ColumnStatistics* stat : stats) {
+        if (stat->null_count >
+            static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) - result) {
+            return Status::Invalid("Mosaic null count exceeds int64 range");
+        }
+        result += stat->null_count;
+    }
+    return std::optional<int64_t>(static_cast<int64_t>(result));
+}
+
+Result<Timestamp> DecodeTimestamp(const std::vector<uint8_t>& bytes,
+                                  const std::shared_ptr<arrow::TimestampType>& type) {
+    if (type->unit() == arrow::TimeUnit::NANO) {
+        if (bytes.size() != 12) {
+            return Status::Invalid(
+                fmt::format("invalid Mosaic nanosecond timestamp statistic size {}", bytes.size()));
+        }
+        std::vector<uint8_t> millis_bytes(bytes.begin(), bytes.begin() + 8);
+        std::vector<uint8_t> nanos_bytes(bytes.begin() + 8, bytes.end());
+        PAIMON_ASSIGN_OR_RAISE(int64_t millis, DecodeBigEndian<int64_t>(millis_bytes));
+        PAIMON_ASSIGN_OR_RAISE(int32_t nanos, DecodeBigEndian<int32_t>(nanos_bytes));
+        if (nanos < 0 || nanos > 999999) {
+            return Status::Invalid("invalid Mosaic nanosecond timestamp statistic");
+        }
+        return Timestamp(millis, nanos);
+    }
+    PAIMON_ASSIGN_OR_RAISE(int64_t value, DecodeBigEndian<int64_t>(bytes));
+    auto [millis, nanos] = DateTimeUtils::TimestampConverter(
+        value, DateTimeUtils::GetTimeTypeFromArrowType(type), DateTimeUtils::TimeType::MILLISECOND,
+        DateTimeUtils::TimeType::NANOSECOND);
+    return Timestamp(millis, static_cast<int32_t>(nanos));
+}
+
+Result<std::unique_ptr<ColumnStats>> ConvertFieldStatistics(
+    const std::shared_ptr<arrow::DataType>& type,
+    const std::vector<const MosaicStatsUtils::ColumnStatistics*>& stats,
+    bool missing_null_count_is_zero) {
+    PAIMON_ASSIGN_OR_RAISE(std::optional<int64_t> null_count,
+                           CollectNullCount(stats, missing_null_count_is_zero));
+    switch (type->id()) {
+        case arrow::Type::BOOL: {
+            auto decoder = [](const std::vector<uint8_t>& bytes) -> Result<bool> {
+                if (bytes.size() != 1 || bytes[0] > 1) {
+                    return Status::Invalid("invalid Mosaic boolean statistic");
+                }
+                return bytes[0] != 0;
+            };
+            PAIMON_ASSIGN_OR_RAISE(OptionalMinMax<bool> min_max,
+                                   CollectMinMax<bool>(stats, decoder));
+            return ColumnStats::CreateBooleanColumnStats(min_max.first, min_max.second, null_count);
+        }
+        case arrow::Type::INT8: {
+            PAIMON_ASSIGN_OR_RAISE(OptionalMinMax<int8_t> min_max,
+                                   CollectMinMax<int8_t>(stats, DecodeBigEndian<int8_t>));
+            return ColumnStats::CreateTinyIntColumnStats(min_max.first, min_max.second, null_count);
+        }
+        case arrow::Type::INT16: {
+            PAIMON_ASSIGN_OR_RAISE(OptionalMinMax<int16_t> min_max,
+                                   CollectMinMax<int16_t>(stats, DecodeBigEndian<int16_t>));
+            return ColumnStats::CreateSmallIntColumnStats(min_max.first, min_max.second,
+                                                          null_count);
+        }
+        case arrow::Type::INT32: {
+            PAIMON_ASSIGN_OR_RAISE(OptionalMinMax<int32_t> min_max,
+                                   CollectMinMax<int32_t>(stats, DecodeBigEndian<int32_t>));
+            return ColumnStats::CreateIntColumnStats(min_max.first, min_max.second, null_count);
+        }
+        case arrow::Type::INT64: {
+            PAIMON_ASSIGN_OR_RAISE(OptionalMinMax<int64_t> min_max,
+                                   CollectMinMax<int64_t>(stats, DecodeBigEndian<int64_t>));
+            return ColumnStats::CreateBigIntColumnStats(min_max.first, min_max.second, null_count);
+        }
+        case arrow::Type::FLOAT: {
+            auto less = [](float lhs, float rhs) {
+                return FieldsComparator::CompareFloatingPoint(lhs, rhs) < 0;
+            };
+            PAIMON_ASSIGN_OR_RAISE(OptionalMinMax<float> min_max,
+                                   CollectMinMax<float>(stats, DecodeBigEndian<float>, less));
+            return ColumnStats::CreateFloatColumnStats(min_max.first, min_max.second, null_count);
+        }
+        case arrow::Type::DOUBLE: {
+            auto less = [](double lhs, double rhs) {
+                return FieldsComparator::CompareFloatingPoint(lhs, rhs) < 0;
+            };
+            PAIMON_ASSIGN_OR_RAISE(OptionalMinMax<double> min_max,
+                                   CollectMinMax<double>(stats, DecodeBigEndian<double>, less));
+            return ColumnStats::CreateDoubleColumnStats(min_max.first, min_max.second, null_count);
+        }
+        case arrow::Type::STRING: {
+            auto decoder = [](const std::vector<uint8_t>& bytes) -> Result<std::string> {
+                if (bytes.empty()) {
+                    return std::string();
+                }
+                return std::string(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+            };
+            PAIMON_ASSIGN_OR_RAISE(OptionalMinMax<std::string> min_max,
+                                   CollectMinMax<std::string>(stats, decoder));
+            return ColumnStats::CreateStringColumnStats(min_max.first, min_max.second, null_count);
+        }
+        case arrow::Type::BINARY:
+            return ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, null_count);
+        case arrow::Type::DATE32: {
+            PAIMON_ASSIGN_OR_RAISE(OptionalMinMax<int32_t> min_max,
+                                   CollectMinMax<int32_t>(stats, DecodeBigEndian<int32_t>));
+            return ColumnStats::CreateDateColumnStats(min_max.first, min_max.second, null_count);
+        }
+        case arrow::Type::TIMESTAMP: {
+            auto timestamp_type = checked_pointer_cast<arrow::TimestampType>(type);
+            auto decoder = [&timestamp_type](const std::vector<uint8_t>& bytes) {
+                return DecodeTimestamp(bytes, timestamp_type);
+            };
+            PAIMON_ASSIGN_OR_RAISE(OptionalMinMax<Timestamp> min_max,
+                                   CollectMinMax<Timestamp>(stats, decoder));
+            return ColumnStats::CreateTimestampColumnStats(
+                min_max.first, min_max.second, null_count,
+                DateTimeUtils::GetPrecisionFromType(timestamp_type));
+        }
+        case arrow::Type::DECIMAL128: {
+            auto decimal_type = checked_pointer_cast<arrow::Decimal128Type>(type);
+            if (decimal_type->precision() > 18) {
+                return ColumnStats::CreateDecimalColumnStats(std::nullopt, std::nullopt, null_count,
+                                                             decimal_type->precision(),
+                                                             decimal_type->scale());
+            }
+            auto decoder = [&decimal_type](const std::vector<uint8_t>& bytes) -> Result<Decimal> {
+                PAIMON_ASSIGN_OR_RAISE(int64_t value, DecodeBigEndian<int64_t>(bytes));
+                return Decimal::FromUnscaledLong(value, decimal_type->precision(),
+                                                 decimal_type->scale());
+            };
+            PAIMON_ASSIGN_OR_RAISE(OptionalMinMax<Decimal> min_max,
+                                   CollectMinMax<Decimal>(stats, decoder));
+            return ColumnStats::CreateDecimalColumnStats(min_max.first, min_max.second, null_count,
+                                                         decimal_type->precision(),
+                                                         decimal_type->scale());
+        }
+        case arrow::Type::LIST:
+            return ColumnStats::CreateNestedColumnStats(FieldType::ARRAY, null_count);
+        case arrow::Type::MAP:
+            return ColumnStats::CreateNestedColumnStats(FieldType::MAP, null_count);
+        case arrow::Type::STRUCT:
+            return ColumnStats::CreateNestedColumnStats(FieldType::STRUCT, null_count);
+        default:
+            return Status::Invalid(
+                fmt::format("cannot fetch Mosaic statistics for type {}", type->ToString()));
+    }
+}
+
+}  // namespace
+
+Result<MosaicStatsUtils::RowGroupStatistics> MosaicStatsUtils::ReadRowGroupStatistics(
+    uint32_t row_group, const MosaicInputContext* input_context, MosaicReaderHandle* reader) {
+    uint32_t num_stats = 0;
+    if (mosaic_reader_row_group_num_stats(reader, row_group, &num_stats) != 0) {
+        return MosaicFfiError("read Mosaic row group statistic count",
+                              input_context->GetCallbackStatus());
+    }
+    if (num_stats == 0) {
+        return RowGroupStatistics();
+    }
+    std::vector<const char*> names(num_stats);
+    std::vector<uint64_t> null_counts(num_stats);
+    std::vector<const uint8_t*> min_ptrs(num_stats);
+    std::vector<uintptr_t> min_lens(num_stats);
+    std::vector<const uint8_t*> max_ptrs(num_stats);
+    std::vector<uintptr_t> max_lens(num_stats);
+    if (mosaic_reader_row_group_stats(reader, row_group, names.data(), null_counts.data(),
+                                      min_ptrs.data(), min_lens.data(), max_ptrs.data(),
+                                      max_lens.data()) != 0) {
+        return MosaicFfiError("read Mosaic row group statistics",
+                              input_context->GetCallbackStatus());
+    }
+    RowGroupStatistics result;
+    result.reserve(num_stats);
+    for (uint32_t i = 0; i < num_stats; ++i) {
+        if (names[i] == nullptr || (min_ptrs[i] == nullptr) != (max_ptrs[i] == nullptr) ||
+            (min_ptrs[i] == nullptr && (min_lens[i] != 0 || max_lens[i] != 0))) {
+            return Status::Invalid("invalid Mosaic row group statistics");
+        }
+        ColumnStatistics stats = {null_counts[i], std::nullopt, std::nullopt};
+        if (min_ptrs[i] != nullptr) {
+            stats.min = min_lens[i] == 0
+                            ? std::vector<uint8_t>()
+                            : std::vector<uint8_t>(min_ptrs[i], min_ptrs[i] + min_lens[i]);
+            stats.max = max_lens[i] == 0
+                            ? std::vector<uint8_t>()
+                            : std::vector<uint8_t>(max_ptrs[i], max_ptrs[i] + max_lens[i]);
+        }
+        auto [iter, inserted] = result.emplace(names[i], std::move(stats));
+        if (!inserted) {
+            return Status::Invalid(
+                fmt::format("duplicate Mosaic statistics for column {}", iter->first));
+        }
+    }
+    return result;
+}
+
+Result<ColumnStatsVector> MosaicStatsUtils::ConvertColumnStatistics(
+    const std::shared_ptr<arrow::Schema>& schema,
+    const std::vector<RowGroupStatistics>& row_group_stats, bool missing_null_count_is_zero) {
+    ColumnStatsVector result;
+    result.reserve(schema->num_fields());
+    for (const std::shared_ptr<arrow::Field>& field : schema->fields()) {
+        std::vector<const ColumnStatistics*> field_stats;
+        field_stats.reserve(row_group_stats.size());
+        for (const RowGroupStatistics& stats : row_group_stats) {
+            auto iter = stats.find(field->name());
+            if (iter != stats.end()) {
+                field_stats.push_back(&iter->second);
+            }
+        }
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr<ColumnStats> column_stats,
+            ConvertFieldStatistics(field->type(), field_stats, missing_null_count_is_zero));
+        result.push_back(std::move(column_stats));
+    }
+    return result;
+}
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_stats.h b/src/paimon/format/mosaic/mosaic_stats.h
new file mode 100644
index 000000000..d5350edd7
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_stats.h
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "paimon/format/mosaic/mosaic_ffi.h"
+#include "paimon/result.h"
+#include "paimon/type_fwd.h"
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
+namespace paimon::mosaic {
+
+class MosaicInputContext;
+
+class MosaicStatsUtils {
+ public:
+    struct ColumnStatistics {
+        uint64_t null_count;
+        std::optional<std::vector<uint8_t>> min;
+        std::optional<std::vector<uint8_t>> max;
+    };
+
+    using RowGroupStatistics = std::unordered_map<std::string, ColumnStatistics>;
+
+    MosaicStatsUtils() = delete;
+    ~MosaicStatsUtils() = delete;
+
+    static Result<RowGroupStatistics> ReadRowGroupStatistics(
+        uint32_t row_group, const MosaicInputContext* input_context, MosaicReaderHandle* reader);
+
+    static Result<ColumnStatsVector> ConvertColumnStatistics(
+        const std::shared_ptr<arrow::Schema>& schema,
+        const std::vector<RowGroupStatistics>& row_group_stats, bool missing_null_count_is_zero);
+};
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_stats_extractor.cpp b/src/paimon/format/mosaic/mosaic_stats_extractor.cpp
new file mode 100644
index 000000000..bef972848
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_stats_extractor.cpp
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/format/mosaic/mosaic_stats_extractor.h"
+
+#include <limits>
+#include <memory>
+#include <vector>
+
+#include "paimon/common/utils/math.h"
+#include "paimon/format/mosaic/mosaic_ffi.h"
+#include "paimon/format/mosaic/mosaic_stats.h"
+#include "paimon/format/mosaic/mosaic_stream.h"
+#include "paimon/fs/file_system.h"
+
+namespace paimon::mosaic {
+
+Result<std::pair<ColumnStatsVector, FormatStatsExtractor::FileInfo>>
+MosaicStatsExtractor::ExtractWithFileInfo(const std::shared_ptr<FileSystem>& file_system,
+                                          const std::string& path,
+                                          const std::shared_ptr<MemoryPool>& pool) {
+    if (file_system == nullptr || pool == nullptr) {
+        return Status::Invalid("Mosaic stats extractor requires file system and memory pool");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> input, file_system->Open(path));
+    PAIMON_ASSIGN_OR_RAISE(int64_t signed_length, input->Length());
+    PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(signed_length, "Mosaic input length"));
+    auto input_context =
+        std::make_unique<MosaicInputContext>(input, static_cast<uint64_t>(signed_length));
+    MosaicInputFile input_file = {};
+    input_file.ctx = input_context.get();
+    input_file.read_at_fn = MosaicInputContext::ReadAt;
+    input_file.length_fn = MosaicInputContext::Length;
+    std::unique_ptr<MosaicReaderHandle, decltype(&mosaic_reader_free)> reader(
+        mosaic_reader_open(input_file), mosaic_reader_free);
+    if (reader == nullptr) {
+        return MosaicFfiError("open Mosaic reader", input_context->GetCallbackStatus());
+    }
+
+    uint32_t num_row_groups = 0;
+    if (mosaic_reader_num_row_groups(reader.get(), &num_row_groups) != 0) {
+        return MosaicFfiError("read Mosaic row group count", input_context->GetCallbackStatus());
+    }
+    int64_t row_count = 0;
+    std::vector<MosaicStatsUtils::RowGroupStatistics> row_group_stats;
+    row_group_stats.reserve(num_row_groups);
+    for (uint32_t row_group = 0; row_group < num_row_groups; ++row_group) {
+        uint32_t row_group_row_count = 0;
+        if (mosaic_reader_row_group_num_rows(reader.get(), row_group, &row_group_row_count) != 0) {
+            return MosaicFfiError("read Mosaic row count", input_context->GetCallbackStatus());
+        }
+        if (row_group_row_count >
+            static_cast<uint64_t>(std::numeric_limits<int64_t>::max() - row_count)) {
+            return Status::Invalid("Mosaic row count exceeds int64 range");
+        }
+        row_count += row_group_row_count;
+        PAIMON_ASSIGN_OR_RAISE(
+            MosaicStatsUtils::RowGroupStatistics stats,
+            MosaicStatsUtils::ReadRowGroupStatistics(row_group, input_context.get(), reader.get()));
+        row_group_stats.push_back(std::move(stats));
+    }
+    PAIMON_ASSIGN_OR_RAISE(ColumnStatsVector stats,
+                           MosaicStatsUtils::ConvertColumnStatistics(
+                               schema_, row_group_stats, /*missing_null_count_is_zero=*/false));
+    return std::make_pair(std::move(stats), FileInfo(row_count));
+}
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_stats_extractor.h b/src/paimon/format/mosaic/mosaic_stats_extractor.h
new file mode 100644
index 000000000..cfa064490
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_stats_extractor.h
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "paimon/format/format_stats_extractor.h"
+#include "paimon/result.h"
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
+namespace paimon::mosaic {
+
+class MosaicStatsExtractor : public FormatStatsExtractor {
+ public:
+    explicit MosaicStatsExtractor(const std::shared_ptr<arrow::Schema>& schema) : schema_(schema) {}
+
+    Result<ColumnStatsVector> Extract(const std::shared_ptr<FileSystem>& file_system,
+                                      const std::string& path,
+                                      const std::shared_ptr<MemoryPool>& pool) override {
+        using ExtractResult = std::pair<ColumnStatsVector, FileInfo>;
+        PAIMON_ASSIGN_OR_RAISE(ExtractResult result, ExtractWithFileInfo(file_system, path, pool));
+        return result.first;
+    }
+
+    Result<std::pair<ColumnStatsVector, FileInfo>> ExtractWithFileInfo(
+        const std::shared_ptr<FileSystem>& file_system, const std::string& path,
+        const std::shared_ptr<MemoryPool>& pool) override;
+
+ private:
+    std::shared_ptr<arrow::Schema> schema_;
+};
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_stream.cpp b/src/paimon/format/mosaic/mosaic_stream.cpp
new file mode 100644
index 000000000..87225fe99
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_stream.cpp
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/format/mosaic/mosaic_stream.h"
+
+#include <utility>
+
+#include "paimon/common/utils/math.h"
+#include "paimon/format/mosaic/mosaic_ffi.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/result.h"
+
+namespace paimon::mosaic {
+
+void MosaicInputContext::SetCallbackStatus(const Status& status) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (callback_status_.ok()) {
+        callback_status_ = status;
+    }
+}
+
+Status MosaicInputContext::GetCallbackStatus() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return callback_status_;
+}
+
+int32_t MosaicInputContext::ReadAt(void* context, uint64_t offset, uint8_t* buffer,
+                                   size_t length) noexcept {
+    auto* input_context = static_cast<MosaicInputContext*>(context);
+    if (input_context == nullptr || buffer == nullptr) {
+        if (input_context != nullptr) {
+            input_context->SetCallbackStatus(Status::Invalid("invalid Mosaic read request"));
+        }
+        return -1;
+    }
+    Status status = ValidateValueInRange<int64_t>(offset, "Mosaic read offset");
+    if (status.ok()) {
+        status = ValidateValueInRange<int64_t>(length, "Mosaic read length");
+    }
+    if (!status.ok()) {
+        input_context->SetCallbackStatus(status);
+        return -1;
+    }
+    auto read_length = static_cast<int64_t>(length);
+    auto read_offset = static_cast<int64_t>(offset);
+    Result<int64_t> result =
+        input_context->input_->Read(reinterpret_cast<char*>(buffer), read_length, read_offset);
+    if (!result.ok()) {
+        input_context->SetCallbackStatus(result.status());
+        return -1;
+    }
+    int64_t bytes_read = std::move(result).value();
+    if (bytes_read != read_length) {
+        input_context->SetCallbackStatus(Status::IOError("short read while reading Mosaic file"));
+        return -1;
+    }
+    return 0;
+}
+
+uint64_t MosaicInputContext::Length(void* context) noexcept {
+    auto* input_context = static_cast<MosaicInputContext*>(context);
+    return input_context == nullptr ? 0 : input_context->length_;
+}
+
+void MosaicOutputContext::SetCallbackStatus(const Status& status) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (callback_status_.ok()) {
+        callback_status_ = status;
+    }
+}
+
+Status MosaicOutputContext::GetCallbackStatus() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return callback_status_;
+}
+
+int32_t MosaicOutputContext::Write(void* context, const uint8_t* data, size_t length) noexcept {
+    auto* output_context = static_cast<MosaicOutputContext*>(context);
+    if (output_context == nullptr || data == nullptr) {
+        if (output_context != nullptr) {
+            output_context->SetCallbackStatus(Status::Invalid("invalid Mosaic write request"));
+        }
+        return -1;
+    }
+    Status status = ValidateValueInRange<int64_t>(length, "Mosaic write length");
+    if (!status.ok()) {
+        output_context->SetCallbackStatus(status);
+        return -1;
+    }
+    auto write_length = static_cast<int64_t>(length);
+    Result<int64_t> result =
+        output_context->output_->Write(reinterpret_cast<const char*>(data), write_length);
+    if (!result.ok()) {
+        output_context->SetCallbackStatus(result.status());
+        return -1;
+    }
+    int64_t bytes_written = std::move(result).value();
+    if (bytes_written != write_length) {
+        output_context->SetCallbackStatus(Status::IOError("short write while writing Mosaic file"));
+        return -1;
+    }
+    return 0;
+}
+
+int32_t MosaicOutputContext::Flush(void* context) noexcept {
+    auto* output_context = static_cast<MosaicOutputContext*>(context);
+    if (output_context == nullptr) {
+        return -1;
+    }
+    Status status = output_context->output_->Flush();
+    if (!status.ok()) {
+        output_context->SetCallbackStatus(status);
+        return -1;
+    }
+    return 0;
+}
+
+int64_t MosaicOutputContext::GetPos(void* context) noexcept {
+    auto* output_context = static_cast<MosaicOutputContext*>(context);
+    if (output_context == nullptr) {
+        return -1;
+    }
+    Result<int64_t> result = output_context->output_->GetPos();
+    if (!result.ok()) {
+        output_context->SetCallbackStatus(result.status());
+        return -1;
+    }
+    return std::move(result).value();
+}
+
+Status MosaicFfiError(const std::string& operation, const Status& callback_status) {
+    if (!callback_status.ok()) {
+        return callback_status.WithMessage(operation, ": ", callback_status.message());
+    }
+    const char* error = mosaic_last_error();
+    return Status::Invalid(operation, ": ", error == nullptr ? "unknown Mosaic error" : error);
+}
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_stream.h b/src/paimon/format/mosaic/mosaic_stream.h
new file mode 100644
index 000000000..93f27a4f1
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_stream.h
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <mutex>
+#include <string>
+
+#include "paimon/status.h"
+
+namespace paimon {
+class InputStream;
+class OutputStream;
+}  // namespace paimon
+
+namespace paimon::mosaic {
+
+class MosaicInputContext {
+ public:
+    MosaicInputContext(const std::shared_ptr<InputStream>& input, uint64_t length)
+        : input_(input), length_(length) {}
+
+    static int32_t ReadAt(void* context, uint64_t offset, uint8_t* buffer, size_t length) noexcept;
+    static uint64_t Length(void* context) noexcept;
+
+    Status GetCallbackStatus() const;
+
+ private:
+    void SetCallbackStatus(const Status& status);
+
+    std::shared_ptr<InputStream> input_;
+    uint64_t length_;
+    mutable std::mutex mutex_;
+    Status callback_status_;
+};
+
+class MosaicOutputContext {
+ public:
+    explicit MosaicOutputContext(const std::shared_ptr<OutputStream>& output) : output_(output) {}
+
+    static int32_t Write(void* context, const uint8_t* data, size_t length) noexcept;
+    static int32_t Flush(void* context) noexcept;
+    static int64_t GetPos(void* context) noexcept;
+
+    Status GetCallbackStatus() const;
+
+ private:
+    void SetCallbackStatus(const Status& status);
+
+    std::shared_ptr<OutputStream> output_;
+    mutable std::mutex mutex_;
+    Status callback_status_;
+};
+
+Status MosaicFfiError(const std::string& operation, const Status& callback_status);
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_writer_builder.cpp b/src/paimon/format/mosaic/mosaic_writer_builder.cpp
new file mode 100644
index 000000000..20b9407fd
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_writer_builder.cpp
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/format/mosaic/mosaic_writer_builder.h"
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/type.h"
+#include "paimon/common/options/memory_size.h"
+#include "paimon/common/utils/math.h"
+#include "paimon/common/utils/options_utils.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/defs.h"
+#include "paimon/format/mosaic/mosaic_format_defs.h"
+
+namespace paimon::mosaic {
+
+Result<std::unique_ptr<FormatWriter>> MosaicWriterBuilder::Build(
+    const std::shared_ptr<OutputStream>& output, const std::string& compression) {
+    if (pool_ == nullptr) {
+        return Status::Invalid("Mosaic writer memory pool is nullptr");
+    }
+    std::string normalized = StringUtils::ToLowerCase(compression);
+    uint8_t compression_id = 0;
+    if (normalized == "zstd" || normalized == "zstandard") {
+        compression_id = 1;
+    } else if (normalized != "none" && normalized != "null" && normalized != "uncompressed") {
+        return Status::Invalid("unknown Mosaic compression ", compression);
+    }
+    MosaicWriterOptions writer_options = mosaic_writer_options_default();
+    writer_options.compression = compression_id;
+    if (compression_id == 1) {
+        PAIMON_ASSIGN_OR_RAISE(
+            writer_options.zstd_level,
+            OptionsUtils::GetValueFromMap<int32_t>(options_, Options::FILE_COMPRESSION_ZSTD_LEVEL,
+                                                   writer_options.zstd_level));
+    }
+    PAIMON_ASSIGN_OR_RAISE(writer_options.num_buckets,
+                           OptionsUtils::GetValueFromMap<uint32_t>(options_, MOSAIC_NUM_BUCKETS,
+                                                                   writer_options.num_buckets));
+    auto max_dict_total_bytes = options_.find(MOSAIC_MAX_DICT_TOTAL_BYTES);
+    if (max_dict_total_bytes != options_.end()) {
+        PAIMON_ASSIGN_OR_RAISE(int64_t value, MemorySize::ParseBytes(max_dict_total_bytes->second));
+        PAIMON_RETURN_NOT_OK(ValidateValueInRange<uint32_t>(value, "Mosaic max dict total bytes"));
+        writer_options.max_dict_total_bytes = static_cast<uint32_t>(value);
+    }
+    PAIMON_ASSIGN_OR_RAISE(writer_options.max_dict_entries,
+                           OptionsUtils::GetValueFromMap<uint32_t>(
+                               options_, MOSAIC_MAX_DICT_ENTRIES, writer_options.max_dict_entries));
+    auto page_size_threshold = options_.find(MOSAIC_PAGE_SIZE_THRESHOLD);
+    if (page_size_threshold != options_.end()) {
+        PAIMON_ASSIGN_OR_RAISE(int64_t value, MemorySize::ParseBytes(page_size_threshold->second));
+        PAIMON_RETURN_NOT_OK(ValidateValueInRange<uint32_t>(value, "Mosaic page size threshold"));
+        writer_options.page_size_threshold = static_cast<uint32_t>(value);
+    }
+    auto block_size = options_.find(Options::FILE_BLOCK_SIZE);
+    if (block_size != options_.end()) {
+        PAIMON_ASSIGN_OR_RAISE(int64_t row_group_max_size,
+                               MemorySize::ParseBytes(block_size->second));
+        PAIMON_RETURN_NOT_OK(
+            ValidateValueInRange<uint64_t>(row_group_max_size, "Mosaic row group max size"));
+        writer_options.row_group_max_size = static_cast<uint64_t>(row_group_max_size);
+    }
+    std::vector<std::string> stats_columns;
+    auto stats_columns_iter = options_.find(MOSAIC_STATS_COLUMNS);
+    if (stats_columns_iter != options_.end()) {
+        for (std::string column : StringUtils::Split(stats_columns_iter->second, ",")) {
+            StringUtils::Trim(&column);
+            if (!column.empty() && schema_->GetFieldByName(column) != nullptr) {
+                stats_columns.push_back(std::move(column));
+            }
+        }
+    }
+    std::vector<const char*> stats_column_pointers;
+    stats_column_pointers.reserve(stats_columns.size());
+    for (const std::string& column : stats_columns) {
+        stats_column_pointers.push_back(column.c_str());
+    }
+    writer_options.stats_columns = stats_column_pointers.data();
+    PAIMON_RETURN_NOT_OK(
+        ValidateValueInRange<uint32_t>(stats_column_pointers.size(), "Mosaic stats column count"));
+    writer_options.num_stats_columns = static_cast<uint32_t>(stats_column_pointers.size());
+    return MosaicFormatWriter::Create(output, schema_, writer_options);
+}
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/mosaic/mosaic_writer_builder.h b/src/paimon/format/mosaic/mosaic_writer_builder.h
new file mode 100644
index 000000000..d3d40ff57
--- /dev/null
+++ b/src/paimon/format/mosaic/mosaic_writer_builder.h
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include "paimon/format/mosaic/mosaic_format_writer.h"
+#include "paimon/format/writer_builder.h"
+#include "paimon/memory/memory_pool.h"
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
+namespace paimon::mosaic {
+
+class MosaicWriterBuilder : public WriterBuilder {
+ public:
+    MosaicWriterBuilder(const std::shared_ptr<arrow::Schema>& schema,
+                        const std::map<std::string, std::string>& options)
+        : schema_(schema), options_(options), pool_(GetDefaultPool()) {}
+
+    WriterBuilder* WithMemoryPool(const std::shared_ptr<MemoryPool>& pool) override {
+        pool_ = pool;
+        return this;
+    }
+
+    Result<std::unique_ptr<FormatWriter>> Build(const std::shared_ptr<OutputStream>& output,
+                                                const std::string& compression) override;
+
+ private:
+    std::shared_ptr<arrow::Schema> schema_;
+    std::map<std::string, std::string> options_;
+    std::shared_ptr<MemoryPool> pool_;
+};
+
+}  // namespace paimon::mosaic
diff --git a/src/paimon/format/orc/orc_format_writer.cpp b/src/paimon/format/orc/orc_format_writer.cpp
index fc2316b21..cc723b9d8 100644
--- a/src/paimon/format/orc/orc_format_writer.cpp
+++ b/src/paimon/format/orc/orc_format_writer.cpp
@@ -40,10 +40,12 @@
 #include "orc/Writer.hh"
 #include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/options/memory_size.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/options_utils.h"
 #include "paimon/common/utils/string_utils.h"
 #include "paimon/core/schema/arrow_schema_validator.h"
+#include "paimon/defs.h"
 #include "paimon/format/orc/orc_adapter.h"
 #include "paimon/format/orc/orc_format_defs.h"
 #include "paimon/format/orc/orc_memory_pool.h"
@@ -248,9 +250,14 @@ Result<::orc::WriterOptions> OrcFormatWriter::PrepareWriterOptions(
         }
     }
     ::orc::WriterOptions writer_options;
-    PAIMON_ASSIGN_OR_RAISE(
-        uint64_t stripe_size,
-        OptionsUtils::GetValueFromMap<uint64_t>(options, ORC_STRIPE_SIZE, DEFAULT_STRIPE_SIZE));
+    int64_t stripe_size = DEFAULT_STRIPE_SIZE;
+    auto file_block_size = options.find(Options::FILE_BLOCK_SIZE);
+    if (file_block_size != options.end()) {
+        PAIMON_ASSIGN_OR_RAISE(stripe_size, MemorySize::ParseBytes(file_block_size->second));
+    } else {
+        PAIMON_ASSIGN_OR_RAISE(stripe_size, OptionsUtils::GetValueFromMap<int64_t>(
+                                                options, ORC_STRIPE_SIZE, DEFAULT_STRIPE_SIZE));
+    }
     writer_options.setStripeSize(stripe_size);
     PAIMON_ASSIGN_OR_RAISE(::orc::CompressionKind compression,
                            ToOrcCompressionKind(StringUtils::ToLowerCase(file_compression)));
diff --git a/src/paimon/format/orc/orc_format_writer_test.cpp b/src/paimon/format/orc/orc_format_writer_test.cpp
index fff6816c0..70719eb51 100644
--- a/src/paimon/format/orc/orc_format_writer_test.cpp
+++ b/src/paimon/format/orc/orc_format_writer_test.cpp
@@ -288,6 +288,13 @@ TEST_F(OrcFormatWriterTest, TestPrepareWriterOptions) {
                              OrcFormatWriter::PrepareWriterOptions(options, "zstd", data_type));
         ASSERT_FALSE(writer_options.getEnableDictionary());
     }
+    {
+        std::map<std::string, std::string> options = {{ORC_STRIPE_SIZE, "4096"},
+                                                      {Options::FILE_BLOCK_SIZE, "8 KB"}};
+        ASSERT_OK_AND_ASSIGN(::orc::WriterOptions writer_options,
+                             OrcFormatWriter::PrepareWriterOptions(options, "zstd", data_type));
+        ASSERT_EQ(writer_options.getStripeSize(), 8 * 1024);
+    }
     {
         // test disable config for timestamp with timezone
         arrow::FieldVector invalid_fields = {
diff --git a/src/paimon/format/parquet/parquet_writer_builder.cpp b/src/paimon/format/parquet/parquet_writer_builder.cpp
index 5946dd704..19d3e7fd4 100644
--- a/src/paimon/format/parquet/parquet_writer_builder.cpp
+++ b/src/paimon/format/parquet/parquet_writer_builder.cpp
@@ -24,10 +24,12 @@
 #include "arrow/util/compression.h"
 #include "arrow/util/type_fwd.h"
 #include "fmt/format.h"
+#include "paimon/common/options/memory_size.h"
 #include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/options_utils.h"
 #include "paimon/core/core_options.h"
+#include "paimon/defs.h"
 #include "paimon/format/parquet/parquet_format_defs.h"
 #include "paimon/format/parquet/parquet_format_writer.h"
 #include "paimon/status.h"
@@ -77,9 +79,14 @@ Result<std::shared_ptr<::parquet::WriterProperties>> ParquetWriterBuilder::Prepa
         builder.compression_level(file_compression_level);
     }
 
-    PAIMON_ASSIGN_OR_RAISE(int64_t row_group_size, OptionsUtils::GetValueFromMap<int64_t>(
-                                                       options_, PARQUET_BLOCK_SIZE,
-                                                       ::parquet::DEFAULT_MAX_ROW_GROUP_SIZE));
+    int64_t row_group_size = ::parquet::DEFAULT_MAX_ROW_GROUP_SIZE;
+    auto file_block_size = options_.find(Options::FILE_BLOCK_SIZE);
+    if (file_block_size != options_.end()) {
+        PAIMON_ASSIGN_OR_RAISE(row_group_size, MemorySize::ParseBytes(file_block_size->second));
+    } else {
+        PAIMON_ASSIGN_OR_RAISE(row_group_size, OptionsUtils::GetValueFromMap<int64_t>(
+                                                   options_, PARQUET_BLOCK_SIZE, row_group_size));
+    }
     builder.max_row_group_size(row_group_size);
 
     PAIMON_ASSIGN_OR_RAISE(int64_t page_size,
diff --git a/src/paimon/format/parquet/parquet_writer_builder_test.cpp b/src/paimon/format/parquet/parquet_writer_builder_test.cpp
index 2c5de2cf3..3a95a8de3 100644
--- a/src/paimon/format/parquet/parquet_writer_builder_test.cpp
+++ b/src/paimon/format/parquet/parquet_writer_builder_test.cpp
@@ -77,6 +77,21 @@ TEST(ParquetWriterBuilderTest, PrepareWriterProperties) {
     ASSERT_EQ(3, properties->default_column_properties().compression_level());
 }
 
+TEST(ParquetWriterBuilderTest, PrepareWriterPropertiesWithFileBlockSize) {
+    arrow::FieldVector fields;
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
+    std::map<std::string, std::string> options = {
+        {Options::FILE_FORMAT, "parquet"},
+        {Options::MANIFEST_FORMAT, "parquet"},
+        {PARQUET_BLOCK_SIZE, "2048"},
+        {Options::FILE_BLOCK_SIZE, "8 KB"},
+    };
+    ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<::parquet::WriterProperties> properties,
+                         builder.PrepareWriterProperties("zstd"));
+    ASSERT_EQ(properties->max_row_group_size(), 8 * 1024);
+}
+
 TEST(ParquetWriterBuilderTest, PrepareWriterPropertiesWithZstdLevelPriority) {
     arrow::FieldVector fields;
     std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
diff --git a/src/paimon/testing/utils/read_result_collector.h b/src/paimon/testing/utils/read_result_collector.h
index 8321a838a..558528662 100644
--- a/src/paimon/testing/utils/read_result_collector.h
+++ b/src/paimon/testing/utils/read_result_collector.h
@@ -138,7 +138,7 @@ class ReadResultCollector {
             return std::shared_ptr<arrow::Array>();
         }
         auto& [c_array, c_schema] = batch;
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto array,
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
                                           arrow::ImportArray(c_array.get(), c_schema.get()));
         return DictArrayConverter::ConvertDictArray(array, arrow::default_memory_pool());
     }
@@ -150,6 +150,11 @@ class ReadResultCollector {
         return std::make_pair(std::move(c_array), std::move(c_schema));
     }
 
+    static Status CheckBatchOffset(const BatchReader::ReadBatch& batch) {
+        assert(!BatchReader::IsEofBatch(batch));
+        return CheckArrayOffset(batch.first.get());
+    }
+
     // Noted that, sort chunked array by multiple key for timestamp type may cause
     // coredump in arrow, refer to https://github.com/apache/arrow/issues/47252
     static Result<std::shared_ptr<arrow::ChunkedArray>> SortArray(
@@ -197,16 +202,17 @@ class ReadResultCollector {
             }
             PAIMON_RETURN_NOT_OK(CheckBatchOffset(batch));
         }
-        auto& [c_array, c_schema] = batch;
-        assert(c_array->length > 0);
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto result_array,
-                                          arrow::ImportArray(c_array.get(), c_schema.get()));
-        return result_array;
+        assert(batch.first->length > 0);
+        return ImportReadBatch(std::move(batch));
     }
 
-    static Status CheckBatchOffset(const BatchReader::ReadBatch& batch) {
-        assert(!BatchReader::IsEofBatch(batch));
-        return CheckArrayOffset(batch.first.get());
+    static Result<std::shared_ptr<arrow::Array>> ImportReadBatch(BatchReader::ReadBatch&& batch) {
+        auto& [c_array, c_schema] = batch;
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::RecordBatch> record_batch,
+                                          arrow::ImportRecordBatch(c_array.get(), c_schema.get()));
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> struct_array,
+                                          record_batch->ToStructArray());
+        return struct_array;
     }
 
     static Status CheckArrayOffset(const ArrowArray* array) {
diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp
index b8a2c87a8..ebeb23361 100644
--- a/test/inte/append_compaction_inte_test.cpp
+++ b/test/inte/append_compaction_inte_test.cpp
@@ -155,6 +155,9 @@ class AppendCompactionInteTest : public testing::Test,
 std::vector<std::string> GetTestValuesForAppendCompactionInteTest() {
     std::vector<std::string> values;
     values.emplace_back("parquet");
+#ifdef PAIMON_ENABLE_MOSAIC
+    values.emplace_back("mosaic");
+#endif
 #ifdef PAIMON_ENABLE_ORC
     values.emplace_back("orc");
 #endif
@@ -254,7 +257,7 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompaction) {
 
 TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithMapSharedShredding) {
     auto file_format = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp
index b3c5cb237..f5d5960b3 100644
--- a/test/inte/blob_table_inte_test.cpp
+++ b/test/inte/blob_table_inte_test.cpp
@@ -63,6 +63,7 @@
 #include "paimon/data/blob.h"
 #include "paimon/defs.h"
 #include "paimon/file_store_write.h"
+#include "paimon/format/mosaic/mosaic_format_defs.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/global_index/bitmap_global_index_result.h"
@@ -589,6 +590,9 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter
 std::vector<std::string> GetTestValuesForBlobTableInteTest() {
     std::vector<std::string> values;
     values.emplace_back("parquet");
+#ifdef PAIMON_ENABLE_MOSAIC
+    values.emplace_back("mosaic");
+#endif
 #ifdef PAIMON_ENABLE_ORC
     values.emplace_back("orc");
 #endif
@@ -2229,7 +2233,16 @@ TEST_P(BlobTableInteTest, TestPredicate) {
         // Avro does not have stats.
         return;
     }
-    CreateTable();
+    if (GetParam() == "mosaic") {
+        CreateTable(/*partition_keys=*/{}, {{Options::MANIFEST_FORMAT, "orc"},
+                                            {Options::FILE_FORMAT, GetParam()},
+                                            {Options::FILE_SYSTEM, "local"},
+                                            {Options::ROW_TRACKING_ENABLED, "true"},
+                                            {Options::DATA_EVOLUTION_ENABLED, "true"},
+                                            {mosaic::MOSAIC_STATS_COLUMNS, "f0,f2"}});
+    } else {
+        CreateTable();
+    }
     std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
     auto schema = arrow::schema(fields_);
 
@@ -2366,7 +2379,7 @@ TEST_P(BlobTableInteTest, TestIOException) {
 
 TEST_P(BlobTableInteTest, TestReadTableWithDenseStats) {
     auto file_format = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
     std::string table_path =
@@ -2430,7 +2443,7 @@ TEST_P(BlobTableInteTest, TestReadTableWithDenseStats) {
 
 TEST_P(BlobTableInteTest, TestDataEvolutionAndAlterTable) {
     auto file_format = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
     std::string table_path = paimon::test::GetDataDir() + file_format +
@@ -2725,7 +2738,7 @@ TEST_P(BlobTableInteTest, TestAppendWriteWithNullBlob) {
 
 TEST_P(BlobTableInteTest, TestReadTableWithMultiBlobFields) {
     auto file_format = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
     std::string table_path = paimon::test::GetDataDir() + file_format +
@@ -2797,6 +2810,9 @@ TEST_P(BlobTableInteTest, TestReadTableWithMultiBlobFields) {
 }
 
 TEST_P(BlobTableInteTest, TestBlobDescriptorField) {
+    if (GetParam() == "mosaic") {
+        return;
+    }
     // Two blob fields configured via BLOB_DESCRIPTOR_FIELD and stored inline as descriptors.
     arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
                                  BlobUtils::ToArrowField("b0", true),
@@ -2850,6 +2866,9 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorField) {
 }
 
 TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialInline) {
+    if (GetParam() == "mosaic") {
+        return;
+    }
     // 4 blob fields: b0,b1 are inline descriptors; b2,b3 are regular blob fields written to
     // .blob files.
     arrow::FieldVector fields = {
@@ -2911,6 +2930,9 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialInline) {
 }
 
 TEST_P(BlobTableInteTest, TestBlobDescriptorMultiCommitAndShuffledReadSchema) {
+    if (GetParam() == "mosaic") {
+        return;
+    }
     // Multiple write+commit rounds with a shuffled read schema: b3, b2, b1, b0, f0.
     arrow::FieldVector fields = {
         arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true),
@@ -3036,7 +3058,7 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorMultiCommitAndShuffledReadSchema) {
 // The shared-shredding map is read from one main data file while the blob payload is read from a
 // separate blob file with the same row-id range.
 TEST_P(BlobTableInteTest, TestSharedShreddingWithBlobDataEvolution) {
-    if (GetParam() == "avro") {
+    if (GetParam() == "avro" || GetParam() == "mosaic") {
         return;
     }
 
@@ -3095,7 +3117,7 @@ TEST_P(BlobTableInteTest, TestSharedShreddingWithBlobDataEvolution) {
 
 // Two independent shared-shredding map columns are written into different main files.
 TEST_P(BlobTableInteTest, TestMultipleSharedShreddingMapsWithBlobDataEvolution) {
-    if (GetParam() == "avro") {
+    if (GetParam() == "avro" || GetParam() == "mosaic") {
         return;
     }
 
@@ -3157,7 +3179,7 @@ TEST_P(BlobTableInteTest, TestMultipleSharedShreddingMapsWithBlobDataEvolution)
 
 // A newer partial data file rewrites only the shared-shredding map for the same row-id range.
 TEST_P(BlobTableInteTest, TestSharedShreddingMapOverrideWithBlobDataEvolution) {
-    if (GetParam() == "avro") {
+    if (GetParam() == "avro" || GetParam() == "mosaic") {
         return;
     }
 
@@ -3281,6 +3303,9 @@ TEST_P(BlobTableInteTest, TestOrcMapStorageLayoutEvolutionWithBlobDataEvolution)
 }
 
 TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) {
+    if (GetParam() == "mosaic") {
+        return;
+    }
     // Test DataEvolution (split-column write) combined with blob descriptor fields.
     // Schema: f0(int32), b0/b1(blob descriptor inline), b2/b3(blob).
     // Commit 1: file A writes (f0, b2, b3)
@@ -3402,6 +3427,9 @@ TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) {
 }
 
 TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWriteRawBytesDirectly) {
+    if (GetParam() == "mosaic") {
+        return;
+    }
     // Similar to TestBlobDescriptorField but writes raw bytes directly without converting to
     // descriptor first. Descriptor fields reject values without the descriptor magic header.
     arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
@@ -3432,7 +3460,7 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWriteRawBytesDirectly) {
 
 TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) {
     auto file_format = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -3596,7 +3624,7 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) {
 
 TEST_P(BlobTableInteTest, TestForwardBlobViewReference) {
     auto file_format = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -3756,6 +3784,9 @@ TEST_P(BlobTableInteTest, TestForwardBlobViewReference) {
 
 TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamDescriptorBlob) {
     auto file_format = GetParam();
+    if (file_format == "mosaic") {
+        return;
+    }
     // Upstream table has two blob descriptor fields. The downstream view references cells from
     // both b0 (field_id=1) and b1 (field_id=2).
     const std::string upstream_db_name = "upstream_two_blob";
@@ -3880,7 +3911,7 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamDescriptorBlob) {
 
 TEST_P(BlobTableInteTest, TestBlobViewFieldWithMultipleUpstreamTables) {
     auto file_format = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -4041,6 +4072,9 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithMultipleUpstreamTables) {
 }
 
 TEST_P(BlobTableInteTest, TestBlobViewFailsWhenBothPathsAbsent) {
+    if (GetParam() == "mosaic") {
+        return;
+    }
     auto upstream_dir = UniqueTestDirectory::Create("local");
     arrow::FieldVector fields =
         CreateBlobViewTable(upstream_dir->Str(), /*deletion_vectors_enabled=*/false);
@@ -4056,6 +4090,9 @@ TEST_P(BlobTableInteTest, TestBlobViewFailsWhenBothPathsAbsent) {
 }
 
 TEST_P(BlobTableInteTest, TestBlobViewSkipsDanglingReferenceOfDeletedRow) {
+    if (GetParam() == "mosaic") {
+        return;
+    }
     auto upstream_dir = UniqueTestDirectory::Create("local");
     arrow::FieldVector fields =
         CreateBlobViewTable(upstream_dir->Str(), /*deletion_vectors_enabled=*/true);
@@ -4096,6 +4133,9 @@ TEST_P(BlobTableInteTest, TestBlobViewSkipsDanglingReferenceOfDeletedRow) {
 }
 
 TEST_P(BlobTableInteTest, TestBlobViewSkipsDanglingReferenceInEveryRowRangeGroup) {
+    if (GetParam() == "mosaic") {
+        return;
+    }
     auto upstream_dir = UniqueTestDirectory::Create("local");
     arrow::FieldVector fields =
         CreateBlobViewTable(upstream_dir->Str(), /*deletion_vectors_enabled=*/true);
@@ -4137,6 +4177,9 @@ TEST_P(BlobTableInteTest, TestBlobViewSkipsDanglingReferenceInEveryRowRangeGroup
 // predicate, so filtering a row out that way does not stop its blob view reference from being
 // resolved. This asserts the current behavior, not a desired one.
 TEST_P(BlobTableInteTest, TestBlobViewPreReadHonorsRowRangesNotPredicate) {
+    if (GetParam() == "mosaic") {
+        return;
+    }
     auto upstream_dir = UniqueTestDirectory::Create("local");
     arrow::FieldVector fields =
         CreateBlobViewTable(upstream_dir->Str(), /*deletion_vectors_enabled=*/false);
@@ -4170,6 +4213,9 @@ TEST_P(BlobTableInteTest, TestBlobViewPreReadHonorsRowRangesNotPredicate) {
 
 TEST_P(BlobTableInteTest, TestBlobViewWithFallbackPath) {
     auto file_format = GetParam();
+    if (file_format == "mosaic") {
+        return;
+    }
     const std::string upstream_db_name = "fallback_db";
     const std::string upstream_table_name = "fallback_table";
     arrow::FieldVector upstream_fields = {arrow::field("f0", arrow::int32()),
@@ -4281,7 +4327,7 @@ TEST_P(BlobTableInteTest, TestBlobViewWithFallbackPath) {
 
 TEST_P(BlobTableInteTest, TestReadBlobDescriptorFieldFromJava) {
     auto file_format = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
     std::string table_path =
diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp
index 13fede801..303c6d700 100644
--- a/test/inte/data_evolution_table_test.cpp
+++ b/test/inte/data_evolution_table_test.cpp
@@ -32,6 +32,7 @@
 #include "paimon/core/global_index/indexed_split_impl.h"
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/defs.h"
+#include "paimon/format/mosaic/mosaic_format_defs.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/global_index/bitmap_global_index_result.h"
 #include "paimon/global_index/indexed_split.h"
@@ -90,22 +91,24 @@ class DataEvolutionTableTest : public ::testing::Test,
         return CreateTable(/*partition_keys=*/{});
     }
 
-    Result<std::vector<std::shared_ptr<CommitMessage>>> WriteArray(
+    Result<std::vector<std::shared_ptr<CommitMessage>>> WriteArrays(
         const std::string& table_path, const std::map<std::string, std::string>& partition,
         const std::vector<std::string>& write_cols,
-        const std::shared_ptr<arrow::Array>& write_array) const {
+        const std::vector<std::shared_ptr<arrow::Array>>& write_arrays) const {
         // write
         WriteContextBuilder write_builder(table_path, "commit_user_1");
         write_builder.WithWriteSchema(write_cols);
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<WriteContext> write_context, write_builder.Finish());
         PAIMON_ASSIGN_OR_RAISE(auto file_store_write,
                                FileStoreWrite::Create(std::move(write_context)));
-        ArrowArray c_array;
-        EXPECT_TRUE(arrow::ExportArray(*write_array, &c_array).ok());
-        auto record_batch = std::make_unique<RecordBatch>(
-            partition, /*bucket=*/0,
-            /*row_kinds=*/std::vector<RecordBatch::RowKind>(), &c_array);
-        PAIMON_RETURN_NOT_OK(file_store_write->Write(std::move(record_batch)));
+        for (const auto& write_array : write_arrays) {
+            ArrowArray c_array;
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array));
+            auto record_batch = std::make_unique<RecordBatch>(
+                partition, /*bucket=*/0,
+                /*row_kinds=*/std::vector<RecordBatch::RowKind>(), &c_array);
+            PAIMON_RETURN_NOT_OK(file_store_write->Write(std::move(record_batch)));
+        }
         PAIMON_ASSIGN_OR_RAISE(auto commit_msgs,
                                file_store_write->PrepareCommit(
                                    /*wait_compaction=*/false, /*commit_identifier=*/0));
@@ -113,6 +116,13 @@ class DataEvolutionTableTest : public ::testing::Test,
         return commit_msgs;
     }
 
+    Result<std::vector<std::shared_ptr<CommitMessage>>> WriteArray(
+        const std::string& table_path, const std::map<std::string, std::string>& partition,
+        const std::vector<std::string>& write_cols,
+        const std::shared_ptr<arrow::Array>& write_array) const {
+        return WriteArrays(table_path, partition, write_cols, {write_array});
+    }
+
     Result<std::vector<std::shared_ptr<CommitMessage>>> WriteArray(
         const std::string& table_path, const std::vector<std::string>& write_cols,
         const std::shared_ptr<arrow::Array>& write_array) const {
@@ -840,6 +850,9 @@ TEST_P(DataEvolutionTableTest, TestOnlySomeColumns) {
 }
 
 TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) {
+    if (FileFormat() == "mosaic") {
+        return;
+    }
     if (FileFormat() == "avro") {
         return;
     }
@@ -1478,6 +1491,9 @@ TEST_P(DataEvolutionTableTest, TestPartitionWithPredicate) {
         {Options::MANIFEST_FORMAT, "orc"},         {Options::FILE_FORMAT, FileFormat()},
         {Options::FILE_SYSTEM, "local"},           {Options::ROW_TRACKING_ENABLED, "true"},
         {Options::DATA_EVOLUTION_ENABLED, "true"}, {"parquet.write.max-row-group-length", "1"}};
+    if (file_format == "mosaic") {
+        options.emplace(mosaic::MOSAIC_STATS_COLUMNS, "f0");
+    }
 
     CreateTable(partition_keys, options);
     std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -1643,6 +1659,9 @@ TEST_P(DataEvolutionTableTest, TestPartitionWithPredicate) {
 
 TEST_P(DataEvolutionTableTest, TestAlterTable) {
     auto file_format = FileFormat();
+    if (file_format == "mosaic") {
+        return;
+    }
     if (file_format == "avro") {
         return;
     }
@@ -1739,6 +1758,9 @@ TEST_P(DataEvolutionTableTest, TestAlterTable) {
 }
 
 TEST_P(DataEvolutionTableTest, TestReadCompactFiles) {
+    if (FileFormat() == "mosaic") {
+        return;
+    }
     auto file_format = FileFormat();
     if (file_format == "avro") {
         return;
@@ -1769,6 +1791,9 @@ TEST_P(DataEvolutionTableTest, TestReadCompactFiles) {
 }
 
 TEST_P(DataEvolutionTableTest, TestReadTableWithDenseStats) {
+    if (FileFormat() == "mosaic") {
+        return;
+    }
     auto file_format = FileFormat();
     if (file_format == "avro") {
         return;
@@ -1850,6 +1875,9 @@ TEST_P(DataEvolutionTableTest, TestReadTableWithDenseStats) {
 }
 
 TEST_P(DataEvolutionTableTest, TestScanAndReadWithIndex) {
+    if (FileFormat() == "mosaic") {
+        return;
+    }
     auto file_format = FileFormat();
     if (file_format == "avro") {
         return;
@@ -1988,6 +2016,9 @@ TEST_P(DataEvolutionTableTest, TestScanAndReadWithIndex) {
 }
 
 TEST_P(DataEvolutionTableTest, TestDataEvolutionPredicatePushDownBoundaries) {
+    if (FileFormat() == "mosaic") {
+        return;
+    }
     auto file_format = FileFormat();
     if (file_format == "avro") {
         return;
@@ -2102,15 +2133,19 @@ TEST_P(DataEvolutionTableTest, TestFormatPredicatePushDownWithoutFileIndex) {
         return;
     }
 
-    CreateDataEvolutionTable(
-        /*deletion_vectors_enabled=*/false, {{Options::FILE_INDEX_READ_ENABLED, "false"},
-                                             {Options::WRITE_BATCH_SIZE, "1"},
-                                             {"parquet.page.size", "1"},
-                                             {"parquet.enable-dictionary", "false"},
-                                             {"parquet.write.enable-page-index", "true"},
-                                             {"parquet.read.enable-page-index-filter", "true"},
-                                             {"orc.stripe.size", "1"},
-                                             {"orc.row.index.stride", "1"}});
+    std::map<std::string, std::string> options = {{Options::FILE_INDEX_READ_ENABLED, "false"},
+                                                  {Options::WRITE_BATCH_SIZE, "1"},
+                                                  {"parquet.page.size", "1"},
+                                                  {"parquet.enable-dictionary", "false"},
+                                                  {"parquet.write.enable-page-index", "true"},
+                                                  {"parquet.read.enable-page-index-filter", "true"},
+                                                  {"orc.stripe.size", "1"},
+                                                  {"orc.row.index.stride", "1"}};
+    if (FileFormat() == "mosaic") {
+        options.emplace(Options::FILE_BLOCK_SIZE, "1 B");
+        options.emplace(mosaic::MOSAIC_STATS_COLUMNS, "f0");
+    }
+    CreateDataEvolutionTable(/*deletion_vectors_enabled=*/false, options);
     std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
 
     auto input = std::dynamic_pointer_cast<arrow::StructArray>(
@@ -2121,7 +2156,16 @@ TEST_P(DataEvolutionTableTest, TestFormatPredicatePushDownWithoutFileIndex) {
         [4, "d", "w"]
     ])")
             .ValueOrDie());
-    ASSERT_OK_AND_ASSIGN(auto commit_messages, WriteArray(table_path, {"f0", "f1", "f2"}, input));
+    std::vector<std::shared_ptr<arrow::Array>> write_arrays;
+    for (int64_t i = 0; i < input->length(); i++) {
+        arrow::ArrayVector children;
+        for (const auto& child : input->fields()) {
+            children.push_back(child->Slice(i, 1));
+        }
+        write_arrays.push_back(arrow::StructArray::Make(children, fields_).ValueOrDie());
+    }
+    ASSERT_OK_AND_ASSIGN(auto commit_messages, WriteArrays(table_path, /*partition=*/{},
+                                                           {"f0", "f1", "f2"}, write_arrays));
     ASSERT_OK(Commit(table_path, commit_messages));
 
     auto predicate =
@@ -2142,7 +2186,16 @@ TEST_P(DataEvolutionTableTest, TestPredicate) {
         // Avro does not have stats.
         return;
     }
-    CreateTable();
+    if (FileFormat() == "mosaic") {
+        CreateTable(/*partition_keys=*/{}, {{Options::MANIFEST_FORMAT, "orc"},
+                                            {Options::FILE_FORMAT, FileFormat()},
+                                            {Options::FILE_SYSTEM, "local"},
+                                            {Options::ROW_TRACKING_ENABLED, "true"},
+                                            {Options::DATA_EVOLUTION_ENABLED, "true"},
+                                            {mosaic::MOSAIC_STATS_COLUMNS, "f0,f1,f2"}});
+    } else {
+        CreateTable();
+    }
     std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
     auto schema = arrow::schema(fields_);
 
@@ -2283,6 +2336,9 @@ TEST_P(DataEvolutionTableTest, TestWithRowIds) {
                                                   {Options::FILE_SYSTEM, "local"},
                                                   {Options::ROW_TRACKING_ENABLED, "true"},
                                                   {Options::DATA_EVOLUTION_ENABLED, "true"}};
+    if (FileFormat() == "mosaic") {
+        options.emplace(mosaic::MOSAIC_STATS_COLUMNS, "f0,f1");
+    }
     CreateTable(/*partition_keys=*/{}, options);
 
     std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -3099,6 +3155,9 @@ std::vector<DataEvolutionTableParam> GetTestValuesForDataEvolutionTableTest() {
     std::vector<DataEvolutionTableParam> values;
     for (bool enable_snapshot_live_manifest_cache : {false, true}) {
         values.emplace_back("parquet", enable_snapshot_live_manifest_cache);
+#ifdef PAIMON_ENABLE_MOSAIC
+        values.emplace_back("mosaic", enable_snapshot_live_manifest_cache);
+#endif
 #ifdef PAIMON_ENABLE_ORC
         values.emplace_back("orc", enable_snapshot_live_manifest_cache);
 #endif
diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp
index 18e77f9de..8e9150aee 100644
--- a/test/inte/pk_compaction_inte_test.cpp
+++ b/test/inte/pk_compaction_inte_test.cpp
@@ -445,7 +445,7 @@ TEST_F(PkCompactionInteTest, TestMetadataOnlyLevelUpgradeKeepsValueStats) {
 // Verify shared-shredding MAP can be read correctly after PK full compaction.
 TEST_P(PkCompactionInteTest, TestKeyValueTableFullCompactionWithMapSharedShredding) {
     auto file_format = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -517,7 +517,7 @@ TEST_P(PkCompactionInteTest, TestKeyValueTableFullCompactionWithMapSharedShreddi
 
 TEST_P(PkCompactionInteTest, TestKeyValueTableDvCompactionWithMapSharedShredding) {
     auto file_format = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -3235,7 +3235,7 @@ TEST_F(PkCompactionInteTest, RemoteLookupFileWithSchemaEvolution) {
 //   6. ScanAndVerify after full compact
 TEST_P(PkCompactionInteTest, TestLookupCompatibility) {
     auto file_format = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
     // Step 1: Copy pk_compact_lookup table to temp dir.
@@ -3597,6 +3597,9 @@ TEST_F(PkCompactionInteTest, AggHllAndThetaSketches) {
 std::vector<std::string> GetTestValuesForCompactionInteTest() {
     std::vector<std::string> values;
     values.emplace_back("parquet");
+#ifdef PAIMON_ENABLE_MOSAIC
+    values.emplace_back("mosaic");
+#endif
 #ifdef PAIMON_ENABLE_ORC
     values.emplace_back("orc");
 #endif
diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp
index 537a4cb36..11080c9d5 100644
--- a/test/inte/scan_and_read_inte_test.cpp
+++ b/test/inte/scan_and_read_inte_test.cpp
@@ -2752,6 +2752,97 @@ TEST_P(ScanAndReadInteTest, TestCastTimestampType) {
     ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString();
 }
 
+#ifdef PAIMON_ENABLE_MOSAIC
+TEST_F(ScanAndReadInteTest, TestMosaicJavaAndPythonCompatibility) {
+    TimezoneGuard timezone_guard("UTC");
+    std::string timezone = DateTimeUtils::GetLocalTimezoneName();
+    arrow::FieldVector fields = {
+        arrow::field("_VALUE_KIND", arrow::int8()),
+        arrow::field("id", arrow::int32()),
+        arrow::field("f_boolean", arrow::boolean()),
+        arrow::field("f_tinyint", arrow::int8()),
+        arrow::field("f_smallint", arrow::int16()),
+        arrow::field("f_bigint", arrow::int64()),
+        arrow::field("f_float", arrow::float32()),
+        arrow::field("f_double", arrow::float64()),
+        arrow::field("f_char", arrow::utf8()),
+        arrow::field("f_varchar", arrow::utf8()),
+        arrow::field("f_binary", arrow::binary()),
+        arrow::field("f_varbinary", arrow::binary()),
+        arrow::field("f_date", arrow::date32()),
+        arrow::field("f_ts_3", arrow::timestamp(arrow::TimeUnit::MILLI)),
+        arrow::field("f_ts_6", arrow::timestamp(arrow::TimeUnit::MICRO)),
+        arrow::field("f_ts_9", arrow::timestamp(arrow::TimeUnit::NANO)),
+        arrow::field("f_ltz_3", arrow::timestamp(arrow::TimeUnit::MILLI, timezone)),
+        arrow::field("f_ltz_6", arrow::timestamp(arrow::TimeUnit::MICRO, timezone)),
+        arrow::field("f_ltz_9", arrow::timestamp(arrow::TimeUnit::NANO, timezone)),
+        arrow::field("f_decimal_1_0", arrow::decimal128(1, 0)),
+        arrow::field("f_decimal_18_2", arrow::decimal128(18, 2)),
+        arrow::field("f_decimal_19_2", arrow::decimal128(19, 2)),
+        arrow::field("f_decimal_38_18", arrow::decimal128(38, 18)),
+        arrow::field("f_array_int", arrow::list(arrow::int32())),
+        arrow::field("f_map_numeric", arrow::map(arrow::int8(), arrow::int16())),
+        arrow::field("f_map_string_bigint", arrow::map(arrow::utf8(), arrow::int64())),
+        arrow::field("f_array_array_int", arrow::list(arrow::list(arrow::int32()))),
+        arrow::field("f_array_map", arrow::list(arrow::map(arrow::utf8(), arrow::int32()))),
+        arrow::field("f_map_array", arrow::map(arrow::utf8(), arrow::list(arrow::int32()))),
+    };
+    std::shared_ptr<arrow::DataType> data_type = arrow::struct_(fields);
+    std::shared_ptr<arrow::Array> expected_array =
+        arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([
+[0, 1, true, -5, -1000, 10000000001, 1.25, 10.5, "char0001", "value-1", "bin00001", "\u0001\u0002\u0003", 20000, "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "1970-01-01 00:01:01.321", "1970-01-01 00:01:01.654321", "1970-01-01 00:01:01.321654987", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [1, 2], [[0, 0], [10, 1]], [["k0", 1000], ["z0", 2000]], [[1, 2], [3]], [[["nested0", 0]], [["nested10", 10]]], [["array0", [0, 1]]]],
+[0, 2, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null],
+[0, 10, true, -3, -998, 10000000010, 3.25, 12.5, "char0010", "value-10", "bin00010", "\u000a\u000b\u000c", 20002, "1970-01-01 00:00:10.123", "1970-01-01 00:00:10.123456", "1970-01-01 00:00:10.123456789", "1970-01-01 00:01:10.321", "1970-01-01 00:01:10.654321", "1970-01-01 00:01:10.321654987", "-1", "10.25", "12345678901234569.89", "12345678901234567892.123456789012345678", [10, 11], [[2, 20], [12, 21]], [["k2", 1002], ["z2", 2002]], [[10, 11], [12]], [[["nested2", 2]], [["nested12", 12]]], [["array2", [2, 3]]]],
+[0, 11, false, -2, -997, 10000000011, 4.25, 13.5, "char0011", "value-11", "bin00011", "\u000b\u000c\u000d", 20003, "1970-01-01 00:00:11.123", "1970-01-01 00:00:11.123456", "1970-01-01 00:00:11.123456789", "1970-01-01 00:01:11.321", "1970-01-01 00:01:11.654321", "1970-01-01 00:01:11.321654987", "0", "11.25", "12345678901234570.89", "12345678901234567893.123456789012345678", [11, 12], [[3, 30], [13, 31]], [["k3", 1003], ["z3", 2003]], [[11, 12], [13]], [[["nested3", 3]], [["nested13", 13]]], [["array3", [3, 4]]]],
+[0, 20, true, -1, -996, 10000000020, 5.25, 14.5, "char0020", "value-20", "bin00020", "\u0014\u0015\u0016", 20004, "1970-01-01 00:00:20.123", "1970-01-01 00:00:20.123456", "1970-01-01 00:00:20.123456789", "1970-01-01 00:01:20.321", "1970-01-01 00:01:20.654321", "1970-01-01 00:01:20.321654987", "1", "20.25", "12345678901234571.89", "12345678901234567894.123456789012345678", [20, 21], [[4, 40], [14, 41]], [["k4", 1004], ["z4", 2004]], [[20, 21], [22]], [[["nested4", 4]], [["nested14", 14]]], [["array4", [4, 5]]]],
+[0, 21, false, 0, -995, 10000000021, 6.25, 15.5, "char0021", "value-21", "bin00021", "\u0015\u0016\u0017", 20005, "1970-01-01 00:00:21.123", "1970-01-01 00:00:21.123456", "1970-01-01 00:00:21.123456789", "1970-01-01 00:01:21.321", "1970-01-01 00:01:21.654321", "1970-01-01 00:01:21.321654987", "2", "21.25", "12345678901234572.89", "12345678901234567895.123456789012345678", [21, 22], [[5, 50], [15, 51]], [["k5", 1005], ["z5", 2005]], [[21, 22], [23]], [[["nested5", 5]], [["nested15", 15]]], [["array5", [5, 6]]]]
+])")
+            .ValueOrDie();
+    auto expected = std::make_shared<arrow::ChunkedArray>(expected_array);
+
+    auto check_compatibility = [](const std::string& table_path,
+                                  const std::shared_ptr<Predicate>& predicate,
+                                  const std::shared_ptr<arrow::ChunkedArray>& expected_result) {
+        ScanContextBuilder scan_context_builder(table_path);
+        ReadContextBuilder read_context_builder(table_path);
+        if (predicate != nullptr) {
+            scan_context_builder.SetPredicate(predicate);
+            read_context_builder.SetPredicate(predicate);
+        }
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<ScanContext> scan_context,
+                             scan_context_builder.Finish());
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableScan> table_scan,
+                             TableScan::Create(std::move(scan_context)));
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, table_scan->CreatePlan());
+
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context,
+                             read_context_builder.Finish());
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+                             TableRead::Create(std::move(read_context)));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> batch_reader,
+                             table_read->CreateReader(plan->Splits()));
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> actual,
+                             ReadResultCollector::CollectResult(batch_reader.get()));
+        ASSERT_TRUE(expected_result->Equals(actual))
+            << "actual: " << (actual == nullptr ? "null" : actual->ToString())
+            << "\nexpected: " << expected_result->ToString();
+    };
+
+    std::shared_ptr<Predicate> predicate = PredicateBuilder::GreaterOrEqual(
+        /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(20));
+    std::shared_ptr<Predicate> predicate_without_stats = PredicateBuilder::GreaterOrEqual(
+        /*field_index=*/4, /*field_name=*/"f_bigint", FieldType::BIGINT,
+        Literal(int64_t{10000000020LL}));
+    for (const std::string table_name : {"append_java_compat", "append_python_compat"}) {
+        SCOPED_TRACE(table_name);
+        std::string table_path = GetDataDir() + "/mosaic/" + table_name + ".db/" + table_name;
+        check_compatibility(table_path, /*predicate=*/nullptr, expected);
+        check_compatibility(table_path, predicate, expected->Slice(4, 2));
+        check_compatibility(table_path, predicate_without_stats, expected);
+    }
+}
+#endif
+
 TEST_F(ScanAndReadInteTest, TestAvroWithAppendTable) {
     auto read_data = [](int64_t snapshot_id, const std::string& result_json) {
         std::string table_path = GetDataDir() + "/avro/append_multiple.db/append_multiple";
diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp
index 5916fc97c..397566d03 100644
--- a/test/inte/write_and_read_inte_test.cpp
+++ b/test/inte/write_and_read_inte_test.cpp
@@ -641,6 +641,9 @@ TEST_P(WriteAndReadInteTest, TestNestedType) {
         arrow::field("f6", arrow::decimal128(2, 2))};
     auto schema = arrow::schema(fields);
     auto [file_format, file_system] = GetParam();
+    if (file_format == "mosaic") {
+        return;
+    }
     std::map<std::string, std::string> options = {
         {Options::MANIFEST_FORMAT, "avro"},  {Options::FILE_FORMAT, file_format},
         {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
@@ -690,7 +693,7 @@ TEST_P(WriteAndReadInteTest, TestNestedType) {
 
 TEST_P(WriteAndReadInteTest, TestSchemaEvolutionAddFieldInsideListAndMap) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "lance" || file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
     auto list_struct =
@@ -976,6 +979,9 @@ TEST_P(WriteAndReadInteTest, TestAppendTimestampType) {
     };
     auto schema = arrow::schema(fields);
     auto [file_format, file_system] = GetParam();
+    if (file_format == "mosaic") {
+        return;
+    }
     std::map<std::string, std::string> options = {
         {Options::MANIFEST_FORMAT, "avro"},  {Options::FILE_FORMAT, file_format},
         {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
@@ -1029,6 +1035,9 @@ TEST_P(WriteAndReadInteTest, TestPkTimestampType) {
     };
     auto schema = arrow::schema(fields);
     auto [file_format, file_system] = GetParam();
+    if (file_format == "mosaic") {
+        return;
+    }
     std::map<std::string, std::string> options = {
         {Options::MANIFEST_FORMAT, "avro"},  {Options::FILE_FORMAT, file_format},
         {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
@@ -1075,6 +1084,9 @@ TEST_P(WriteAndReadInteTest, TestPkTimestampType) {
 /// the reader has to convert milli back to second for every nested leaf.
 TEST_P(WriteAndReadInteTest, TestAppendNestedTimestampSecondPrecision) {
     auto [file_format, file_system] = GetParam();
+    if (file_format == "mosaic") {
+        return;
+    }
     TimezoneGuard timezone_guard("Asia/Shanghai");
     auto timezone = DateTimeUtils::GetLocalTimezoneName();
     auto event_type = arrow::struct_({
@@ -1130,6 +1142,9 @@ TEST_P(WriteAndReadInteTest, TestAppendNestedTimestampSecondPrecision) {
 /// file schema differ only in the timezone of those leaves; the micro precision stays unchanged.
 TEST_P(WriteAndReadInteTest, TestAppendNestedTimestampLtzMicroTimezoneOnly) {
     auto [file_format, file_system] = GetParam();
+    if (file_format == "mosaic") {
+        return;
+    }
     // Pin a non-UTC timezone so the read schema really differs from what the file reports.
     TimezoneGuard timezone_guard("Asia/Shanghai");
     auto timezone = DateTimeUtils::GetLocalTimezoneName();
@@ -1558,6 +1573,9 @@ TEST_P(WriteAndReadInteTest, TestCharVarcharBinaryVarbinaryTypes) {
 
 std::vector<std::pair<std::string, std::string>> GetTestValuesForWriteAndReadInteTest() {
     std::vector<std::pair<std::string, std::string>> values = {{"parquet", "local"}};
+#ifdef PAIMON_ENABLE_MOSAIC
+    values.emplace_back("mosaic", "local");
+#endif
 #if defined(PAIMON_ENABLE_NETWORK_TESTS) && defined(PAIMON_ENABLE_JINDO)
     values.emplace_back("parquet", "jindo");
 #endif
@@ -1974,7 +1992,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) {
 
 TEST_P(WriteAndReadInteTest, TestAppendSharedShreddingMap) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -2031,7 +2049,7 @@ TEST_P(WriteAndReadInteTest, TestAppendSharedShreddingMap) {
 
 TEST_P(WriteAndReadInteTest, TestMapSharedShreddingColumnPlacementPolicies) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -2115,7 +2133,7 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingColumnPlacementPolicies) {
 
 TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPartitionAndBucket) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -2223,7 +2241,7 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPartitionAndBucket)
 
 TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -2317,7 +2335,7 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) {
 
 TEST_P(WriteAndReadInteTest, TestMapSharedShreddingNewWriterStartsWithMaxColumnCount) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -2386,7 +2404,7 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingNewWriterStartsWithMaxColumnC
 
 TEST_P(WriteAndReadInteTest, TestMapSharedShreddingAdaptsAcrossRollingFiles) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -2459,7 +2477,7 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingAdaptsAcrossRollingFiles) {
 
 TEST_P(WriteAndReadInteTest, TestMapSharedShreddingSwitchMapLayoutAndUseMaxColumnsWithoutMetadata) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -2542,7 +2560,7 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingSwitchMapLayoutAndUseMaxColum
 
 TEST_P(WriteAndReadInteTest, TestMapSharedShreddingReadAfterRenameColumn) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -2626,7 +2644,7 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingReadAfterRenameColumn) {
 
 TEST_P(WriteAndReadInteTest, TestSharedShreddingWithSchemaEvolution) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -2737,7 +2755,7 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingWithSchemaEvolution) {
 // Verify storage-layout evolution: default->shared-shredding.
 TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShredding) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -2815,7 +2833,7 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShredding) {
 // Verify storage-layout evolution: shared-shredding->default.
 TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefault) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -2879,7 +2897,7 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefault) {
 
 TEST_P(WriteAndReadInteTest, TestAppendMapStorageLayoutSharedShreddingToDefaultCompaction) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -2983,7 +3001,7 @@ TEST_P(WriteAndReadInteTest, TestAppendMapStorageLayoutSharedShreddingToDefaultC
 // Nested map values through both selected physical columns and overflow.
 TEST_P(WriteAndReadInteTest, TestSharedShreddingWithStructValue) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -3054,7 +3072,7 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingWithStructValue) {
 
 TEST_P(WriteAndReadInteTest, TestMapSharedShreddingWithComplexValue) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -3168,7 +3186,7 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingWithComplexValue) {
 
 TEST_P(WriteAndReadInteTest, TestMapSharedShreddingWithAllSupportedComplexValueTypes) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -3318,7 +3336,7 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingWithAllSupportedComplexValueT
 
 TEST_P(WriteAndReadInteTest, TestMapSharedShreddingStructValueSchemaEvolutionReadFails) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -3530,7 +3548,7 @@ TEST_P(WriteAndReadInteTest, TestOrcDictionaryLazyDecodingWithSharedShredding) {
 // Verify shared-shredding in the PK read path.
 TEST_P(WriteAndReadInteTest, TestPkSharedShreddingMap) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -3600,7 +3618,7 @@ TEST_P(WriteAndReadInteTest, TestPkSharedShreddingMap) {
 
 TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithOverflow) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -3735,7 +3753,7 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithOverflow) {
 
 TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithNullOrMissingKey) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -3846,7 +3864,7 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithNullOrMissin
 
 TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallMultipleColumns) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -3958,7 +3976,7 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallMultipleColumns)
 
 TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShreddingPartialKeyRecall) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -4048,7 +4066,7 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShreddingPartial
 
 TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefaultPartialKeyRecall) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -4143,7 +4161,7 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefaultPartial
 
 TEST_P(WriteAndReadInteTest, TestSharedShreddingDuplicateSelectedKeys) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
@@ -4191,7 +4209,7 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingDuplicateSelectedKeys) {
 
 TEST_P(WriteAndReadInteTest, TestSharedShreddingAllNullMapColumn) {
     auto [file_format, file_system] = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp
index 9035ae6e5..5b0b8a025 100644
--- a/test/inte/write_inte_test.cpp
+++ b/test/inte/write_inte_test.cpp
@@ -197,10 +197,10 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface
 
     std::shared_ptr<DataFileMeta> ReconstructDataFileMeta(
         const std::shared_ptr<DataFileMeta>& file_meta) const {
-        if (GetParam() != "avro") {
+        if (GetParam() != "avro" && GetParam() != "mosaic") {
             return file_meta;
         }
-        // For the avro format, all stats are null.
+        // Avro and Mosaic without configured statistics have null statistics.
         auto new_meta = std::make_shared<DataFileMeta>(
             file_meta->file_name, file_meta->file_size, file_meta->row_count, file_meta->min_key,
             file_meta->max_key, file_meta->key_stats, file_meta->value_stats,
@@ -383,6 +383,9 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface
 std::vector<std::string> GetTestValuesForWriteInteTest() {
     std::vector<std::string> values;
     values.emplace_back("parquet");
+#ifdef PAIMON_ENABLE_MOSAIC
+    values.emplace_back("mosaic");
+#endif
 #ifdef PAIMON_ENABLE_ORC
     values.emplace_back("orc");
 #endif
@@ -805,6 +808,9 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithPartitionAndMultiBuckets) {
 }
 
 TEST_P(WriteInteTest, TestAppendTableWriteWithComplexType) {
+    if (GetParam() == "mosaic") {
+        return;
+    }
     auto dir = UniqueTestDirectory::Create();
     arrow::FieldVector fields = {
         arrow::field("f1", arrow::map(arrow::int8(), arrow::int16())),
@@ -1626,6 +1632,9 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) {
 }
 
 TEST_P(WriteInteTest, TestPkTableWriteWithComplexType) {
+    if (GetParam() == "mosaic") {
+        return;
+    }
     auto dir = UniqueTestDirectory::Create();
     ASSERT_TRUE(dir);
     arrow::FieldVector fields = {
@@ -2473,7 +2482,7 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) {
 
 TEST_P(WriteInteTest, TestWriteWithFieldId) {
     auto file_format = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
     // prepare write schema and write data
@@ -3076,6 +3085,9 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) {
 }
 
 TEST_P(WriteInteTest, TestWriteWithNestedSchema) {
+    if (GetParam() == "mosaic") {
+        return;
+    }
     arrow::FieldVector fields = {
         arrow::field("f0", arrow::struct_({arrow::field("v0", arrow::boolean()),
                                            arrow::field("v1", arrow::int64())}))};
@@ -3337,6 +3349,9 @@ TEST_P(WriteInteTest, TestWriteMemoryUse) {
 }
 
 TEST_P(WriteInteTest, TestAppendTableWithAllNull) {
+    if (GetParam() == "mosaic") {
+        return;
+    }
     auto dir = UniqueTestDirectory::Create();
     arrow::FieldVector fields = {
         arrow::field("f0", arrow::boolean()),
@@ -4011,7 +4026,7 @@ TEST_P(WriteInteTest, TestNullabilityCheck) {
 
 TEST_P(WriteInteTest, TestPkSpillableMapSharedShreddingReadWrite) {
     auto file_format = GetParam();
-    if (file_format == "avro") {
+    if (file_format == "avro" || file_format == "mosaic") {
         return;
     }
 
diff --git a/test/test_data/mosaic/append_java_compat.db/append_java_compat/README.md b/test/test_data/mosaic/append_java_compat.db/append_java_compat/README.md
new file mode 100644
index 000000000..92c7d2089
--- /dev/null
+++ b/test/test_data/mosaic/append_java_compat.db/append_java_compat/README.md
@@ -0,0 +1,60 @@
+Table: append_java_compat
+Writer: Paimon Java at commit 0043a70fd88ac75dcb83a8f2da5e72ce91e22b1f
+Mosaic version: 0.2.0
+
+Schema:
+id INT NOT NULL
+f_boolean BOOLEAN
+f_tinyint TINYINT
+f_smallint SMALLINT
+f_bigint BIGINT
+f_float FLOAT
+f_double DOUBLE
+f_char CHAR(8)
+f_varchar VARCHAR(64)
+f_binary BINARY(8)
+f_varbinary VARBINARY(64)
+f_date DATE
+f_ts_3 TIMESTAMP(3)
+f_ts_6 TIMESTAMP(6)
+f_ts_9 TIMESTAMP(9)
+f_ltz_3 TIMESTAMP(3) WITH LOCAL TIME ZONE
+f_ltz_6 TIMESTAMP(6) WITH LOCAL TIME ZONE
+f_ltz_9 TIMESTAMP(9) WITH LOCAL TIME ZONE
+f_decimal_1_0 DECIMAL(1, 0)
+f_decimal_18_2 DECIMAL(18, 2)
+f_decimal_19_2 DECIMAL(19, 2)
+f_decimal_38_18 DECIMAL(38, 18)
+f_array_int ARRAY<INT>
+f_map_numeric MAP<TINYINT, SMALLINT>
+f_map_string_bigint MAP<STRING, BIGINT>
+f_array_array_int ARRAY<ARRAY<INT>>
+f_array_map ARRAY<MAP<STRING, INT>>
+f_map_array MAP<STRING, ARRAY<INT>>
+
+Options:
+bucket = -1
+file.block-size = 1 B
+file.format = mosaic
+manifest.format = avro
+mosaic.num-buckets = 4
+mosaic.stats-columns = id,f_varchar,f_date,f_ts_3,f_ltz_9,f_decimal_18_2
+target-file-size = 64 MB
+write.batch-size = 2
+
+Data is written as three row groups with two rows in each row group. Timestamp values are shown in
+UTC. The second row has NULL in every nullable field.
+
+Row group 0:
+Add: (1, true, -5, -1000, 10000000001, 1.25, 10.5, "char0001", "value-1", "bin00001", 0x010203, 2024-10-04, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, 1970-01-01 00:01:01.321, 1970-01-01 00:01:01.654321, 1970-01-01 00:01:01.321654987, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [1, 2], {0: 0, 10: 1}, {"k0": 1000, "z0": 2000}, [[1, 2], [3]], [{"nested0": 0}, {"nested10": 10}], {"array0": [0, 1]})
+Add: (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)
+
+Row group 1:
+Add: (10, true, -3, -998, 10000000010, 3.25, 12.5, "char0010", "value-10", "bin00010", 0x0a0b0c, 2024-10-06, 1970-01-01 00:00:10.123, 1970-01-01 00:00:10.123456, 1970-01-01 00:00:10.123456789, 1970-01-01 00:01:10.321, 1970-01-01 00:01:10.654321, 1970-01-01 00:01:10.321654987, -1, 10.25, 12345678901234569.89, 12345678901234567892.123456789012345678, [10, 11], {2: 20, 12: 21}, {"k2": 1002, "z2": 2002}, [[10, 11], [12]], [{"nested2": 2}, {"nested12": 12}], {"array2": [2, 3]})
+Add: (11, false, -2, -997, 10000000011, 4.25, 13.5, "char0011", "value-11", "bin00011", 0x0b0c0d, 2024-10-07, 1970-01-01 00:00:11.123, 1970-01-01 00:00:11.123456, 1970-01-01 00:00:11.123456789, 1970-01-01 00:01:11.321, 1970-01-01 00:01:11.654321, 1970-01-01 00:01:11.321654987, 0, 11.25, 12345678901234570.89, 12345678901234567893.123456789012345678, [11, 12], {3: 30, 13: 31}, {"k3": 1003, "z3": 2003}, [[11, 12], [13]], [{"nested3": 3}, {"nested13": 13}], {"array3": [3, 4]})
+
+Row group 2:
+Add: (20, true, -1, -996, 10000000020, 5.25, 14.5, "char0020", "value-20", "bin00020", 0x141516, 2024-10-08, 1970-01-01 00:00:20.123, 1970-01-01 00:00:20.123456, 1970-01-01 00:00:20.123456789, 1970-01-01 00:01:20.321, 1970-01-01 00:01:20.654321, 1970-01-01 00:01:20.321654987, 1, 20.25, 12345678901234571.89, 12345678901234567894.123456789012345678, [20, 21], {4: 40, 14: 41}, {"k4": 1004, "z4": 2004}, [[20, 21], [22]], [{"nested4": 4}, {"nested14": 14}], {"array4": [4, 5]})
+Add: (21, false, 0, -995, 10000000021, 6.25, 15.5, "char0021", "value-21", "bin00021", 0x151617, 2024-10-09, 1970-01-01 00:00:21.123, 1970-01-01 00:00:21.123456, 1970-01-01 00:00:21.123456789, 1970-01-01 00:01:21.321, 1970-01-01 00:01:21.654321, 1970-01-01 00:01:21.321654987, 2, 21.25, 12345678901234572.89, 12345678901234567895.123456789012345678, [21, 22], {5: 50, 15: 51}, {"k5": 1005, "z5": 2005}, [[21, 22], [23]], [{"nested5": 5}, {"nested15": 15}], {"array5": [5, 6]})
+
+Commit - snapshot 1
diff --git a/test/test_data/mosaic/append_java_compat.db/append_java_compat/bucket-0/data-2c5a05a4-4c30-4777-81fa-43a43f7c260d-0.mosaic b/test/test_data/mosaic/append_java_compat.db/append_java_compat/bucket-0/data-2c5a05a4-4c30-4777-81fa-43a43f7c260d-0.mosaic
new file mode 100644
index 0000000000000000000000000000000000000000..8c0babf5163cda54b8f7ec271e36d0659cb9fe48
GIT binary patch
literal 2007
zcmZ8h2~<-@6n%fmPBb8ivQ!ZR5dkF$B7z<hz$z-BEENSM2oYpa2~|Kr*~A@Du^JEy
zLJ%y<=7z{IRSsI0mV?@=H7#y!sZv3~DhTxdpM>`Go%1sDXWpB+d3WBOdVa=`%_a>B
zDj%M8>CCPX!eZs0a)VY>ZOvmSOa>H%iRy5B)v4<vziMvPLFW0~Z4=)_Bz|47;{KTn
ziTNkj%d9_*51+vFwPicd!SVSOo=%xREo@_s^!IC1$-{lR5pcyq_}c;eZ6JebYRl`(
z=*r@c4DndLb?<vn>9|(kf~CK>zw?*b!MsucRBsp$Xja=gQR<Wpn^1U$^Q^SvS7J@G
z=RS<MdRWBR@Y2<s`D9!FU3Rl(39|3)tcA($LGI0^Emqhf7DCS-)SXDhPammUTCm*l
zWJ&T`4Nm8Q*NwTW-i10jQ^$H7K_D6pq0JivJjzs<N&Eu+LkNbT7-Mr}!X~A&7zF@?
za1_VUjS0#sMJbI+t>2G<kj9|nbeu4LrX`>VPQ<YZ=MZi}I2S{MIWO28=9+cuU0htm
zE@A|jQL&LKF*|xpA!EHV`AcQAgejItrDQlprkDWjFs)#*G|JIio?X1Zan8ACajshx
z4h@Z4vxL;mDGyKxCa)qYYD@SOz!AWe#>(ol<s&d!+fMe-C1-f&-~H-b{QYR6#ttBJ
z1ldvm*)9Mk$mo9!xZciAPfDw+U?ehaSkx7@G#pxK3<iqgG8#+&|1lCCZdNZ)<8Bs7
z^TW*Qfi=8Wp#Eg}$%-DSQbndoq=l#n#szcW!$g>b#!JvG5;R_lZjl1^Connydqu$p
z#=Ws1%e%!rALmUI)paYiC68n=bv^7hFRt>tgau>6FJuZRAQ4Iw3dxIyhC#F!9FvPu
zwbji)&Iz7~Z;CE9#{}b;aG3_mT*O2I#RCKC+BY_FABP?+Q%yKGIn!28T?qz<=JRb!
zPg!8Zdynborq@=feAbFG2hRlTTykev<j*nfsF>x8WJP@ZsdirO57px}<&$HwYc9FB
z$qz2Iy0}AE!pV(uLS2qW_i%>aH%8o5_r#t)y>J<u1DVVsi!cke*^`9hFU$DL%O7Ej
z50>aSLPSp<4TA~;c|w9bJ79DH)6YO&wkC%~JvVfez`;B5leXS`=WotBGx^>WCxSs|
zkj(&D6~W+5lEG<`qRSKg);P>cb-VLyPC=64mDGNQgBN{*u+(8clCjogPy92vr!1w`
zbJ5Y15&F^FkGu05dv#ILri<Z^0uK&tUu4-3(DA+Rjp*bCv3f%N(`zWdJuiOmL)RqJ
zlkz@U+YW9^&iufnIel5}VOIzG13qGF7nP}-kum?{@wry1=)I-QwcNhI(z@QrVg?HL
zNft-IzZe2@$VY%k$I85xWs(adlf)v~c;<Wo#utn?Fe#y!G$*3@bc)F{jr;lX&U_@6
z=t$%`Sy07wNMu3VIJrG^g)+q1H5uaKZ^CEaemGs1B0TGTp%Z!CsZ?kMflmWo+XtGl
z-M|fILfwH6u^k>MDbBif&Wpca2Y02l_o+=<{7$d(ERV3>-Tzd#bC&mNb)?qutY%X6
zI!T-pQ-vVu?yoe5TfVTqa>)H8{?`7FT65c=^p-8HzCksq{kzN-)R>hY^X)HBQ?~Vr
z{H80cCpD;kbmD}p`8n%N<)!-IW-Ii>fwIS~9&Xti@>_+K!A{P)BJ03uU*%o2-C4Lt
zGbj+cJsJ4$=2lJIYEFz6Kl;4#*o@!i%e}BH53QDFX<8*MEL$+6sBfXuT=B=80H5Ij
z*1{*3N8-C~bXfI%T)7e(12)C9>Keo*O4D(OySbpgQdLpeeWx%)bGN{opC=MI+1WYz
zqWb4YlCWwMDQ%>!`rL<9I?~FQZy>oMZNSrm&LBbv<tChI49=i70Wx@ol{dRdEzqy(
z=K{kHA*ht$QI?2NQjMb(xkJVgGD8IsUo*GVBZgF@HY&w{2{qe|)FK8{XmLmcok1k%
z)sJaNB}9-iT*=c<5WPZCABaFrsg-_*ln=@rlPzIXHH;H6g^SXUQCw(G`Y0|eVVw~d
z_F&(?xbVnO11_qujkt*Zri|%l#5Mcd(P|-ctN)9OXa>pxb4GDVT#U`2$3=C6K8n_-
SAMqw)BrAxKrSl62lK%tjre~l4

literal 0
HcmV?d00001

diff --git a/test/test_data/mosaic/append_java_compat.db/append_java_compat/manifest/manifest-9d1301ba-d0b3-4d85-ae44-e93f6194876f-0 b/test/test_data/mosaic/append_java_compat.db/append_java_compat/manifest/manifest-9d1301ba-d0b3-4d85-ae44-e93f6194876f-0
new file mode 100644
index 0000000000000000000000000000000000000000..548f4da29e2ee87f725acf886dfc84ad4ac1c547
GIT binary patch
literal 2291
zcmeZI%3@>@ODrqO*DFrWNX<?BCs(ahQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD
zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@
z=9MVb>L3(&d-}QHmJM(W3h@j<F(9e3B()gD2&YhIZ`TmqhJ^TsIQqmxR0ZQ!=H}_+
z3YR4iHu3RL)mXd?l8N_o^hFq^R9sS&nU{{@Xt1ha&nP5SIr(U6g8akdo&7`oko<<B
z#@Ev?-rF@2?r<f{@bPtwAfyCnLVR$DV+bO6hzfvskhyS;Si%D2x-dtdP*;S(I2=e=
zm0zfj4<tAchD9somFDCq0n-ySc_kJVB~}6hBeNtm7nBmA@=D-vi>-~-QA$ZoODxSP
zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI
zW-0_GG2&GRJA1hLI>vh<Wqb@1eO$v_eGoRH7aOh-AwiDdOo&Jf@R&kN1n}&K9M#T2
zu8yFBB|gLxDW}5H3*0EMyL5obCp|McF((94UY2C$rWThZ<`(GYX6EE%0!t%|bcIyH
zxw!hchPcLKW@eaW(8P(JG?9#O1qQ#1i>nLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a
z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8>
zGE%yNxs7lThX;8ABZ>MRLzoNCAS47GBwJv38(3<^){21Z-{kz1)MTM5U~<e$Ni0fX
zSUx$G+sFN`-i(~t>XQSv?qj;Ztg%)9FN4QlHik#O`i2GGmnCY}t%~?>e`Wy(W1Ago
zyjWz@#mobj^0xb=P7~2TGAZM>Z`JDwHye-q-M;Lkn&y9ZFUP9fqg_+4WFA-`d}C>_
z!zJ!zmqTxO%{{kiO2$o}=M7tAuIJs?5R;eW?f=GJvwP>a`pX9xUDqFqne8c8*WS_9
zGXIw<i}Zi~Nsb?bnOjoI9{qP#h;abPo7O+cH^_UpKsl;<;tCBLe>?8LIm=kWoAT9N
z_GZp{P`!jh{?5tGlO^^Z4LkL$X)Z5AVQXXHfny5tDdwEdb*8FkG#*V=FqmN^)5L#*
IQ3>7a0KnM;$N&HU

literal 0
HcmV?d00001

diff --git a/test/test_data/mosaic/append_java_compat.db/append_java_compat/manifest/manifest-list-1572ca97-622c-452d-8a80-a992a0684230-0 b/test/test_data/mosaic/append_java_compat.db/append_java_compat/manifest/manifest-list-1572ca97-622c-452d-8a80-a992a0684230-0
new file mode 100644
index 0000000000000000000000000000000000000000..7730dd6f654b46a736d26b40cb4354e11540898a
GIT binary patch
literal 1006
zcmbVL&rX9d9M)v<rU#?m?agH20d~VtBe8%PFpUdoI_NeGl%;egbK&Cqc=Uxldhivr
zopaz=bnZs!_ov_YYn}f6ZsUYJ@J!3Y5lzY4!3>D$n1UAIG|YGent+kCw$8^SR6jF5
z)QJ2ZQEg1(G-KLaY)Bd9L{LQ^#59R=(CW>;m2ek!T+^}T=E5sx0u;>>aA6{dZQX(;
zBo{np!#c_}?<%MyTfx}Ag>}P#23DGGskatsKm@%Fu-AIWGd&pryPocqs(+M+ah<rP
z9?P_GS4V#Jxc#Z1`H%FLe!Bt6_7TEu$G0n|>;X1O5+M6t)K54kQ&~+cXsRZ)*a!2D
zf+A>whz`gk5ul}*MNm|S<+d-OrDN0hrCb&1+;3k&ZwXqusNvTvDwjb8-I^<X5$r;=
zo$a<%>DqB_s+BC$es#Lq=t|uGk%QuB{_4UkqT%6VE(nVVk9JNQ&%xQ#{>w3Ye=R=t
F&?hHILudd1

literal 0
HcmV?d00001

diff --git a/test/test_data/mosaic/append_java_compat.db/append_java_compat/manifest/manifest-list-1572ca97-622c-452d-8a80-a992a0684230-1 b/test/test_data/mosaic/append_java_compat.db/append_java_compat/manifest/manifest-list-1572ca97-622c-452d-8a80-a992a0684230-1
new file mode 100644
index 0000000000000000000000000000000000000000..de8c63592fcd6c16219fd389693b74cbdf012d7a
GIT binary patch
literal 1113
zcmbVLF^|(Q7|m5>Wk4syY!-&5v}!r76-#K0F1jQ|O;Ql2V!5%?3rd{I@gen2xu3!x
zU}0f^=_W>g0231fKY$GhyI0!Kp5UnL!HM7dp5OcI{`l2S^)`OZvw9vL(-eQdc?QI6
zN<kNJ8fKh;8ellx_|B&|R6jF5sbjf5ru8X~(~Q;U#U_+djs;cpD5eR?LHFqFcO^80
z1JCNaa&w^-GXaX)3h>0TA#x1|t{}PKF`I0nJnKaXm1JueH*k<)n$Se5rnjnhCDMd8
z^sj+^Y7BhKmllxc8~&>5Kf*(3lXyiuj^(1EVF%^oj%R}Af4Fyy=hr~Ffo-E+KX6N@
zJOb<_Nr23I5kKY}&tx>QpsDKAVjs*i8Wp1&AT+`!i2z;2EJj7SSzhk|ItH@Lzl5tS
zJq~(@&|iX<DysP#5~a(ajBd@9x;7j_dpp@}sZw>&e^!oU>Gm?y^+ea~{vUZ%?9N|Z
zm=PM@I?V-U1aqRDe|Yox%TT;M|MBVL1-tR6@^0te`};q&gQB95YV~D`Ssc+^=pE8*
zHJW3rlg7BEx5@qk9n*GOr=3=`*X*?SAMQnZ<J;Yec1MniCg<f)tBbEKZFgPj7mfO3
A`~Uy|

literal 0
HcmV?d00001

diff --git a/test/test_data/mosaic/append_java_compat.db/append_java_compat/schema/schema-0 b/test/test_data/mosaic/append_java_compat.db/append_java_compat/schema/schema-0
new file mode 100644
index 000000000..00a4d8cde
--- /dev/null
+++ b/test/test_data/mosaic/append_java_compat.db/append_java_compat/schema/schema-0
@@ -0,0 +1,162 @@
+{
+  "version" : 3,
+  "id" : 0,
+  "fields" : [ {
+    "id" : 0,
+    "name" : "id",
+    "type" : "INT NOT NULL"
+  }, {
+    "id" : 1,
+    "name" : "f_boolean",
+    "type" : "BOOLEAN"
+  }, {
+    "id" : 2,
+    "name" : "f_tinyint",
+    "type" : "TINYINT"
+  }, {
+    "id" : 3,
+    "name" : "f_smallint",
+    "type" : "SMALLINT"
+  }, {
+    "id" : 4,
+    "name" : "f_bigint",
+    "type" : "BIGINT"
+  }, {
+    "id" : 5,
+    "name" : "f_float",
+    "type" : "FLOAT"
+  }, {
+    "id" : 6,
+    "name" : "f_double",
+    "type" : "DOUBLE"
+  }, {
+    "id" : 7,
+    "name" : "f_char",
+    "type" : "CHAR(8)"
+  }, {
+    "id" : 8,
+    "name" : "f_varchar",
+    "type" : "VARCHAR(64)"
+  }, {
+    "id" : 9,
+    "name" : "f_binary",
+    "type" : "BINARY(8)"
+  }, {
+    "id" : 10,
+    "name" : "f_varbinary",
+    "type" : "VARBINARY(64)"
+  }, {
+    "id" : 11,
+    "name" : "f_date",
+    "type" : "DATE"
+  }, {
+    "id" : 12,
+    "name" : "f_ts_3",
+    "type" : "TIMESTAMP(3)"
+  }, {
+    "id" : 13,
+    "name" : "f_ts_6",
+    "type" : "TIMESTAMP(6)"
+  }, {
+    "id" : 14,
+    "name" : "f_ts_9",
+    "type" : "TIMESTAMP(9)"
+  }, {
+    "id" : 15,
+    "name" : "f_ltz_3",
+    "type" : "TIMESTAMP(3) WITH LOCAL TIME ZONE"
+  }, {
+    "id" : 16,
+    "name" : "f_ltz_6",
+    "type" : "TIMESTAMP(6) WITH LOCAL TIME ZONE"
+  }, {
+    "id" : 17,
+    "name" : "f_ltz_9",
+    "type" : "TIMESTAMP(9) WITH LOCAL TIME ZONE"
+  }, {
+    "id" : 18,
+    "name" : "f_decimal_1_0",
+    "type" : "DECIMAL(1, 0)"
+  }, {
+    "id" : 19,
+    "name" : "f_decimal_18_2",
+    "type" : "DECIMAL(18, 2)"
+  }, {
+    "id" : 20,
+    "name" : "f_decimal_19_2",
+    "type" : "DECIMAL(19, 2)"
+  }, {
+    "id" : 21,
+    "name" : "f_decimal_38_18",
+    "type" : "DECIMAL(38, 18)"
+  }, {
+    "id" : 22,
+    "name" : "f_array_int",
+    "type" : {
+      "type" : "ARRAY",
+      "element" : "INT"
+    }
+  }, {
+    "id" : 23,
+    "name" : "f_map_numeric",
+    "type" : {
+      "type" : "MAP",
+      "key" : "TINYINT",
+      "value" : "SMALLINT"
+    }
+  }, {
+    "id" : 24,
+    "name" : "f_map_string_bigint",
+    "type" : {
+      "type" : "MAP",
+      "key" : "STRING",
+      "value" : "BIGINT"
+    }
+  }, {
+    "id" : 25,
+    "name" : "f_array_array_int",
+    "type" : {
+      "type" : "ARRAY",
+      "element" : {
+        "type" : "ARRAY",
+        "element" : "INT"
+      }
+    }
+  }, {
+    "id" : 26,
+    "name" : "f_array_map",
+    "type" : {
+      "type" : "ARRAY",
+      "element" : {
+        "type" : "MAP",
+        "key" : "STRING",
+        "value" : "INT"
+      }
+    }
+  }, {
+    "id" : 27,
+    "name" : "f_map_array",
+    "type" : {
+      "type" : "MAP",
+      "key" : "STRING",
+      "value" : {
+        "type" : "ARRAY",
+        "element" : "INT"
+      }
+    }
+  } ],
+  "highestFieldId" : 27,
+  "partitionKeys" : [ ],
+  "primaryKeys" : [ ],
+  "options" : {
+    "bucket" : "-1",
+    "file.block-size" : "1 B",
+    "mosaic.num-buckets" : "4",
+    "mosaic.stats-columns" : "id,f_varchar,f_date,f_ts_3,f_ltz_9,f_decimal_18_2",
+    "target-file-size" : "64 MB",
+    "manifest.format" : "avro",
+    "file.format" : "mosaic",
+    "write.batch-size" : "2"
+  },
+  "timeMillis" : 1787565827926
+}
\ No newline at end of file
diff --git a/test/test_data/mosaic/append_java_compat.db/append_java_compat/snapshot/EARLIEST b/test/test_data/mosaic/append_java_compat.db/append_java_compat/snapshot/EARLIEST
new file mode 100644
index 000000000..56a6051ca
--- /dev/null
+++ b/test/test_data/mosaic/append_java_compat.db/append_java_compat/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git a/test/test_data/mosaic/append_java_compat.db/append_java_compat/snapshot/LATEST b/test/test_data/mosaic/append_java_compat.db/append_java_compat/snapshot/LATEST
new file mode 100644
index 000000000..56a6051ca
--- /dev/null
+++ b/test/test_data/mosaic/append_java_compat.db/append_java_compat/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git a/test/test_data/mosaic/append_java_compat.db/append_java_compat/snapshot/snapshot-1 b/test/test_data/mosaic/append_java_compat.db/append_java_compat/snapshot/snapshot-1
new file mode 100644
index 000000000..946d52e7c
--- /dev/null
+++ b/test/test_data/mosaic/append_java_compat.db/append_java_compat/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+  "version" : 3,
+  "uuid" : "81405eea-75ce-455b-b5ad-32e5bb9c010d",
+  "id" : 1,
+  "schemaId" : 0,
+  "baseManifestList" : "manifest-list-1572ca97-622c-452d-8a80-a992a0684230-0",
+  "baseManifestListSize" : 1006,
+  "deltaManifestList" : "manifest-list-1572ca97-622c-452d-8a80-a992a0684230-1",
+  "deltaManifestListSize" : 1113,
+  "commitUser" : "a1009c67-ff1d-4292-a67d-f69edbb1ab45",
+  "writerVersion" : "java-2.1-SNAPSHOT-UNKNOWN",
+  "commitIdentifier" : 9223372036854775807,
+  "commitKind" : "APPEND",
+  "timeMillis" : 1787565830612,
+  "totalRecordCount" : 6,
+  "deltaRecordCount" : 6,
+  "nextRowId" : 0
+}
\ No newline at end of file
diff --git a/test/test_data/mosaic/append_python_compat.db/append_python_compat/README.md b/test/test_data/mosaic/append_python_compat.db/append_python_compat/README.md
new file mode 100644
index 000000000..4971ae2a7
--- /dev/null
+++ b/test/test_data/mosaic/append_python_compat.db/append_python_compat/README.md
@@ -0,0 +1,62 @@
+Table: append_python_compat
+Writer: Paimon Python at commit 0043a70fd88ac75dcb83a8f2da5e72ce91e22b1f
+Mosaic version: 0.2.0
+
+This table was created and committed through Paimon Python's batch table-write pipeline.
+
+Schema:
+id INT NOT NULL
+f_boolean BOOLEAN
+f_tinyint TINYINT
+f_smallint SMALLINT
+f_bigint BIGINT
+f_float FLOAT
+f_double DOUBLE
+f_char CHAR(8)
+f_varchar VARCHAR(64)
+f_binary BINARY(8)
+f_varbinary VARBINARY(64)
+f_date DATE
+f_ts_3 TIMESTAMP(3)
+f_ts_6 TIMESTAMP(6)
+f_ts_9 TIMESTAMP(9)
+f_ltz_3 TIMESTAMP(3) WITH LOCAL TIME ZONE
+f_ltz_6 TIMESTAMP(6) WITH LOCAL TIME ZONE
+f_ltz_9 TIMESTAMP(9) WITH LOCAL TIME ZONE
+f_decimal_1_0 DECIMAL(1, 0)
+f_decimal_18_2 DECIMAL(18, 2)
+f_decimal_19_2 DECIMAL(19, 2)
+f_decimal_38_18 DECIMAL(38, 18)
+f_array_int ARRAY<INT>
+f_map_numeric MAP<TINYINT, SMALLINT>
+f_map_string_bigint MAP<STRING, BIGINT>
+f_array_array_int ARRAY<ARRAY<INT>>
+f_array_map ARRAY<MAP<STRING, INT>>
+f_map_array MAP<STRING, ARRAY<INT>>
+
+Options:
+bucket = -1
+file.block-size = 1 B
+file.format = mosaic
+manifest.format = avro
+mosaic.num-buckets = 4
+mosaic.stats-columns = id,f_varchar,f_date,f_ts_3,f_ltz_9,f_decimal_18_2
+target-file-size = 64 MB
+write.batch-size = 2
+
+Data is written as three row groups with two rows in each row group. Timestamp values are shown in
+UTC. The second row has NULL in every nullable field.
+
+Row group 0:
+Add: (1, true, -5, -1000, 10000000001, 1.25, 10.5, "char0001", "value-1", "bin00001", 0x010203, 2024-10-04, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, 1970-01-01 00:01:01.321, 1970-01-01 00:01:01.654321, 1970-01-01 00:01:01.321654987, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [1, 2], {0: 0, 10: 1}, {"k0": 1000, "z0": 2000}, [[1, 2], [3]], [{"nested0": 0}, {"nested10": 10}], {"array0": [0, 1]})
+Add: (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)
+
+Row group 1:
+Add: (10, true, -3, -998, 10000000010, 3.25, 12.5, "char0010", "value-10", "bin00010", 0x0a0b0c, 2024-10-06, 1970-01-01 00:00:10.123, 1970-01-01 00:00:10.123456, 1970-01-01 00:00:10.123456789, 1970-01-01 00:01:10.321, 1970-01-01 00:01:10.654321, 1970-01-01 00:01:10.321654987, -1, 10.25, 12345678901234569.89, 12345678901234567892.123456789012345678, [10, 11], {2: 20, 12: 21}, {"k2": 1002, "z2": 2002}, [[10, 11], [12]], [{"nested2": 2}, {"nested12": 12}], {"array2": [2, 3]})
+Add: (11, false, -2, -997, 10000000011, 4.25, 13.5, "char0011", "value-11", "bin00011", 0x0b0c0d, 2024-10-07, 1970-01-01 00:00:11.123, 1970-01-01 00:00:11.123456, 1970-01-01 00:00:11.123456789, 1970-01-01 00:01:11.321, 1970-01-01 00:01:11.654321, 1970-01-01 00:01:11.321654987, 0, 11.25, 12345678901234570.89, 12345678901234567893.123456789012345678, [11, 12], {3: 30, 13: 31}, {"k3": 1003, "z3": 2003}, [[11, 12], [13]], [{"nested3": 3}, {"nested13": 13}], {"array3": [3, 4]})
+
+Row group 2:
+Add: (20, true, -1, -996, 10000000020, 5.25, 14.5, "char0020", "value-20", "bin00020", 0x141516, 2024-10-08, 1970-01-01 00:00:20.123, 1970-01-01 00:00:20.123456, 1970-01-01 00:00:20.123456789, 1970-01-01 00:01:20.321, 1970-01-01 00:01:20.654321, 1970-01-01 00:01:20.321654987, 1, 20.25, 12345678901234571.89, 12345678901234567894.123456789012345678, [20, 21], {4: 40, 14: 41}, {"k4": 1004, "z4": 2004}, [[20, 21], [22]], [{"nested4": 4}, {"nested14": 14}], {"array4": [4, 5]})
+Add: (21, false, 0, -995, 10000000021, 6.25, 15.5, "char0021", "value-21", "bin00021", 0x151617, 2024-10-09, 1970-01-01 00:00:21.123, 1970-01-01 00:00:21.123456, 1970-01-01 00:00:21.123456789, 1970-01-01 00:01:21.321, 1970-01-01 00:01:21.654321, 1970-01-01 00:01:21.321654987, 2, 21.25, 12345678901234572.89, 12345678901234567895.123456789012345678, [21, 22], {5: 50, 15: 51}, {"k5": 1005, "z5": 2005}, [[21, 22], [23]], [{"nested5": 5}, {"nested15": 15}], {"array5": [5, 6]})
+
+Commit - snapshot 1
diff --git a/test/test_data/mosaic/append_python_compat.db/append_python_compat/bucket-0/data-087c1f82-5909-45cd-a6a8-ff0314c8e365-0.mosaic b/test/test_data/mosaic/append_python_compat.db/append_python_compat/bucket-0/data-087c1f82-5909-45cd-a6a8-ff0314c8e365-0.mosaic
new file mode 100644
index 0000000000000000000000000000000000000000..6740a11a7e5c9ebe9b1d30c36caafc38b3851931
GIT binary patch
literal 2004
zcmZ8h3pA8z7=HiDT+Emn)`(%<Qqp8IGm37f8Ih<cvQ!+zkOpN`<d%C^mTkHWDT<Z0
z7STm2lvKKGM=EvFwp4p;DMg(k?En8WWB2TP&O6`tf8YCl-!t#?z0b6>Ge<18l%t?<
z;hL7r>K!9&z4Tj_&*~FNIgDW4p<rdSDqyOj;@a3B<!#lFb~Y>3?0r!5(X!PYr_V*_
z9gC5eew!3v#B{e}+0sGKbBWs}%bUJ-Q>OT@tJBECUHTAkz<hbz0X!`rg{f!79Zcy>
z=Z%eUb=<2z_oL$PTHNzjJa-!KluUtzB><?}FfPzcQeU9dDI2$-(2a9VnkKEm>gO!z
z3~JnG!`S%B!GQTJ_0wHeL-{gf??=ZaaZWx?4JDUNFb^Gso;TceBmqCUzj{T!pWs+=
zTzEN7=YsQ%1%U(pGwi5i=k7-!4E3cg90oj8&pM6x1^O3X3_&r*){v+zGCLs(00`kI
zj-#8RWG8}4XjE!VE(X>#1|6s4gz?h&fFd{%$0D3fxGv!w3|+;3$zn6tMaJ0M+Y9Z5
z2rxs}hbV-sun9^=j4W=mEKI}{io{|v93xY7fwqs9zf>G5SR~CT%B`JW*S*0ZDcJT*
zZBn{5b#wef6jqT}5f!x}d@|q&;4*DxyE3F>FkagitdR?LFyPturcT}wM%36ABm$7E
z0FdkiV1k7HZph(wMsn=7>M}+&(?W;3qKbw?3yr}*QCvdPQU8C8goo?b7^?8u#<8wz
z^=m+9bdRCx8~>XLJw%}h*(MSzQC*A!2EYR|Fh}JgG*N`g#b}}!u)l%P4%izCHZUGc
z1W8UC=j?_>G@I%^SyS;7Nknx&tJRq!yA|a&G2mR9Aq6BzhJup3cw`i8_JAO+Frlfk
z0mwPQ6Y)*a#bWDXY+VjhLz%shNT7INK=u7i^_-{v1*Hn3I?FUGHFd=p{OroJDycBW
zi1!}xiIZG)Lg5l_lQw+XYsa!Xqc)ywz4kIkcO*UN>*K10S-)3KDnD)+k#XvRQ>(OK
zg~|Esu3~o9h8d{+;jn)8=;zv?yQ=>6Cr>U}$znqqv(R|0F-!kh)Ztg9Jip^lu%!jX
zU2JPaO&$${G7Wjw1bI_{(F@Gw8uBvAGj*uvhK>_Bd`E7cbaTMdK<BjOgUK@p2JJvH
z6C@`H25*uKP8StkG+VyT)-hrBo$mShv6@#>^*s*GbR+8$+gv0iTz9v;TRKk?Up3F;
zVEh>U;O(b<d9@$9LdEsx1D<#njBNMdpYdwH=YAtB?u<}nRP*Omly^TzzNgb6R_~Z}
zP|}*sxt!_h9Xo$8{r=j<p-*04v2c%4RRc2Ne?0ay$$~$$HD&R)yYWqHv@E8e&_S{|
z2A)OWok>0dL_AUAyfTekAekf<$;LC_8Dc!cNg5_a6q5!-G>=X(dAc^2Cw-BJtS34W
zsV55>To1lF@QaURD)jm@?3Jfdghjsw%(>n9Lsz`@Z;Q^oKnAV}P0tk?Gz%^iDYMX7
zFdJrp5q)^4-M&Zl4f4vIH5QFF6>ayD%A!Sevm>|mWyM6le7Gat{Q2Wc9f7fSU10}q
z9jZKXgNCba?5JJi)G5E-T)AZOVl%JAR1eFKEst#7l!Gn1ee#SVTc*!URQZ?}`mHk!
zYI}M0`Ui7C1gGxDZIO&C`;N7?zumx07|jkJ==Nt`as0SD!MSqtpohY#bQS+fhN<VY
z{qOvK>fH3Er%_PDXGb*Q2YQYivU}|+MX`{%O-(eT{0gD+sGFVYt&8ab%deSUlF^}E
zZXFlL<h|j&*<X^QCxBTIt@IRP5nA59o3k~)rd(0-`rRF+Z}#2%g|*G~yF3=(^3X@s
zZ;xaka0{txqy;W^A!UxV<9-`SqDUK>H<!*JLI~L=oT&{?p*8_hxSEysyGS8Wud3&U
znjJz-DZ@kgh*nRvqY1e~#t|Ar1rcAf5^E4mDpD01ufc@s??S2&4Jx$AH;B$464cVi
z)T9y$NEt5UswaqAo~RGRP(?|VdWX~w$^>zI7?%v~L`dPH#A6&6+Mhg*3yX@>;=&&8
z{TCM=;;+F)b+r~3(cF|Vmuqp&X&$c@vY_(6xQI@m%sX=&m&jh*32IzaE2yJrAJijW
QPmE**F|u^ay?mtq031|n&j0`b

literal 0
HcmV?d00001

diff --git a/test/test_data/mosaic/append_python_compat.db/append_python_compat/manifest/manifest-82c6d797-6335-462b-b86e-a076ae20578a-0 b/test/test_data/mosaic/append_python_compat.db/append_python_compat/manifest/manifest-82c6d797-6335-462b-b86e-a076ae20578a-0
new file mode 100644
index 0000000000000000000000000000000000000000..de32901f50da704cb5a90badd14151183568652d
GIT binary patch
literal 2209
zcmds2&ui0Q7*1pHq$naoP*}}LVVRv<XXh<zzp~K$n8}x}I!pYPq+29ON0JG36!b9v
z3h}&~2l3)b@IMd{&$62b!IKxkFKO3&X=7W#lkF{ezUO=2=Y8HM^3LO2{xW*vcNUF~
zX&INE1|jlH<ePKx!@$_HT=aSVkO{l{7IU}2_?FS}O=h9Mc*u>Ph{&_sRuBR&^t;iE
zwrx3PfWg~`Ly+D8iptAU^!}8}_Clt&P#{2cUaC$;Y;p?Z;Rw-AH?)Egs&%bW2XHbO
zlp!bReWW@WTH^&kASUTiMIox$jwM<lijHK;5%K8~Cz7O05c;;aJJLvuRQU%fq|+J0
zDe{(Hku@pRa|TZ2CA|*r6U{JL`iWd~78qvIRmee7zDeTr?GrI4$1(+q%4~3g21w>V
zRyrF}(gdNe5lOnucn6My^+@VQzK>!n+o9#gPCmsjv7)VB3lG+`+UURuvC$)fUaOZj
zb6iu><-qmHCF7ek5NG7ze=1KjU=`fcfK<Vopo!~1Aw8MpTx`j|JE2xKfXL~5DpWF<
z1<(KjdHZ-YfhJTqJz7T6JIOT1EHm8Zv@0qK;G+D}A)i|Q3>U_QVxKvkUE4qo93*gP
zyEtu;yU)6|<JdU(GR&BY)hZAG1bS9N46hv+d)(G3CV<COt%7PSgpSi1Gzr-fvVNjW
ztE(SvvZhqXa7$RCDNN%amIbf2Kn`a}G|QBx-Gk>36bXMWHaWPF+4CU1rxDe7MTPM!
zQnnlxJKG?(6dq6XA2*B>tkt95y}bG`_<p3Gd^`R+Cm+@6Pq`a!Z~ZE~K7WQf$>*sn
zRL(T<H;yfq?-)z%@@=-XR$OCON`}dzRa9o%?c(y%icz+fS4(Vh(d`7tHjd}d(KP+)
k%bSlc_GpTt@gM)rQjac{-(UFg{N{)2Kc8JI(4&|C26~X3;{X5v

literal 0
HcmV?d00001

diff --git a/test/test_data/mosaic/append_python_compat.db/append_python_compat/manifest/manifest-list-16d5961d-78cd-498a-9cff-80ef60258224-0 b/test/test_data/mosaic/append_python_compat.db/append_python_compat/manifest/manifest-list-16d5961d-78cd-498a-9cff-80ef60258224-0
new file mode 100644
index 0000000000000000000000000000000000000000..da8b9dfe059460dd2913ab345d6352c340ab51ff
GIT binary patch
literal 799
zcmbV~OHRWu5QdAyjs+4ztRt^04#3W8ToEZwAdgb0BI70w1m%Gow^FEbJ0v!&dIT06
zfD;f0S`r$dV(a<*d*&OP)4QGBLptZ#v7ZIZKX}LmO#{k<qxvxSZ&^a0_ZC1D_Y9mU
zfHObi0q83rrAd8)X&Qwr7sDuKm<cLtVZ`DfulUJg&5;qZY|S)eeU%oa0+fBF0Sz@B
z5d&h>#B#x-^rnlnwW}5xXB(X1VggkasYGVm&D<Yo73s*?l0So%qdC=B#CD+5O#jW<
zhegtHxzilh3^Ibc+wOi^2$p}7g5h{e(r|U13{2N(H9Z08ERHKjU)@vAX(6pef+h7%
zR}85Ad8Jns0~XR*EUJ~ufYK}5YqbZb2t%T&f9RK!TINL?TsQUk;O29!FC-oOBf~3u
T{__0w`ss1`Hs61_?2YYrKsEGb

literal 0
HcmV?d00001

diff --git a/test/test_data/mosaic/append_python_compat.db/append_python_compat/manifest/manifest-list-16d5961d-78cd-498a-9cff-80ef60258224-1 b/test/test_data/mosaic/append_python_compat.db/append_python_compat/manifest/manifest-list-16d5961d-78cd-498a-9cff-80ef60258224-1
new file mode 100644
index 0000000000000000000000000000000000000000..b89a3a145e10f3a65fa3d2f2fa6cf8727dc68209
GIT binary patch
literal 897
zcmbV~Pfo%>6o*T3=R#wQabrSeqYJ7HN{c2YHneDC{}@`(NN_T=gVvN{+Q9@(ui?^_
zSMUPHrAP4!CbkGj0hQS0ysy9a=6j<zu2XZ%=+<v<T5TI!OLqZ5o{fBarFR%uSGa}V
z=I;UN-e9no0Y0|cz74V&;GtIUL`9y{!~rQeE><x?ajofK*A61Sc0c4`RVo{@uEq61
zn&S}=W-}vDNmeAN392-T1;lr}%W0$`pC?GSJ;7-e6$+v#i7?JKMst6lMM;s&8Tk{T
zY|3VIEohj6Ihy{PvoDKq%Fk2Hsj>#Ef>KFN-{=w?j7d?2vl&UPqA0MaSG2^?HQ;qz
zH*)knJ@S3jjjcHZw|YArFreqJ9@e9xjhm?Bl4#{|Kv)lxYc+~Tk}5!1{6l|y)3Sb=
z1cyWQ8&0-MJfYaZKQihe#qJ*;KJjZVt?n(ZKRvwDkExC4t&fbEr>WQU+(oPZ)R{ug
x;%t7GXSnU{JhQ{)8cd_WVFs~0hj5O~^997PFMy_2DVmBtnqn6U>4VgGhfiqf35@^%

literal 0
HcmV?d00001

diff --git a/test/test_data/mosaic/append_python_compat.db/append_python_compat/schema/schema-0 b/test/test_data/mosaic/append_python_compat.db/append_python_compat/schema/schema-0
new file mode 100644
index 000000000..55e18d15a
--- /dev/null
+++ b/test/test_data/mosaic/append_python_compat.db/append_python_compat/schema/schema-0
@@ -0,0 +1,201 @@
+{
+  "version": 3,
+  "id": 0,
+  "fields": [
+    {
+      "id": 0,
+      "name": "id",
+      "type": "INT NOT NULL"
+    },
+    {
+      "id": 1,
+      "name": "f_boolean",
+      "type": "BOOLEAN"
+    },
+    {
+      "id": 2,
+      "name": "f_tinyint",
+      "type": "TINYINT"
+    },
+    {
+      "id": 3,
+      "name": "f_smallint",
+      "type": "SMALLINT"
+    },
+    {
+      "id": 4,
+      "name": "f_bigint",
+      "type": "BIGINT"
+    },
+    {
+      "id": 5,
+      "name": "f_float",
+      "type": "FLOAT"
+    },
+    {
+      "id": 6,
+      "name": "f_double",
+      "type": "DOUBLE"
+    },
+    {
+      "id": 7,
+      "name": "f_char",
+      "type": "CHAR(8)"
+    },
+    {
+      "id": 8,
+      "name": "f_varchar",
+      "type": "VARCHAR(64)"
+    },
+    {
+      "id": 9,
+      "name": "f_binary",
+      "type": "BINARY(8)"
+    },
+    {
+      "id": 10,
+      "name": "f_varbinary",
+      "type": "VARBINARY(64)"
+    },
+    {
+      "id": 11,
+      "name": "f_date",
+      "type": "DATE"
+    },
+    {
+      "id": 12,
+      "name": "f_ts_3",
+      "type": "TIMESTAMP(3)"
+    },
+    {
+      "id": 13,
+      "name": "f_ts_6",
+      "type": "TIMESTAMP(6)"
+    },
+    {
+      "id": 14,
+      "name": "f_ts_9",
+      "type": "TIMESTAMP(9)"
+    },
+    {
+      "id": 15,
+      "name": "f_ltz_3",
+      "type": "TIMESTAMP(3) WITH LOCAL TIME ZONE"
+    },
+    {
+      "id": 16,
+      "name": "f_ltz_6",
+      "type": "TIMESTAMP(6) WITH LOCAL TIME ZONE"
+    },
+    {
+      "id": 17,
+      "name": "f_ltz_9",
+      "type": "TIMESTAMP(9) WITH LOCAL TIME ZONE"
+    },
+    {
+      "id": 18,
+      "name": "f_decimal_1_0",
+      "type": "DECIMAL(1, 0)"
+    },
+    {
+      "id": 19,
+      "name": "f_decimal_18_2",
+      "type": "DECIMAL(18, 2)"
+    },
+    {
+      "id": 20,
+      "name": "f_decimal_19_2",
+      "type": "DECIMAL(19, 2)"
+    },
+    {
+      "id": 21,
+      "name": "f_decimal_38_18",
+      "type": "DECIMAL(38, 18)"
+    },
+    {
+      "id": 22,
+      "name": "f_array_int",
+      "type": {
+        "type": "ARRAY",
+        "element": "INT",
+        "nullable": true
+      }
+    },
+    {
+      "id": 23,
+      "name": "f_map_numeric",
+      "type": {
+        "type": "MAP<TINYINT NOT NULL, SMALLINT>",
+        "key": "TINYINT NOT NULL",
+        "value": "SMALLINT",
+        "nullable": true
+      }
+    },
+    {
+      "id": 24,
+      "name": "f_map_string_bigint",
+      "type": {
+        "type": "MAP<STRING NOT NULL, BIGINT>",
+        "key": "STRING NOT NULL",
+        "value": "BIGINT",
+        "nullable": true
+      }
+    },
+    {
+      "id": 25,
+      "name": "f_array_array_int",
+      "type": {
+        "type": "ARRAY",
+        "element": {
+          "type": "ARRAY",
+          "element": "INT",
+          "nullable": true
+        },
+        "nullable": true
+      }
+    },
+    {
+      "id": 26,
+      "name": "f_array_map",
+      "type": {
+        "type": "ARRAY",
+        "element": {
+          "type": "MAP<STRING NOT NULL, INT>",
+          "key": "STRING NOT NULL",
+          "value": "INT",
+          "nullable": true
+        },
+        "nullable": true
+      }
+    },
+    {
+      "id": 27,
+      "name": "f_map_array",
+      "type": {
+        "type": "MAP<STRING NOT NULL, ARRAY<INT>>",
+        "key": "STRING NOT NULL",
+        "value": {
+          "type": "ARRAY",
+          "element": "INT",
+          "nullable": true
+        },
+        "nullable": true
+      }
+    }
+  ],
+  "highestFieldId": 27,
+  "partitionKeys": [],
+  "primaryKeys": [],
+  "options": {
+    "bucket": "-1",
+    "file.block-size": "1 B",
+    "file.format": "mosaic",
+    "manifest.format": "avro",
+    "mosaic.num-buckets": "4",
+    "mosaic.stats-columns": "id,f_varchar,f_date,f_ts_3,f_ltz_9,f_decimal_18_2",
+    "target-file-size": "64 MB",
+    "write.batch-size": "2"
+  },
+  "comment": null,
+  "timeMillis": 1787569308159
+}
\ No newline at end of file
diff --git a/test/test_data/mosaic/append_python_compat.db/append_python_compat/snapshot/LATEST b/test/test_data/mosaic/append_python_compat.db/append_python_compat/snapshot/LATEST
new file mode 100644
index 000000000..56a6051ca
--- /dev/null
+++ b/test/test_data/mosaic/append_python_compat.db/append_python_compat/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git a/test/test_data/mosaic/append_python_compat.db/append_python_compat/snapshot/snapshot-1 b/test/test_data/mosaic/append_python_compat.db/append_python_compat/snapshot/snapshot-1
new file mode 100644
index 000000000..9a7fb6f06
--- /dev/null
+++ b/test/test_data/mosaic/append_python_compat.db/append_python_compat/snapshot/snapshot-1
@@ -0,0 +1,15 @@
+{
+  "version": 3,
+  "id": 1,
+  "schemaId": 0,
+  "baseManifestList": "manifest-list-16d5961d-78cd-498a-9cff-80ef60258224-0",
+  "deltaManifestList": "manifest-list-16d5961d-78cd-498a-9cff-80ef60258224-1",
+  "totalRecordCount": 6,
+  "deltaRecordCount": 6,
+  "commitUser": "c6489507-9e13-4fec-abfa-74b1957f8008",
+  "commitIdentifier": 9223372036854775807,
+  "commitKind": "APPEND",
+  "timeMillis": 1787569308179,
+  "uuid": "1d6d2d3e-abe1-45ac-88d0-d90429aafbb5",
+  "writerVersion": "python-2.1.dev-0043a70fd88ac75dcb83a8f2da5e72ce91e22b1f"
+}
\ No newline at end of file
diff --git a/third_party/versions.txt b/third_party/versions.txt
index 2e20dc72f..ffb1bbf3b 100644
--- a/third_party/versions.txt
+++ b/third_party/versions.txt
@@ -68,6 +68,10 @@ PAIMON_ARROW_BUILD_VERSION=17.0.0
 PAIMON_ARROW_BUILD_SHA256_CHECKSUM=9d280d8042e7cf526f8c28d170d93bfab65e50f94569f6a790982a878d8d898d
 PAIMON_ARROW_PKG_NAME=apache-arrow-${PAIMON_ARROW_BUILD_VERSION}.tar.gz
 
+PAIMON_MOSAIC_BUILD_VERSION=0.2.0
+PAIMON_MOSAIC_BUILD_SHA256_CHECKSUM=8123eaadd293a7904b692eff900484691711dc0cb24878166a2f90a378d37a22
+PAIMON_MOSAIC_PKG_NAME=apache-paimon-mosaic-${PAIMON_MOSAIC_BUILD_VERSION}-src.tgz
+
 PAIMON_AVRO_BUILD_VERSION=c499eefb48aa2db906c7bca14a047223806f36db
 PAIMON_AVRO_BUILD_SHA256_CHECKSUM=9771f1dcfe3c01aff7ff670e873e66d3406362f71941821d482de65f3d32d780
 PAIMON_AVRO_PKG_NAME=avro-${PAIMON_AVRO_BUILD_VERSION}.tar.gz
@@ -162,6 +166,7 @@ DEPENDENCIES=(
   "PAIMON_GTEST_URL ${PAIMON_GTEST_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/google/googletest/archive/release-${PAIMON_GTEST_BUILD_VERSION}.tar.gz"
   "PAIMON_BENCHMARK_URL ${PAIMON_BENCHMARK_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/google/benchmark/archive/refs/tags/v${PAIMON_BENCHMARK_BUILD_VERSION}.tar.gz"
   "PAIMON_ARROW_URL ${PAIMON_ARROW_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/apache/arrow/releases/download/apache-arrow-${PAIMON_ARROW_BUILD_VERSION}/apache-arrow-${PAIMON_ARROW_BUILD_VERSION}.tar.gz"
+  "PAIMON_MOSAIC_URL ${PAIMON_MOSAIC_PKG_NAME} https://downloads.apache.org/paimon/paimon-mosaic-${PAIMON_MOSAIC_BUILD_VERSION}/${PAIMON_MOSAIC_PKG_NAME}"
   "PAIMON_AVRO_URL ${PAIMON_AVRO_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/apache/avro/archive/${PAIMON_AVRO_BUILD_VERSION}.tar.gz"
   "PAIMON_AWS_C_AUTH_URL ${PAIMON_AWS_C_AUTH_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-auth/archive/refs/tags/${PAIMON_AWS_C_AUTH_BUILD_VERSION}.tar.gz"
   "PAIMON_AWS_C_CAL_URL ${PAIMON_AWS_C_CAL_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-cal/archive/refs/tags/${PAIMON_AWS_C_CAL_BUILD_VERSION}.tar.gz"

From 813c6444e39dba4076018a34126a3e3b3dfc176c Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 28 Aug 2026 14:57:16 +0800
Subject: [PATCH 84/93] test(realtime): streamline realtime test coverage

---
 .../core/mergetree/merge_tree_writer_test.cpp |  47 -
 .../operation/merge_file_split_read_test.cpp  |  45 -
 .../primary_key_realtime_store_test.cpp       |  20 -
 .../core/realtime/realtime_context_test.cpp   |   6 +-
 .../realtime_primary_key_reader_test.cpp      |   1 -
 test/inte/realtime_write_inte_test.cpp        | 849 +-----------------
 6 files changed, 5 insertions(+), 963 deletions(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index 427c113e1..db2533fae 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -345,29 +345,6 @@ TEST_P(MergeTreeWriterTest, TestSimple) {
     DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{},
                                           /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
-
-    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
-      [2, 0, "Alice", 10, 0, 13.1],
-      [0, 0, "Lucy", 20, 1, 14.1],
-      [1, 0, "Paul", 20, 1, null]
-    ])")
-            .ValueOrDie());
-    auto sorted_reader_path_factory = std::make_shared<DataFilePathFactory>();
-    ASSERT_OK(sorted_reader_path_factory->Init(dir->Str() + "/sorted-readers", "orc",
-                                               options.DataFilePrefix(), nullptr));
-    ASSERT_OK_AND_ASSIGN(auto sorted_reader_writer,
-                         CreateMergeWriter(-1, dir->Str(), sorted_reader_path_factory, 1, options));
-    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
-    sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
-    ASSERT_OK(sorted_reader_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
-    ASSERT_OK_AND_ASSIGN(CommitIncrement sorted_reader_commit_increment,
-                         sorted_reader_writer->PrepareCommit(false));
-    ASSERT_OK(sorted_reader_writer->Close());
-    ASSERT_EQ(1, sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles().size());
-    std::string sorted_reader_path = sorted_reader_path_factory->ToPath(
-        sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles()[0]);
-    CheckFileContent(sorted_reader_path, expected_array);
 }
 
 TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) {
@@ -449,30 +426,6 @@ TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) {
     DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{},
                                           /*changelog_files=*/{});
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
-
-    auto sorted_reader_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(write_type_, R"([
-      [16, 0, "Alice", 10, 0, 113.1],
-      [14, 0, "Lucy", 20, 1, 114.1],
-      [13, 0, "Paul", 20, 1, 15.1],
-      [15, 0, "Skye", 10, 0, 118.1]
-    ])")
-            .ValueOrDie());
-    auto sorted_reader_path_factory = std::make_shared<DataFilePathFactory>();
-    ASSERT_OK(sorted_reader_path_factory->Init(dir->Str() + "/sorted-readers", "orc",
-                                               options.DataFilePrefix(), nullptr));
-    ASSERT_OK_AND_ASSIGN(auto sorted_reader_writer,
-                         CreateMergeWriter(9, dir->Str(), sorted_reader_path_factory, 0, options));
-    std::vector<std::unique_ptr<KeyValueRecordReader>> sorted_readers;
-    sorted_readers.push_back(CreateSingleReader(sorted_reader_array));
-    ASSERT_OK(sorted_reader_writer->WriteSortedReadersToFiles(std::move(sorted_readers)));
-    ASSERT_OK_AND_ASSIGN(CommitIncrement sorted_reader_commit_increment,
-                         sorted_reader_writer->PrepareCommit(false));
-    ASSERT_OK(sorted_reader_writer->Close());
-    ASSERT_EQ(1, sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles().size());
-    std::string sorted_reader_path = sorted_reader_path_factory->ToPath(
-        sorted_reader_commit_increment.GetNewFilesIncrement().NewFiles()[0]);
-    CheckFileContent(sorted_reader_path, expected_array);
 }
 
 TEST_P(MergeTreeWriterTest, TestSortedReaders) {
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index df9f5a02f..d02120de4 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -52,7 +52,6 @@
 #include "paimon/executor.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/memory/memory_pool.h"
-#include "paimon/metrics.h"
 #include "paimon/predicate/literal.h"
 #include "paimon/predicate/predicate_builder.h"
 #include "paimon/read_context.h"
@@ -67,29 +66,6 @@ class FileSystem;
 }  // namespace paimon
 
 namespace paimon::test {
-namespace {
-
-class TrackingKeyValueRecordReader : public KeyValueRecordReader {
- public:
-    explicit TrackingKeyValueRecordReader(int32_t* close_count) : close_count_(close_count) {}
-
-    Result<std::unique_ptr<KeyValueRecordReader::Iterator>> NextBatch() override {
-        return std::unique_ptr<KeyValueRecordReader::Iterator>();
-    }
-
-    void Close() override {
-        ++(*close_count_);
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return nullptr;
-    }
-
- private:
-    int32_t* close_count_;
-};
-
-}  // namespace
 
 // Parameter: min_heap/loser_tree; enable/disable IO prefetch; enable/disable multi thread row to
 // batch
@@ -762,27 +738,6 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections)
     batch_reader->Close();
 }
 
-TEST_F(MergeFileSplitReadTest, TestRealtimeReaderFailureClosesPluginReader) {
-    std::string path =
-        paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
-    ReadContextBuilder context_builder(path);
-    context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"});
-    context_builder.SetOptions(
-        {{Options::MERGE_ENGINE, "aggregation"}, {"fields.v0.aggregate-function", "unsupported"}});
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, context_builder.Finish());
-    std::shared_ptr<InternalReadContext> internal_context = CreateInternalReadContext(read_context);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<MergeFileSplitRead> split_read,
-                         CreateMergeFileSplitRead(internal_context));
-
-    int32_t close_count = 0;
-    std::vector<std::unique_ptr<KeyValueRecordReader>> plugin_readers;
-    plugin_readers.push_back(std::make_unique<TrackingKeyValueRecordReader>(&close_count));
-
-    ASSERT_NOK_WITH_MSG(split_read->CreateRealtimeReader({}, std::move(plugin_readers)),
-                        "unsupported");
-    ASSERT_EQ(1, close_count);
-}
-
 TEST_P(MergeFileSplitReadTest, TestLookUp) {
     std::string path =
         paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor";
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index d3d5dff5e..7d6fa72f5 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -263,26 +263,6 @@ TEST(PrimaryKeyRealtimeStoreTest, TestSlicedReadersExportZeroOffsets) {
     AssertSlicedBatch(readers[0].get());
 }
 
-TEST(PrimaryKeyRealtimeStoreTest, TestCloseUnreadBatchReaders) {
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
-                         PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
-    ASSERT_OK(
-        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 10, 0, 1, "a"]])"), OffsetRange(0, 1)}));
-    ASSERT_OK(
-        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 20, 1, 2, "b"]])"), OffsetRange(1, 2)}));
-    ASSERT_OK(
-        store->Write(RealtimeWriteBatch{MakeBatch(R"([[0, 30, 2, 3, "c"]])"), OffsetRange(2, 3)}));
-    ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
-                         store->SealForCommit());
-    ASSERT_TRUE(segment.has_value());
-    ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
-                         store->CreateCommitReaders(segment.value()));
-    ASSERT_EQ(3, readers.size());
-    for (const std::unique_ptr<BatchReader>& reader : readers) {
-        reader->Close();
-    }
-}
-
 TEST(PrimaryKeyRealtimeStoreTest, TestReclaimKeepsReadView) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
                          PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 82f831f6a..636cf12fc 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -217,8 +217,10 @@ TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
     const std::map<std::string, std::string> partition = {{"dt", "2026-08-02"}};
     const RealtimePartitionBucket partition_bucket(partition, /*bucket=*/0);
 
-    ASSERT_OK(
-        GetOrCreateAppendStore(context, partition, 0, MakeWriteSchema(), {}, GetDefaultPool()));
+    ASSERT_OK(context->GetOrCreateRealtimeStore(
+        RealtimeStoreCreateRequest{
+            MakeWriteSchema(), /*options=*/{}, GetDefaultPool(), RealtimeStoreMode::PRIMARY_KEY},
+        partition_bucket));
 
     ASSERT_OK_AND_ASSIGN(int64_t first, context->AdvanceMaterializedMaxSequenceNumber(
                                             partition_bucket, /*max_sequence_number=*/4));
diff --git a/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
index beb1a6435..89a39cd63 100644
--- a/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
@@ -20,7 +20,6 @@
 
 #include <cstdint>
 #include <memory>
-#include <optional>
 #include <string>
 #include <utility>
 #include <vector>
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 6b611300e..55227d841 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -29,7 +29,6 @@
 #include <map>
 #include <memory>
 #include <mutex>
-#include <new>
 #include <set>
 #include <string>
 #include <thread>
@@ -44,8 +43,6 @@
 #include "paimon/catalog/catalog.h"
 #include "paimon/catalog/identifier.h"
 #include "paimon/commit_context.h"
-#include "paimon/common/factories/io_hook.h"
-#include "paimon/common/reader/reader_utils.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/path_util.h"
@@ -84,88 +81,6 @@
 namespace paimon::test {
 namespace {
 
-bool HasSuffix(const std::string& value, const std::string& suffix) {
-    return value.size() >= suffix.size() &&
-           value.compare(value.size() - suffix.size(), suffix.size(), suffix) == 0;
-}
-
-Result<std::set<std::string>> ListPhysicalArtifacts(const std::shared_ptr<FileSystem>& file_system,
-                                                    const std::string& root) {
-    std::set<std::string> artifacts;
-    std::vector<std::string> directories = {root};
-    while (!directories.empty()) {
-        std::string directory = std::move(directories.back());
-        directories.pop_back();
-        std::vector<BasicFileStatus> statuses;
-        PAIMON_RETURN_NOT_OK(file_system->ListDir(directory, &statuses));
-        for (const BasicFileStatus& status : statuses) {
-            if (status.IsDir()) {
-                directories.push_back(status.GetPath());
-            } else if (HasSuffix(status.GetPath(), ".orc") ||
-                       HasSuffix(status.GetPath(), ".index") ||
-                       HasSuffix(status.GetPath(), ".channel")) {
-                artifacts.insert(status.GetPath());
-            }
-        }
-    }
-    return artifacts;
-}
-
-class FailAllocationMemoryPool final : public MemoryPool {
- public:
-    explicit FailAllocationMemoryPool(const std::shared_ptr<MemoryPool>& delegate)
-        : delegate_(delegate) {}
-
-    void FailAfterAllocations(int64_t successful_allocations) {
-        allocations_before_failure_.store(successful_allocations, std::memory_order_release);
-    }
-
-    void* Malloc(uint64_t size, uint64_t alignment = 0) override {
-        if (ShouldFail()) {
-            throw std::bad_alloc();
-        }
-        return delegate_->Malloc(size, alignment);
-    }
-
-    void* Realloc(void* p, size_t old_size, size_t new_size, uint64_t alignment = 0) override {
-        if (ShouldFail()) {
-            throw std::bad_alloc();
-        }
-        return delegate_->Realloc(p, old_size, new_size, alignment);
-    }
-
-    void Free(void* p, uint64_t size) override {
-        delegate_->Free(p, size);
-    }
-
-    void Free(void* p, uint64_t size, uint64_t alignment) override {
-        delegate_->Free(p, size, alignment);
-    }
-
-    uint64_t CurrentUsage() const override {
-        return delegate_->CurrentUsage();
-    }
-
-    uint64_t MaxMemoryUsage() const override {
-        return delegate_->MaxMemoryUsage();
-    }
-
- private:
-    bool ShouldFail() {
-        int64_t remaining = allocations_before_failure_.load(std::memory_order_acquire);
-        while (remaining >= 0) {
-            if (allocations_before_failure_.compare_exchange_weak(remaining, remaining - 1,
-                                                                  std::memory_order_acq_rel)) {
-                return remaining == 0;
-            }
-        }
-        return false;
-    }
-
-    std::shared_ptr<MemoryPool> delegate_;
-    std::atomic<int64_t> allocations_before_failure_{-1};
-};
-
 class TrackingRealtimeReadView final : public RealtimeReadView {
  public:
     explicit TrackingRealtimeReadView(std::shared_ptr<RealtimeReadView> delegate)
@@ -345,10 +260,6 @@ class CloseTrackingBatchReader final : public BatchReader {
 struct CloseTrackingReaderState {
     std::shared_ptr<std::atomic<int32_t>> query_close_count =
         std::make_shared<std::atomic<int32_t>>(0);
-    std::shared_ptr<std::atomic<int32_t>> commit_close_count =
-        std::make_shared<std::atomic<int32_t>>(0);
-    int32_t query_null_index = -1;
-    int32_t commit_null_index = -1;
 };
 
 class CloseTrackingRealtimeStore final : public DelegatingRealtimeStore {
@@ -357,18 +268,6 @@ class CloseTrackingRealtimeStore final : public DelegatingRealtimeStore {
                                const std::shared_ptr<CloseTrackingReaderState>& state)
         : DelegatingRealtimeStore(delegate), state_(state) {}
 
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               delegate_->CreateCommitReaders(segment));
-        for (std::unique_ptr<BatchReader>& reader : readers) {
-            reader = std::make_unique<CloseTrackingBatchReader>(std::move(reader),
-                                                                state_->commit_close_count);
-        }
-        PAIMON_RETURN_NOT_OK(InsertNullReader(state_->commit_null_index, &readers));
-        return readers;
-    }
-
     Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
         const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
         const RealtimeQueryContext& context) override {
@@ -378,302 +277,13 @@ class CloseTrackingRealtimeStore final : public DelegatingRealtimeStore {
             reader = std::make_unique<CloseTrackingBatchReader>(std::move(reader),
                                                                 state_->query_close_count);
         }
-        PAIMON_RETURN_NOT_OK(InsertNullReader(state_->query_null_index, &readers));
         return readers;
     }
 
  private:
-    static Status InsertNullReader(int32_t index,
-                                   std::vector<std::unique_ptr<BatchReader>>* readers) {
-        if (index < 0) {
-            return Status::OK();
-        }
-        if (index > static_cast<int32_t>(readers->size())) {
-            return Status::Invalid("null reader index exceeds reader count");
-        }
-        readers->insert(readers->begin() + index, nullptr);
-        return Status::OK();
-    }
-
     std::shared_ptr<CloseTrackingReaderState> state_;
 };
 
-class SplitBatchReader final : public BatchReader {
- public:
-    explicit SplitBatchReader(std::unique_ptr<BatchReader> delegate)
-        : delegate_(std::move(delegate)) {}
-
-    Result<ReadBatch> NextBatch() override {
-        while (!current_batch_ || next_row_ == current_batch_->length()) {
-            PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
-            if (BatchReader::IsEofBatch(batch)) {
-                return MakeEofBatch();
-            }
-            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-                std::shared_ptr<arrow::Array> array,
-                arrow::ImportArray(batch.first.get(), batch.second.get()));
-            if (!array || array->type_id() != arrow::Type::STRUCT) {
-                return Status::Invalid("split batch reader received a non-struct batch");
-            }
-            current_batch_ = std::dynamic_pointer_cast<arrow::StructArray>(array);
-            next_row_ = 0;
-        }
-        std::shared_ptr<arrow::Array> slice = current_batch_->Slice(next_row_, /*length=*/1);
-        ++next_row_;
-        auto output = std::make_unique<ArrowArray>();
-        auto schema = std::make_unique<ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*slice, output.get(), schema.get()));
-        return ReadBatch(std::move(output), std::move(schema));
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        current_batch_.reset();
-        delegate_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> delegate_;
-    std::shared_ptr<arrow::StructArray> current_batch_;
-    int64_t next_row_ = 0;
-};
-
-class SplitCommitReaderRealtimeStore final : public DelegatingRealtimeStore {
- public:
-    explicit SplitCommitReaderRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
-        : DelegatingRealtimeStore(delegate) {}
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               delegate_->CreateCommitReaders(segment));
-        for (std::unique_ptr<BatchReader>& reader : readers) {
-            reader = std::make_unique<SplitBatchReader>(std::move(reader));
-        }
-        return readers;
-    }
-};
-
-class FailAfterPhysicalFileBatchReader final : public BatchReader {
- public:
-    FailAfterPhysicalFileBatchReader(std::unique_ptr<BatchReader> delegate,
-                                     const std::shared_ptr<FileSystem>& file_system,
-                                     std::string root, size_t baseline_artifact_count,
-                                     const std::shared_ptr<std::atomic<bool>>& saw_artifacts)
-        : delegate_(std::move(delegate)),
-          file_system_(file_system),
-          root_(std::move(root)),
-          baseline_artifact_count_(baseline_artifact_count),
-          saw_artifacts_(saw_artifacts) {}
-
-    Result<ReadBatch> NextBatch() override {
-        if (returned_batch_count_ < 4) {
-            ++returned_batch_count_;
-            return delegate_->NextBatch();
-        }
-        const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30);
-        while (std::chrono::steady_clock::now() < deadline) {
-            PAIMON_ASSIGN_OR_RAISE(std::set<std::string> artifacts,
-                                   ListPhysicalArtifacts(file_system_, root_));
-            bool has_data = false;
-            for (const std::string& artifact : artifacts) {
-                has_data = has_data || HasSuffix(artifact, ".orc");
-            }
-            if (artifacts.size() > baseline_artifact_count_ && has_data) {
-                saw_artifacts_->store(true, std::memory_order_release);
-                return Status::IOError(
-                    "injected commit reader failure after physical file creation");
-            }
-            std::this_thread::sleep_for(std::chrono::milliseconds(1));
-        }
-        return Status::IOError("timed out waiting for partial physical files");
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        delegate_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> delegate_;
-    std::shared_ptr<FileSystem> file_system_;
-    std::string root_;
-    size_t baseline_artifact_count_;
-    std::shared_ptr<std::atomic<bool>> saw_artifacts_;
-    int32_t returned_batch_count_ = 0;
-};
-
-class FailAfterPhysicalFileRealtimeStore final : public DelegatingRealtimeStore {
- public:
-    FailAfterPhysicalFileRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                                       const std::shared_ptr<FileSystem>& file_system,
-                                       const std::string& root, size_t baseline_artifact_count,
-                                       const std::shared_ptr<std::atomic<bool>>& saw_artifacts)
-        : DelegatingRealtimeStore(delegate),
-          file_system_(file_system),
-          root_(root),
-          baseline_artifact_count_(baseline_artifact_count),
-          saw_artifacts_(saw_artifacts) {}
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               delegate_->CreateCommitReaders(segment));
-        if (readers.empty()) {
-            return Status::Invalid("commit reader failure test requires a reader");
-        }
-        readers[0] = std::make_unique<FailAfterPhysicalFileBatchReader>(
-            std::make_unique<SplitBatchReader>(std::move(readers[0])), file_system_, root_,
-            baseline_artifact_count_, saw_artifacts_);
-        return readers;
-    }
-
- private:
-    std::shared_ptr<FileSystem> file_system_;
-    std::string root_;
-    size_t baseline_artifact_count_;
-    std::shared_ptr<std::atomic<bool>> saw_artifacts_;
-};
-
-enum class ReaderMalformation { DROP_LAST, DUPLICATE_OFFSET, OUT_OF_RANGE_OFFSET };
-
-class CorruptingBatchReader final : public BatchReader {
- public:
-    CorruptingBatchReader(std::unique_ptr<BatchReader> delegate, ReaderMalformation malformation)
-        : delegate_(std::move(delegate)), malformation_(malformation) {}
-
-    Result<ReadBatch> NextBatch() override {
-        switch (malformation_) {
-            case ReaderMalformation::DROP_LAST:
-                return DropLast();
-            case ReaderMalformation::DUPLICATE_OFFSET:
-                return SubstituteOffset(/*offset=*/0);
-            case ReaderMalformation::OUT_OF_RANGE_OFFSET:
-                return SubstituteOffset(/*offset=*/-1);
-        }
-        return Status::Invalid("unknown commit reader malformation");
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        ReleaseBuffered();
-        delegate_->Close();
-    }
-
- private:
-    void ReleaseBuffered() {
-        if (buffered_.has_value()) {
-            ReaderUtils::ReleaseReadBatch(std::move(buffered_.value()));
-            buffered_.reset();
-        }
-    }
-
-    Result<ReadBatch> DropLast() {
-        if (!buffered_.has_value()) {
-            PAIMON_ASSIGN_OR_RAISE(ReadBatch first, delegate_->NextBatch());
-            if (BatchReader::IsEofBatch(first)) {
-                return MakeEofBatch();
-            }
-            buffered_ = std::move(first);
-        }
-        PAIMON_ASSIGN_OR_RAISE(ReadBatch next, delegate_->NextBatch());
-        if (BatchReader::IsEofBatch(next)) {
-            ReleaseBuffered();
-            return MakeEofBatch();
-        }
-        ReadBatch result = std::move(buffered_.value());
-        buffered_ = std::move(next);
-        return result;
-    }
-
-    Result<ReadBatch> SubstituteOffset(int64_t offset) {
-        PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, delegate_->NextBatch());
-        if (BatchReader::IsEofBatch(batch)) {
-            return batch;
-        }
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-            std::shared_ptr<arrow::Array> array,
-            arrow::ImportArray(batch.first.get(), batch.second.get()));
-        if (!array || array->type_id() != arrow::Type::STRUCT || array->length() == 0) {
-            return Status::Invalid("offset substitution requires a non-empty struct batch");
-        }
-        std::shared_ptr<arrow::StructArray> struct_array =
-            std::dynamic_pointer_cast<arrow::StructArray>(array);
-        std::shared_ptr<arrow::Int64Array> offsets =
-            std::dynamic_pointer_cast<arrow::Int64Array>(struct_array->field(2));
-        if (!offsets) {
-            return Status::Invalid("offset substitution requires an int64 REALTIME_OFFSET");
-        }
-        arrow::Int64Builder builder;
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(offsets->length()));
-        for (int64_t row = 0; row < offsets->length(); ++row) {
-            builder.UnsafeAppend(offset);
-        }
-        std::shared_ptr<arrow::Array> substituted_offsets;
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&substituted_offsets));
-        std::shared_ptr<arrow::ArrayData> substituted_data = struct_array->data()->Copy();
-        substituted_data->child_data[2] = substituted_offsets->data();
-        std::shared_ptr<arrow::Array> substituted = arrow::MakeArray(std::move(substituted_data));
-        auto output = std::make_unique<ArrowArray>();
-        auto schema = std::make_unique<ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(
-            arrow::ExportArray(*substituted, output.get(), schema.get()));
-        return ReadBatch(std::move(output), std::move(schema));
-    }
-
-    std::unique_ptr<BatchReader> delegate_;
-    ReaderMalformation malformation_;
-    std::optional<ReadBatch> buffered_;
-};
-
-class MalformedCoverageRealtimeStore final : public DelegatingRealtimeStore {
- public:
-    MalformedCoverageRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                                   ReaderMalformation malformation)
-        : DelegatingRealtimeStore(delegate), malformation_(malformation) {}
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
-        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               delegate_->CreateCommitReaders(segment));
-        for (std::unique_ptr<BatchReader>& reader : readers) {
-            reader = std::make_unique<CorruptingBatchReader>(std::move(reader), malformation_);
-        }
-        return readers;
-    }
-
- private:
-    ReaderMalformation malformation_;
-};
-
-class MissingQueryOffsetRealtimeStore final : public DelegatingRealtimeStore {
- public:
-    explicit MissingQueryOffsetRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
-        : DelegatingRealtimeStore(delegate) {}
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
-        const RealtimeQueryContext& context) override {
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               delegate_->CreateQueryReaders(view, offset_begin, context));
-        if (readers.empty()) {
-            return Status::Invalid("query offset drop requires a reader");
-        }
-        readers[0] = std::make_unique<CorruptingBatchReader>(std::move(readers[0]),
-                                                             ReaderMalformation::DROP_LAST);
-        return readers;
-    }
-};
-
 }  // namespace
 
 namespace {
@@ -1419,77 +1029,6 @@ class RealtimeWriteInteTest : public ::testing::Test {
         ASSERT_OK(writer->Close());
     }
 
-    void CheckVectorReaderRetry(bool primary_key) {
-        if (primary_key) {
-            CreatePkTable(/*partition_keys=*/{"pt"});
-        } else {
-            CreateTable(/*partition_keys=*/{"pt"});
-        }
-        auto close_state = std::make_shared<CloseTrackingReaderState>();
-        auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(close_state);
-        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                             RealtimeContext::Create(factory));
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                             CreateRealtimeWriter(realtime_context));
-        std::vector<Row> p0_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0_batch,
-                             MakeBatch(p0_rows, /*partitioned=*/true));
-        ASSERT_OK(writer->Write(std::move(p0_batch)));
-        std::vector<Row> p1_rows = MakeRows(/*first_id=*/10, /*count=*/3, /*partition=*/"p1");
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p1_batch,
-                             MakeBatch(p1_rows, /*partitioned=*/true));
-        ASSERT_OK(writer->Write(std::move(p1_batch)));
-        ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
-                             CreatePlan(realtime_context, /*predicate=*/nullptr));
-        ASSERT_EQ(2, plan->Splits().size());
-
-        std::vector<std::shared_ptr<Split>> invalid_splits = plan->Splits();
-        std::shared_ptr<RealtimeSplit> second_split =
-            std::dynamic_pointer_cast<RealtimeSplit>(invalid_splits[1]);
-        ASSERT_NE(nullptr, second_split);
-        std::vector<std::shared_ptr<Split>> second_disk_splits = second_split->DiskSplits();
-        invalid_splits[1] = std::make_shared<RealtimeSplit>(
-            RealtimeSplit::kCurrentVersion + 1, second_split->SnapshotId(),
-            second_split->Partition(), second_split->Bucket(), std::move(second_disk_splits),
-            second_split->CommittedEndOffset(), second_split->MemoryEndOffset(),
-            second_split->OpaqueTicket());
-
-        ReadContextBuilder read_builder(table_path_);
-        read_builder.SetOptions(options_)
-            .SetReadFieldNames({"id", "payload", "pt"})
-            .WithRealtimeContext(realtime_context)
-            .WithMemoryPool(pool_);
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
-                             TableRead::Create(std::move(read_context)));
-        ASSERT_NOK_WITH_MSG(table_read->CreateReader(invalid_splits),
-                            "unsupported real-time split version");
-        ASSERT_EQ(1, close_state->query_close_count->load(std::memory_order_acquire));
-
-        std::vector<Row> expected_rows = p0_rows;
-        expected_rows.insert(expected_rows.end(), p1_rows.begin(), p1_rows.end());
-        ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
-        ASSERT_EQ(expected_rows, actual_rows);
-        ASSERT_OK(writer->Close());
-    }
-
-    void CheckPkRejectsReaderMalformation(ReaderMalformation malformation,
-                                          const std::string& expected_error) {
-        CreatePkTable();
-        auto factory = MakeDecoratingFactory<MalformedCoverageRealtimeStore>(malformation);
-        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                             RealtimeContext::Create(factory));
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                             CreateRealtimeWriter(realtime_context));
-        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                             MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
-                                       /*partitioned=*/false));
-        ASSERT_OK(writer->Write(std::move(batch)));
-        ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
-                            expected_error);
-        ASSERT_OK(writer->Close());
-    }
-
     std::unique_ptr<UniqueTestDirectory> dir_;
     std::string table_path_;
     std::string commit_user_ = "realtime_commit_user";
@@ -2197,26 +1736,6 @@ TEST_F(RealtimeWriteInteTest, TestPkRecovery) {
         RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
         RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT};
 
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
-                         CreateRealtimeWriter(failed_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
-                         MakeBatch(mutations, /*partitioned=*/false, /*bucket=*/0, mutation_kinds));
-    ASSERT_OK(failed_writer->Write(std::move(failed_batch)));
-    IOHook* io_hook = IOHook::GetInstance();
-    ScopeGuard hook_guard([io_hook]() { io_hook->Clear(); });
-    io_hook->Reset(/*pos=*/0, IOHook::Mode::RETURN_ERROR);
-    Result<std::vector<RealtimeCommitProgress>> failed_prepare =
-        failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/1);
-    io_hook->Clear();
-    ASSERT_TRUE(failed_prepare.status().IsIOError()) << failed_prepare.status().ToString();
-    ASSERT_OK(failed_writer->Close());
-    failed_writer.reset();
-    failed_context.reset();
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failed_prepare, ReadRows());
-    ASSERT_EQ((std::vector<Row>{{99, "seed", "p0"}}), rows_after_failed_prepare);
-
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> first_context, RealtimeContext::Create());
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
                          CreateRealtimeWriter(first_context));
@@ -2379,9 +1898,8 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
 
 TEST_F(RealtimeWriteInteTest, TestPkMultipleStoredBatchesMergeForQueryAndCommit) {
     CreatePkTable();
-    auto factory = MakeDecoratingFactory<SplitCommitReaderRealtimeStore>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
+                         RealtimeContext::Create());
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                          CreateRealtimeWriter(realtime_context));
     ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
@@ -2410,37 +1928,6 @@ TEST_F(RealtimeWriteInteTest, TestPkMultipleStoredBatchesMergeForQueryAndCommit)
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkRejectsMalformedCoverage) {
-    CheckPkRejectsReaderMalformation(ReaderMalformation::DROP_LAST,
-                                     "commit readers did not cover the sealed range");
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkRejectsDuplicateOffset) {
-    CheckPkRejectsReaderMalformation(ReaderMalformation::DUPLICATE_OFFSET,
-                                     "commit readers did not cover the sealed range");
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkRejectsOutOfRangeOffset) {
-    CheckPkRejectsReaderMalformation(ReaderMalformation::OUT_OF_RANGE_OFFSET,
-                                     "offset is outside the sealed range");
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkRejectsMissingQueryOffset) {
-    CreatePkTable();
-    auto factory = MakeDecoratingFactory<MissingQueryOffsetRealtimeStore>();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}},
-                                   /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-    ASSERT_NOK_WITH_MSG(ReadRows(realtime_context),
-                        "query readers did not cover the visible range");
-    ASSERT_OK(writer->Close());
-}
-
 TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
     CreatePkTable();
     auto state = std::make_shared<CloseTrackingReaderState>();
@@ -2459,131 +1946,6 @@ TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkQueryReaderCloseFailure) {
-    CreatePkTable();
-    auto state = std::make_shared<CloseTrackingReaderState>();
-    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
-                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(first_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
-                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
-    ASSERT_EQ(1, progress.size());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
-                         MakeBatch({Row{2, "two", "p0"}}, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(second_batch)));
-
-    for (int32_t null_index = 0; null_index <= 1; ++null_index) {
-        state->query_null_index = null_index;
-        ASSERT_NOK_WITH_MSG(CreateQueryReader(realtime_context),
-                            "PK real-time store returned a null query reader");
-        ASSERT_EQ((null_index + 1) * 2, state->query_close_count->load(std::memory_order_acquire));
-    }
-    ASSERT_OK(writer->Close());
-}
-
-TEST_F(RealtimeWriteInteTest, TestAppendQueryReaderCloseFailure) {
-    CreateTable(/*partition_keys=*/{});
-    auto state = std::make_shared<CloseTrackingReaderState>();
-    state->query_null_index = 1;
-    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    std::vector<Row> rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch(rows, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
-                         CreatePlan(realtime_context, /*predicate=*/nullptr));
-
-    ASSERT_NOK_WITH_MSG(CreateQueryReader(plan, realtime_context),
-                        "append-only real-time store returned a null query reader");
-    ASSERT_EQ(1, state->query_close_count->load(std::memory_order_acquire));
-
-    state->query_null_index = -1;
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
-    ASSERT_EQ(rows, actual_rows);
-    ASSERT_OK(writer->Close());
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkCommitReaderCloseFailure) {
-    CreatePkTable();
-    auto state = std::make_shared<CloseTrackingReaderState>();
-    state->commit_null_index = 1;
-    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-
-    ASSERT_NOK_WITH_MSG(writer->PrepareCommitWithProgress(/*commit_identifier=*/0),
-                        "PK real-time store returned a null commit reader");
-    ASSERT_EQ(1, state->commit_close_count->load(std::memory_order_acquire));
-    ASSERT_OK(writer->Close());
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkPrepareFailureCleansPartialPhysicalFiles) {
-    options_[Options::WRITE_BATCH_SIZE] = "1";
-    options_[Options::TARGET_FILE_ROW_NUM] = "1";
-    options_["file-index.bitmap.columns"] = "payload";
-    options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
-    CreatePkTable();
-    std::shared_ptr<FileSystem> file_system = dir_->GetFileSystem();
-    ASSERT_OK_AND_ASSIGN(std::set<std::string> baseline_artifacts,
-                         ListPhysicalArtifacts(file_system, dir_->Str()));
-    auto saw_artifacts = std::make_shared<std::atomic<bool>>(false);
-    auto factory = MakeDecoratingFactory<FailAfterPhysicalFileRealtimeStore>(
-        file_system, dir_->Str(), baseline_artifacts.size(), saw_artifacts);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
-                         RealtimeContext::Create(factory));
-    WriteContextBuilder failed_builder(table_path_, commit_user_);
-    failed_builder.SetOptions(options_)
-        .WithStreamingMode(true)
-        .WithRealtimeContext(failed_context)
-        .WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp"));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> failed_write_context,
-                         failed_builder.Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
-                         FileStoreWrite::Create(std::move(failed_write_context)));
-
-    const std::vector<Row> wal = {
-        {1, "old", "p0"}, {1, "new", "p0"}, {2, "two", "p0"}, {2, "gone", "p0"}};
-    const std::vector<RecordBatch::RowKind> row_kinds = {
-        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
-        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE};
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
-                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
-    ASSERT_OK(failed_writer->Write(std::move(failed_batch)));
-    Result<std::vector<RealtimeCommitProgress>> failed_prepare =
-        failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/0);
-    ASSERT_TRUE(failed_prepare.status().IsIOError()) << failed_prepare.status().ToString();
-    ASSERT_NE(std::string::npos,
-              failed_prepare.status().ToString().find(
-                  "injected commit reader failure after physical file creation"));
-    ASSERT_TRUE(saw_artifacts->load(std::memory_order_acquire));
-    ASSERT_OK_AND_ASSIGN(std::set<std::string> artifacts_after_abort,
-                         ListPhysicalArtifacts(file_system, dir_->Str()));
-    ASSERT_EQ(baseline_artifacts, artifacts_after_abort);
-
-    ASSERT_OK(failed_writer->Close());
-    failed_writer.reset();
-    failed_context.reset();
-
-    const std::vector<Row> expected_rows = {{1, "new", "p0"}};
-    ReplayPkWalAndCommit(wal, row_kinds, /*commit_identifier=*/0, expected_rows);
-    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
-    ASSERT_EQ(4, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
-}
-
 TEST_F(RealtimeWriteInteTest, TestRollingFilesPreserveProgress) {
     options_[Options::TARGET_FILE_ROW_NUM] = "10";
     CreateTable(/*partition_keys=*/{});
@@ -3015,44 +2377,6 @@ TEST_F(RealtimeWriteInteTest, TestPlanExcludesRowsWrittenAfterMemoryEndOffset) {
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkRejectsReversedVisibleOffsets) {
-    CreatePkTable();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    std::vector<Row> rows = {{1, "one", "p0"}, {2, "two", "p0"}};
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch(rows, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
-                         CreatePlan(realtime_context, /*predicate=*/nullptr));
-    ASSERT_EQ(1, plan->Splits().size());
-    std::shared_ptr<RealtimeSplit> split =
-        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
-    ASSERT_NE(nullptr, split);
-    std::vector<std::shared_ptr<Split>> disk_splits = split->DiskSplits();
-    std::vector<std::shared_ptr<Split>> invalid_splits = {std::make_shared<RealtimeSplit>(
-        split->Version(), split->SnapshotId(), split->Partition(), split->Bucket(),
-        std::move(disk_splits), split->MemoryEndOffset() + 1, split->MemoryEndOffset(),
-        split->OpaqueTicket())};
-
-    ReadContextBuilder read_builder(table_path_);
-    read_builder.SetOptions(options_)
-        .SetReadFieldNames({"id", "payload", "pt"})
-        .WithRealtimeContext(realtime_context)
-        .WithMemoryPool(pool_);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
-                         TableRead::Create(std::move(read_context)));
-    ASSERT_NOK_WITH_MSG(table_read->CreateReader(invalid_splits),
-                        "memory end offset precedes committed end offset");
-
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
-    ASSERT_EQ(rows, actual_rows);
-    ASSERT_OK(writer->Close());
-}
-
 TEST_F(RealtimeWriteInteTest, TestReadFailsAfterRealtimeSplitTicketExpires) {
     options_[Options::REALTIME_READ_VIEW_TTL] = "10 ms";
     CreateTable(/*partition_keys=*/{});
@@ -3134,14 +2458,6 @@ TEST_F(RealtimeWriteInteTest, TestFailedReaderCreationPreservesRealtimeSplitTick
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestAppendVectorRetry) {
-    CheckVectorReaderRetry(/*primary_key=*/false);
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkVectorRetry) {
-    CheckVectorReaderRetry(/*primary_key=*/true);
-}
-
 TEST_F(RealtimeWriteInteTest, TestCloseWriterKeepsContextReadable) {
     CreateTable(/*partition_keys=*/{});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
@@ -4630,167 +3946,4 @@ TEST_F(RealtimeWriteInteTest, TestRestoreOffsetFromCommittedSnapshot) {
     ASSERT_EQ(5, second_committed_offsets.at(partition_bucket));
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkWriteFailureRecovery) {
-    CreatePkTable();
-    const std::vector<Row> seed_rows = {{99, "seed", "p0"}};
-    ReplayPkWalAndCommit(seed_rows, /*row_kinds=*/{}, /*commit_identifier=*/0, seed_rows);
-
-    const std::vector<Row> wal = {
-        {1, "one", "p0"}, {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
-    const std::vector<RecordBatch::RowKind> row_kinds = {
-        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
-        RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT};
-    std::shared_ptr<FailAllocationMemoryPool> failing_pool =
-        std::make_shared<FailAllocationMemoryPool>(pool_);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
-                         RealtimeContext::Create());
-    WriteContextBuilder failed_builder(table_path_, commit_user_);
-    failed_builder.SetOptions(options_)
-        .WithStreamingMode(true)
-        .WithRealtimeContext(failed_context)
-        .WithMemoryPool(failing_pool);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> failed_write_context,
-                         failed_builder.Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
-                         FileStoreWrite::Create(std::move(failed_write_context)));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> empty_batch,
-                         MakeUnpartitionedBatchFromJson("[]"));
-    ASSERT_OK(failed_writer->Write(std::move(empty_batch)));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
-                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
-    failing_pool->FailAfterAllocations(1);
-    Status failed_write = failed_writer->Write(std::move(failed_batch));
-    ASSERT_TRUE(failed_write.IsOutOfMemory()) << failed_write.ToString();
-    ASSERT_OK(failed_writer->Close());
-    failed_writer.reset();
-    failed_context.reset();
-
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failure, ReadRows());
-    ASSERT_EQ(seed_rows, rows_after_failure);
-
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> replay_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> replay_writer,
-                         CreateRealtimeWriter(replay_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> replay_batch,
-                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
-    ASSERT_OK(replay_writer->Write(std::move(replay_batch)));
-    const std::vector<Row> expected_rows = {
-        {1, "one-new", "p0"}, {3, "three", "p0"}, {99, "seed", "p0"}};
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> replayed_rows, ReadRows(replay_context));
-    ASSERT_EQ(expected_rows, replayed_rows);
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
-                         replay_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_EQ(1, progress.size());
-    ASSERT_EQ(OffsetRange(1, 5), progress[0].offset_range);
-    ASSERT_EQ(1, NewFiles(progress).size());
-    ASSERT_EQ(1, NewFiles(progress)[0]->delete_row_count);
-    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/1));
-    ASSERT_OK(replay_writer->RefreshCommittedSnapshot(snapshot_id));
-    ASSERT_OK(replay_writer->Close());
-    replay_writer.reset();
-    replay_context.reset();
-
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> persisted_rows, ReadRows());
-    ASSERT_EQ(expected_rows, persisted_rows);
-    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
-    ASSERT_EQ(5, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkExternalCommitRecovery) {
-    CreatePkTable();
-    const std::vector<Row> seed_rows = {{99, "seed", "p0"}};
-    ReplayPkWalAndCommit(seed_rows, /*row_kinds=*/{}, /*commit_identifier=*/0, seed_rows);
-
-    const std::vector<Row> wal = {
-        {1, "one", "p0"}, {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
-    const std::vector<RecordBatch::RowKind> row_kinds = {
-        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
-        RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT};
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
-                         CreateRealtimeWriter(failed_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> failed_batch,
-                         MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
-    ASSERT_OK(failed_writer->Write(std::move(failed_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> failed_progress,
-                         failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_EQ(1, failed_progress.size());
-    CommitContextBuilder commit_builder(table_path_, commit_user_);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
-                         commit_builder.SetOptions(options_).Finish());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
-                         FileStoreCommit::Create(std::move(commit_context)));
-    IOHook* io_hook = IOHook::GetInstance();
-    ScopeGuard hook_guard([io_hook]() { io_hook->Clear(); });
-    io_hook->Reset(/*pos=*/0, IOHook::Mode::RETURN_ERROR);
-    Result<int64_t> failed_commit =
-        commit->CommitWithProgress(failed_progress, /*commit_identifier=*/1,
-                                   /*watermark=*/std::nullopt);
-    io_hook->Clear();
-    ASSERT_TRUE(failed_commit.status().IsIOError()) << failed_commit.status().ToString();
-    ASSERT_OK(failed_writer->Close());
-    failed_writer.reset();
-    failed_context.reset();
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failure, ReadRows());
-    ASSERT_EQ(seed_rows, rows_after_failure);
-
-    const std::vector<Row> expected_rows = {
-        {1, "one-new", "p0"}, {3, "three", "p0"}, {99, "seed", "p0"}};
-    ReplayPkWalAndCommit(wal, row_kinds, /*commit_identifier=*/1, expected_rows);
-}
-
-TEST_F(RealtimeWriteInteTest, TestPkRefreshRecovery) {
-    CreatePkTable();
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> failed_context,
-                         RealtimeContext::Create());
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> failed_writer,
-                         CreateRealtimeWriter(failed_context));
-
-    const std::vector<Row> base_rows = {{1, "one", "p0"}, {2, "two", "p0"}};
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
-                         MakeBatch(base_rows, /*partitioned=*/false));
-    ASSERT_OK(failed_writer->Write(std::move(base_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
-                         failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
-    ASSERT_OK_AND_ASSIGN(int64_t base_snapshot_id, Commit(base_progress, /*commit_identifier=*/0));
-    ASSERT_OK(failed_writer->RefreshCommittedSnapshot(base_snapshot_id));
-
-    const std::vector<Row> committed_wal = {
-        {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
-    const std::vector<RecordBatch::RowKind> committed_kinds = {RecordBatch::RowKind::UPDATE_AFTER,
-                                                               RecordBatch::RowKind::DELETE,
-                                                               RecordBatch::RowKind::INSERT};
-    ASSERT_OK_AND_ASSIGN(
-        std::unique_ptr<RecordBatch> committed_batch,
-        MakeBatch(committed_wal, /*partitioned=*/false, /*bucket=*/0, committed_kinds));
-    ASSERT_OK(failed_writer->Write(std::move(committed_batch)));
-    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> committed_progress,
-                         failed_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
-    ASSERT_OK_AND_ASSIGN(int64_t committed_snapshot_id,
-                         Commit(committed_progress, /*commit_identifier=*/1));
-
-    const std::vector<Row> replay_wal = {{4, "four", "p0"}};
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> replay_batch,
-                         MakeBatch(replay_wal, /*partitioned=*/false));
-    ASSERT_OK(failed_writer->Write(std::move(replay_batch)));
-    IOHook* io_hook = IOHook::GetInstance();
-    ScopeGuard hook_guard([io_hook]() { io_hook->Clear(); });
-    io_hook->Reset(/*pos=*/0, IOHook::Mode::RETURN_ERROR);
-    Status failed_refresh = failed_writer->RefreshCommittedSnapshot(committed_snapshot_id);
-    io_hook->Clear();
-    ASSERT_TRUE(failed_refresh.IsIOError()) << failed_refresh.ToString();
-    ASSERT_OK(failed_writer->Close());
-    failed_writer.reset();
-    failed_context.reset();
-    const std::vector<Row> committed_rows = {{1, "one-new", "p0"}, {3, "three", "p0"}};
-    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_failure, ReadRows());
-    ASSERT_EQ(committed_rows, rows_after_failure);
-
-    const std::vector<Row> expected_rows = {
-        {1, "one-new", "p0"}, {3, "three", "p0"}, {4, "four", "p0"}};
-    ReplayPkWalAndCommit(replay_wal, /*row_kinds=*/{}, /*commit_identifier=*/2, expected_rows);
-}
-
 }  // namespace paimon::test

From e3574185c75c8b01a985c9e50917fdd0026f5401 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 28 Aug 2026 16:09:29 +0800
Subject: [PATCH 85/93] fix(style): apply clang-format

---
 src/paimon/core/realtime/realtime_context_test.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index 636cf12fc..c9531eda5 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -218,8 +218,8 @@ TEST(RealtimeContextTest, TestReconcilesPrimaryKeyInitialSequence) {
     const RealtimePartitionBucket partition_bucket(partition, /*bucket=*/0);
 
     ASSERT_OK(context->GetOrCreateRealtimeStore(
-        RealtimeStoreCreateRequest{
-            MakeWriteSchema(), /*options=*/{}, GetDefaultPool(), RealtimeStoreMode::PRIMARY_KEY},
+        RealtimeStoreCreateRequest{MakeWriteSchema(), /*options=*/{}, GetDefaultPool(),
+                                   RealtimeStoreMode::PRIMARY_KEY},
         partition_bucket));
 
     ASSERT_OK_AND_ASSIGN(int64_t first, context->AdvanceMaterializedMaxSequenceNumber(

From feb511e858ad858be39d79901780f5e105cb7f86 Mon Sep 17 00:00:00 2001
From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com>
Date: Fri, 28 Aug 2026 16:19:55 +0800
Subject: [PATCH 86/93] fix: canonicalize floating-point NaN values (#255)

---
 include/paimon/predicate/literal.h            | 13 +++---
 .../data/variant/generic_variant_test.cpp     | 19 ++++++++
 .../common/data/variant/variant_builder.cpp   |  7 ++-
 .../file_index/bloomfilter/fast_hash.cpp      | 23 ++--------
 .../file_index/bloomfilter/fast_hash_test.cpp | 20 ++------
 .../global_index/btree/key_serializer.cpp     | 11 ++---
 .../btree/key_serializer_test.cpp             | 28 +++++++++++
 .../global_index/global_index_result.cpp      |  5 +-
 .../global_index/global_index_result_test.cpp | 26 +++++++++++
 src/paimon/common/predicate/literal.cpp       |  6 +--
 src/paimon/common/utils/math.h                | 46 +++++++++++++++++++
 src/paimon/common/utils/math_test.cpp         | 23 ++++++++++
 .../core/bucket/hive_bucket_function.cpp      | 11 ++---
 .../core/bucket/hive_bucket_function_test.cpp | 33 ++++++-------
 .../core/global_index/indexed_split_test.cpp  | 31 +++++++++++++
 src/paimon/core/table/source/split.cpp        |  5 +-
 16 files changed, 218 insertions(+), 89 deletions(-)

diff --git a/include/paimon/predicate/literal.h b/include/paimon/predicate/literal.h
index ef5864573..168d6a451 100644
--- a/include/paimon/predicate/literal.h
+++ b/include/paimon/predicate/literal.h
@@ -91,13 +91,12 @@ class PAIMON_EXPORT Literal {
     std::string ToString() const;
 
     /// Gets the hash code for this literal.
-    /// @note HashCode() hashes the exact bit representation (including Decimal scale), while
-    /// operator== delegates to CompareTo() which uses numeric equality (e.g. decimals with
-    /// different scales can compare equal). This means the hash-equality contract (equal objects
-    /// must have equal hashes) may be violated for Decimal literals with different scales. In
-    /// practice this is safe because all current std::unordered_map<Literal, ...> usages (bitmap
-    /// file index) only store values from the same column, which guarantees a fixed precision and
-    /// scale.
+    /// @note HashCode() canonicalizes all floating-point NaNs so that values considered equal by
+    /// CompareTo() have the same hash. Decimal values include their scale in the hash, while
+    /// CompareTo() uses numeric equality, so Decimal literals with different scales can still
+    /// violate the hash-equality contract. In practice this is safe because all current
+    /// std::unordered_map<Literal, ...> usages only store values from the same column, which has a
+    /// fixed precision and scale.
     size_t HashCode() const;
 
     /// Compares this literal with another literal. The comparison follows SQL semantics for the
diff --git a/src/paimon/common/data/variant/generic_variant_test.cpp b/src/paimon/common/data/variant/generic_variant_test.cpp
index 11386a28a..bc46a5730 100644
--- a/src/paimon/common/data/variant/generic_variant_test.cpp
+++ b/src/paimon/common/data/variant/generic_variant_test.cpp
@@ -19,6 +19,7 @@
 
 #include "paimon/common/data/variant/generic_variant.h"
 
+#include <cstdint>
 #include <functional>
 #include <limits>
 #include <string>
@@ -27,6 +28,7 @@
 #include "gtest/gtest.h"
 #include "paimon/common/data/variant/variant_builder.h"
 #include "paimon/common/data/variant/variant_defs.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/testing/utils/testharness.h"
 
@@ -350,6 +352,23 @@ TEST_F(GenericVariantTest, NonFiniteDoubleToJson) {
     ASSERT_EQ(json, "\"Infinity\"");
 }
 
+TEST_F(GenericVariantTest, CanonicalizesFloatingPointNaN) {
+    {
+        VariantBuilder builder(false);
+        ASSERT_OK(builder.AppendFloat(FloatingPointFromBits<float>(0xffc12345U)));
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> variant, builder.Build(pool_));
+        ASSERT_OK_AND_ASSIGN(std::string_view value, variant->Value());
+        ASSERT_EQ(ToHex(value), "380000c07f");
+    }
+    {
+        VariantBuilder builder(false);
+        ASSERT_OK(builder.AppendDouble(FloatingPointFromBits<double>(0xfff8123456789abcULL)));
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> variant, builder.Build(pool_));
+        ASSERT_OK_AND_ASSIGN(std::string_view value, variant->Value());
+        ASSERT_EQ(ToHex(value), "1c000000000000f87f");
+    }
+}
+
 TEST_F(GenericVariantTest, GetTypeInfoReturnsHeaderBits) {
     // GetTypeInfo exposes the primitive header's type-info bits; 42 is encoded as an int1.
     auto v = FromJson("42");
diff --git a/src/paimon/common/data/variant/variant_builder.cpp b/src/paimon/common/data/variant/variant_builder.cpp
index b3cf7ee5d..51f704175 100644
--- a/src/paimon/common/data/variant/variant_builder.cpp
+++ b/src/paimon/common/data/variant/variant_builder.cpp
@@ -30,6 +30,7 @@
 
 #include "fmt/format.h"
 #include "paimon/common/data/variant/variant_defs.h"
+#include "paimon/common/utils/math.h"
 #include "rapidjson/error/en.h"
 #include "rapidjson/memorystream.h"
 #include "rapidjson/reader.h"
@@ -339,8 +340,7 @@ Status VariantBuilder::AppendLong(int64_t l) {
 Status VariantBuilder::AppendDouble(double d) {
     PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 8));
     write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kDouble);
-    int64_t bits;
-    memcpy(&bits, &d, sizeof(bits));
+    const int64_t bits = CanonicalizeDoubleToLongBits(d);
     VariantBinaryUtil::WriteLong(bits, 8, write_buffer_.data(), write_pos_);
     write_pos_ += 8;
     return Status::OK();
@@ -409,8 +409,7 @@ Status VariantBuilder::AppendTimestampNtz(int64_t micros_since_epoch) {
 Status VariantBuilder::AppendFloat(float f) {
     PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 4));
     write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kFloat);
-    int32_t bits;
-    memcpy(&bits, &f, sizeof(bits));
+    const int32_t bits = CanonicalizeFloatToIntBits(f);
     VariantBinaryUtil::WriteLong(bits, 4, write_buffer_.data(), write_pos_);
     write_pos_ += 4;
     return Status::OK();
diff --git a/src/paimon/common/file_index/bloomfilter/fast_hash.cpp b/src/paimon/common/file_index/bloomfilter/fast_hash.cpp
index b1d8784fc..ef63f0fc1 100644
--- a/src/paimon/common/file_index/bloomfilter/fast_hash.cpp
+++ b/src/paimon/common/file_index/bloomfilter/fast_hash.cpp
@@ -19,7 +19,6 @@
 #include "paimon/common/file_index/bloomfilter/fast_hash.h"
 
 #include <cassert>
-#include <cmath>
 #include <cstring>
 #include <string>
 #include <utility>
@@ -28,6 +27,7 @@
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/common/utils/field_type_utils.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/data/timestamp.h"
 #include "paimon/defs.h"
 #include "paimon/file_index/file_index_result.h"
@@ -35,11 +35,6 @@
 #include "xxhash.h"  // NOLINT(build/include_subdir)
 
 namespace paimon {
-namespace {
-constexpr int32_t kCanonicalFloatNaNBits = 0x7fc00000;
-constexpr int64_t kCanonicalDoubleNaNBits = 0x7ff8000000000000L;
-}  // namespace
-
 Result<FastHash::HashFunction> FastHash::GetHashFunction(
     const std::shared_ptr<arrow::DataType>& arrow_type) {
     PAIMON_ASSIGN_OR_RAISE(FieldType field_type,
@@ -64,23 +59,11 @@ Result<FastHash::HashFunction> FastHash::GetHashFunction(
             });
         case FieldType::FLOAT:
             return HashFunction([](const Literal& literal) -> int64_t {
-                const auto raw_value = literal.GetValue<float>();
-                if (std::isnan(raw_value)) {
-                    return GetLongHash(kCanonicalFloatNaNBits);
-                }
-                int32_t bits = 0;
-                std::memcpy(&bits, &raw_value, sizeof(raw_value));
-                return GetLongHash(bits);
+                return GetLongHash(CanonicalizeFloatToIntBits(literal.GetValue<float>()));
             });
         case FieldType::DOUBLE:
             return HashFunction([](const Literal& literal) -> int64_t {
-                const auto raw_value = literal.GetValue<double>();
-                if (std::isnan(raw_value)) {
-                    return GetLongHash(kCanonicalDoubleNaNBits);
-                }
-                int64_t bits;
-                std::memcpy(&bits, &raw_value, sizeof(raw_value));
-                return GetLongHash(bits);
+                return GetLongHash(CanonicalizeDoubleToLongBits(literal.GetValue<double>()));
             });
         case FieldType::TIMESTAMP: {
             auto ts_type = checked_pointer_cast<arrow::TimestampType>(arrow_type);
diff --git a/src/paimon/common/file_index/bloomfilter/fast_hash_test.cpp b/src/paimon/common/file_index/bloomfilter/fast_hash_test.cpp
index 8a528e443..6ffbc7964 100644
--- a/src/paimon/common/file_index/bloomfilter/fast_hash_test.cpp
+++ b/src/paimon/common/file_index/bloomfilter/fast_hash_test.cpp
@@ -20,12 +20,12 @@
 
 #include <cmath>
 #include <cstdint>
-#include <cstring>
 #include <limits>
 #include <string>
 #include <vector>
 
 #include "gtest/gtest.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/data/timestamp.h"
 #include "paimon/defs.h"
 #include "paimon/file_index/file_index_result.h"
@@ -168,26 +168,16 @@ TEST_F(FastHashTest, TestCompatibleWithJava) {
 }
 
 TEST_F(FastHashTest, TestNaNCompatibleWithJava) {
-    auto float_from_bits = [](uint32_t bits) {
-        float value;
-        std::memcpy(&value, &bits, sizeof(value));
-        return value;
-    };
-    const float float_nan = float_from_bits(0x7fc12345);
-    const float negative_float_nan = float_from_bits(0xffc54321);
+    const auto float_nan = FloatingPointFromBits<float>(0x7fc12345U);
+    const auto negative_float_nan = FloatingPointFromBits<float>(0xffc54321U);
     ASSERT_TRUE(std::isnan(float_nan));
     ASSERT_TRUE(std::isnan(negative_float_nan));
     ASSERT_OK_AND_ASSIGN(auto float_hash_function, FastHash::GetHashFunction(arrow::float32()));
     CheckResult(float_hash_function, {Literal(float_nan), Literal(negative_float_nan)},
                 {0x67c27c6d9936ae63, 0x67c27c6d9936ae63});
 
-    auto double_from_bits = [](uint64_t bits) {
-        double value;
-        std::memcpy(&value, &bits, sizeof(value));
-        return value;
-    };
-    const double double_nan = double_from_bits(0x7ff8123456789abc);
-    const double negative_double_nan = double_from_bits(0xfff8abcdef012345);
+    const auto double_nan = FloatingPointFromBits<double>(0x7ff8123456789abcULL);
+    const auto negative_double_nan = FloatingPointFromBits<double>(0xfff8abcdef012345ULL);
     ASSERT_TRUE(std::isnan(double_nan));
     ASSERT_TRUE(std::isnan(negative_double_nan));
     ASSERT_OK_AND_ASSIGN(auto double_hash_function, FastHash::GetHashFunction(arrow::float64()));
diff --git a/src/paimon/common/global_index/btree/key_serializer.cpp b/src/paimon/common/global_index/btree/key_serializer.cpp
index 464f37ea4..4b66d1d1f 100644
--- a/src/paimon/common/global_index/btree/key_serializer.cpp
+++ b/src/paimon/common/global_index/btree/key_serializer.cpp
@@ -27,6 +27,7 @@
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/common/utils/field_type_utils.h"
 #include "paimon/common/utils/fields_comparator.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/common/utils/preconditions.h"
 #include "paimon/common/utils/var_length_int_utils.h"
 #include "paimon/data/decimal.h"
@@ -164,19 +165,13 @@ Result<std::shared_ptr<Bytes>> KeySerializer::SerializeKey(
         case FieldType::FLOAT: {
             MemorySliceOutput output(4, pool);
             output.Reset();
-            auto fvalue = literal.GetValue<float>();
-            int32_t ivalue;
-            memcpy(&ivalue, &fvalue, sizeof(float));
-            output.WriteValue<int32_t>(ivalue);
+            output.WriteValue<int32_t>(CanonicalizeFloatToIntBits(literal.GetValue<float>()));
             return output.ToSlice().CopyBytes(pool);
         }
         case FieldType::DOUBLE: {
             MemorySliceOutput output(8, pool);
             output.Reset();
-            auto dvalue = literal.GetValue<double>();
-            int64_t ivalue;
-            memcpy(&ivalue, &dvalue, sizeof(double));
-            output.WriteValue<int64_t>(ivalue);
+            output.WriteValue<int64_t>(CanonicalizeDoubleToLongBits(literal.GetValue<double>()));
             return output.ToSlice().CopyBytes(pool);
         }
         case FieldType::STRING: {
diff --git a/src/paimon/common/global_index/btree/key_serializer_test.cpp b/src/paimon/common/global_index/btree/key_serializer_test.cpp
index e36e72ed6..61322fde4 100644
--- a/src/paimon/common/global_index/btree/key_serializer_test.cpp
+++ b/src/paimon/common/global_index/btree/key_serializer_test.cpp
@@ -19,7 +19,11 @@
 
 #include "paimon/common/global_index/btree/key_serializer.h"
 
+#include <cstdint>
+#include <string>
+
 #include "gtest/gtest.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/data/decimal.h"
 #include "paimon/data/timestamp.h"
 #include "paimon/testing/utils/testharness.h"
@@ -208,6 +212,30 @@ TEST_F(KeySerializerTest, SerializeAndDeserializeAllTypes) {
     }
 }
 
+TEST_F(KeySerializerTest, CanonicalizesFloatingPointNaN) {
+    const auto float_nan = FloatingPointFromBits<float>(0xffc12345U);
+    const auto canonical_float_nan = FloatingPointFromBits<float>(kCanonicalFloatNaNBits);
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<Bytes> float_bytes,
+        KeySerializer::SerializeKey(Literal(float_nan), arrow::float32(), pool_.get()));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<Bytes> canonical_float_bytes,
+        KeySerializer::SerializeKey(Literal(canonical_float_nan), arrow::float32(), pool_.get()));
+    ASSERT_EQ(std::string(float_bytes->data(), float_bytes->size()),
+              std::string(canonical_float_bytes->data(), canonical_float_bytes->size()));
+
+    const auto double_nan = FloatingPointFromBits<double>(0xfff8123456789abcULL);
+    const auto canonical_double_nan = FloatingPointFromBits<double>(kCanonicalDoubleNaNBits);
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<Bytes> double_bytes,
+        KeySerializer::SerializeKey(Literal(double_nan), arrow::float64(), pool_.get()));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<Bytes> canonical_double_bytes,
+        KeySerializer::SerializeKey(Literal(canonical_double_nan), arrow::float64(), pool_.get()));
+    ASSERT_EQ(std::string(double_bytes->data(), double_bytes->size()),
+              std::string(canonical_double_bytes->data(), canonical_double_bytes->size()));
+}
+
 TEST_F(KeySerializerTest, RejectsMalformedSerializedKeys) {
     auto wrap = [this](const std::string& value) {
         return MemorySlice::Wrap(std::make_shared<Bytes>(value, pool_.get()));
diff --git a/src/paimon/common/global_index/global_index_result.cpp b/src/paimon/common/global_index/global_index_result.cpp
index f329b0361..d2b94f216 100644
--- a/src/paimon/common/global_index/global_index_result.cpp
+++ b/src/paimon/common/global_index/global_index_result.cpp
@@ -22,6 +22,7 @@
 #include "fmt/format.h"
 #include "paimon/common/io/memory_segment_output_stream.h"
 #include "paimon/common/memory/memory_segment_utils.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/global_index/bitmap_global_index_result.h"
 #include "paimon/global_index/bitmap_scored_global_index_result.h"
 #include "paimon/io/byte_array_input_stream.h"
@@ -37,8 +38,8 @@ void WriteBitmapAndScores(const RoaringBitmap64* bitmap, const std::vector<float
     out->WriteBytes(bitmap_bytes);
 
     out->WriteValue<int32_t>(scores.size());
-    for (auto score : scores) {
-        out->WriteValue<float>(score);
+    for (float score : scores) {
+        out->WriteValue<float>(CanonicalizeFloatingPoint(score));
     }
 }
 
diff --git a/src/paimon/common/global_index/global_index_result_test.cpp b/src/paimon/common/global_index/global_index_result_test.cpp
index 73c6d05ed..3179e6710 100644
--- a/src/paimon/common/global_index/global_index_result_test.cpp
+++ b/src/paimon/common/global_index/global_index_result_test.cpp
@@ -19,9 +19,12 @@
 
 #include "paimon/global_index/global_index_result.h"
 
+#include <cmath>
+#include <cstdint>
 #include <utility>
 
 #include "gtest/gtest.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/global_index/bitmap_global_index_result.h"
 #include "paimon/global_index/bitmap_scored_global_index_result.h"
 #include "paimon/testing/utils/testharness.h"
@@ -144,6 +147,29 @@ TEST_F(GlobalIndexResultTest, TestSerializeAndDeserializeWithScore) {
                                                 serialize_bytes->data() + serialize_bytes->size()));
 }
 
+TEST_F(GlobalIndexResultTest, TestSerializeCanonicalizesNaNScore) {
+    auto pool = GetDefaultPool();
+    const auto payload_nan = FloatingPointFromBits<float>(0xffc12345U);
+    const auto canonical_nan = FloatingPointFromBits<float>(kCanonicalFloatNaNBits);
+    auto index_result = std::make_shared<BitmapScoredGlobalIndexResult>(
+        RoaringBitmap64::From({1}), std::vector<float>{payload_nan});
+    auto canonical_index_result = std::make_shared<BitmapScoredGlobalIndexResult>(
+        RoaringBitmap64::From({1}), std::vector<float>{canonical_nan});
+
+    ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> serialized,
+                         GlobalIndexResult::Serialize(index_result, pool));
+    ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> canonical_serialized,
+                         GlobalIndexResult::Serialize(canonical_index_result, pool));
+    ASSERT_EQ(*serialized, *canonical_serialized);
+
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr<GlobalIndexResult> deserialized,
+        GlobalIndexResult::Deserialize(serialized->data(), serialized->size(), pool));
+    auto scored_result = std::dynamic_pointer_cast<BitmapScoredGlobalIndexResult>(deserialized);
+    ASSERT_TRUE(scored_result);
+    ASSERT_TRUE(std::isnan(scored_result->GetScores()[0]));
+}
+
 TEST_F(GlobalIndexResultTest, TestInvalidSerialize) {
     auto pool = GetDefaultPool();
     auto result = std::make_shared<FakeGlobalIndexResult>(std::vector<int64_t>({1, 3, 5, 100}));
diff --git a/src/paimon/common/predicate/literal.cpp b/src/paimon/common/predicate/literal.cpp
index 3b2bcc0e6..d679c2ccf 100644
--- a/src/paimon/common/predicate/literal.cpp
+++ b/src/paimon/common/predicate/literal.cpp
@@ -18,7 +18,6 @@
 
 #include "paimon/predicate/literal.h"
 
-#include <cmath>
 #include <cstring>
 #include <functional>
 #include <sstream>
@@ -29,6 +28,7 @@
 #include "fmt/format.h"
 #include "paimon/common/utils/field_type_utils.h"
 #include "paimon/common/utils/fields_comparator.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/data/decimal.h"
 #include "paimon/data/timestamp.h"
 #include "paimon/status.h"
@@ -63,9 +63,9 @@ class Literal::Impl {
             case FieldType::BIGINT:
                 return std::hash<int64_t>{}(value_.BigIntVal);
             case FieldType::FLOAT:
-                return std::hash<float>{}(value_.FloatVal);
+                return std::hash<float>{}(CanonicalizeFloatingPoint(value_.FloatVal));
             case FieldType::DOUBLE:
-                return std::hash<double>{}(value_.DoubleVal);
+                return std::hash<double>{}(CanonicalizeFloatingPoint(value_.DoubleVal));
             case FieldType::STRING:
             case FieldType::BINARY:
                 return std::hash<std::string_view>{}(std::string_view(value_.Buffer, size_));
diff --git a/src/paimon/common/utils/math.h b/src/paimon/common/utils/math.h
index 6aba6523f..b9e2e3103 100644
--- a/src/paimon/common/utils/math.h
+++ b/src/paimon/common/utils/math.h
@@ -28,6 +28,7 @@
 #pragma once
 
 #include <cassert>
+#include <cmath>
 #include <cstddef>
 #include <cstdint>
 #include <cstring>
@@ -41,6 +42,51 @@
 
 namespace paimon {
 
+inline constexpr uint32_t kCanonicalFloatNaNBits = 0x7fc00000;
+inline constexpr uint64_t kCanonicalDoubleNaNBits = 0x7ff8000000000000;
+
+template <typename FloatingPoint, typename Bits>
+inline FloatingPoint FloatingPointFromBits(Bits bits) {
+    static_assert(std::is_floating_point_v<FloatingPoint>);
+    static_assert(std::is_integral_v<Bits>);
+    static_assert(sizeof(FloatingPoint) == sizeof(Bits));
+    FloatingPoint value;
+    std::memcpy(&value, &bits, sizeof(value));
+    return value;
+}
+
+inline float CanonicalizeFloatingPoint(float value) {
+    if (std::isnan(value)) {
+        return FloatingPointFromBits<float>(kCanonicalFloatNaNBits);
+    }
+    return value;
+}
+
+inline double CanonicalizeFloatingPoint(double value) {
+    if (std::isnan(value)) {
+        return FloatingPointFromBits<double>(kCanonicalDoubleNaNBits);
+    }
+    return value;
+}
+
+inline int32_t CanonicalizeFloatToIntBits(float value) {
+    if (std::isnan(value)) {
+        return static_cast<int32_t>(kCanonicalFloatNaNBits);
+    }
+    int32_t bits;
+    std::memcpy(&bits, &value, sizeof(bits));
+    return bits;
+}
+
+inline int64_t CanonicalizeDoubleToLongBits(double value) {
+    if (std::isnan(value)) {
+        return static_cast<int64_t>(kCanonicalDoubleNaNBits);
+    }
+    int64_t bits;
+    std::memcpy(&bits, &value, sizeof(bits));
+    return bits;
+}
+
 template <typename To, typename From>
 constexpr bool InRange(From value) {
     static_assert(std::is_integral_v<To> && std::is_integral_v<From>,
diff --git a/src/paimon/common/utils/math_test.cpp b/src/paimon/common/utils/math_test.cpp
index 49d31d472..ce9379f6c 100644
--- a/src/paimon/common/utils/math_test.cpp
+++ b/src/paimon/common/utils/math_test.cpp
@@ -28,6 +28,29 @@
 
 namespace paimon::test {
 
+TEST(MathTest, FloatingPointNaNCanonicalization) {
+    const auto float_nan = CanonicalizeFloatingPoint(FloatingPointFromBits<float>(0xffc12345U));
+    uint32_t float_nan_bits;
+    std::memcpy(&float_nan_bits, &float_nan, sizeof(float_nan_bits));
+    ASSERT_EQ(kCanonicalFloatNaNBits, float_nan_bits);
+    ASSERT_EQ(static_cast<int32_t>(kCanonicalFloatNaNBits),
+              CanonicalizeFloatToIntBits(FloatingPointFromBits<float>(0x7fa12345U)));
+
+    const auto double_nan =
+        CanonicalizeFloatingPoint(FloatingPointFromBits<double>(0xfff8123456789abcULL));
+    uint64_t double_nan_bits;
+    std::memcpy(&double_nan_bits, &double_nan, sizeof(double_nan_bits));
+    ASSERT_EQ(kCanonicalDoubleNaNBits, double_nan_bits);
+    ASSERT_EQ(static_cast<int64_t>(kCanonicalDoubleNaNBits),
+              CanonicalizeDoubleToLongBits(FloatingPointFromBits<double>(0x7ff123456789abcdULL)));
+
+    const float negative_zero = CanonicalizeFloatingPoint(-0.0f);
+    uint32_t negative_zero_bits;
+    std::memcpy(&negative_zero_bits, &negative_zero, sizeof(negative_zero_bits));
+    ASSERT_EQ(0x80000000U, negative_zero_bits);
+    ASSERT_EQ(0x3ff0000000000000, CanonicalizeDoubleToLongBits(1.0));
+}
+
 // Test case: Test EndianSwapValue for different integral types
 TEST(MathTest, EndianSwapValue) {
     // Test 16-bit value
diff --git a/src/paimon/core/bucket/hive_bucket_function.cpp b/src/paimon/core/bucket/hive_bucket_function.cpp
index 913053c1f..e87c292d9 100644
--- a/src/paimon/core/bucket/hive_bucket_function.cpp
+++ b/src/paimon/core/bucket/hive_bucket_function.cpp
@@ -19,13 +19,12 @@
 #include "paimon/core/bucket/hive_bucket_function.h"
 
 #include <cassert>
-#include <cmath>
-#include <cstring>
 #include <limits>
 
 #include "fmt/format.h"
 #include "paimon/common/data/binary_row.h"
 #include "paimon/common/utils/field_type_utils.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/core/bucket/hive_hasher.h"
 #include "paimon/status.h"
 
@@ -105,10 +104,8 @@ uint32_t HiveBucketFunction::ComputeHash(const BinaryRow& row, int32_t field_ind
             uint32_t bits;
             if (float_value == -0.0f) {
                 bits = 0;
-            } else if (std::isnan(float_value)) {
-                bits = 0x7FC00000U;
             } else {
-                std::memcpy(&bits, &float_value, sizeof(bits));
+                bits = static_cast<uint32_t>(CanonicalizeFloatToIntBits(float_value));
             }
             return HiveHasher::HashInt(bits);
         }
@@ -117,10 +114,8 @@ uint32_t HiveBucketFunction::ComputeHash(const BinaryRow& row, int32_t field_ind
             uint64_t bits;
             if (double_value == -0.0) {
                 bits = 0;
-            } else if (std::isnan(double_value)) {
-                bits = 0x7FF8000000000000ULL;
             } else {
-                std::memcpy(&bits, &double_value, sizeof(bits));
+                bits = static_cast<uint64_t>(CanonicalizeDoubleToLongBits(double_value));
             }
             return HiveHasher::HashLong(bits);
         }
diff --git a/src/paimon/core/bucket/hive_bucket_function_test.cpp b/src/paimon/core/bucket/hive_bucket_function_test.cpp
index 21f2a9843..d97a0294b 100644
--- a/src/paimon/core/bucket/hive_bucket_function_test.cpp
+++ b/src/paimon/core/bucket/hive_bucket_function_test.cpp
@@ -18,12 +18,12 @@
 
 #include "paimon/core/bucket/hive_bucket_function.h"
 
-#include <cstring>
 #include <limits>
 
 #include "gtest/gtest.h"
 #include "paimon/common/data/binary_row.h"
 #include "paimon/common/data/binary_row_writer.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/core/bucket/hive_hasher.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/testing/utils/binary_row_generator.h"
@@ -111,18 +111,6 @@ class HiveBucketFunctionTest : public ::testing::Test {
         auto pool = GetDefaultPool();
         return BinaryRowGenerator::GenerateRow({value}, pool.get());
     }
-
-    float FloatFromBits(uint32_t bits) {
-        float value;
-        std::memcpy(&value, &bits, sizeof(value));
-        return value;
-    }
-
-    double DoubleFromBits(uint64_t bits) {
-        double value;
-        std::memcpy(&value, &bits, sizeof(value));
-        return value;
-    }
 };
 
 /// Test matching Java: testHiveBucketFunction
@@ -235,11 +223,12 @@ TEST_F(HiveBucketFunctionTest, TestFloatNaNCanonicalizationCompatibleWithJava) {
     ASSERT_OK_AND_ASSIGN(auto func, HiveBucketFunction::Create(field_types));
 
     // Verified with Java HiveBucketFunction:
-    // Float.NaN, Float.intBitsToFloat(0x7fa12345), and Float.intBitsToFloat(0x7fc00000)
-    // all hash through Float.floatToIntBits(...) = 0x7fc00000.
+    // Float.NaN, a payload NaN, and the canonical NaN all hash through
+    // Float.floatToIntBits(...) to kCanonicalFloatNaNBits.
     ASSERT_EQ(344, func->Bucket(CreateFloatRow(std::numeric_limits<float>::quiet_NaN()), 1000));
-    ASSERT_EQ(344, func->Bucket(CreateFloatRow(FloatFromBits(0x7FA12345U)), 1000));
-    ASSERT_EQ(344, func->Bucket(CreateFloatRow(FloatFromBits(0x7FC00000U)), 1000));
+    ASSERT_EQ(344, func->Bucket(CreateFloatRow(FloatingPointFromBits<float>(0x7FA12345U)), 1000));
+    ASSERT_EQ(344, func->Bucket(
+                       CreateFloatRow(FloatingPointFromBits<float>(kCanonicalFloatNaNBits)), 1000));
 }
 
 TEST_F(HiveBucketFunctionTest, TestDoubleNaNCanonicalizationCompatibleWithJava) {
@@ -248,10 +237,14 @@ TEST_F(HiveBucketFunctionTest, TestDoubleNaNCanonicalizationCompatibleWithJava)
 
     // Verified with Java HiveBucketFunction:
     // Double.NaN, Double.longBitsToDouble(0x7ff123456789abcd), and canonical NaN
-    // all hash through Double.doubleToLongBits(...) = 0x7ff8000000000000.
+    // All NaNs hash through Double.doubleToLongBits(...) to kCanonicalDoubleNaNBits.
     ASSERT_EQ(360, func->Bucket(CreateDoubleRow(std::numeric_limits<double>::quiet_NaN()), 1000));
-    ASSERT_EQ(360, func->Bucket(CreateDoubleRow(DoubleFromBits(0x7FF123456789ABCDULL)), 1000));
-    ASSERT_EQ(360, func->Bucket(CreateDoubleRow(DoubleFromBits(0x7FF8000000000000ULL)), 1000));
+    ASSERT_EQ(
+        360,
+        func->Bucket(CreateDoubleRow(FloatingPointFromBits<double>(0x7FF123456789ABCDULL)), 1000));
+    ASSERT_EQ(360,
+              func->Bucket(CreateDoubleRow(FloatingPointFromBits<double>(kCanonicalDoubleNaNBits)),
+                           1000));
 }
 
 TEST_F(HiveBucketFunctionTest, TestTinyintNegativeValuesCompatibleWithJava) {
diff --git a/src/paimon/core/global_index/indexed_split_test.cpp b/src/paimon/core/global_index/indexed_split_test.cpp
index 7cd121257..03cd2976d 100644
--- a/src/paimon/core/global_index/indexed_split_test.cpp
+++ b/src/paimon/core/global_index/indexed_split_test.cpp
@@ -17,6 +17,8 @@
  * under the License.
  */
 
+#include <cmath>
+#include <cstdint>
 #include <memory>
 #include <string>
 #include <utility>
@@ -26,6 +28,7 @@
 #include "gtest/gtest.h"
 #include "paimon/common/data/binary_row.h"
 #include "paimon/common/data/data_define.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/core/global_index/indexed_split_impl.h"
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/fs/local/local_file_system.h"
@@ -155,6 +158,34 @@ TEST(IndexedSplitTest, TestIndexedSplitWithScore) {
         << roundtrip_indexed_split->ToString();
 }
 
+TEST(IndexedSplitTest, TestSerializeCanonicalizesNaNScore) {
+    auto pool = GetDefaultPool();
+    DataSplitImpl::Builder builder(
+        /*partition=*/BinaryRow::EmptyRow(),
+        /*bucket=*/0, /*bucket_path=*/"bucket-0",
+        /*data_files=*/{});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<DataSplitImpl> data_split, builder.Build());
+
+    const auto payload_nan = FloatingPointFromBits<float>(0xffc12345U);
+    const auto canonical_nan = FloatingPointFromBits<float>(kCanonicalFloatNaNBits);
+    auto indexed_split = std::make_shared<IndexedSplitImpl>(
+        data_split, std::vector<Range>{Range(0, 0)}, std::vector<float>{payload_nan});
+    auto canonical_indexed_split = std::make_shared<IndexedSplitImpl>(
+        data_split, std::vector<Range>{Range(0, 0)}, std::vector<float>{canonical_nan});
+
+    ASSERT_OK_AND_ASSIGN(std::string serialized, Split::Serialize(indexed_split, pool));
+    ASSERT_OK_AND_ASSIGN(std::string canonical_serialized,
+                         Split::Serialize(canonical_indexed_split, pool));
+    ASSERT_EQ(serialized, canonical_serialized);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Split> roundtrip,
+                         Split::Deserialize(serialized.data(), serialized.size(), pool));
+    auto roundtrip_indexed_split = std::dynamic_pointer_cast<IndexedSplitImpl>(roundtrip);
+    ASSERT_TRUE(roundtrip_indexed_split);
+    ASSERT_EQ(roundtrip_indexed_split->Scores().size(), 1);
+    ASSERT_TRUE(std::isnan(roundtrip_indexed_split->Scores()[0]));
+}
+
 TEST(IndexedSplitTest, TestValidate) {
     auto meta = std::make_shared<DataFileMeta>(
         "file.orc", 1l, 200l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(),
diff --git a/src/paimon/core/table/source/split.cpp b/src/paimon/core/table/source/split.cpp
index 007df3cb1..d6117a899 100644
--- a/src/paimon/core/table/source/split.cpp
+++ b/src/paimon/core/table/source/split.cpp
@@ -23,6 +23,7 @@
 #include "paimon/common/data/binary_row.h"
 #include "paimon/common/io/memory_segment_output_stream.h"
 #include "paimon/common/memory/memory_segment_utils.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/common/utils/serialization_utils.h"
 #include "paimon/core/global_index/indexed_split_impl.h"
 #include "paimon/core/io/data_file_meta_serializer.h"
@@ -159,8 +160,8 @@ Result<std::string> Split::Serialize(const std::shared_ptr<Split>& split,
         if (!scores.empty()) {
             out.WriteValue<bool>(true);
             out.WriteValue<int32_t>(scores.size());
-            for (const auto& score : scores) {
-                out.WriteValue<float>(score);
+            for (float score : scores) {
+                out.WriteValue<float>(CanonicalizeFloatingPoint(score));
             }
         } else {
             out.WriteValue<bool>(false);

From 71e415cf9c281da641a7f0452d820b836c0c27ae Mon Sep 17 00:00:00 2001
From: gripleaf <425797155@qq.com>
Date: Fri, 28 Aug 2026 16:47:00 +0800
Subject: [PATCH 87/93] feat(metrics): expose scan and prefetch metrics (#211)

---
 docs/source/user_guide.rst                    |   1 +
 docs/source/user_guide/metrics.rst            | 116 +++++++
 include/paimon/defs.h                         |   4 +
 .../reader/prefetch_file_batch_reader.h       |  41 +++
 include/paimon/table/source/scan_metrics.h    |  54 ++++
 include/paimon/table/source/table_scan.h      |   7 +
 src/paimon/common/defs.cpp                    |   1 +
 .../apply_bitmap_index_batch_reader_test.cpp  |   3 +-
 .../reader/delegating_prefetch_reader.h       |   2 +-
 ...e_materializing_file_batch_reader_test.cpp |   6 +-
 .../prefetch_file_batch_reader_impl.cpp       | 293 ++++++++++++++++--
 .../reader/prefetch_file_batch_reader_impl.h  |   7 +-
 .../prefetch_file_batch_reader_impl_test.cpp  | 235 ++++++++++++--
 src/paimon/core/core_options.cpp              |   7 +
 src/paimon/core/core_options.h                |   2 +
 src/paimon/core/core_options_test.cpp         |   3 +
 ...pply_deletion_vector_batch_reader_test.cpp |   3 +-
 .../core/operation/abstract_split_read.cpp    |   2 +-
 .../append_only_file_store_scan_test.cpp      |  43 ++-
 src/paimon/core/operation/file_store_scan.cpp |  50 ++-
 src/paimon/core/operation/file_store_scan.h   |   7 +-
 .../key_value_file_store_scan_test.cpp        |   2 +-
 .../core/operation/metrics/scan_metrics.h     |  35 ---
 .../core/table/source/abstract_table_scan.h   |   4 +
 .../core/table/source/realtime_table_scan.h   |   4 +
 .../table/source/snapshot/snapshot_reader.h   |   4 +
 src/paimon/core/table/source/table_scan.cpp   |   5 +
 .../core/table/source/table_scan_test.cpp     |  46 +++
 .../core/table/system/system_table_test.cpp   |  13 +
 29 files changed, 895 insertions(+), 105 deletions(-)
 create mode 100644 docs/source/user_guide/metrics.rst
 create mode 100644 include/paimon/table/source/scan_metrics.h
 delete mode 100644 src/paimon/core/operation/metrics/scan_metrics.h

diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst
index 38ef0fc64..fdee89d9a 100644
--- a/docs/source/user_guide.rst
+++ b/docs/source/user_guide.rst
@@ -38,6 +38,7 @@ User Guide
    user_guide/commit
    user_guide/compaction
    user_guide/read
+   user_guide/metrics
    user_guide/clean
    user_guide/prefetch
    user_guide/arrow
diff --git a/docs/source/user_guide/metrics.rst b/docs/source/user_guide/metrics.rst
new file mode 100644
index 000000000..72ee2a176
--- /dev/null
+++ b/docs/source/user_guide/metrics.rst
@@ -0,0 +1,116 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+Metrics reference
+=================
+
+``Metrics`` contains counters, gauges, and histogram snapshots. A counter is a non-negative
+integer, a gauge represents current state, and a histogram records a distribution of observed
+values. The scan and prefetch-reader metrics described below are returned as point-in-time
+snapshots. Modifying a returned snapshot does not modify the component that produced it.
+
+Scan planning
+-------------
+
+The names below are declared by ``ScanMetrics``. ``last*`` counters are replaced after each
+successful ``CreatePlan()`` call. Histograms and the cache hit/miss counters accumulate for the
+lifetime of the scan. They are unrelated to ``SetReadSchema()`` and ``ReadAheadCache::Reset()``.
+The first six names match Java ``ScanMetrics``; the remaining names are C++-only.
+
+.. csv-table::
+   :header: "Name", "Type", "Unit", "Meaning"
+   :widths: 34, 12, 12, 52
+
+   "lastScanDuration", "counter", "milliseconds", "Duration of the last successful plan"
+   "scanDuration", "histogram", "milliseconds", "Distribution of successful plan durations"
+   "lastScannedSnapshotId", "counter", "snapshot ID", "Snapshot used by the last plan, or 0"
+   "lastScannedManifests", "counter", "files", "Manifest files selected by the last plan"
+   "lastScanSkippedTableFiles", "counter", "files", "Table files skipped by the last plan"
+   "lastScanResultedTableFiles", "counter", "files", "Table files returned by the last plan"
+   "lastManifestReadDuration", "counter", "milliseconds", "Manifest-list and entry read time for the last plan"
+   "manifestReadDuration", "histogram", "milliseconds", "Distribution of manifest read times"
+   "lastSnapshotCacheEnabled", "counter", "boolean", "Whether snapshot manifest-entry cache was eligible"
+   "lastSnapshotCacheHit", "counter", "boolean", "Whether the last eligible lookup hit"
+   "snapshotCacheHits", "counter", "lookups", "Cumulative exact-snapshot cache hits"
+   "snapshotCacheMisses", "counter", "lookups", "Cumulative eligible cache misses"
+   "lastSnapshotCacheLoadDuration", "counter", "milliseconds", "Cache load time for the last plan"
+   "snapshotCacheLoadDuration", "histogram", "milliseconds", "Distribution of cache load times"
+   "lastSnapshotCacheStoreDuration", "counter", "milliseconds", "Cache store time for the last plan; 0 when not stored"
+   "snapshotCacheStoreDuration", "histogram", "milliseconds", "Distribution of cache store times"
+   "lastLazyDecodeScannedRows", "counter", "manifest rows", "Candidate manifest rows inspected by the last plan"
+   "lastLazyDecodeMaterializedRows", "counter", "manifest rows", "Manifest rows retained after lazy filtering"
+
+Prefetch reader
+---------------
+
+The names below are declared by ``PrefetchMetrics``. Counters and histograms accumulate for the
+lifetime of the prefetch reader, including across ``SetReadSchema()``. ``enabled`` and
+``parallelism`` describe the most recently initialized schema. ``queue-depth`` is reset by
+``SetReadSchema()`` and ``Close()``; ``queue-depth.max`` remains the lifetime maximum.
+These metrics are C++-only and have no counterparts in Java Paimon.
+
+.. csv-table::
+   :header: "Name", "Type", "Unit", "Meaning"
+   :widths: 38, 12, 12, 48
+
+   "prefetch.enabled", "gauge", "boolean", "Whether the most recently initialized schema selected prefetch"
+   "prefetch.parallelism", "gauge", "readers", "Effective reader parallelism"
+   "prefetch.read-ranges.total", "counter", "ranges", "Generated ranges before bitmap filtering"
+   "prefetch.read-ranges.after-bitmap", "counter", "ranges", "Ranges retained after bitmap filtering"
+   "prefetch.seek.count", "counter", "operations", "Underlying reader seek operations"
+   "prefetch.produced-batches", "counter", "batches", "Data batches placed into prefetch queues"
+   "prefetch.consumed-batches", "counter", "batches", "Data batches returned to the consumer"
+   "prefetch.discarded-batches", "counter", "batches", "Data batches released without consumption, plus EOF entries released during cleanup"
+   "prefetch.errors", "counter", "errors", "Errors recorded by the background prefetch loop"
+   "prefetch.adaptive-disabled-count", "counter", "decisions", "Times adaptive strategy disabled prefetch"
+   "prefetch.queue-full-count", "counter", "events", "Times production found a full queue"
+   "prefetch.queue-depth", "gauge", "queue entries", "Current queued entries, including retained EOF markers"
+   "prefetch.queue-depth.max", "gauge", "queue entries", "Maximum queued entries in the reader lifetime"
+   "prefetch.reader-read-latency-us", "histogram", "microseconds", "Underlying reader batch latency"
+   "prefetch.consumer-wait-latency-us", "histogram", "microseconds", "Consumer wait latency per returned batch or EOF"
+
+Prefetch I/O
+------------
+
+``PrefetchIoMetrics`` describes only I/O that passes through the prefetch reader's instrumented
+input streams. It is not a whole-query or whole-table I/O total. All counters accumulate for the
+reader lifetime and are retained across ``SetReadSchema()`` and cache reset. Latency uses relaxed
+atomic count and sum counters instead of per-I/O histograms to reduce hot-path cost. Collection is
+disabled by default; set ``prefetch.io-metrics.enabled`` to ``true`` in the read options to enable
+it. When disabled, these per-I/O metrics are absent and the input streams have no metrics
+instrumentation.
+``io.async.pending`` is current state and returns to zero when all callbacks complete.
+These metrics are C++-only and have no counterparts in Java Paimon.
+
+.. csv-table::
+   :header: "Name", "Type", "Unit", "Meaning"
+   :widths: 34, 12, 12, 52
+
+   "io.read.requests", "counter", "requests", "Synchronous read requests"
+   "io.read.requested-bytes", "counter", "bytes", "Bytes requested by synchronous reads"
+   "io.read.physical-bytes", "counter", "bytes", "Bytes returned by successful synchronous reads"
+   "io.read.failed", "counter", "requests", "Failed synchronous reads"
+   "io.read.latency.count", "counter", "requests", "Completed synchronous read latency samples"
+   "io.read.latency.sum-us", "counter", "microseconds", "Sum of synchronous read latency"
+   "io.async.requests", "counter", "requests", "Asynchronous read requests"
+   "io.async.requested-bytes", "counter", "bytes", "Bytes requested by asynchronous reads"
+   "io.async.physical-bytes", "counter", "bytes", "Bytes attributed to successful asynchronous reads"
+   "io.async.completed", "counter", "requests", "Successful asynchronous reads"
+   "io.async.failed", "counter", "requests", "Failed asynchronous reads"
+   "io.async.pending", "gauge", "requests", "Asynchronous callbacks not yet completed"
+   "io.async.latency.count", "counter", "requests", "Completed asynchronous callback latency samples"
+   "io.async.latency.sum-us", "counter", "microseconds", "Sum of asynchronous callback latency"
diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index c96cf5d4f..0ca67e3a7 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -208,6 +208,10 @@ struct PAIMON_EXPORT Options {
     /// for the target bucket when rebuilding the cache. Default value is true.
     static const char SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED[];
 
+    /// "prefetch.io-metrics.enabled" - Whether to collect per-I/O metrics for prefetch reads.
+    /// Default value is false.
+    static const char PREFETCH_IO_METRICS_ENABLED[];
+
     /// "read.batch-size" - Read batch size for any file format if it supports.
     /// The default value is 1024.
     static const char READ_BATCH_SIZE[];
diff --git a/include/paimon/reader/prefetch_file_batch_reader.h b/include/paimon/reader/prefetch_file_batch_reader.h
index 5e6313e8f..842279956 100644
--- a/include/paimon/reader/prefetch_file_batch_reader.h
+++ b/include/paimon/reader/prefetch_file_batch_reader.h
@@ -27,6 +27,47 @@
 
 namespace paimon {
 
+/// C++-only prefetch reader metrics. Java Paimon has no corresponding metrics.
+class PAIMON_EXPORT PrefetchMetrics {
+ public:
+    static constexpr char ENABLED[] = "prefetch.enabled";
+    static constexpr char PARALLELISM[] = "prefetch.parallelism";
+    static constexpr char READ_RANGES_TOTAL[] = "prefetch.read-ranges.total";
+    static constexpr char READ_RANGES_AFTER_BITMAP[] = "prefetch.read-ranges.after-bitmap";
+    static constexpr char SEEK_COUNT[] = "prefetch.seek.count";
+    static constexpr char PRODUCED_BATCHES[] = "prefetch.produced-batches";
+    static constexpr char CONSUMED_BATCHES[] = "prefetch.consumed-batches";
+    static constexpr char DISCARDED_BATCHES[] = "prefetch.discarded-batches";
+    static constexpr char ERRORS[] = "prefetch.errors";
+    static constexpr char ADAPTIVE_DISABLED_COUNT[] = "prefetch.adaptive-disabled-count";
+    static constexpr char QUEUE_FULL_COUNT[] = "prefetch.queue-full-count";
+    static constexpr char QUEUE_DEPTH[] = "prefetch.queue-depth";
+    static constexpr char QUEUE_DEPTH_MAX[] = "prefetch.queue-depth.max";
+    static constexpr char READER_READ_LATENCY_US[] = "prefetch.reader-read-latency-us";
+    static constexpr char CONSUMER_WAIT_LATENCY_US[] = "prefetch.consumer-wait-latency-us";
+};
+
+/// C++-only metric names for I/O observed by the prefetch reader's instrumented input streams.
+/// Java Paimon has no corresponding metrics.
+/// These metrics do not represent whole-query or whole-table I/O.
+class PAIMON_EXPORT PrefetchIoMetrics {
+ public:
+    static constexpr char READ_REQUESTS[] = "io.read.requests";
+    static constexpr char READ_REQUESTED_BYTES[] = "io.read.requested-bytes";
+    static constexpr char READ_PHYSICAL_BYTES[] = "io.read.physical-bytes";
+    static constexpr char READ_FAILED[] = "io.read.failed";
+    static constexpr char READ_LATENCY_COUNT[] = "io.read.latency.count";
+    static constexpr char READ_LATENCY_SUM_US[] = "io.read.latency.sum-us";
+    static constexpr char ASYNC_REQUESTS[] = "io.async.requests";
+    static constexpr char ASYNC_REQUESTED_BYTES[] = "io.async.requested-bytes";
+    static constexpr char ASYNC_PHYSICAL_BYTES[] = "io.async.physical-bytes";
+    static constexpr char ASYNC_COMPLETED[] = "io.async.completed";
+    static constexpr char ASYNC_FAILED[] = "io.async.failed";
+    static constexpr char ASYNC_PENDING[] = "io.async.pending";
+    static constexpr char ASYNC_LATENCY_COUNT[] = "io.async.latency.count";
+    static constexpr char ASYNC_LATENCY_SUM_US[] = "io.async.latency.sum-us";
+};
+
 /// The prefetch file batch reader extends the basic FileBatchReader interface for prefetch read,
 /// if a format implementation inherits from this class, it will automatically support the C++
 /// Paimon prefetch capability and integrate with the Paimon prefetch framework.
diff --git a/include/paimon/table/source/scan_metrics.h b/include/paimon/table/source/scan_metrics.h
new file mode 100644
index 000000000..15bc41ff1
--- /dev/null
+++ b/include/paimon/table/source/scan_metrics.h
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "paimon/visibility.h"
+
+namespace paimon {
+
+/// Metric names for scan planning operations.
+class PAIMON_EXPORT ScanMetrics {
+ public:
+    static constexpr char LAST_SCAN_DURATION[] = "lastScanDuration";
+    // Histogram metric for scan plan duration (milliseconds).
+    static constexpr char SCAN_DURATION[] = "scanDuration";
+    static constexpr char LAST_SCANNED_SNAPSHOT_ID[] = "lastScannedSnapshotId";
+    static constexpr char LAST_SCANNED_MANIFESTS[] = "lastScannedManifests";
+    static constexpr char LAST_SCAN_SKIPPED_TABLE_FILES[] = "lastScanSkippedTableFiles";
+    static constexpr char LAST_SCAN_RESULTED_TABLE_FILES[] = "lastScanResultedTableFiles";
+
+    // The metrics below are C++-only and do not have counterparts in Java ScanMetrics.
+    static constexpr char LAST_MANIFEST_READ_DURATION[] = "lastManifestReadDuration";
+    // Histogram metric for manifest-list and manifest-entry read duration (milliseconds).
+    static constexpr char MANIFEST_READ_DURATION[] = "manifestReadDuration";
+    static constexpr char LAST_SNAPSHOT_CACHE_ENABLED[] = "lastSnapshotCacheEnabled";
+    static constexpr char LAST_SNAPSHOT_CACHE_HIT[] = "lastSnapshotCacheHit";
+    static constexpr char SNAPSHOT_CACHE_HITS[] = "snapshotCacheHits";
+    static constexpr char SNAPSHOT_CACHE_MISSES[] = "snapshotCacheMisses";
+    static constexpr char LAST_SNAPSHOT_CACHE_LOAD_DURATION[] = "lastSnapshotCacheLoadDuration";
+    static constexpr char SNAPSHOT_CACHE_LOAD_DURATION[] = "snapshotCacheLoadDuration";
+    static constexpr char LAST_SNAPSHOT_CACHE_STORE_DURATION[] = "lastSnapshotCacheStoreDuration";
+    static constexpr char SNAPSHOT_CACHE_STORE_DURATION[] = "snapshotCacheStoreDuration";
+    // Candidate manifest-entry rows inspected by lazy scan filtering.
+    static constexpr char LAST_LAZY_DECODE_SCANNED_ROWS[] = "lastLazyDecodeScannedRows";
+    // Full manifest entries retained after lazy scan filtering.
+    static constexpr char LAST_LAZY_DECODE_MATERIALIZED_ROWS[] = "lastLazyDecodeMaterializedRows";
+};
+
+}  // namespace paimon
diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h
index c9b42915f..14c447568 100644
--- a/include/paimon/table/source/table_scan.h
+++ b/include/paimon/table/source/table_scan.h
@@ -23,6 +23,7 @@
 
 #include "paimon/result.h"
 #include "paimon/table/source/plan.h"
+#include "paimon/table/source/scan_metrics.h"
 #include "paimon/type_fwd.h"
 #include "paimon/visibility.h"
 
@@ -44,5 +45,11 @@ class PAIMON_EXPORT TableScan {
     ///
     /// @return A Result containing a shared pointer to the created `Plan` or an error status.
     virtual Result<std::shared_ptr<Plan>> CreatePlan() = 0;
+
+    /// Retrieve metrics related to scan planning operations.
+    ///
+    /// @return A point-in-time snapshot of scan metrics. Mutating the returned object does not
+    /// affect metrics collected by this scan.
+    virtual std::shared_ptr<Metrics> GetMetrics() const;
 };
 }  // namespace paimon
diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp
index 454961461..39e6d494c 100644
--- a/src/paimon/common/defs.cpp
+++ b/src/paimon/common/defs.cpp
@@ -62,6 +62,7 @@ const char Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS[] =
     "scan.manifest-entry-cache.max-snapshots";
 const char Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED[] =
     "scan.manifest-entry.lazy-decode.enabled";
+const char Options::PREFETCH_IO_METRICS_ENABLED[] = "prefetch.io-metrics.enabled";
 const char Options::READ_BATCH_SIZE[] = "read.batch-size";
 const char Options::WRITE_BATCH_SIZE[] = "write.batch-size";
 const char Options::WRITE_BUFFER_SIZE[] = "write-buffer-size";
diff --git a/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp b/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp
index 1082e6695..ea45c9791 100644
--- a/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp
+++ b/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp
@@ -97,7 +97,8 @@ class ApplyBitmapIndexBatchReaderTest : public ::testing::Test,
                         prefetch_batch_count, batch_size, prefetch_batch_count * 2,
                         /*enable_adaptive_prefetch_strategy=*/false, executor_,
                         /*initialize_read_ranges=*/true,
-                        /*read_ahead_cache_enabled=*/true, CacheConfig(), pool_));
+                        /*read_ahead_cache_enabled=*/true, CacheConfig(),
+                        /*enable_io_metrics=*/false, pool_));
             } else {
                 file_batch_reader =
                     std::make_unique<MockFileBatchReader>(data, target_type_, batch_size);
diff --git a/src/paimon/common/reader/delegating_prefetch_reader.h b/src/paimon/common/reader/delegating_prefetch_reader.h
index 3cbcb08bd..78396e200 100644
--- a/src/paimon/common/reader/delegating_prefetch_reader.h
+++ b/src/paimon/common/reader/delegating_prefetch_reader.h
@@ -45,7 +45,7 @@ class DelegatingPrefetchReader : public FileBatchReader {
     }
 
     std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return GetReader()->GetReaderMetrics();
+        return prefetch_reader_->GetReaderMetrics();
     }
 
     Result<std::unique_ptr<::ArrowSchema>> GetFileSchema() const override {
diff --git a/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp b/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp
index ff571a284..075fbe107 100644
--- a/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp
+++ b/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp
@@ -532,7 +532,7 @@ TEST_F(LateMaterializingFileBatchReaderTest, WorksAsInnerOfPrefetchReader) {
             /*prefetch_max_parallel_num=*/1, /*batch_size=*/3, /*prefetch_batch_count=*/2,
             /*enable_adaptive_prefetch_strategy=*/false, executor,
             /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/false, CacheConfig(),
-            GetDefaultPool()));
+            /*enable_io_metrics=*/false, GetDefaultPool()));
     auto predicate =
         PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, Literal(4l));
     ::ArrowSchema c_schema;
@@ -568,7 +568,7 @@ TEST_F(LateMaterializingFileBatchReaderTest, PrefetchInnerReentrantSetReadSchema
             /*prefetch_max_parallel_num=*/1, /*batch_size=*/3, /*prefetch_batch_count=*/2,
             /*enable_adaptive_prefetch_strategy=*/false, executor,
             /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/false, CacheConfig(),
-            GetDefaultPool()));
+            /*enable_io_metrics=*/false, GetDefaultPool()));
 
     auto full_schema = arrow::schema(full_fields_);
     auto predicate1 =
@@ -631,7 +631,7 @@ TEST_F(LateMaterializingFileBatchReaderTest, PrefetchInnerParallelReadersWithSee
             /*prefetch_max_parallel_num=*/3, /*batch_size=*/3, /*prefetch_batch_count=*/6,
             /*enable_adaptive_prefetch_strategy=*/false, executor,
             /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/false, CacheConfig(),
-            GetDefaultPool()));
+            /*enable_io_metrics=*/false, GetDefaultPool()));
     auto predicate =
         PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, Literal(5l));
     ::ArrowSchema c_schema;
diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp
index 4b6411b7b..5285a2d80 100644
--- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp
+++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp
@@ -21,6 +21,7 @@
 #include <algorithm>
 #include <chrono>
 #include <future>
+#include <limits>
 #include <thread>
 
 #include "arrow/array/array_base.h"
@@ -44,8 +45,149 @@ class Schema;
 
 namespace paimon {
 
+struct PrefetchMetricsState {
+    std::atomic<uint64_t> read_ranges_total{0};
+    std::atomic<uint64_t> read_ranges_after_bitmap{0};
+    std::atomic<uint64_t> seek_count{0};
+    std::atomic<uint64_t> produced_batches{0};
+    std::atomic<uint64_t> consumed_batches{0};
+    std::atomic<uint64_t> discarded_batches{0};
+    std::atomic<uint64_t> errors{0};
+    std::atomic<uint64_t> adaptive_disabled_count{0};
+    std::atomic<uint64_t> queue_full_count{0};
+    std::atomic<uint64_t> queue_depth{0};
+    std::atomic<uint64_t> queue_depth_max{0};
+    std::atomic<bool> enabled{false};
+    std::shared_ptr<MetricsImpl> histograms = std::make_shared<MetricsImpl>();
+};
+
+struct PrefetchIoMetricsState {
+    std::atomic<uint64_t> read_requests{0};
+    std::atomic<uint64_t> read_requested_bytes{0};
+    std::atomic<uint64_t> read_physical_bytes{0};
+    std::atomic<uint64_t> read_failed{0};
+    std::atomic<uint64_t> read_latency_count{0};
+    std::atomic<uint64_t> read_latency_sum_us{0};
+    std::atomic<uint64_t> async_requests{0};
+    std::atomic<uint64_t> async_requested_bytes{0};
+    std::atomic<uint64_t> async_physical_bytes{0};
+    std::atomic<uint64_t> async_completed{0};
+    std::atomic<uint64_t> async_failed{0};
+    std::atomic<uint64_t> async_pending{0};
+    std::atomic<uint64_t> async_latency_count{0};
+    std::atomic<uint64_t> async_latency_sum_us{0};
+};
+
 namespace {
 
+// Metrics do not synchronize reader state. A concurrently collected snapshot may be approximate.
+constexpr std::memory_order kMetricsMemoryOrder = std::memory_order_relaxed;
+
+uint64_t ElapsedMicros(const std::chrono::steady_clock::time_point& start) {
+    return static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::microseconds>(
+                                     std::chrono::steady_clock::now() - start)
+                                     .count());
+}
+
+void UpdateMax(std::atomic<uint64_t>* target, uint64_t value) {
+    uint64_t current = target->load(kMetricsMemoryOrder);
+    while (current < value && !target->compare_exchange_weak(current, value, kMetricsMemoryOrder,
+                                                             kMetricsMemoryOrder)) {
+    }
+}
+
+void RecordLatency(uint64_t latency_us, std::atomic<uint64_t>* count,
+                   std::atomic<uint64_t>* sum_us) {
+    count->fetch_add(1, kMetricsMemoryOrder);
+    sum_us->fetch_add(latency_us, kMetricsMemoryOrder);
+}
+
+class MetricsInputStream : public InputStream {
+ public:
+    MetricsInputStream(const std::shared_ptr<InputStream>& stream,
+                       const std::shared_ptr<PrefetchIoMetricsState>& metrics)
+        : stream_(stream), metrics_(metrics) {}
+
+    MetricsInputStream(std::unique_ptr<InputStream>&& stream,
+                       const std::shared_ptr<PrefetchIoMetricsState>& metrics)
+        : stream_(std::move(stream)), metrics_(metrics) {}
+
+    Status Seek(int64_t offset, SeekOrigin origin) override {
+        return stream_->Seek(offset, origin);
+    }
+
+    Result<int64_t> GetPos() const override {
+        return stream_->GetPos();
+    }
+
+    Result<int64_t> Read(char* buffer, int64_t size) override {
+        return RecordRead([&]() { return stream_->Read(buffer, size); }, size);
+    }
+
+    Result<int64_t> Read(char* buffer, int64_t size, int64_t offset) override {
+        return RecordRead([&]() { return stream_->Read(buffer, size, offset); }, size);
+    }
+
+    void ReadAsync(char* buffer, int64_t size, int64_t offset,
+                   std::function<void(Status)>&& callback) override {
+        metrics_->async_requests.fetch_add(1, kMetricsMemoryOrder);
+        metrics_->async_requested_bytes.fetch_add(static_cast<uint64_t>(std::max<int64_t>(0, size)),
+                                                  kMetricsMemoryOrder);
+        metrics_->async_pending.fetch_add(1, kMetricsMemoryOrder);
+        std::shared_ptr<PrefetchIoMetricsState> metrics = metrics_;
+        const auto start = std::chrono::steady_clock::now();
+        stream_->ReadAsync(
+            buffer, size, offset,
+            [metrics, size, start, callback = std::move(callback)](Status status) mutable {
+                metrics->async_pending.fetch_sub(1, kMetricsMemoryOrder);
+                if (status.ok()) {
+                    metrics->async_completed.fetch_add(1, kMetricsMemoryOrder);
+                    metrics->async_physical_bytes.fetch_add(
+                        static_cast<uint64_t>(std::max<int64_t>(0, size)), kMetricsMemoryOrder);
+                } else {
+                    metrics->async_failed.fetch_add(1, kMetricsMemoryOrder);
+                }
+                RecordLatency(ElapsedMicros(start), &metrics->async_latency_count,
+                              &metrics->async_latency_sum_us);
+                callback(status);
+            });
+    }
+
+    Status Close() override {
+        return stream_->Close();
+    }
+
+    Result<std::string> GetUri() const override {
+        return stream_->GetUri();
+    }
+
+    Result<int64_t> Length() const override {
+        return stream_->Length();
+    }
+
+ private:
+    template <typename ReadFunction>
+    Result<int64_t> RecordRead(ReadFunction&& read, int64_t size) {
+        metrics_->read_requests.fetch_add(1, kMetricsMemoryOrder);
+        metrics_->read_requested_bytes.fetch_add(static_cast<uint64_t>(std::max<int64_t>(0, size)),
+                                                 kMetricsMemoryOrder);
+        const auto start = std::chrono::steady_clock::now();
+        Result<int64_t> result = read();
+        if (result.ok()) {
+            metrics_->read_physical_bytes.fetch_add(
+                static_cast<uint64_t>(std::max<int64_t>(0, result.value())), kMetricsMemoryOrder);
+        } else {
+            metrics_->read_failed.fetch_add(1, kMetricsMemoryOrder);
+        }
+        RecordLatency(ElapsedMicros(start), &metrics_->read_latency_count,
+                      &metrics_->read_latency_sum_us);
+        return result;
+    }
+
+    std::shared_ptr<InputStream> stream_;
+    std::shared_ptr<PrefetchIoMetricsState> metrics_;
+};
+
 std::pair<int64_t, int64_t> ComputeBatchSliceByReadRange(
     const std::vector<uint64_t>& global_row_ids, const std::pair<uint64_t, uint64_t>& read_range) {
     auto begin_it =
@@ -62,7 +204,7 @@ Result<std::unique_ptr<PrefetchFileBatchReaderImpl>> PrefetchFileBatchReaderImpl
     const std::shared_ptr<FileSystem>& fs, uint32_t prefetch_max_parallel_num, int32_t batch_size,
     uint32_t prefetch_batch_count, bool enable_adaptive_prefetch_strategy,
     const std::shared_ptr<Executor>& executor, bool initialize_read_ranges,
-    bool read_ahead_cache_enabled, const CacheConfig& cache_config,
+    bool read_ahead_cache_enabled, const CacheConfig& cache_config, bool enable_io_metrics,
     const std::shared_ptr<MemoryPool>& pool) {
     if (prefetch_max_parallel_num == 0) {
         return Status::Invalid("prefetch max parallel num should be greater than 0.");
@@ -83,24 +225,35 @@ Result<std::unique_ptr<PrefetchFileBatchReaderImpl>> PrefetchFileBatchReaderImpl
         return Status::Invalid("executor should not be nullptr.");
     }
 
+    std::shared_ptr<PrefetchIoMetricsState> io_metrics;
+    if (enable_io_metrics) {
+        io_metrics = std::make_shared<PrefetchIoMetricsState>();
+    }
     std::shared_ptr<ReadAheadCache> cache;
     if (read_ahead_cache_enabled) {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> input_stream,
                                fs->Open(FileStatus(data_file_path, data_file_size)));
+        if (io_metrics) {
+            input_stream = std::make_shared<MetricsInputStream>(input_stream, io_metrics);
+        }
         cache = std::make_shared<ReadAheadCache>(input_stream, cache_config, pool);
     }
     std::vector<std::future<Result<std::unique_ptr<FileBatchReader>>>> futures;
     for (uint32_t i = 0; i < prefetch_max_parallel_num; i++) {
-        futures.push_back(
-            Via(executor.get(),
-                [&fs, &data_file_path, data_file_size, &reader_builder,
-                 &cache]() -> Result<std::unique_ptr<FileBatchReader>> {
-                    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<InputStream> input_stream,
-                                           fs->Open(FileStatus(data_file_path, data_file_size)));
-                    auto cache_input_stream =
-                        std::make_shared<CacheInputStream>(std::move(input_stream), cache);
-                    return reader_builder->Build(cache_input_stream);
-                }));
+        futures.push_back(Via(
+            executor.get(),
+            [&fs, &data_file_path, data_file_size, &reader_builder, &cache,
+             io_metrics]() -> Result<std::unique_ptr<FileBatchReader>> {
+                PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<InputStream> input_stream,
+                                       fs->Open(FileStatus(data_file_path, data_file_size)));
+                if (io_metrics) {
+                    input_stream =
+                        std::make_unique<MetricsInputStream>(std::move(input_stream), io_metrics);
+                }
+                auto cache_input_stream =
+                    std::make_shared<CacheInputStream>(std::move(input_stream), cache);
+                return reader_builder->Build(cache_input_stream);
+            }));
     }
     std::vector<std::shared_ptr<PrefetchFileBatchReader>> readers;
     for (auto& file_batch_reader : CollectAll(futures)) {
@@ -121,9 +274,9 @@ Result<std::unique_ptr<PrefetchFileBatchReaderImpl>> PrefetchFileBatchReaderImpl
     }
     uint32_t prefetch_queue_capacity = prefetch_batch_count / readers.size();
 
-    auto reader = std::unique_ptr<PrefetchFileBatchReaderImpl>(
-        new PrefetchFileBatchReaderImpl(readers, batch_size, prefetch_queue_capacity,
-                                        enable_adaptive_prefetch_strategy, executor, cache, pool));
+    auto reader = std::unique_ptr<PrefetchFileBatchReaderImpl>(new PrefetchFileBatchReaderImpl(
+        readers, batch_size, prefetch_queue_capacity, enable_adaptive_prefetch_strategy, executor,
+        cache, io_metrics, pool));
     if (initialize_read_ranges) {
         // normally initialize read ranges should be false, as set read schema will refresh read
         // ranges, and set read schema will always be called before read.
@@ -136,6 +289,7 @@ PrefetchFileBatchReaderImpl::PrefetchFileBatchReaderImpl(
     const std::vector<std::shared_ptr<PrefetchFileBatchReader>>& readers, int32_t batch_size,
     uint32_t prefetch_queue_capacity, bool enable_adaptive_prefetch_strategy,
     const std::shared_ptr<Executor>& executor, const std::shared_ptr<ReadAheadCache>& cache,
+    const std::shared_ptr<PrefetchIoMetricsState>& io_metrics,
     const std::shared_ptr<MemoryPool>& pool)
     : readers_(std::move(readers)),
       batch_size_(batch_size),
@@ -143,7 +297,9 @@ PrefetchFileBatchReaderImpl::PrefetchFileBatchReaderImpl(
       cache_(cache),
       arrow_pool_(GetArrowPool(pool)),
       prefetch_queue_capacity_(prefetch_queue_capacity),
-      enable_adaptive_prefetch_strategy_(enable_adaptive_prefetch_strategy) {
+      enable_adaptive_prefetch_strategy_(enable_adaptive_prefetch_strategy),
+      prefetch_metrics_(std::make_shared<PrefetchMetricsState>()),
+      io_metrics_(io_metrics) {
     for (size_t i = 0; i < readers_.size(); i++) {
         prefetch_queues_.emplace_back(std::make_unique<ThreadsafeQueue<PrefetchBatch>>());
         readers_pos_.emplace_back(std::make_unique<std::atomic<uint64_t>>(0));
@@ -186,6 +342,7 @@ Status PrefetchFileBatchReaderImpl::RefreshReadRanges() {
 Status PrefetchFileBatchReaderImpl::RefreshReadRangesAfterCleanUp() {
     bool need_prefetch;
     PAIMON_ASSIGN_OR_RAISE(auto read_ranges, readers_[0]->GenReadRanges(&need_prefetch));
+    const bool format_requested_prefetch = need_prefetch;
 
     if (!enable_adaptive_prefetch_strategy_) {
         need_prefetch = true;
@@ -197,8 +354,17 @@ Status PrefetchFileBatchReaderImpl::RefreshReadRangesAfterCleanUp() {
         }
     }
 
+    if (format_requested_prefetch && !need_prefetch) {
+        prefetch_metrics_->adaptive_disabled_count.fetch_add(1, kMetricsMemoryOrder);
+    }
     need_prefetch_ = need_prefetch;
-    PAIMON_RETURN_NOT_OK(SetReadRanges(FilterReadRanges(read_ranges, selection_bitmap_)));
+    prefetch_metrics_->enabled.store(need_prefetch_, kMetricsMemoryOrder);
+    prefetch_metrics_->read_ranges_total.fetch_add(read_ranges.size(), kMetricsMemoryOrder);
+    std::vector<std::pair<uint64_t, uint64_t>> filtered_ranges =
+        FilterReadRanges(read_ranges, selection_bitmap_);
+    prefetch_metrics_->read_ranges_after_bitmap.fetch_add(filtered_ranges.size(),
+                                                          kMetricsMemoryOrder);
+    PAIMON_RETURN_NOT_OK(SetReadRanges(filtered_ranges));
     return Status::OK();
 }
 
@@ -272,6 +438,7 @@ Status PrefetchFileBatchReaderImpl::CleanUp() {
                 if (batch == std::nullopt) {
                     break;
                 }
+                prefetch_metrics_->discarded_batches.fetch_add(1, kMetricsMemoryOrder);
                 ReaderUtils::ReleaseReadBatch(std::move(batch.value().batch.first));
             }
         }
@@ -297,6 +464,7 @@ Status PrefetchFileBatchReaderImpl::CleanUp() {
     current_batch_global_row_ids_.clear();
     read_ranges_freshed_ = false;
     clean_prefetch_queue();
+    prefetch_metrics_->queue_depth.store(0, kMetricsMemoryOrder);
     for (size_t i = 0; i < readers_pos_.size(); i++) {
         readers_pos_[i]->store(0);
         reader_is_working_[i] = false;
@@ -357,6 +525,7 @@ void PrefetchFileBatchReaderImpl::Workloop() {
                 }
                 if (prefetch_queues_[reader_idx]->size() >= prefetch_queue_capacity_) {
                     // queue is full, skip
+                    prefetch_metrics_->queue_full_count.fetch_add(1, kMetricsMemoryOrder);
                     continue;
                 }
                 if (readers_pos_[reader_idx]->load() != std::numeric_limits<uint64_t>::max()) {
@@ -417,6 +586,7 @@ Status PrefetchFileBatchReaderImpl::EnsureReaderPosition(
     uint64_t pos = std::max(readers_pos_[reader_idx]->load(), current_read_range.first);
     PAIMON_ASSIGN_OR_RAISE(uint64_t next_row_to_read, readers_[reader_idx]->GetNextRowToRead());
     if (next_row_to_read != pos) {
+        prefetch_metrics_->seek_count.fetch_add(1, kMetricsMemoryOrder);
         return readers_[reader_idx]->SeekToRow(pos);
     }
     return Status::OK();
@@ -447,6 +617,7 @@ Status PrefetchFileBatchReaderImpl::HandleReadResult(
             global_row_ids.push_back(global_row_id);
         }
         if (global_row_ids.empty()) {
+            prefetch_metrics_->discarded_batches.fetch_add(1, kMetricsMemoryOrder);
             ReaderUtils::ReleaseReadBatch(std::move(read_batch));
             return Status::OK();
         }
@@ -464,6 +635,7 @@ Status PrefetchFileBatchReaderImpl::HandleReadResult(
                 FindReadRangeContaining(reader_idx, global_row_ids[0]);
             if (owner_range == std::nullopt) {
                 readers_pos_[reader_idx]->store(global_row_ids[0]);
+                prefetch_metrics_->discarded_batches.fetch_add(1, kMetricsMemoryOrder);
                 ReaderUtils::ReleaseReadBatch(std::move(read_batch));
                 return Status::OK();
             }
@@ -493,15 +665,23 @@ Status PrefetchFileBatchReaderImpl::HandleReadResult(
             readers_pos_[reader_idx]->store(next_row_to_read);
         }
         if (bitmap.IsEmpty()) {
+            prefetch_metrics_->discarded_batches.fetch_add(1, kMetricsMemoryOrder);
             ReaderUtils::ReleaseReadBatch(std::move(read_batch));
             return Status::OK();
         }
         prefetch_queue->push(
             {read_range, std::move(read_batch_with_bitmap), std::move(global_row_ids)});
+        prefetch_metrics_->produced_batches.fetch_add(1, kMetricsMemoryOrder);
+        const uint64_t queue_depth =
+            prefetch_metrics_->queue_depth.fetch_add(1, kMetricsMemoryOrder) + 1;
+        UpdateMax(&prefetch_metrics_->queue_depth_max, queue_depth);
     } else {
         std::pair<uint64_t, uint64_t> eof_range;
         PAIMON_ASSIGN_OR_RAISE(eof_range, EofRange());
         prefetch_queue->push({eof_range, std::move(read_batch_with_bitmap), {}});
+        const uint64_t queue_depth =
+            prefetch_metrics_->queue_depth.fetch_add(1, kMetricsMemoryOrder) + 1;
+        UpdateMax(&prefetch_metrics_->queue_depth_max, queue_depth);
         readers_pos_[reader_idx]->store(std::numeric_limits<uint64_t>::max());
     }
     return Status::OK();
@@ -532,8 +712,11 @@ Status PrefetchFileBatchReaderImpl::DoReadBatch(size_t reader_idx) {
     FileBatchReader* reader = readers_[reader_idx].get();
     PAIMON_RETURN_NOT_OK(EnsureReaderPosition(reader_idx, read_range));
 
-    PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap read_batch_with_bitmap,
-                           reader->NextBatchWithBitmap());
+    const auto read_start = std::chrono::steady_clock::now();
+    Result<ReadBatchWithBitmap> read_result = reader->NextBatchWithBitmap();
+    prefetch_metrics_->histograms->ObserveHistogram(PrefetchMetrics::READER_READ_LATENCY_US,
+                                                    ElapsedMicros(read_start));
+    PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap read_batch_with_bitmap, std::move(read_result));
 
     return HandleReadResult(reader_idx, read_range, std::move(read_batch_with_bitmap));
 }
@@ -547,6 +730,7 @@ Result<BatchReader::ReadBatchWithBitmap> PrefetchFileBatchReaderImpl::NextBatchW
             std::make_unique<std::thread>(&PrefetchFileBatchReaderImpl::Workloop, this);
     }
 
+    const auto wait_start = std::chrono::steady_clock::now();
     while (true) {
         PAIMON_RETURN_NOT_OK(GetReadStatus());
         if (is_shutdown_) {
@@ -585,6 +769,10 @@ Result<BatchReader::ReadBatchWithBitmap> PrefetchFileBatchReaderImpl::NextBatchW
                     cv_.notify_one();
                 }
                 current_batch_global_row_ids_ = std::move(prefetch_batch.value().global_row_ids);
+                prefetch_metrics_->consumed_batches.fetch_add(1, kMetricsMemoryOrder);
+                prefetch_metrics_->queue_depth.fetch_sub(1, kMetricsMemoryOrder);
+                prefetch_metrics_->histograms->ObserveHistogram(
+                    PrefetchMetrics::CONSUMER_WAIT_LATENCY_US, ElapsedMicros(wait_start));
                 return std::move(prefetch_batch).value().batch;
             }
         }
@@ -595,6 +783,8 @@ Result<BatchReader::ReadBatchWithBitmap> PrefetchFileBatchReaderImpl::NextBatchW
                 return Status::Invalid("peek batch not suppose to be nullptr");
             }
             current_batch_global_row_ids_.clear();
+            prefetch_metrics_->histograms->ObserveHistogram(
+                PrefetchMetrics::CONSUMER_WAIT_LATENCY_US, ElapsedMicros(wait_start));
             return BatchReader::MakeEofBatchWithBitmap();
         }
         if (value_count == prefetch_queues_.size()) {
@@ -620,15 +810,69 @@ Status PrefetchFileBatchReaderImpl::SeekToRow(uint64_t row_number) {
 }
 
 std::shared_ptr<Metrics> PrefetchFileBatchReaderImpl::GetReaderMetrics() const {
-    auto res_metrics = MetricsImpl::CollectReadMetrics(readers_);
+    auto result = std::make_shared<MetricsImpl>();
+    if (need_prefetch_) {
+        result->Merge(MetricsImpl::CollectReadMetrics(readers_));
+    } else if (!readers_.empty()) {
+        result->Merge(readers_[0]->GetReaderMetrics());
+    }
+
+    auto set_prefetch_counter = [&result](const char* name, const std::atomic<uint64_t>& value) {
+        result->SetCounter(name, value.load(kMetricsMemoryOrder));
+    };
+    set_prefetch_counter(PrefetchMetrics::READ_RANGES_TOTAL, prefetch_metrics_->read_ranges_total);
+    set_prefetch_counter(PrefetchMetrics::READ_RANGES_AFTER_BITMAP,
+                         prefetch_metrics_->read_ranges_after_bitmap);
+    set_prefetch_counter(PrefetchMetrics::SEEK_COUNT, prefetch_metrics_->seek_count);
+    set_prefetch_counter(PrefetchMetrics::PRODUCED_BATCHES, prefetch_metrics_->produced_batches);
+    set_prefetch_counter(PrefetchMetrics::CONSUMED_BATCHES, prefetch_metrics_->consumed_batches);
+    set_prefetch_counter(PrefetchMetrics::DISCARDED_BATCHES, prefetch_metrics_->discarded_batches);
+    set_prefetch_counter(PrefetchMetrics::ERRORS, prefetch_metrics_->errors);
+    set_prefetch_counter(PrefetchMetrics::ADAPTIVE_DISABLED_COUNT,
+                         prefetch_metrics_->adaptive_disabled_count);
+    set_prefetch_counter(PrefetchMetrics::QUEUE_FULL_COUNT, prefetch_metrics_->queue_full_count);
+    result->SetGauge(PrefetchMetrics::ENABLED,
+                     prefetch_metrics_->enabled.load(kMetricsMemoryOrder) ? 1.0 : 0.0);
+    result->SetGauge(PrefetchMetrics::PARALLELISM,
+                     prefetch_metrics_->enabled.load(kMetricsMemoryOrder)
+                         ? static_cast<double>(parallel_num_)
+                         : 1.0);
+    result->SetGauge(PrefetchMetrics::QUEUE_DEPTH,
+                     static_cast<double>(prefetch_metrics_->queue_depth.load(kMetricsMemoryOrder)));
+    result->SetGauge(
+        PrefetchMetrics::QUEUE_DEPTH_MAX,
+        static_cast<double>(prefetch_metrics_->queue_depth_max.load(kMetricsMemoryOrder)));
+    result->Merge(prefetch_metrics_->histograms);
     if (cache_) {
-        // The shared read-ahead cache serves reads of all sub-readers, so its
-        // hit/miss counters are file-level and merge into the reader metrics.
+        // PR #209 owns the read-ahead cache metrics. Keep collecting its file-level
+        // hit/miss counters without defining another C++ metrics surface here.
         std::shared_ptr<Metrics> cache_metrics = std::make_shared<MetricsImpl>();
         cache_->CollectMetrics(&cache_metrics);
-        res_metrics->Merge(cache_metrics);
+        result->Merge(cache_metrics);
     }
-    return res_metrics;
+
+    if (!io_metrics_) {
+        return result;
+    }
+    auto set_io_counter = [&result](const char* name, const std::atomic<uint64_t>& value) {
+        result->SetCounter(name, value.load(kMetricsMemoryOrder));
+    };
+    set_io_counter(PrefetchIoMetrics::READ_REQUESTS, io_metrics_->read_requests);
+    set_io_counter(PrefetchIoMetrics::READ_REQUESTED_BYTES, io_metrics_->read_requested_bytes);
+    set_io_counter(PrefetchIoMetrics::READ_PHYSICAL_BYTES, io_metrics_->read_physical_bytes);
+    set_io_counter(PrefetchIoMetrics::READ_FAILED, io_metrics_->read_failed);
+    set_io_counter(PrefetchIoMetrics::READ_LATENCY_COUNT, io_metrics_->read_latency_count);
+    set_io_counter(PrefetchIoMetrics::READ_LATENCY_SUM_US, io_metrics_->read_latency_sum_us);
+    set_io_counter(PrefetchIoMetrics::ASYNC_REQUESTS, io_metrics_->async_requests);
+    set_io_counter(PrefetchIoMetrics::ASYNC_REQUESTED_BYTES, io_metrics_->async_requested_bytes);
+    set_io_counter(PrefetchIoMetrics::ASYNC_PHYSICAL_BYTES, io_metrics_->async_physical_bytes);
+    set_io_counter(PrefetchIoMetrics::ASYNC_COMPLETED, io_metrics_->async_completed);
+    set_io_counter(PrefetchIoMetrics::ASYNC_FAILED, io_metrics_->async_failed);
+    set_io_counter(PrefetchIoMetrics::ASYNC_LATENCY_COUNT, io_metrics_->async_latency_count);
+    set_io_counter(PrefetchIoMetrics::ASYNC_LATENCY_SUM_US, io_metrics_->async_latency_sum_us);
+    result->SetGauge(PrefetchIoMetrics::ASYNC_PENDING,
+                     static_cast<double>(io_metrics_->async_pending.load(kMetricsMemoryOrder)));
+    return result;
 }
 
 Result<std::unique_ptr<::ArrowSchema>> PrefetchFileBatchReaderImpl::GetFileSchema() const {
@@ -662,6 +906,9 @@ Result<uint64_t> PrefetchFileBatchReaderImpl::GetNextRowToRead() const {
 }
 
 void PrefetchFileBatchReaderImpl::SetReadStatus(const Status& status) {
+    if (!status.ok()) {
+        prefetch_metrics_->errors.fetch_add(1, kMetricsMemoryOrder);
+    }
     std::unique_lock<std::shared_mutex> lock(rw_mutex_);
     read_status_ = status;
 }
diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h
index 08d3bd370..c99323698 100644
--- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h
+++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h
@@ -57,6 +57,8 @@ class FileSystem;
 class Executor;
 class Predicate;
 class Metrics;
+struct PrefetchMetricsState;
+struct PrefetchIoMetricsState;
 
 class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader {
  public:
@@ -66,7 +68,7 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader {
         uint32_t prefetch_max_parallel_num, int32_t batch_size, uint32_t prefetch_batch_count,
         bool enable_adaptive_prefetch_strategy, const std::shared_ptr<Executor>& executor,
         bool initialize_read_ranges, bool read_ahead_cache_enabled, const CacheConfig& cache_config,
-        const std::shared_ptr<MemoryPool>& pool);
+        bool enable_io_metrics, const std::shared_ptr<MemoryPool>& pool);
 
     ~PrefetchFileBatchReaderImpl() override;
 
@@ -119,6 +121,7 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader {
         const std::vector<std::shared_ptr<PrefetchFileBatchReader>>& readers, int32_t batch_size,
         uint32_t prefetch_queue_capacity, bool enable_adaptive_prefetch_strategy,
         const std::shared_ptr<Executor>& executor, const std::shared_ptr<ReadAheadCache>& cache,
+        const std::shared_ptr<PrefetchIoMetricsState>& io_metrics,
         const std::shared_ptr<MemoryPool>& pool);
 
     Status CleanUp();
@@ -178,5 +181,7 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader {
     const uint32_t prefetch_queue_capacity_;
     const bool enable_adaptive_prefetch_strategy_;
     int32_t parallel_num_;
+    std::shared_ptr<PrefetchMetricsState> prefetch_metrics_;
+    std::shared_ptr<PrefetchIoMetricsState> io_metrics_;
 };
 }  // namespace paimon
diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp
index 192028ac4..514e7e49d 100644
--- a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp
+++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp
@@ -34,6 +34,7 @@
 #include "paimon/format/format_writer.h"
 #include "paimon/fs/file_system_factory.h"
 #include "paimon/fs/local/local_file_system.h"
+#include "paimon/metrics.h"
 #include "paimon/predicate/predicate_builder.h"
 #include "paimon/testing/mock/mock_file_batch_reader.h"
 #include "paimon/testing/mock/mock_file_system.h"
@@ -113,6 +114,62 @@ class ControlledMockFormatReaderBuilder : public ReaderBuilder {
     mutable std::atomic<size_t> build_count_{0};
 };
 
+class FailingInputStream : public MockInputStream {
+ public:
+    Result<int64_t> Read(char* buffer, int64_t size, int64_t offset) override {
+        return Status::IOError("injected synchronous read failure");
+    }
+};
+
+class FailingFileSystem : public MockFileSystem {
+ public:
+    Result<std::unique_ptr<InputStream>> Open(const std::string& path) const override {
+        return std::make_unique<FailingInputStream>();
+    }
+};
+
+class IoReadingMockFileBatchReader : public MockFileBatchReader {
+ public:
+    IoReadingMockFileBatchReader(const std::shared_ptr<arrow::Array>& data,
+                                 const std::shared_ptr<arrow::DataType>& schema,
+                                 int32_t read_batch_size,
+                                 const std::shared_ptr<InputStream>& input_stream)
+        : MockFileBatchReader(data, schema, read_batch_size), input_stream_(input_stream) {}
+
+    Result<ReadBatchWithBitmap> NextBatchWithBitmap() override {
+        char value = 0;
+        PAIMON_ASSIGN_OR_RAISE(int64_t read_size, input_stream_->Read(&value, 1, 0));
+        (void)read_size;
+        return MockFileBatchReader::NextBatchWithBitmap();
+    }
+
+ private:
+    std::shared_ptr<InputStream> input_stream_;
+};
+
+class IoReadingMockFormatReaderBuilder : public ReaderBuilder {
+ public:
+    IoReadingMockFormatReaderBuilder(const std::shared_ptr<arrow::Array>& data,
+                                     const std::shared_ptr<arrow::DataType>& schema,
+                                     int32_t read_batch_size)
+        : data_(data), schema_(schema), read_batch_size_(read_batch_size) {}
+
+    ReaderBuilder* WithMemoryPool(const std::shared_ptr<MemoryPool>& pool) override {
+        return this;
+    }
+
+    Result<std::unique_ptr<FileBatchReader>> Build(
+        const std::shared_ptr<InputStream>& input_stream) const override {
+        return std::make_unique<IoReadingMockFileBatchReader>(data_, schema_, read_batch_size_,
+                                                              input_stream);
+    }
+
+ private:
+    std::shared_ptr<arrow::Array> data_;
+    std::shared_ptr<arrow::DataType> schema_;
+    int32_t read_batch_size_ = 0;
+};
+
 struct TestParam {
     std::string file_format;
     bool read_ahead_cache_enabled;
@@ -210,7 +267,7 @@ class PrefetchFileBatchReaderImplTest : public ::testing::Test,
                 prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                 /*enable_adaptive_prefetch_strategy=*/false, executor,
                 /*initialize_read_ranges=*/false, read_ahead_cache_enabled, CacheConfig(),
-                GetDefaultPool()));
+                /*enable_io_metrics=*/true, GetDefaultPool()));
         std::unique_ptr<ArrowSchema> c_schema = std::make_unique<ArrowSchema>();
         auto arrow_status = arrow::ExportSchema(*read_schema, c_schema.get());
         EXPECT_TRUE(arrow_status.ok());
@@ -299,7 +356,11 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestSimple) {
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
+        if (prefetch_max_parallel_num == 1) {
+            ASSERT_NOK(
+                reader->GetReaderMetrics()->GetCounter(PrefetchIoMetrics::READ_LATENCY_COUNT));
+        }
         ASSERT_NOK(reader->GetPreviousBatchFileRowId(0));
         ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get()));
 
@@ -322,7 +383,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestReadWithLimits) {
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/true, GetDefaultPool()));
     // simulate read limits, only read 8 batches
     for (int32_t i = 0; i < 8; i++) {
         ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch_with_bitmap,
@@ -337,6 +398,30 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestReadWithLimits) {
     // test metrics
     auto read_metrics = reader->GetReaderMetrics();
     ASSERT_TRUE(read_metrics);
+    ASSERT_OK_AND_ASSIGN(double prefetch_enabled, read_metrics->GetGauge(PrefetchMetrics::ENABLED));
+    ASSERT_OK_AND_ASSIGN(uint64_t produced_batches,
+                         read_metrics->GetCounter(PrefetchMetrics::PRODUCED_BATCHES));
+    ASSERT_OK_AND_ASSIGN(uint64_t consumed_batches,
+                         read_metrics->GetCounter(PrefetchMetrics::CONSUMED_BATCHES));
+    ASSERT_OK_AND_ASSIGN(uint64_t read_ranges,
+                         read_metrics->GetCounter(PrefetchMetrics::READ_RANGES_TOTAL));
+    ASSERT_EQ(prefetch_enabled, 1.0);
+    ASSERT_GT(produced_batches, 0);
+    ASSERT_EQ(consumed_batches, 8);
+    ASSERT_GT(read_ranges, 0);
+    ASSERT_OK(read_metrics->GetGauge(PrefetchMetrics::QUEUE_DEPTH));
+    ASSERT_OK_AND_ASSIGN(uint64_t async_completed,
+                         read_metrics->GetCounter(PrefetchIoMetrics::ASYNC_COMPLETED));
+    ASSERT_OK_AND_ASSIGN(uint64_t async_failed,
+                         read_metrics->GetCounter(PrefetchIoMetrics::ASYNC_FAILED));
+    ASSERT_OK_AND_ASSIGN(uint64_t async_latency_count,
+                         read_metrics->GetCounter(PrefetchIoMetrics::ASYNC_LATENCY_COUNT));
+    ASSERT_EQ(async_latency_count, async_completed + async_failed);
+
+    std::shared_ptr<Metrics> second_metrics = reader->GetReaderMetrics();
+    ASSERT_OK_AND_ASSIGN(uint64_t second_consumed_batches,
+                         second_metrics->GetCounter(PrefetchMetrics::CONSUMED_BATCHES));
+    ASSERT_EQ(second_consumed_batches, consumed_batches);
 }
 
 TEST_F(PrefetchFileBatchReaderImplTest, TestReadWithoutInitializeReadRanges) {
@@ -351,13 +436,43 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestReadWithoutInitializeReadRanges) {
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
     // simulate read limits, only read 8 batches
     ASSERT_NOK_WITH_MSG(reader->NextBatchWithBitmap(),
                         "prefetch reader read ranges are not initialized");
     reader->Close();
 }
 
+TEST_F(PrefetchFileBatchReaderImplTest, TestFailedIoMetrics) {
+    auto data_array = PrepareArray(10);
+    IoReadingMockFormatReaderBuilder reader_builder(data_array, data_type_, /*read_batch_size=*/10);
+    auto failing_fs = std::make_shared<FailingFileSystem>();
+    ASSERT_OK_AND_ASSIGN(
+        auto reader,
+        PrefetchFileBatchReaderImpl::Create(
+            /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, failing_fs,
+            /*prefetch_max_parallel_num=*/1, /*batch_size=*/10,
+            /*prefetch_batch_count=*/2, /*enable_adaptive_prefetch_strategy=*/false, executor_,
+            /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/false, CacheConfig(),
+            /*enable_io_metrics=*/true, GetDefaultPool()));
+
+    ASSERT_NOK_WITH_MSG(reader->NextBatchWithBitmap(), "injected synchronous read failure");
+    std::shared_ptr<Metrics> metrics = reader->GetReaderMetrics();
+    ASSERT_OK_AND_ASSIGN(uint64_t read_requests,
+                         metrics->GetCounter(PrefetchIoMetrics::READ_REQUESTS));
+    ASSERT_OK_AND_ASSIGN(uint64_t failed_requests,
+                         metrics->GetCounter(PrefetchIoMetrics::READ_FAILED));
+    ASSERT_OK_AND_ASSIGN(uint64_t physical_bytes,
+                         metrics->GetCounter(PrefetchIoMetrics::READ_PHYSICAL_BYTES));
+    ASSERT_EQ(read_requests, 1);
+    ASSERT_EQ(failed_requests, 1);
+    ASSERT_EQ(physical_bytes, 0);
+    ASSERT_OK_AND_ASSIGN(uint64_t latency_count,
+                         metrics->GetCounter(PrefetchIoMetrics::READ_LATENCY_COUNT));
+    ASSERT_OK(metrics->GetCounter(PrefetchIoMetrics::READ_LATENCY_SUM_US));
+    ASSERT_EQ(latency_count, read_requests);
+}
+
 TEST_F(PrefetchFileBatchReaderImplTest, FilterReadRangesWithoutBitmap) {
     std::vector<std::pair<uint64_t, uint64_t>> read_ranges = {
         {0, 1000},    {1000, 2000}, {2000, 3000}, {3000, 4000}, {4000, 5000},
@@ -427,7 +542,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, RefreshReadRanges) {
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     ASSERT_OK(prefetch_reader->RefreshReadRanges());
     std::vector<std::pair<uint64_t, uint64_t>> read_ranges_0 = {{0, 30}, {90, 101}};
@@ -457,9 +572,17 @@ TEST_F(PrefetchFileBatchReaderImplTest, RefreshReadRangesDisablePrefetchByAdapti
                              /*prefetch_batch_count=*/2,
                              /*enable_adaptive_prefetch_strategy=*/true, executor_,
                              /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
 
     ASSERT_FALSE(reader->NeedPrefetch());
+    std::shared_ptr<Metrics> metrics = reader->GetReaderMetrics();
+    ASSERT_OK_AND_ASSIGN(double enabled, metrics->GetGauge(PrefetchMetrics::ENABLED));
+    ASSERT_OK_AND_ASSIGN(double parallelism, metrics->GetGauge(PrefetchMetrics::PARALLELISM));
+    ASSERT_OK_AND_ASSIGN(uint64_t adaptive_disabled_count,
+                         metrics->GetCounter(PrefetchMetrics::ADAPTIVE_DISABLED_COUNT));
+    ASSERT_EQ(enabled, 0.0);
+    ASSERT_EQ(parallelism, 1.0);
+    ASSERT_EQ(adaptive_disabled_count, 1);
 }
 
 TEST_F(PrefetchFileBatchReaderImplTest, SetReadRanges) {
@@ -473,7 +596,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, SetReadRanges) {
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     ASSERT_FALSE(prefetch_reader->need_prefetch_);
     prefetch_reader->need_prefetch_ = true;
@@ -516,7 +639,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, SetReadRangesReturnErrorWhenPushDownFail
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
 
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     prefetch_reader->need_prefetch_ = true;
@@ -542,7 +665,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, WorkloopSetReadStatusWhenCacheInitFailed
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true,
-                             invalid_cache_config, GetDefaultPool()));
+                             invalid_cache_config, /*enable_io_metrics=*/false, GetDefaultPool()));
 
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     prefetch_reader->Workloop();
@@ -562,7 +685,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, DoReadBatchReturnOkWhenShutdown) {
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
 
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     prefetch_reader->is_shutdown_ = true;
@@ -580,7 +703,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, DoReadBatchReturnOkWhenNoCurrentReadRang
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
 
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     prefetch_reader->read_ranges_in_group_ = {{}};
@@ -598,7 +721,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestReadWithLargeBatchSize) {
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
     ASSERT_NOK(reader->GetPreviousBatchFileRowId(0));
     ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get()));
     auto row_ids = array_and_row_ids.second;
@@ -618,7 +741,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPartialReaderSuccessRead) {
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     for (int32_t i = 0; i < prefetch_max_parallel_num; i++) {
         dynamic_cast<MockFileBatchReader*>(prefetch_reader->readers_[i].get())
@@ -663,7 +786,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestAllReaderFailedWithIOError) {
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
 
     auto prefetch_reader = dynamic_cast<PrefetchFileBatchReaderImpl*>(reader.get());
     for (int32_t i = 0; i < prefetch_max_parallel_num; i++) {
@@ -679,6 +802,9 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestAllReaderFailedWithIOError) {
     ASSERT_FALSE(prefetch_reader->is_shutdown_);
     ASSERT_NOK(prefetch_reader->GetReadStatus());
     ASSERT_FALSE(HasValue(prefetch_reader->prefetch_queues_));
+    std::shared_ptr<Metrics> metrics = reader->GetReaderMetrics();
+    ASSERT_OK_AND_ASSIGN(uint64_t prefetch_errors, metrics->GetCounter(PrefetchMetrics::ERRORS));
+    ASSERT_GT(prefetch_errors, 0);
 
     // call NextBatch again, will still return error status
     auto batch_result2 = reader->NextBatchWithBitmap();
@@ -698,7 +824,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPrefetchWithEmptyData) {
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
     ASSERT_NOK(reader->GetPreviousBatchFileRowId(0));
     ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get()));
     auto row_ids = array_and_row_ids.second;
@@ -717,7 +843,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestCallNextBatchAfterReadingEof) {
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
     ASSERT_NOK(reader->GetPreviousBatchFileRowId(0));
     ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get()));
     auto row_ids = array_and_row_ids.second;
@@ -725,10 +851,28 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestCallNextBatchAfterReadingEof) {
     auto expected_array = std::make_shared<arrow::ChunkedArray>(data_array);
     ASSERT_TRUE(array_and_row_ids.first->Equals(expected_array));
 
+    std::shared_ptr<Metrics> eof_metrics = reader->GetReaderMetrics();
+    ASSERT_OK_AND_ASSIGN(uint64_t produced_batches,
+                         eof_metrics->GetCounter(PrefetchMetrics::PRODUCED_BATCHES));
+    ASSERT_OK_AND_ASSIGN(uint64_t consumed_batches,
+                         eof_metrics->GetCounter(PrefetchMetrics::CONSUMED_BATCHES));
+    ASSERT_OK_AND_ASSIGN(uint64_t discarded_batches,
+                         eof_metrics->GetCounter(PrefetchMetrics::DISCARDED_BATCHES));
+
     // continue to call NextBatch() after reading eof
     ASSERT_OK_AND_ASSIGN(auto batch_with_bitmap, reader->NextBatchWithBitmap());
     ASSERT_NOK(reader->GetPreviousBatchFileRowId(0));
     ASSERT_TRUE(BatchReader::IsEofBatch(batch_with_bitmap));
+    std::shared_ptr<Metrics> repeated_eof_metrics = reader->GetReaderMetrics();
+    ASSERT_OK_AND_ASSIGN(uint64_t produced_batches_after,
+                         repeated_eof_metrics->GetCounter(PrefetchMetrics::PRODUCED_BATCHES));
+    ASSERT_OK_AND_ASSIGN(uint64_t consumed_batches_after,
+                         repeated_eof_metrics->GetCounter(PrefetchMetrics::CONSUMED_BATCHES));
+    ASSERT_OK_AND_ASSIGN(uint64_t discarded_batches_after,
+                         repeated_eof_metrics->GetCounter(PrefetchMetrics::DISCARDED_BATCHES));
+    ASSERT_EQ(produced_batches_after, produced_batches);
+    ASSERT_EQ(consumed_batches_after, consumed_batches);
+    ASSERT_EQ(discarded_batches_after, discarded_batches);
 }
 
 TEST_F(PrefetchFileBatchReaderImplTest, TestCreateReaderWithoutNextBatch) {
@@ -742,7 +886,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestCreateReaderWithoutNextBatch) {
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
 }
 
 TEST_F(PrefetchFileBatchReaderImplTest, TestInvalidCase) {
@@ -757,7 +901,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestInvalidCase) {
             /*prefetch_max_parallel_num=*/0, batch_size, 2,
             /*enable_adaptive_prefetch_strategy=*/false, executor_,
             /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true, CacheConfig(),
-            GetDefaultPool()));
+            /*enable_io_metrics=*/false, GetDefaultPool()));
     }
     {
         ASSERT_NOK(PrefetchFileBatchReaderImpl::Create(
@@ -765,7 +909,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestInvalidCase) {
             prefetch_max_parallel_num, /*batch_size=*/-1, prefetch_max_parallel_num * 2,
             /*enable_adaptive_prefetch_strategy=*/false, executor_,
             /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true, CacheConfig(),
-            GetDefaultPool()));
+            /*enable_io_metrics=*/false, GetDefaultPool()));
     }
     {
         ASSERT_NOK(PrefetchFileBatchReaderImpl::Create(
@@ -773,7 +917,8 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestInvalidCase) {
             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
             /*enable_adaptive_prefetch_strategy=*/false,
             /*executor=*/nullptr, /*initialize_read_ranges=*/true,
-            /*read_ahead_cache_enabled=*/true, CacheConfig(), GetDefaultPool()));
+            /*read_ahead_cache_enabled=*/true, CacheConfig(), /*enable_io_metrics=*/false,
+            GetDefaultPool()));
     }
     {
         ASSERT_NOK(PrefetchFileBatchReaderImpl::Create(
@@ -781,7 +926,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestInvalidCase) {
             prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
             /*enable_adaptive_prefetch_strategy=*/false, executor_,
             /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true, CacheConfig(),
-            GetDefaultPool()));
+            /*enable_io_metrics=*/false, GetDefaultPool()));
     }
     {
         ASSERT_NOK(PrefetchFileBatchReaderImpl::Create(
@@ -789,7 +934,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestInvalidCase) {
             /*fs=*/nullptr, prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
             /*enable_adaptive_prefetch_strategy=*/false, executor_,
             /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true, CacheConfig(),
-            GetDefaultPool()));
+            /*enable_io_metrics=*/false, GetDefaultPool()));
     }
     {
         ASSERT_OK_AND_ASSIGN(
@@ -798,7 +943,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestInvalidCase) {
                              prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2,
                              /*enable_adaptive_prefetch_strategy=*/false, executor_,
                              /*initialize_read_ranges=*/true, /*read_ahead_cache_enabled=*/true,
-                             CacheConfig(), GetDefaultPool()));
+                             CacheConfig(), /*enable_io_metrics=*/false, GetDefaultPool()));
         ASSERT_NOK_WITH_MSG(reader->SeekToRow(/*row_number=*/101),
                             "not support seek to row for prefetch reader");
     }
@@ -834,6 +979,18 @@ TEST_P(PrefetchFileBatchReaderImplTest, TestPrefetchWithPredicatePushdownWithCom
     auto expected_array = std::make_shared<arrow::ChunkedArray>(expected_array_vector);
     ASSERT_TRUE(expected_array->Equals(array_and_row_ids.first));
     ASSERT_EQ(expected_row_ids, array_and_row_ids.second);
+
+    std::shared_ptr<Metrics> metrics = reader->GetReaderMetrics();
+    ASSERT_OK_AND_ASSIGN(uint64_t sync_requests,
+                         metrics->GetCounter(PrefetchIoMetrics::READ_REQUESTS));
+    ASSERT_OK_AND_ASSIGN(uint64_t async_requests,
+                         metrics->GetCounter(PrefetchIoMetrics::ASYNC_REQUESTS));
+    ASSERT_OK_AND_ASSIGN(uint64_t sync_physical_bytes,
+                         metrics->GetCounter(PrefetchIoMetrics::READ_PHYSICAL_BYTES));
+    ASSERT_OK_AND_ASSIGN(uint64_t async_physical_bytes,
+                         metrics->GetCounter(PrefetchIoMetrics::ASYNC_PHYSICAL_BYTES));
+    ASSERT_GT(sync_requests + async_requests, 0);
+    ASSERT_GT(sync_physical_bytes + async_physical_bytes, 0);
 }
 
 /// There are three stripes: [0,30), [30,60), [60,90). Each stripe has 3 row groups.
@@ -883,14 +1040,14 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPrefetchWithBitmap) {
     MockFormatReaderBuilder reader_builder(data_array, data_type_, bitmap,
                                            /*read_batch_size=*/100);
     int32_t prefetch_max_parallel_num = 3;
-    ASSERT_OK_AND_ASSIGN(auto reader,
-                         PrefetchFileBatchReaderImpl::Create(
-                             /*data_file_path=*/"", /*data_file_size=*/0, &reader_builder, mock_fs_,
-                             prefetch_max_parallel_num,
-                             /*batch_size=*/100, prefetch_max_parallel_num * 2,
-                             /*enable_adaptive_prefetch_strategy=*/false, executor_,
-                             /*initialize_read_ranges=*/true,
-                             /*read_ahead_cache_enabled=*/true, CacheConfig(), GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto reader, PrefetchFileBatchReaderImpl::Create(
+                                          /*data_file_path=*/"", /*data_file_size=*/0,
+                                          &reader_builder, mock_fs_, prefetch_max_parallel_num,
+                                          /*batch_size=*/100, prefetch_max_parallel_num * 2,
+                                          /*enable_adaptive_prefetch_strategy=*/false, executor_,
+                                          /*initialize_read_ranges=*/true,
+                                          /*read_ahead_cache_enabled=*/true, CacheConfig(),
+                                          /*enable_io_metrics=*/false, GetDefaultPool()));
     ASSERT_OK_AND_ASSIGN(auto result_chunk_array, ReadResultCollector::CollectResult(reader.get()));
 
     ASSERT_OK_AND_ASSIGN(auto data_batch, ReadResultCollector::GetReadBatch(data_array));
@@ -934,12 +1091,28 @@ TEST_P(PrefetchFileBatchReaderImplTest, TestRowMapping) {
         ASSERT_EQ(reader->GetPreviousBatchFileRowId(i).value(), 70 + i);
     }
 
+    std::shared_ptr<Metrics> metrics_before_schema = reader->GetReaderMetrics();
+    ASSERT_OK_AND_ASSIGN(uint64_t consumed_before_schema,
+                         metrics_before_schema->GetCounter(PrefetchMetrics::CONSUMED_BATCHES));
+    ASSERT_OK_AND_ASSIGN(uint64_t ranges_before_schema,
+                         metrics_before_schema->GetCounter(PrefetchMetrics::READ_RANGES_TOTAL));
+
     // Set read schema again
     std::unique_ptr<ArrowSchema> c_schema = std::make_unique<ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok());
     predicate = PredicateBuilder::Between(/*field_index=*/1, /*field_name=*/"f1", FieldType::BIGINT,
                                           Literal(30l), Literal(49l));
     ASSERT_OK(reader->SetReadSchema(c_schema.get(), predicate, std::nullopt));
+    std::shared_ptr<Metrics> metrics_after_schema = reader->GetReaderMetrics();
+    ASSERT_OK_AND_ASSIGN(uint64_t consumed_after_schema,
+                         metrics_after_schema->GetCounter(PrefetchMetrics::CONSUMED_BATCHES));
+    ASSERT_OK_AND_ASSIGN(uint64_t ranges_after_schema,
+                         metrics_after_schema->GetCounter(PrefetchMetrics::READ_RANGES_TOTAL));
+    ASSERT_OK_AND_ASSIGN(double queue_depth_after_schema,
+                         metrics_after_schema->GetGauge(PrefetchMetrics::QUEUE_DEPTH));
+    ASSERT_EQ(consumed_after_schema, consumed_before_schema);
+    ASSERT_GT(ranges_after_schema, ranges_before_schema);
+    ASSERT_EQ(queue_depth_after_schema, 0.0);
 
     ASSERT_NOK(reader->GetPreviousBatchFileRowId(0));
     ASSERT_OK_AND_ASSIGN(batch,
diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp
index a5b32b09a..dfa0aa7ff 100644
--- a/src/paimon/core/core_options.cpp
+++ b/src/paimon/core/core_options.cpp
@@ -421,6 +421,7 @@ struct CoreOptions::Impl {
     bool key_value_sequence_number_enabled = false;
     bool file_index_read_enabled = true;
     bool enable_adaptive_prefetch_strategy = true;
+    bool prefetch_io_metrics_enabled = false;
     bool index_file_in_data_file_dir = false;
     bool row_tracking_enabled = false;
     bool row_tracking_partition_group_on_commit = true;
@@ -779,6 +780,8 @@ struct CoreOptions::Impl {
         }
         PAIMON_RETURN_NOT_OK(parser.Parse<bool>(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED,
                                                 &scan_manifest_entry_lazy_decode_enabled));
+        PAIMON_RETURN_NOT_OK(
+            parser.Parse<bool>(Options::PREFETCH_IO_METRICS_ENABLED, &prefetch_io_metrics_enabled));
         // Parse scan.fallback-branch - fallback branch when partition not found
         PAIMON_RETURN_NOT_OK(parser.Parse(Options::SCAN_FALLBACK_BRANCH, &scan_fallback_branch));
         // Parse branch - branch name, default "main"
@@ -1376,6 +1379,10 @@ bool CoreOptions::EnableAdaptivePrefetchStrategy() const {
     return impl_->enable_adaptive_prefetch_strategy;
 }
 
+bool CoreOptions::PrefetchIoMetricsEnabled() const {
+    return impl_->prefetch_io_metrics_enabled;
+}
+
 Result<std::optional<std::string>> CoreOptions::GetFieldAggFunc(
     const std::string& field_name) const {
     ConfigParser parser(impl_->raw_options);
diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h
index 85a4a7fdf..f4e7964f7 100644
--- a/src/paimon/core/core_options.h
+++ b/src/paimon/core/core_options.h
@@ -230,6 +230,8 @@ class PAIMON_EXPORT CoreOptions {
 
     std::string DataFilePrefix() const;
 
+    bool PrefetchIoMetricsEnabled() const;
+
     bool IndexFileInDataFileDir() const;
 
     bool RowTrackingEnabled() const;
diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp
index c110623d7..b0a3274ba 100644
--- a/src/paimon/core/core_options_test.cpp
+++ b/src/paimon/core/core_options_test.cpp
@@ -142,6 +142,7 @@ TEST(CoreOptionsTest, TestDefaultValue) {
     ASSERT_EQ(std::nullopt, core_options.GetDataFileExternalPaths());
     ASSERT_EQ(ExternalPathStrategy::NONE, core_options.GetExternalPathStrategy());
     ASSERT_TRUE(core_options.EnableAdaptivePrefetchStrategy());
+    ASSERT_FALSE(core_options.PrefetchIoMetricsEnabled());
     ASSERT_EQ(core_options.DataFilePrefix(), "data-");
     ASSERT_FALSE(core_options.IndexFileInDataFileDir());
     ASSERT_FALSE(core_options.RowTrackingEnabled());
@@ -223,6 +224,7 @@ TEST(CoreOptionsTest, TestFromMap) {
         {Options::SCAN_MODE, "from-snapshot-full"},
         {Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "7"},
         {Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, "false"},
+        {Options::PREFETCH_IO_METRICS_ENABLED, "true"},
         {Options::SNAPSHOT_NUM_RETAINED_MIN, "15"},
         {Options::SNAPSHOT_NUM_RETAINED_MAX, "30"},
         {Options::SNAPSHOT_EXPIRE_LIMIT, "20"},
@@ -362,6 +364,7 @@ TEST(CoreOptionsTest, TestFromMap) {
     ASSERT_EQ(5, core_options.GetScanSnapshotId().value_or(-1));
     ASSERT_EQ(7, core_options.GetScanManifestEntryCacheMaxSnapshots());
     ASSERT_FALSE(core_options.ScanManifestEntryLazyDecodeEnabled());
+    ASSERT_TRUE(core_options.PrefetchIoMetricsEnabled());
     ExpireConfig expire_config = core_options.GetExpireConfig();
     ASSERT_EQ(15, expire_config.GetSnapshotRetainMin());
     ASSERT_EQ(30, expire_config.GetSnapshotRetainMax());
diff --git a/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp b/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp
index 5451c9d29..6c0d23be0 100644
--- a/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp
+++ b/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp
@@ -87,7 +87,8 @@ class ApplyDeletionVectorBatchReaderTest : public ::testing::Test,
                         prefetch_batch_count, batch_size, prefetch_batch_count * 2,
                         /*enable_adaptive_prefetch_strategy=*/false, executor_,
                         /*initialize_read_ranges=*/true,
-                        /*read_ahead_cache_enabled=*/true, CacheConfig(), pool_));
+                        /*read_ahead_cache_enabled=*/true, CacheConfig(),
+                        /*enable_io_metrics=*/false, pool_));
             } else {
                 file_batch_reader =
                     std::make_unique<MockFileBatchReader>(data, target_type_, batch_size);
diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp
index 91157f71a..5beee34ba 100644
--- a/src/paimon/core/operation/abstract_split_read.cpp
+++ b/src/paimon/core/operation/abstract_split_read.cpp
@@ -168,7 +168,7 @@ Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::CreateFileBatchReade
                 context_->GetPrefetchBatchCount(), options_.EnableAdaptivePrefetchStrategy(),
                 executor_,
                 /*initialize_read_ranges=*/false, context_->ReadAheadCacheEnabled(),
-                context_->GetCacheConfig(), pool_));
+                context_->GetCacheConfig(), options_.PrefetchIoMetricsEnabled(), pool_));
         return std::make_unique<DelegatingPrefetchReader>(std::move(prefetch_reader));
     } else {
         PAIMON_ASSIGN_OR_RAISE(
diff --git a/src/paimon/core/operation/append_only_file_store_scan_test.cpp b/src/paimon/core/operation/append_only_file_store_scan_test.cpp
index e1fb5a43a..d72dcaf0d 100644
--- a/src/paimon/core/operation/append_only_file_store_scan_test.cpp
+++ b/src/paimon/core/operation/append_only_file_store_scan_test.cpp
@@ -31,7 +31,6 @@
 #include "paimon/common/io/cache/lru_cache.h"
 #include "paimon/core/manifest/manifest_entry.h"
 #include "paimon/core/manifest/partition_entry.h"
-#include "paimon/core/operation/metrics/scan_metrics.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/stats/simple_stats_evolution.h"
@@ -45,6 +44,7 @@
 #include "paimon/predicate/predicate_builder.h"
 #include "paimon/scan_context.h"
 #include "paimon/status.h"
+#include "paimon/table/source/scan_metrics.h"
 #include "paimon/table/source/table_scan.h"
 #include "paimon/testing/utils/testharness.h"
 #include "paimon/testing/utils/timezone_guard.h"
@@ -173,7 +173,20 @@ TEST(AppendOnlyFileStoreScanTest, TestScanDurationMetric) {
                          metrics->GetCounter(ScanMetrics::LAST_SCAN_DURATION));
     ASSERT_OK_AND_ASSIGN(HistogramStats stats,
                          metrics->GetHistogramStats(ScanMetrics::SCAN_DURATION));
+    ASSERT_OK_AND_ASSIGN(uint64_t manifest_read_duration,
+                         metrics->GetCounter(ScanMetrics::LAST_MANIFEST_READ_DURATION));
+    ASSERT_OK_AND_ASSIGN(HistogramStats manifest_stats,
+                         metrics->GetHistogramStats(ScanMetrics::MANIFEST_READ_DURATION));
+    ASSERT_OK_AND_ASSIGN(uint64_t scanned_rows,
+                         metrics->GetCounter(ScanMetrics::LAST_LAZY_DECODE_SCANNED_ROWS));
+    ASSERT_OK_AND_ASSIGN(uint64_t materialized_rows,
+                         metrics->GetCounter(ScanMetrics::LAST_LAZY_DECODE_MATERIALIZED_ROWS));
     ASSERT_EQ(stats.count, kPlanCount);
+    ASSERT_EQ(manifest_stats.count, kPlanCount);
+    ASSERT_LE(manifest_stats.min, static_cast<double>(manifest_read_duration));
+    ASSERT_LE(static_cast<double>(manifest_read_duration), manifest_stats.max);
+    ASSERT_GT(scanned_rows, 0);
+    ASSERT_GE(scanned_rows, materialized_rows);
     ASSERT_LE(stats.min, stats.max);
     ASSERT_LE(stats.min, static_cast<double>(last_scan_duration));
     ASSERT_LE(static_cast<double>(last_scan_duration), stats.max);
@@ -268,12 +281,40 @@ TEST(AppendOnlyFileStoreScanTest, TestSnapshotLiveManifestCachePath) {
     scan_first->WithSnapshot(snapshot_5);
     ASSERT_OK_AND_ASSIGN(auto plan_first, scan_first->CreatePlan());
     std::vector<std::string> first_file_names = SortedFileNames(plan_first->Files());
+    std::shared_ptr<Metrics> first_metrics = scan_first->GetScanMetrics();
+    ASSERT_OK_AND_ASSIGN(uint64_t first_cache_enabled,
+                         first_metrics->GetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED));
+    ASSERT_OK_AND_ASSIGN(uint64_t first_cache_hit,
+                         first_metrics->GetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_HIT));
+    ASSERT_OK_AND_ASSIGN(uint64_t first_cache_misses,
+                         first_metrics->GetCounter(ScanMetrics::SNAPSHOT_CACHE_MISSES));
+    ASSERT_OK(first_metrics->GetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_LOAD_DURATION));
+    ASSERT_OK(first_metrics->GetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_STORE_DURATION));
+    ASSERT_OK(first_metrics->GetHistogramStats(ScanMetrics::SNAPSHOT_CACHE_LOAD_DURATION));
+    ASSERT_OK(first_metrics->GetHistogramStats(ScanMetrics::SNAPSHOT_CACHE_STORE_DURATION));
+    ASSERT_EQ(first_cache_enabled, 1);
+    ASSERT_EQ(first_cache_hit, 0);
+    ASSERT_EQ(first_cache_misses, 1);
 
     // Second scan on the same snapshot should read the same bucket live entries from cache.
     auto scan_second = BuildScan(table_path, cache, /*bucket=*/0);
     scan_second->WithSnapshot(snapshot_5);
     ASSERT_OK_AND_ASSIGN(auto plan_second, scan_second->CreatePlan());
     ASSERT_EQ(first_file_names, SortedFileNames(plan_second->Files()));
+    std::shared_ptr<Metrics> second_metrics = scan_second->GetScanMetrics();
+    ASSERT_OK_AND_ASSIGN(uint64_t second_cache_hit,
+                         second_metrics->GetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_HIT));
+    ASSERT_OK_AND_ASSIGN(uint64_t second_cache_hits,
+                         second_metrics->GetCounter(ScanMetrics::SNAPSHOT_CACHE_HITS));
+    ASSERT_OK_AND_ASSIGN(uint64_t scanned_rows,
+                         second_metrics->GetCounter(ScanMetrics::LAST_LAZY_DECODE_SCANNED_ROWS));
+    ASSERT_OK_AND_ASSIGN(
+        uint64_t materialized_rows,
+        second_metrics->GetCounter(ScanMetrics::LAST_LAZY_DECODE_MATERIALIZED_ROWS));
+    ASSERT_EQ(second_cache_hit, 1);
+    ASSERT_EQ(second_cache_hits, 1);
+    ASSERT_GE(scanned_rows, materialized_rows);
+    ASSERT_OK(second_metrics->GetHistogramStats(ScanMetrics::SNAPSHOT_CACHE_LOAD_DURATION));
 }
 
 TEST(AppendOnlyFileStoreScanTest, TestSnapshotLiveManifestCacheRebuildOnMiss) {
diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp
index f21b0bb7f..467ba6b27 100644
--- a/src/paimon/core/operation/file_store_scan.cpp
+++ b/src/paimon/core/operation/file_store_scan.cpp
@@ -43,7 +43,6 @@
 #include "paimon/core/manifest/manifest_file_meta.h"
 #include "paimon/core/manifest/manifest_list.h"
 #include "paimon/core/manifest/snapshot_live_manifest_entries.h"
-#include "paimon/core/operation/metrics/scan_metrics.h"
 #include "paimon/core/partition/partition_info.h"
 #include "paimon/core/stats/simple_stats.h"
 #include "paimon/core/stats/simple_stats_evolution.h"
@@ -58,6 +57,7 @@
 #include "paimon/predicate/predicate_builder.h"
 #include "paimon/predicate/predicate_utils.h"
 #include "paimon/scan_context.h"
+#include "paimon/table/source/scan_metrics.h"
 
 namespace paimon {
 enum class FieldType;
@@ -136,6 +136,9 @@ Result<std::vector<PartitionEntry>> FileStoreScan::ReadPartitionEntries() const
 
 Result<std::shared_ptr<FileStoreScan::RawPlan>> FileStoreScan::CreatePlan() const {
     Duration duration;
+    Duration manifest_read_duration;
+    metrics_->SetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_LOAD_DURATION, 0);
+    metrics_->SetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_STORE_DURATION, 0);
     std::optional<Snapshot> snapshot;
     std::vector<ManifestFileMeta> all_manifest_file_metas;
     std::vector<ManifestFileMeta> filtered_manifest_file_metas;
@@ -148,9 +151,13 @@ Result<std::shared_ptr<FileStoreScan::RawPlan>> FileStoreScan::CreatePlan() cons
         core_options_.GetScanManifestEntryCacheMaxSnapshots() > 0 &&
         core_options_.GetCache() != nullptr && !table_path_.empty() &&
         !row_range_index_.has_value() && bucket_filter_.has_value();
+    uint64_t lazy_decode_scanned_rows = 0;
+    bool snapshot_cache_hit = false;
     if (use_snapshot_live_manifest_cache) {
-        PAIMON_RETURN_NOT_OK(ReadManifestEntriesWithCache(
-            snapshot.value(), all_manifest_file_metas, bucket_filter_.value(), &manifest_entries));
+        PAIMON_RETURN_NOT_OK(ReadManifestEntriesWithCache(snapshot.value(), all_manifest_file_metas,
+                                                          bucket_filter_.value(), &manifest_entries,
+                                                          &snapshot_cache_hit));
+        lazy_decode_scanned_rows = manifest_entries.size();
         std::vector<ManifestEntry> filtered_entries;
         filtered_entries.reserve(manifest_entries.size());
         for (auto& entry : manifest_entries) {
@@ -161,8 +168,15 @@ Result<std::shared_ptr<FileStoreScan::RawPlan>> FileStoreScan::CreatePlan() cons
         }
         manifest_entries = std::move(filtered_entries);
     } else {
+        lazy_decode_scanned_rows = std::accumulate(
+            filtered_manifest_file_metas.begin(), filtered_manifest_file_metas.end(), uint64_t{0},
+            [](uint64_t sum, const ManifestFileMeta& meta) {
+                return sum + static_cast<uint64_t>(meta.NumAddedFiles() + meta.NumDeletedFiles());
+            });
         PAIMON_RETURN_NOT_OK(ReadManifestEntries(filtered_manifest_file_metas, &manifest_entries));
     }
+    const uint64_t lazy_decode_materialized_rows = manifest_entries.size();
+    const uint64_t manifest_read_duration_ms = manifest_read_duration.Get();
     PAIMON_ASSIGN_OR_RAISE(manifest_entries,
                            PostFilterManifestEntries(std::move(manifest_entries)));
 
@@ -208,6 +222,22 @@ Result<std::shared_ptr<FileStoreScan::RawPlan>> FileStoreScan::CreatePlan() cons
         ScanMetrics::LAST_SCAN_SKIPPED_TABLE_FILES,
         std::max(int64_t{0}, all_data_files - static_cast<int64_t>(manifest_entries.size())));
     metrics_->SetCounter(ScanMetrics::LAST_SCAN_RESULTED_TABLE_FILES, manifest_entries.size());
+    metrics_->SetCounter(ScanMetrics::LAST_MANIFEST_READ_DURATION, manifest_read_duration_ms);
+    metrics_->ObserveHistogram(ScanMetrics::MANIFEST_READ_DURATION,
+                               static_cast<double>(manifest_read_duration_ms));
+    metrics_->SetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED,
+                         use_snapshot_live_manifest_cache ? 1 : 0);
+    metrics_->SetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_HIT, snapshot_cache_hit ? 1 : 0);
+    Result<uint64_t> cache_hits = metrics_->GetCounter(ScanMetrics::SNAPSHOT_CACHE_HITS);
+    Result<uint64_t> cache_misses = metrics_->GetCounter(ScanMetrics::SNAPSHOT_CACHE_MISSES);
+    metrics_->SetCounter(ScanMetrics::SNAPSHOT_CACHE_HITS,
+                         (cache_hits.ok() ? cache_hits.value() : 0) + (snapshot_cache_hit ? 1 : 0));
+    metrics_->SetCounter(ScanMetrics::SNAPSHOT_CACHE_MISSES,
+                         (cache_misses.ok() ? cache_misses.value() : 0) +
+                             (use_snapshot_live_manifest_cache && !snapshot_cache_hit ? 1 : 0));
+    metrics_->SetCounter(ScanMetrics::LAST_LAZY_DECODE_SCANNED_ROWS, lazy_decode_scanned_rows);
+    metrics_->SetCounter(ScanMetrics::LAST_LAZY_DECODE_MATERIALIZED_ROWS,
+                         lazy_decode_materialized_rows);
     return std::make_shared<FileStoreScan::RawPlan>(scan_mode_, snapshot,
                                                     std::move(manifest_entries));
 }
@@ -298,15 +328,22 @@ Status FileStoreScan::ReadManifestEntries(const std::vector<ManifestFileMeta>& m
 // snapshot's data manifests.
 Status FileStoreScan::ReadManifestEntriesWithCache(
     const Snapshot& snapshot, const std::vector<ManifestFileMeta>& all_manifest_metas,
-    int32_t bucket, std::vector<ManifestEntry>* manifest_entries) const {
+    int32_t bucket, std::vector<ManifestEntry>* manifest_entries, bool* cache_hit) const {
+    Duration cache_load_duration;
     PAIMON_ASSIGN_OR_RAISE(SnapshotLiveManifestEntries cached_entries,
                            LoadSnapshotLiveManifestEntries(bucket));
+    const uint64_t cache_load_duration_ms = cache_load_duration.Get();
+    metrics_->SetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_LOAD_DURATION, cache_load_duration_ms);
+    metrics_->ObserveHistogram(ScanMetrics::SNAPSHOT_CACHE_LOAD_DURATION,
+                               static_cast<double>(cache_load_duration_ms));
     std::optional<SnapshotLiveManifestEntries::Entry> cached =
         cached_entries.LatestBeforeOrEqual(snapshot.Id());
     if (cached && cached->snapshot_id == snapshot.Id()) {
+        *cache_hit = true;
         *manifest_entries = *cached->entries;
         return Status::OK();
     }
+    *cache_hit = false;
 
     // Rebuild the target snapshot bucket from all manifests and write the live entries back to the
     // cache.
@@ -320,7 +357,12 @@ Status FileStoreScan::ReadManifestEntriesWithCache(
         ReadAndMergeBucketFileEntries(bucket_manifest_metas, bucket, manifest_entries));
     std::vector<ManifestEntry> cache_entries = *manifest_entries;
     cached_entries.Put(snapshot.Id(), std::move(cache_entries));
+    Duration cache_store_duration;
     PAIMON_RETURN_NOT_OK(StoreSnapshotLiveManifestEntries(bucket, cached_entries));
+    const uint64_t cache_store_duration_ms = cache_store_duration.Get();
+    metrics_->SetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_STORE_DURATION, cache_store_duration_ms);
+    metrics_->ObserveHistogram(ScanMetrics::SNAPSHOT_CACHE_STORE_DURATION,
+                               static_cast<double>(cache_store_duration_ms));
     return Status::OK();
 }
 
diff --git a/src/paimon/core/operation/file_store_scan.h b/src/paimon/core/operation/file_store_scan.h
index 155f36e74..ff53e0c9e 100644
--- a/src/paimon/core/operation/file_store_scan.h
+++ b/src/paimon/core/operation/file_store_scan.h
@@ -161,7 +161,9 @@ class FileStoreScan {
     }
 
     std::shared_ptr<Metrics> GetScanMetrics() const {
-        return metrics_;
+        auto snapshot = std::make_shared<MetricsImpl>();
+        snapshot->Overwrite(metrics_);
+        return snapshot;
     }
 
     static Result<std::shared_ptr<PredicateFilter>> CreatePartitionPredicate(
@@ -260,7 +262,8 @@ class FileStoreScan {
     Status ReadManifestEntriesWithCache(const Snapshot& snapshot,
                                         const std::vector<ManifestFileMeta>& bucket_manifest_metas,
                                         int32_t bucket,
-                                        std::vector<ManifestEntry>* manifest_entries) const;
+                                        std::vector<ManifestEntry>* manifest_entries,
+                                        bool* cache_hit) const;
     std::shared_ptr<CacheKey> SnapshotLiveManifestEntriesCacheKey(int32_t bucket) const;
     Result<SnapshotLiveManifestEntries> LoadSnapshotLiveManifestEntries(int32_t bucket) const;
     Status StoreSnapshotLiveManifestEntries(int32_t bucket,
diff --git a/src/paimon/core/operation/key_value_file_store_scan_test.cpp b/src/paimon/core/operation/key_value_file_store_scan_test.cpp
index 703d7e493..f9fd2ea2f 100644
--- a/src/paimon/core/operation/key_value_file_store_scan_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_scan_test.cpp
@@ -33,7 +33,6 @@
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/manifest/manifest_file.h"
 #include "paimon/core/manifest/manifest_list.h"
-#include "paimon/core/operation/metrics/scan_metrics.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/snapshot.h"
@@ -50,6 +49,7 @@
 #include "paimon/predicate/literal.h"
 #include "paimon/predicate/predicate_builder.h"
 #include "paimon/scan_context.h"
+#include "paimon/table/source/scan_metrics.h"
 #include "paimon/testing/utils/binary_row_generator.h"
 #include "paimon/testing/utils/testharness.h"
 
diff --git a/src/paimon/core/operation/metrics/scan_metrics.h b/src/paimon/core/operation/metrics/scan_metrics.h
deleted file mode 100644
index 483ab7afc..000000000
--- a/src/paimon/core/operation/metrics/scan_metrics.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-namespace paimon {
-
-/// Metrics to measure scan operation.
-class ScanMetrics {
- public:
-    static constexpr char LAST_SCAN_DURATION[] = "lastScanDuration";
-    // Histogram metric for scan plan duration (milliseconds).
-    static constexpr char SCAN_DURATION[] = "scanDuration";
-    static constexpr char LAST_SCANNED_SNAPSHOT_ID[] = "lastScannedSnapshotId";
-    static constexpr char LAST_SCANNED_MANIFESTS[] = "lastScannedManifests";
-    static constexpr char LAST_SCAN_SKIPPED_TABLE_FILES[] = "lastScanSkippedTableFiles";
-    static constexpr char LAST_SCAN_RESULTED_TABLE_FILES[] = "lastScanResultedTableFiles";
-};
-
-}  // namespace paimon
diff --git a/src/paimon/core/table/source/abstract_table_scan.h b/src/paimon/core/table/source/abstract_table_scan.h
index 363fe0dce..f8dd15aac 100644
--- a/src/paimon/core/table/source/abstract_table_scan.h
+++ b/src/paimon/core/table/source/abstract_table_scan.h
@@ -39,6 +39,10 @@ class AbstractTableScan : public TableScan {
                       const std::shared_ptr<SnapshotReader>& snapshot_reader)
         : core_options_(core_options), snapshot_reader_(snapshot_reader) {}
 
+    std::shared_ptr<Metrics> GetMetrics() const override {
+        return snapshot_reader_->GetMetrics();
+    }
+
  protected:
     Result<std::shared_ptr<StartingScanner>> CreateStartingScanner(bool is_streaming) const {
         const auto& snapshot_manager = snapshot_reader_->GetSnapshotManager();
diff --git a/src/paimon/core/table/source/realtime_table_scan.h b/src/paimon/core/table/source/realtime_table_scan.h
index 7d036d420..2b635faf6 100644
--- a/src/paimon/core/table/source/realtime_table_scan.h
+++ b/src/paimon/core/table/source/realtime_table_scan.h
@@ -47,6 +47,10 @@ class RealtimeTableScan : public TableScan {
 
     Result<std::shared_ptr<Plan>> CreatePlan() override;
 
+    std::shared_ptr<Metrics> GetMetrics() const override {
+        return disk_scan_->GetMetrics();
+    }
+
  private:
     using MemoryViewMap = std::map<RealtimePartitionBucket, RealtimePartitionBucketView>;
 
diff --git a/src/paimon/core/table/source/snapshot/snapshot_reader.h b/src/paimon/core/table/source/snapshot/snapshot_reader.h
index c425fd863..35134d22a 100644
--- a/src/paimon/core/table/source/snapshot/snapshot_reader.h
+++ b/src/paimon/core/table/source/snapshot/snapshot_reader.h
@@ -108,6 +108,10 @@ class SnapshotReader {
         return scan_->GetPartitionPredicate();
     }
 
+    std::shared_ptr<Metrics> GetMetrics() const {
+        return scan_->GetScanMetrics();
+    }
+
     /// Get splits from `FileKind::ADD` files.
     Result<std::shared_ptr<Plan>> Read() const;
 
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 2dda955ac..8111a94e3 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -27,6 +27,7 @@
 #include <vector>
 
 #include "fmt/format.h"
+#include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/predicate/predicate_validator.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/fields_comparator.h"
@@ -187,6 +188,10 @@ Result<std::unique_ptr<TableScan>> NewDataTableScan(const std::shared_ptr<ScanCo
 
 }  // namespace
 
+std::shared_ptr<Metrics> TableScan::GetMetrics() const {
+    return std::make_shared<MetricsImpl>();
+}
+
 Result<std::unique_ptr<TableScan>> TableScan::Create(std::unique_ptr<ScanContext> context) {
     if (context == nullptr) {
         return Status::Invalid("scan context is null pointer");
diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp
index 358dfd469..dbc5b9926 100644
--- a/src/paimon/core/table/source/table_scan_test.cpp
+++ b/src/paimon/core/table/source/table_scan_test.cpp
@@ -26,11 +26,36 @@
 
 #include "gtest/gtest.h"
 #include "paimon/defs.h"
+#include "paimon/metrics.h"
 #include "paimon/scan_context.h"
 #include "paimon/status.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
+namespace {
+
+class DefaultMetricsTableScan : public TableScan {
+ public:
+    Result<std::shared_ptr<Plan>> CreatePlan() override {
+        return Status::NotImplemented("not implemented");
+    }
+};
+
+}  // namespace
+
+TEST(TableScanTest, TestDefaultMetricsSnapshot) {
+    DefaultMetricsTableScan table_scan;
+    std::shared_ptr<Metrics> metrics = table_scan.GetMetrics();
+    ASSERT_TRUE(metrics);
+    metrics->SetCounter("external", 1);
+
+    std::shared_ptr<Metrics> second_metrics = table_scan.GetMetrics();
+    ASSERT_TRUE(second_metrics);
+    Result<uint64_t> external_counter = second_metrics->GetCounter("external");
+    ASSERT_FALSE(external_counter.ok());
+    ASSERT_EQ(external_counter.status().code(), StatusCode::KeyError);
+}
+
 TEST(TableScanTest, TestNoSnapshot) {
     std::string path = paimon::test::GetDataDir() +
                        "/orc/append_table_with_nested_type.db/append_table_with_nested_type/";
@@ -61,6 +86,27 @@ TEST(TableScanTest, TestPkSchemaEvolutionScan) {
     ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan());
     ASSERT_TRUE(plan->SnapshotId());
     ASSERT_FALSE(plan->Splits().empty());
+
+    std::shared_ptr<Metrics> metrics = table_scan->GetMetrics();
+    ASSERT_TRUE(metrics);
+    ASSERT_OK_AND_ASSIGN(uint64_t scanned_snapshot_id,
+                         metrics->GetCounter(ScanMetrics::LAST_SCANNED_SNAPSHOT_ID));
+    ASSERT_EQ(scanned_snapshot_id, static_cast<uint64_t>(plan->SnapshotId().value()));
+    ASSERT_OK_AND_ASSIGN(uint64_t resulted_table_files,
+                         metrics->GetCounter(ScanMetrics::LAST_SCAN_RESULTED_TABLE_FILES));
+    ASSERT_GT(resulted_table_files, 0);
+    ASSERT_OK(metrics->GetCounter(ScanMetrics::LAST_MANIFEST_READ_DURATION));
+    ASSERT_OK(metrics->GetHistogramStats(ScanMetrics::MANIFEST_READ_DURATION));
+    ASSERT_OK_AND_ASSIGN(uint64_t lazy_decode_scanned_rows,
+                         metrics->GetCounter(ScanMetrics::LAST_LAZY_DECODE_SCANNED_ROWS));
+    ASSERT_OK_AND_ASSIGN(uint64_t lazy_decode_materialized_rows,
+                         metrics->GetCounter(ScanMetrics::LAST_LAZY_DECODE_MATERIALIZED_ROWS));
+    ASSERT_GE(lazy_decode_scanned_rows, lazy_decode_materialized_rows);
+
+    metrics->SetCounter(ScanMetrics::LAST_SCANNED_SNAPSHOT_ID, 0);
+    ASSERT_OK_AND_ASSIGN(uint64_t internal_snapshot_id, table_scan->GetMetrics()->GetCounter(
+                                                            ScanMetrics::LAST_SCANNED_SNAPSHOT_ID));
+    ASSERT_EQ(internal_snapshot_id, static_cast<uint64_t>(plan->SnapshotId().value()));
 }
 
 TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) {
diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp
index 0c67ae3ac..7e5847dbf 100644
--- a/src/paimon/core/table/system/system_table_test.cpp
+++ b/src/paimon/core/table/system/system_table_test.cpp
@@ -33,10 +33,12 @@
 #include "paimon/core/table/system/audit_log_system_table.h"
 #include "paimon/core/table/system/binlog_system_table.h"
 #include "paimon/core/table/system/read_optimized_system_table.h"
+#include "paimon/core/table/system/system_table_scan.h"
 #include "paimon/defs.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/fs/file_system_factory.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/metrics.h"
 #include "paimon/reader/batch_reader.h"
 #include "paimon/result.h"
 #include "paimon/status.h"
@@ -199,4 +201,15 @@ TEST(SystemTableTest, TestGlobalSystemTableWithoutCatalogReturnsNotImplemented)
                         "global system table requires catalog context: tables");
 }
 
+TEST(SystemTableTest, TestScanMetricsAreSnapshots) {
+    SystemTableScan scan("/tmp/table");
+    std::shared_ptr<Metrics> metrics = scan.GetMetrics();
+    ASSERT_TRUE(metrics);
+    metrics->SetCounter("external", 1);
+
+    std::shared_ptr<Metrics> second_metrics = scan.GetMetrics();
+    ASSERT_TRUE(second_metrics);
+    ASSERT_NOK_WITH_MSG(second_metrics->GetCounter("external"), "metric 'external' not found");
+}
+
 }  // namespace paimon::test

From 864497581488637094990366aeb5d21a05c7c1f3 Mon Sep 17 00:00:00 2001
From: "Mr Dk." <mrdrivingduck@gmail.com>
Date: Fri, 28 Aug 2026 16:47:52 +0800
Subject: [PATCH 88/93] fix(executor): support destruction from worker threads
 (#233)

---
 .../common/executor/default_executor_test.cpp | 51 +++++++++++++++
 src/paimon/common/executor/executor.cpp       | 63 ++++++++++---------
 src/paimon/fs/s3/s3_file_system_test.cpp      | 50 +++++++++++++++
 3 files changed, 133 insertions(+), 31 deletions(-)

diff --git a/src/paimon/common/executor/default_executor_test.cpp b/src/paimon/common/executor/default_executor_test.cpp
index 91f2c94c2..07e78681b 100644
--- a/src/paimon/common/executor/default_executor_test.cpp
+++ b/src/paimon/common/executor/default_executor_test.cpp
@@ -125,6 +125,57 @@ TEST(DefaultExecutorTest, TestAddTaskAfterShutdownNowIgnored) {
     ASSERT_EQ(executed_count.load(), 0);
 }
 
+TEST(DefaultExecutorTest, TestConcurrentShutdownNow) {
+    constexpr int32_t kShutdownThreadCount = 2;
+    constexpr int32_t kAttempts = 50;
+    for (int32_t attempt = 0; attempt < kAttempts; ++attempt) {
+        ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(/*thread_count=*/4));
+        std::atomic<int32_t> ready_shutdown_count = 0;
+        std::promise<void> start_signal;
+        std::shared_future<void> start_future = start_signal.get_future().share();
+        std::vector<std::thread> shutdown_threads;
+        shutdown_threads.reserve(kShutdownThreadCount);
+
+        for (int32_t thread_index = 0; thread_index < kShutdownThreadCount; ++thread_index) {
+            shutdown_threads.emplace_back([&]() {
+                ++ready_shutdown_count;
+                start_future.wait();
+                executor->ShutdownNow();
+            });
+        }
+        for (int32_t retry = 0; retry < 100 && ready_shutdown_count.load() < kShutdownThreadCount;
+             ++retry) {
+            std::this_thread::sleep_for(std::chrono::milliseconds(10));
+        }
+        const int32_t ready_count_before_start = ready_shutdown_count.load();
+        start_signal.set_value();
+        for (std::thread& shutdown_thread : shutdown_threads) {
+            shutdown_thread.join();
+        }
+        ASSERT_EQ(kShutdownThreadCount, ready_count_before_start);
+    }
+}
+
+TEST(DefaultExecutorTest, TestDestroyFromWorkerThread) {
+    std::unique_ptr<Executor> created = CreateDefaultExecutor();
+    std::shared_ptr<Executor> executor(std::move(created));
+    std::shared_ptr<Executor> task_executor = executor;
+    auto release = std::make_shared<std::promise<void>>();
+    std::shared_future<void> release_future = release->get_future().share();
+    auto destroyed = std::make_shared<std::promise<void>>();
+    std::future<void> future = destroyed->get_future();
+
+    executor->Add([executor = std::move(task_executor), release_future, destroyed]() mutable {
+        release_future.wait();
+        executor.reset();
+        destroyed->set_value();
+    });
+
+    executor.reset();
+    release->set_value();
+    ASSERT_EQ(std::future_status::ready, future.wait_for(std::chrono::seconds(5)));
+}
+
 TEST(DefaultExecutorTest, TestAddTaskFromMultipleThreads) {
     ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(/*thread_count=*/4));
 
diff --git a/src/paimon/common/executor/executor.cpp b/src/paimon/common/executor/executor.cpp
index cd3f699d3..45253fe1f 100644
--- a/src/paimon/common/executor/executor.cpp
+++ b/src/paimon/common/executor/executor.cpp
@@ -40,23 +40,26 @@ class DefaultExecutor : public Executor {
     uint32_t GetThreadNum() const override;
 
  private:
-    void WorkerThread();
+    struct State {
+        std::queue<std::function<void()>> tasks;
+        std::mutex mutex;
+        std::condition_variable condition;
+        bool stop = false;
+    };
+
+    static void WorkerThread(std::shared_ptr<State> state);
     void ShutdownInternal(bool wait_for_pending_tasks);
 
  private:
     uint32_t thread_count_;
     std::vector<std::thread> workers_;
-    std::queue<std::function<void()>> tasks_;
-    std::mutex queue_mutex_;
-    std::condition_variable condition_;
-    bool stop_ = false;
-    int32_t active_tasks_ = 0;
+    std::shared_ptr<State> state_ = std::make_shared<State>();
 };
 
 DefaultExecutor::DefaultExecutor(uint32_t thread_count) : thread_count_(thread_count) {
     assert(thread_count > 0);
     for (uint32_t i = 0; i < thread_count_; ++i) {
-        workers_.emplace_back(&DefaultExecutor::WorkerThread, this);
+        workers_.emplace_back(&DefaultExecutor::WorkerThread, state_);
     }
 }
 
@@ -66,21 +69,25 @@ uint32_t DefaultExecutor::GetThreadNum() const {
 
 void DefaultExecutor::ShutdownInternal(bool wait_for_pending_tasks) {
     {
-        std::unique_lock<std::mutex> lock(queue_mutex_);
-        if (stop_) {
+        std::unique_lock<std::mutex> lock(state_->mutex);
+        if (state_->stop) {
             return;
         }
-        stop_ = true;
+        state_->stop = true;
         if (!wait_for_pending_tasks) {
             // Discard all pending tasks immediately.
             std::queue<std::function<void()>> empty;
-            tasks_.swap(empty);
+            state_->tasks.swap(empty);
         }
-        condition_.notify_all();
+        state_->condition.notify_all();
     }
     for (std::thread& worker : workers_) {
         if (worker.joinable()) {
-            worker.join();
+            if (worker.get_id() == std::this_thread::get_id()) {
+                worker.detach();
+            } else {
+                worker.join();
+            }
         }
     }
 }
@@ -100,38 +107,32 @@ void DefaultExecutor::Add(std::function<void()> func) {
         return;
     }
     {
-        std::unique_lock<std::mutex> lock(queue_mutex_);
-        if (stop_) {
+        std::unique_lock<std::mutex> lock(state_->mutex);
+        if (state_->stop) {
             return;
         }
-        tasks_.emplace(std::move(func));
+        state_->tasks.emplace(std::move(func));
     }
-    condition_.notify_one();
+    state_->condition.notify_one();
 }
 
-void DefaultExecutor::WorkerThread() {
+void DefaultExecutor::WorkerThread(std::shared_ptr<State> state) {
     while (true) {
         std::function<void()> task;
         {
-            std::unique_lock<std::mutex> lock(queue_mutex_);
-            condition_.wait(lock, [this] { return stop_ || !tasks_.empty(); });
-            if (stop_ && tasks_.empty() && active_tasks_ == 0) {
-                condition_.notify_all();
+            std::unique_lock<std::mutex> lock(state->mutex);
+            state->condition.wait(lock, [&state] { return state->stop || !state->tasks.empty(); });
+            if (state->stop && state->tasks.empty()) {
+                state->condition.notify_all();
                 return;
             }
-            if (!tasks_.empty()) {
-                task = std::move(tasks_.front());
-                tasks_.pop();
-                ++active_tasks_;
+            if (!state->tasks.empty()) {
+                task = std::move(state->tasks.front());
+                state->tasks.pop();
             }
         }
         if (task) {
             task();
-            std::unique_lock<std::mutex> lock(queue_mutex_);
-            --active_tasks_;
-            if (tasks_.empty() && active_tasks_ == 0) {
-                condition_.notify_all();
-            }
         }
     }
 }
diff --git a/src/paimon/fs/s3/s3_file_system_test.cpp b/src/paimon/fs/s3/s3_file_system_test.cpp
index 09b1bb4ba..063a7dcb5 100644
--- a/src/paimon/fs/s3/s3_file_system_test.cpp
+++ b/src/paimon/fs/s3/s3_file_system_test.cpp
@@ -21,11 +21,17 @@
 
 #include <gtest/gtest.h>
 
+#include <chrono>
+#include <condition_variable>
 #include <cstdlib>
 #include <cstring>
 #include <filesystem>
 #include <fstream>
+#include <functional>
+#include <future>
+#include <mutex>
 #include <optional>
+#include <thread>
 #include <utility>
 #include <vector>
 
@@ -40,6 +46,9 @@ class MockHttpClient : public HttpClient {
  public:
     Result<HttpResponse> Execute(const HttpRequest& request,
                                  const HttpBodyConsumer& consumer) const override {
+        if (before_execute_) {
+            before_execute_();
+        }
         request_ = request;
         HttpResponse response;
         response.status_code = status_code_;
@@ -55,6 +64,7 @@ class MockHttpClient : public HttpClient {
     int32_t status_code_ = 200;
     HttpHeaders response_headers_;
     std::string body_;
+    std::function<void()> before_execute_;
 };
 
 class ScopedEnvironmentVariable {
@@ -435,6 +445,46 @@ TEST(S3ObjectStoreClientTest, TestRangeAndListObjects) {
     ASSERT_NE(http->request_.url.find("continuation-token=old%20token"), std::string::npos);
 }
 
+TEST(S3ObjectStoreClientTest, TestGetObjectRangeAsyncClientLifetime) {
+    auto http = std::make_shared<MockHttpClient>();
+    http->body_ = "data";
+    std::mutex mutex;
+    std::condition_variable condition;
+    bool request_started = false;
+    bool release_request = false;
+    http->before_execute_ = [&] {
+        std::unique_lock<std::mutex> lock(mutex);
+        request_started = true;
+        condition.notify_one();
+        condition.wait(lock, [&] { return release_request; });
+    };
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<ObjectStoreClient> client,
+                         MakeS3ObjectStoreClient(StaticOptions(), http));
+    char buffer[4];
+    auto promise = std::make_shared<std::promise<Status>>();
+    std::future<Status> future = promise->get_future();
+
+    client->GetObjectRangeAsync({"bucket", "key"}, 0, 4, buffer, [promise](Status status) {
+        promise->set_value(std::move(status));
+    });
+    {
+        std::unique_lock<std::mutex> lock(mutex);
+        ASSERT_TRUE(
+            condition.wait_for(lock, std::chrono::seconds(5), [&] { return request_started; }));
+    }
+    std::thread destruction_thread([client = std::move(client)]() mutable { client.reset(); });
+    {
+        std::lock_guard<std::mutex> lock(mutex);
+        release_request = true;
+    }
+    condition.notify_one();
+
+    destruction_thread.join();
+    ASSERT_EQ(std::future_status::ready, future.wait_for(std::chrono::seconds(5)));
+    ASSERT_OK(future.get());
+    ASSERT_EQ("data", std::string(buffer, sizeof(buffer)));
+}
+
 TEST(S3ObjectStoreClientTest, TestUrlEncodedListObjects) {
     auto http = std::make_shared<MockHttpClient>();
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<ObjectStoreClient> client,

From 020a14dc9dfad35b4bc07034684ffe599a8382a7 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 28 Aug 2026 18:01:18 +0800
Subject: [PATCH 89/93] docs: preserve comments across refactoring

---
 include/paimon/realtime/arrow_realtime_store_factory.h | 1 +
 include/paimon/realtime/realtime_store.h               | 1 +
 src/paimon/core/mergetree/merge_tree_writer.cpp        | 7 ++++++-
 src/paimon/core/operation/merge_file_split_read.cpp    | 2 ++
 4 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/include/paimon/realtime/arrow_realtime_store_factory.h b/include/paimon/realtime/arrow_realtime_store_factory.h
index 153d524d4..b3fa630de 100644
--- a/include/paimon/realtime/arrow_realtime_store_factory.h
+++ b/include/paimon/realtime/arrow_realtime_store_factory.h
@@ -26,6 +26,7 @@ namespace paimon {
 /// Factory for Paimon's default Arrow-backed `RealtimeStore`.
 class PAIMON_EXPORT ArrowRealtimeStoreFactory : public RealtimeStoreFactory {
  public:
+    /// Creates an Arrow-backed store for one partition and bucket.
     Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override;
 };
 
diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 81e64b9fd..6ae81f1f4 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -189,6 +189,7 @@ class PAIMON_EXPORT RealtimeStoreFactory {
  public:
     virtual ~RealtimeStoreFactory() = default;
 
+    /// Creates a store configured with the supplied schema, statistics, options, and memory pool.
     /// Creates a store for the requested table mode.
     /// The factory consumes `request`, including ownership of `request.write_schema`.
     virtual Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) = 0;
diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp
index 75623fd96..c9eb44bba 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer.cpp
@@ -172,16 +172,20 @@ Status MergeTreeWriter::WriteSortedReadersToFiles(
         }
     }
 
+    // 2. prepare loser tree sort merge reader
     auto sort_merge_reader = std::make_unique<SortMergeReaderWithLoserTree>(
         std::move(readers), key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_);
     raw_readers_guard.Release();
+    // 3. project key value to arrow array
     auto create_consumer = [target_schema = write_schema_, pool = pool_]()
         -> Result<std::unique_ptr<RowToArrowArrayConverter<KeyValue, KeyValueBatch>>> {
         return KeyValueMetaProjectionConsumer::Create(target_schema, pool);
     };
+    // consumer batch size is WriteBatchSize
     auto async_key_value_producer_consumer =
         std::make_unique<AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>>(
-            std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(), 1, pool_);
+            std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(),
+            /*projection_thread_num=*/1, pool_);
     ScopeGuard async_readers_guard([&]() -> void { async_key_value_producer_consumer->Close(); });
     std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>> rolling_writer;
     PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter());
@@ -309,6 +313,7 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction,
             wait_for_latest_compaction = true;
         }
         auto cleanup_guard = ScopeGuard([&]() { write_buffer_->Clear(); });
+        // 1. flush write buffer to get sorted readers
         PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
                                write_buffer_->CreateReaders());
         PAIMON_RETURN_NOT_OK(WriteSortedReadersToFiles(std::move(readers)));
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 3bcc705aa..1f96c195c 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -663,6 +663,7 @@ MergeFileSplitRead::CreateRecordReadersForSection(
     std::vector<std::unique_ptr<KeyValueRecordReader>> record_readers;
     record_readers.reserve(section.size());
     for (const SortedRun& run : section) {
+        // no overlap in a run
         PAIMON_ASSIGN_OR_RAISE(
             std::unique_ptr<KeyValueRecordReader> run_reader,
             CreateReaderForRun(partition, run, dv_factory, predicate, data_file_path_factory));
@@ -722,6 +723,7 @@ Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeRead
     DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
     const std::shared_ptr<DataFilePathFactory>& data_file_path_factory, bool drop_delete,
     const std::shared_ptr<MergeFunctionWrapper<KeyValue>>& merge_function_wrapper) {
+    // with overlap in one section
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> record_readers,
                            CreateRecordReadersForSection(section, partition, dv_factory, predicate,
                                                          data_file_path_factory));

From a77b5f4608f60f0731311680cb2f4c095c035e56 Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 28 Aug 2026 18:47:30 +0800
Subject: [PATCH 90/93] test(realtime): streamline primary key reader coverage

---
 .../key_value_file_store_write_test.cpp       |  18 ++-
 .../primary_key_realtime_store_test.cpp       |  85 +++----------
 .../realtime_primary_key_reader_test.cpp      |  23 ++++
 test/inte/realtime_write_inte_test.cpp        | 116 +-----------------
 4 files changed, 54 insertions(+), 188 deletions(-)

diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index 01ba8459d..88b848eea 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -50,6 +50,7 @@
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/operation/restore_files.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/core/stats/simple_stats.h"
 #include "paimon/core/table/sink/commit_message_impl.h"
 #include "paimon/file_store_commit.h"
@@ -257,17 +258,12 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
         if (views.size() != 1) {
             return Status::Invalid("expected exactly one real-time store");
         }
-        std::shared_ptr<arrow::Schema> transport_schema = arrow::schema({
-            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
-                ->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-                ->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
-            DataField::ConvertDataFieldToArrowField(
-                DataField(0, arrow::field("id", arrow::int64(), false))),
-            DataField::ConvertDataFieldToArrowField(
-                DataField(1, arrow::field("value", arrow::utf8()))),
-        });
+        arrow::FieldVector value_fields = {DataField::ConvertDataFieldToArrowField(DataField(
+                                               0, arrow::field("id", arrow::int64(), false))),
+                                           DataField::ConvertDataFieldToArrowField(
+                                               DataField(1, arrow::field("value", arrow::utf8())))};
+        std::shared_ptr<arrow::Schema> transport_schema =
+            RealtimePrimaryKeyLayout::CreateSchema(value_fields);
         auto c_schema = std::make_unique<ArrowSchema>();
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*transport_schema, c_schema.get()));
         RealtimeQueryContext query_context{c_schema.get(), nullptr, false};
diff --git a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
index 7d6fa72f5..ed80db275 100644
--- a/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/primary_key_realtime_store_test.cpp
@@ -18,7 +18,6 @@
 
 #include "paimon/core/realtime/primary_key_realtime_store.h"
 
-#include <cstddef>
 #include <cstdint>
 #include <memory>
 #include <optional>
@@ -29,10 +28,10 @@
 #include "arrow/api.h"
 #include "arrow/c/bridge.h"
 #include "arrow/ipc/json_simple.h"
-#include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/macros.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/realtime/arrow_realtime_store_factory.h"
@@ -50,28 +49,17 @@ std::shared_ptr<arrow::Field> FieldWithId(const std::string& name,
 }
 
 std::shared_ptr<arrow::Schema> TransportSchema() {
-    return arrow::schema(
-        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-             ->WithNullable(false),
-         DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
-         DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64()))),
-         DataField::ConvertDataFieldToArrowField(
-             DataField(1, arrow::field("value", arrow::utf8())))});
+    return RealtimePrimaryKeyLayout::CreateSchema(
+        {FieldWithId("id", arrow::int64(), 0), FieldWithId("value", arrow::utf8(), 1)});
 }
 
 std::shared_ptr<arrow::Schema> NestedTransportSchema() {
-    return arrow::schema(
-        {DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-         DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-             ->WithNullable(false),
-         DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
-         DataField::ConvertDataFieldToArrowField(DataField(0, arrow::field("id", arrow::int64()))),
-         DataField::ConvertDataFieldToArrowField(DataField(
-             1,
-             arrow::field("value",
-                          arrow::struct_({arrow::field("name", arrow::utf8()),
-                                          arrow::field("items", arrow::list(arrow::int32()))}))))});
+    return RealtimePrimaryKeyLayout::CreateSchema(
+        {FieldWithId("id", arrow::int64(), 0),
+         FieldWithId("value",
+                     arrow::struct_({arrow::field("name", arrow::utf8()),
+                                     arrow::field("items", arrow::list(arrow::int32()))}),
+                     1)});
 }
 
 std::unique_ptr<RecordBatch> MakeBatch(const std::string& json) {
@@ -125,36 +113,6 @@ Result<std::string> ReadJson(const std::vector<std::unique_ptr<BatchReader>>& re
     return result->ToString();
 }
 
-class TestingMemoryPool final : public MemoryPool {
- public:
-    void* Malloc(uint64_t size, uint64_t alignment) override {
-        return delegate_->Malloc(size, alignment);
-    }
-
-    void* Realloc(void* pointer, size_t old_size, size_t new_size, uint64_t alignment) override {
-        return delegate_->Realloc(pointer, old_size, new_size, alignment);
-    }
-
-    void Free(void* pointer, uint64_t size) override {
-        delegate_->Free(pointer, size);
-    }
-
-    void Free(void* pointer, uint64_t size, uint64_t alignment) override {
-        delegate_->Free(pointer, size, alignment);
-    }
-
-    uint64_t CurrentUsage() const override {
-        return delegate_->CurrentUsage();
-    }
-
-    uint64_t MaxMemoryUsage() const override {
-        return delegate_->MaxMemoryUsage();
-    }
-
- private:
-    std::unique_ptr<MemoryPool> delegate_ = GetMemoryPool();
-};
-
 TEST(PrimaryKeyRealtimeStoreTest, TestWriteAndSealValidation) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
                          PrimaryKeyRealtimeStore::Create(TransportSchema(), GetDefaultPool()));
@@ -337,8 +295,8 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderPerStoredBatch) {
 
 TEST(PrimaryKeyRealtimeStoreTest, TestQueryPoolOutlivesStoreReaderAndExport) {
     const std::shared_ptr<arrow::Schema> stored_schema = TransportSchema();
-    std::shared_ptr<TestingMemoryPool> pool = std::make_shared<TestingMemoryPool>();
-    std::weak_ptr<TestingMemoryPool> pool_lifetime = pool;
+    std::shared_ptr<MemoryPool> pool = GetMemoryPool();
+    std::weak_ptr<MemoryPool> pool_lifetime = pool;
     auto write_schema = std::make_unique<ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*stored_schema, write_schema.get()).ok());
     ArrowRealtimeStoreFactory factory;
@@ -381,16 +339,13 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsNestedFields) {
     const std::shared_ptr<arrow::Field> stored_b = FieldWithId("b", arrow::int32(), 11);
     const std::shared_ptr<arrow::Field> stored_x = FieldWithId("x", arrow::int32(), 20);
     const std::shared_ptr<arrow::Field> stored_y = FieldWithId("y", arrow::int32(), 21);
-    arrow::FieldVector stored_fields = {
-        DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-            ->WithNullable(false),
-        DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
+    arrow::FieldVector stored_value_fields = {
         FieldWithId("id", arrow::int64(), 0),
         FieldWithId("profile", arrow::struct_({stored_profile_a}), 1),
         FieldWithId("items", arrow::list(arrow::struct_({stored_a, stored_b})), 2),
         FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_x, stored_y})), 3)};
-    std::shared_ptr<arrow::Schema> stored_schema = arrow::schema(std::move(stored_fields));
+    std::shared_ptr<arrow::Schema> stored_schema =
+        RealtimePrimaryKeyLayout::CreateSchema(stored_value_fields);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<PrimaryKeyRealtimeStore> store,
                          PrimaryKeyRealtimeStore::Create(stored_schema, GetDefaultPool()));
     ASSERT_OK(store->Write(RealtimeWriteBatch{
@@ -401,14 +356,14 @@ TEST(PrimaryKeyRealtimeStoreTest, TestQueryReaderProjectsNestedFields) {
         OffsetRange(0, 1)}));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> view, store->AcquireReadView());
 
-    arrow::FieldVector requested_fields(stored_schema->fields().begin(),
-                                        stored_schema->fields().begin() + 3);
-    requested_fields.push_back(FieldWithId("profile", arrow::struct_({stored_profile_a}), 1));
-    requested_fields.push_back(
+    arrow::FieldVector requested_value_fields;
+    requested_value_fields.push_back(FieldWithId("profile", arrow::struct_({stored_profile_a}), 1));
+    requested_value_fields.push_back(
         FieldWithId("items", arrow::list(arrow::struct_({stored_b, stored_a})), 2));
-    requested_fields.push_back(
+    requested_value_fields.push_back(
         FieldWithId("attrs", arrow::map(arrow::utf8(), arrow::struct_({stored_y, stored_x})), 3));
-    std::shared_ptr<arrow::Schema> requested_schema = arrow::schema(std::move(requested_fields));
+    std::shared_ptr<arrow::Schema> requested_schema =
+        RealtimePrimaryKeyLayout::CreateSchema(requested_value_fields);
     auto c_schema = std::make_unique<ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*requested_schema, c_schema.get()).ok());
     RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr,
diff --git a/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
index 89a39cd63..bd925cd96 100644
--- a/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
@@ -685,4 +685,27 @@ TEST_F(RealtimePrimaryKeyReaderTest, TestFactoryFailureClosesReaders) {
     ASSERT_EQ(factory_failure_close_count, 1);
 }
 
+TEST_F(RealtimePrimaryKeyReaderTest, TestQueryReaderClose) {
+    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
+                                           DataField(1, arrow::field("v0", arrow::int32()))};
+    std::shared_ptr<arrow::Schema> value_schema =
+        DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
+    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema(value_schema->fields());
+    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
+    std::shared_ptr<arrow::Array> transport_array =
+        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([[0, 10, 0, 1, 100]])")
+            .ValueOrDie();
+
+    int32_t close_count = 0;
+    auto batch_reader = std::make_unique<TrackingBatchReader>(
+        std::make_unique<MockFileBatchReader>(transport_array, transport_type, 1), &close_count);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
+                         CreateRealtimePrimaryKeyQueryReaderForTest(
+                             std::move(batch_reader), transport_schema, OffsetRange(0, 1),
+                             key_schema, value_schema, pool_));
+    reader->Close();
+    ASSERT_EQ(close_count, 1);
+}
+
 }  // namespace paimon::test
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 55227d841..4e28f286f 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -50,6 +50,7 @@
 #include "paimon/core/core_options.h"
 #include "paimon/core/operation/commit/realtime_commit_properties.h"
 #include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_primary_key_reader.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/table/sink/commit_message_impl.h"
@@ -98,32 +99,6 @@ class TrackingRealtimeReadView final : public RealtimeReadView {
     std::shared_ptr<RealtimeReadView> delegate_;
 };
 
-class ReadViewCheckingBatchReader final : public BatchReader {
- public:
-    ReadViewCheckingBatchReader(std::unique_ptr<BatchReader> delegate,
-                                std::weak_ptr<RealtimeReadView> read_view)
-        : delegate_(std::move(delegate)), read_view_(std::move(read_view)) {}
-
-    Result<ReadBatch> NextBatch() override {
-        if (read_view_.expired()) {
-            return Status::Invalid("real-time read view was released before reader completion");
-        }
-        return delegate_->NextBatch();
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        delegate_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> delegate_;
-    std::weak_ptr<RealtimeReadView> read_view_;
-};
-
 class DelegatingRealtimeStore : public RealtimeStore {
  public:
     explicit DelegatingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
@@ -219,13 +194,7 @@ class QueryTrackingRealtimeStore final : public DelegatingRealtimeStore {
         if (!tracking_view) {
             return Status::Invalid("query tracking store received an unexpected read view");
         }
-        PAIMON_ASSIGN_OR_RAISE(
-            std::vector<std::unique_ptr<BatchReader>> readers,
-            delegate_->CreateQueryReaders(tracking_view->Delegate(), offset_begin, context));
-        for (std::unique_ptr<BatchReader>& reader : readers) {
-            reader = std::make_unique<ReadViewCheckingBatchReader>(std::move(reader), view);
-        }
-        return readers;
+        return delegate_->CreateQueryReaders(tracking_view->Delegate(), offset_begin, context);
     }
 
  private:
@@ -233,57 +202,6 @@ class QueryTrackingRealtimeStore final : public DelegatingRealtimeStore {
     std::shared_ptr<std::weak_ptr<RealtimeReadView>> query_view_;
 };
 
-class CloseTrackingBatchReader final : public BatchReader {
- public:
-    CloseTrackingBatchReader(std::unique_ptr<BatchReader> delegate,
-                             const std::shared_ptr<std::atomic<int32_t>>& close_count)
-        : delegate_(std::move(delegate)), close_count_(close_count) {}
-
-    Result<ReadBatch> NextBatch() override {
-        return delegate_->NextBatch();
-    }
-
-    std::shared_ptr<Metrics> GetReaderMetrics() const override {
-        return delegate_->GetReaderMetrics();
-    }
-
-    void Close() override {
-        close_count_->fetch_add(1, std::memory_order_release);
-        delegate_->Close();
-    }
-
- private:
-    std::unique_ptr<BatchReader> delegate_;
-    std::shared_ptr<std::atomic<int32_t>> close_count_;
-};
-
-struct CloseTrackingReaderState {
-    std::shared_ptr<std::atomic<int32_t>> query_close_count =
-        std::make_shared<std::atomic<int32_t>>(0);
-};
-
-class CloseTrackingRealtimeStore final : public DelegatingRealtimeStore {
- public:
-    CloseTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
-                               const std::shared_ptr<CloseTrackingReaderState>& state)
-        : DelegatingRealtimeStore(delegate), state_(state) {}
-
-    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
-        const std::shared_ptr<RealtimeReadView>& view, int64_t offset_begin,
-        const RealtimeQueryContext& context) override {
-        PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
-                               delegate_->CreateQueryReaders(view, offset_begin, context));
-        for (std::unique_ptr<BatchReader>& reader : readers) {
-            reader = std::make_unique<CloseTrackingBatchReader>(std::move(reader),
-                                                                state_->query_close_count);
-        }
-        return readers;
-    }
-
- private:
-    std::shared_ptr<CloseTrackingReaderState> state_;
-};
-
 }  // namespace
 
 namespace {
@@ -846,18 +764,10 @@ class RealtimeWriteInteTest : public ::testing::Test {
             return Status::Invalid("expected a table schema");
         }
         auto read_schema = std::make_unique<ArrowSchema>();
-        arrow::FieldVector requested_fields = {
-            DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())
-                ->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())
-                ->WithNullable(false),
-            DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset())};
         std::shared_ptr<arrow::Schema> value_schema =
             DataField::ConvertDataFieldsToArrowSchema(table_schema.value()->Fields());
-        requested_fields.insert(requested_fields.end(), value_schema->fields().begin(),
-                                value_schema->fields().end());
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(
-            arrow::ExportSchema(*arrow::schema(requested_fields), read_schema.get()));
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(
+            *RealtimePrimaryKeyLayout::CreateSchema(value_schema->fields()), read_schema.get()));
         ScopeGuard schema_guard([schema = read_schema.get()]() { ArrowSchemaRelease(schema); });
         RealtimeQueryContext query_context{read_schema.get(), /*predicate=*/nullptr,
                                            /*enable_predicate_pushdown=*/false};
@@ -1928,24 +1838,6 @@ TEST_F(RealtimeWriteInteTest, TestPkMultipleStoredBatchesMergeForQueryAndCommit)
     ASSERT_OK(writer->Close());
 }
 
-TEST_F(RealtimeWriteInteTest, TestPkQueryReaderClose) {
-    CreatePkTable();
-    auto state = std::make_shared<CloseTrackingReaderState>();
-    auto factory = MakeDecoratingFactory<CloseTrackingRealtimeStore>(state);
-    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
-                         RealtimeContext::Create(factory));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
-                         CreateRealtimeWriter(realtime_context));
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
-                         MakeBatch({Row{1, "one", "p0"}}, /*partitioned=*/false));
-    ASSERT_OK(writer->Write(std::move(batch)));
-
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader, CreateQueryReader(realtime_context));
-    reader->Close();
-    ASSERT_EQ(1, state->query_close_count->load(std::memory_order_acquire));
-    ASSERT_OK(writer->Close());
-}
-
 TEST_F(RealtimeWriteInteTest, TestRollingFilesPreserveProgress) {
     options_[Options::TARGET_FILE_ROW_NUM] = "10";
     CreateTable(/*partition_keys=*/{});

From a51b207c72e260e1e145b0194517afce069333bb Mon Sep 17 00:00:00 2001
From: JeffZhou <17023790+HaHaJeff@users.noreply.github.com>
Date: Fri, 28 Aug 2026 19:43:09 +0800
Subject: [PATCH 91/93] test(realtime): remove redundant close coverage

---
 .../core/mergetree/merge_tree_writer.cpp      |  4 ++--
 .../realtime_primary_key_reader_test.cpp      | 23 -------------------
 2 files changed, 2 insertions(+), 25 deletions(-)

diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp
index c9eb44bba..dde9aaeba 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer.cpp
@@ -172,11 +172,11 @@ Status MergeTreeWriter::WriteSortedReadersToFiles(
         }
     }
 
-    // 2. prepare loser tree sort merge reader
+    // prepare loser tree sort merge reader
     auto sort_merge_reader = std::make_unique<SortMergeReaderWithLoserTree>(
         std::move(readers), key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_);
     raw_readers_guard.Release();
-    // 3. project key value to arrow array
+    // project key value to arrow array
     auto create_consumer = [target_schema = write_schema_, pool = pool_]()
         -> Result<std::unique_ptr<RowToArrowArrayConverter<KeyValue, KeyValueBatch>>> {
         return KeyValueMetaProjectionConsumer::Create(target_schema, pool);
diff --git a/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
index bd925cd96..89a39cd63 100644
--- a/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
+++ b/src/paimon/core/realtime/realtime_primary_key_reader_test.cpp
@@ -685,27 +685,4 @@ TEST_F(RealtimePrimaryKeyReaderTest, TestFactoryFailureClosesReaders) {
     ASSERT_EQ(factory_failure_close_count, 1);
 }
 
-TEST_F(RealtimePrimaryKeyReaderTest, TestQueryReaderClose) {
-    std::vector<DataField> value_fields = {DataField(0, arrow::field("k0", arrow::int32())),
-                                           DataField(1, arrow::field("v0", arrow::int32()))};
-    std::shared_ptr<arrow::Schema> value_schema =
-        DataField::ConvertDataFieldsToArrowSchema(value_fields);
-    std::shared_ptr<arrow::Schema> key_schema = arrow::schema({value_schema->field(0)});
-    std::shared_ptr<arrow::Schema> transport_schema = MakeTransportSchema(value_schema->fields());
-    std::shared_ptr<arrow::DataType> transport_type = arrow::struct_(transport_schema->fields());
-    std::shared_ptr<arrow::Array> transport_array =
-        arrow::ipc::internal::json::ArrayFromJSON(transport_type, R"([[0, 10, 0, 1, 100]])")
-            .ValueOrDie();
-
-    int32_t close_count = 0;
-    auto batch_reader = std::make_unique<TrackingBatchReader>(
-        std::make_unique<MockFileBatchReader>(transport_array, transport_type, 1), &close_count);
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyValueRecordReader> reader,
-                         CreateRealtimePrimaryKeyQueryReaderForTest(
-                             std::move(batch_reader), transport_schema, OffsetRange(0, 1),
-                             key_schema, value_schema, pool_));
-    reader->Close();
-    ASSERT_EQ(close_count, 1);
-}
-
 }  // namespace paimon::test

From 19cba8d1212bcb6f5b5d9cd605fe893c42e8a6a6 Mon Sep 17 00:00:00 2001
From: "Mr Dk." <mrdrivingduck@gmail.com>
Date: Sat, 29 Aug 2026 15:41:15 +0800
Subject: [PATCH 92/93] feat(fs): add OSS file system (#205)

---
 CMakeLists.txt                                |  18 +-
 README.md                                     |   2 +-
 ci/scripts/build_paimon.sh                    |   1 +
 cmake_modules/DefineOptions.cmake             |   5 +
 cmake_modules/ThirdpartyToolchain.cmake       | 119 +++++++
 docs/source/build_system.rst                  |   2 +
 docs/source/building.rst                      |   2 +
 src/paimon/CMakeLists.txt                     |   4 +-
 .../common/fs/object_store_file_system.cpp    |   1 +
 .../common/fs/object_store_file_system.h      |  72 +++-
 .../fs/object_store_file_system_test.cpp      |  11 +
 src/paimon/fs/oss/CMakeLists.txt              |  50 +++
 src/paimon/fs/oss/oss_file_system.cpp         | 192 ++++++++++
 src/paimon/fs/oss/oss_file_system.h           |  51 +++
 src/paimon/fs/oss/oss_file_system_factory.cpp | 234 +++++++++++++
 src/paimon/fs/oss/oss_file_system_factory.h   |  38 ++
 src/paimon/fs/oss/oss_file_system_test.cpp    | 327 ++++++++++++++++++
 src/paimon/fs/s3/s3_file_system.cpp           |  13 +-
 src/paimon/fs/s3/s3_file_system_test.cpp      |   6 +-
 third_party/versions.txt                      |   5 +
 20 files changed, 1136 insertions(+), 17 deletions(-)
 create mode 100644 src/paimon/fs/oss/CMakeLists.txt
 create mode 100644 src/paimon/fs/oss/oss_file_system.cpp
 create mode 100644 src/paimon/fs/oss/oss_file_system.h
 create mode 100644 src/paimon/fs/oss/oss_file_system_factory.cpp
 create mode 100644 src/paimon/fs/oss/oss_file_system_factory.h
 create mode 100644 src/paimon/fs/oss/oss_file_system_test.cpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 46a55426a..2fe2df8e7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -62,6 +62,7 @@ option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
 option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
 option(PAIMON_ENABLE_MOSAIC "Whether to enable mosaic file format (Rust FFI)" OFF)
 option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
+option(PAIMON_ENABLE_OSS "Whether to enable OSS SDK v2 file system" OFF)
 option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF)
 option(PAIMON_ENABLE_NETWORK_TESTS
        "Whether to enable tests that access real remote services over the network" OFF)
@@ -77,8 +78,10 @@ endif()
 if(PAIMON_ENABLE_REST)
     add_definitions(-DPAIMON_ENABLE_REST)
 endif()
-# libcurl backs the HTTP client shared by the S3 file system and the rest catalog.
-if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST)
+# libcurl supports the REST and S3 HTTP clients, the OSS SDK transport, and object store timestamps.
+if(PAIMON_ENABLE_OSS
+   OR PAIMON_ENABLE_S3
+   OR PAIMON_ENABLE_REST)
     find_package(CURL REQUIRED)
 endif()
 if(PAIMON_ENABLE_REST)
@@ -93,6 +96,9 @@ endif()
 if(PAIMON_ENABLE_JINDO)
     add_definitions(-DPAIMON_ENABLE_JINDO)
 endif()
+if(PAIMON_ENABLE_OSS)
+    add_definitions(-DPAIMON_ENABLE_OSS)
+endif()
 if(PAIMON_ENABLE_S3)
     add_definitions(-DPAIMON_ENABLE_S3)
 endif()
@@ -473,6 +479,13 @@ if(PAIMON_BUILD_TESTS)
                                            paimon_jindo_file_system_shared)
         list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
     endif()
+    if(PAIMON_ENABLE_OSS)
+        paimon_link_libraries_whole_archive(PAIMON_OSS_FILE_SYSTEM_STATIC_LINK_LIBS
+                                            paimon_oss_file_system_static)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_oss_file_system_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
+    endif()
     if(PAIMON_ENABLE_S3)
         paimon_link_libraries_whole_archive(PAIMON_S3_FILE_SYSTEM_STATIC_LINK_LIBS
                                             paimon_s3_file_system_static)
@@ -532,6 +545,7 @@ add_subdirectory(src/paimon/fs/local)
 if(PAIMON_ENABLE_JINDO)
     add_subdirectory(src/paimon/fs/jindo)
 endif()
+add_subdirectory(src/paimon/fs/oss)
 add_subdirectory(src/paimon/fs/s3)
 add_subdirectory(src/paimon/format/blob)
 add_subdirectory(src/paimon/format/orc)
diff --git a/README.md b/README.md
index 00a391c03..da928e713 100644
--- a/README.md
+++ b/README.md
@@ -35,7 +35,7 @@ Paimon C++ currently provides:
 - **Scan**: batch and stream scan for append tables and primary key tables without changelog.
 - **Read**: append table read, primary key table read with deletion vector, and primary key table merge-on-read.
 - **Arrow integration**: batch read and write interfaces based on the [Arrow Columnar In-Memory Format](https://arrow.apache.org).
-- **File systems**: file system abstraction with built-in local and Jindo file system support.
+- **File systems**: file system abstraction with built-in local, Jindo, OSS, and S3 file system support.
 - **File formats**: file format abstraction with built-in ORC, Parquet, and Avro support.
 - **Runtime utilities**: memory pool and thread pool abstractions with default implementations.
 - **AI-Oriented Features**: supports RowTracking and DataEvolution mode and provides Global Index
diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh
index 58ed9b0a0..cea59da8b 100755
--- a/ci/scripts/build_paimon.sh
+++ b/ci/scripts/build_paimon.sh
@@ -149,6 +149,7 @@ CMAKE_ARGS=(
     "-DPAIMON_BUILD_TESTS=ON"
     "-DPAIMON_ENABLE_MOSAIC=ON"
     "-DPAIMON_ENABLE_JINDO=ON"
+    "-DPAIMON_ENABLE_OSS=ON"
     "-DPAIMON_ENABLE_S3=ON"
     "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}"
     "-DPAIMON_ENABLE_LUCENE=ON"
diff --git a/cmake_modules/DefineOptions.cmake b/cmake_modules/DefineOptions.cmake
index 21f653155..9f099208f 100644
--- a/cmake_modules/DefineOptions.cmake
+++ b/cmake_modules/DefineOptions.cmake
@@ -219,6 +219,11 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
                          ""
                          AUTO
                          BUNDLED)
+    define_option_string(OSS_SDK_V2_SOURCE
+                         "Dependency source for OSS SDK v2; SYSTEM is unsupported"
+                         ""
+                         AUTO
+                         BUNDLED)
     define_option_string(fmt_SOURCE
                          "Dependency source for fmt"
                          ""
diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
index a35ecd497..0c64819c2 100644
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
@@ -320,6 +320,16 @@ else()
     endif()
 endif()
 
+if(DEFINED ENV{PAIMON_OSS_SDK_V2_URL})
+    set(OSS_SDK_V2_SOURCE_URL "$ENV{PAIMON_OSS_SDK_V2_URL}")
+elseif(EXISTS "${THIRDPARTY_DIR}/${PAIMON_OSS_SDK_V2_PKG_NAME}")
+    set_urls(OSS_SDK_V2_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_OSS_SDK_V2_PKG_NAME}")
+else()
+    set_urls(OSS_SDK_V2_SOURCE_URL
+             "${THIRDPARTY_MIRROR_URL}https://github.com/aliyun/alibabacloud-oss-cpp-sdk-v2/archive/refs/tags/${PAIMON_OSS_SDK_V2_BUILD_VERSION}.tar.gz"
+    )
+endif()
+
 if(DEFINED ENV{PAIMON_LUMINA_URL})
     set(LUMINA_SOURCE_URL "$ENV{PAIMON_LUMINA_URL}")
 elseif(EXISTS "${THIRDPARTY_DIR}/${PAIMON_LUMINA_PKG_NAME}")
@@ -504,6 +514,25 @@ function(paimon_enforce_patched_dependency_policy)
                 PARENT_SCOPE)
         endif()
     endif()
+
+    if(PAIMON_ENABLE_OSS)
+        paimon_set_dependency_source_default(
+            OSS_SDK_V2 BUNDLED "OSS SDK v2 is only supported as a bundled dependency")
+        paimon_get_dependency_source(OSS_SDK_V2 _oss_sdk_v2_source)
+        if(_oss_sdk_v2_source STREQUAL "SYSTEM")
+            message(FATAL_ERROR "OSS_SDK_V2_SOURCE=SYSTEM is not supported. "
+                                "Use OSS_SDK_V2_SOURCE=BUNDLED.")
+        elseif(_oss_sdk_v2_source STREQUAL "AUTO")
+            message(STATUS "Forcing OSS_SDK_V2_SOURCE to BUNDLED because paimon-cpp "
+                           "only supports the bundled OSS SDK v2")
+            set(OSS_SDK_V2_SOURCE
+                "BUNDLED"
+                CACHE STRING "Dependency source for OSS SDK v2" FORCE)
+            set(OSS_SDK_V2_SOURCE
+                "BUNDLED"
+                PARENT_SCOPE)
+        endif()
+    endif()
 endfunction()
 
 function(paimon_apply_dependency_source_defaults)
@@ -609,6 +638,8 @@ function(paimon_get_dependency_compat_target DEPENDENCY_NAME OUT_VAR)
         set(_target tbb)
     elseif("${DEPENDENCY_NAME}" STREQUAL "Avro")
         set(_target avro)
+    elseif("${DEPENDENCY_NAME}" STREQUAL "OSS_SDK_V2")
+        set(_target alibabacloud_oss_v2::oss)
     else()
         set(_target "${DEPENDENCY_NAME}")
     endif()
@@ -683,6 +714,8 @@ macro(paimon_build_dependency DEPENDENCY_NAME)
         build_glog()
     elseif("${DEPENDENCY_NAME}" STREQUAL "Avro")
         build_avro()
+    elseif("${DEPENDENCY_NAME}" STREQUAL "OSS_SDK_V2")
+        build_oss_sdk_v2()
     elseif("${DEPENDENCY_NAME}" STREQUAL "GTest")
         build_gtest()
     elseif("${DEPENDENCY_NAME}" STREQUAL "benchmark")
@@ -1415,6 +1448,89 @@ macro(build_jindosdk_nextarch)
     add_dependencies(jindosdk::nextarch jindosdk-nextarch_ep)
 endmacro()
 
+macro(build_oss_sdk_v2)
+    message(STATUS "Building Alibaba Cloud OSS C++ SDK v2 from source")
+    find_package(CURL REQUIRED)
+    find_package(Threads REQUIRED)
+
+    set(OSS_SDK_V2_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/oss_sdk_v2_ep-install")
+    set(OSS_SDK_V2_INCLUDE_DIR "${OSS_SDK_V2_PREFIX}/include")
+    set(OSS_SDK_V2_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}")
+    set(OSS_SDK_V2_LIB_DIR "${OSS_SDK_V2_PREFIX}/${OSS_SDK_V2_INSTALL_LIBDIR}")
+    set(OSS_SDK_V2_STATIC_LIB
+        "${OSS_SDK_V2_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}alibabacloud-oss-cpp-sdk-v2${CMAKE_STATIC_LIBRARY_SUFFIX}"
+    )
+
+    set(OSS_SDK_V2_CMAKE_ARGS
+        ${EP_COMMON_CMAKE_ARGS}
+        "-DCMAKE_INSTALL_PREFIX=${OSS_SDK_V2_PREFIX}"
+        "-DCMAKE_INSTALL_LIBDIR=${OSS_SDK_V2_INSTALL_LIBDIR}"
+        -DCMAKE_PARENT_CXX_STANDARD=17
+        -DBUILD_SHARED_LIBS=OFF
+        -DBUILD_TESTS=OFF
+        -DBUILD_SAMPLES=OFF
+        -DENABLE_RTTI=OFF
+        -DUSE_CURL_TRANSPORT=ON
+        -DUSE_SYSTEM_CURL=ON
+        -DUSE_SYSTEM_OPENSSL=OFF
+        -DUSE_SYSTEM_MBEDTLS=OFF
+        -DUSE_SYSTEM_TINYXML2=OFF
+        -DUSE_STD_EXPECTED=OFF
+        -DENABLE_ENCRYPTION=OFF)
+    set(OSS_SDK_V2_CURL_INCLUDE_DIR "${CURL_INCLUDE_DIR}")
+    if(NOT OSS_SDK_V2_CURL_INCLUDE_DIR AND CURL_INCLUDE_DIRS)
+        list(GET CURL_INCLUDE_DIRS 0 OSS_SDK_V2_CURL_INCLUDE_DIR)
+    endif()
+    set(OSS_SDK_V2_CURL_LIBRARY "${CURL_LIBRARY_RELEASE}")
+    if(NOT OSS_SDK_V2_CURL_LIBRARY)
+        set(OSS_SDK_V2_CURL_LIBRARY "${CURL_LIBRARY}")
+    endif()
+    if(TARGET CURL::libcurl)
+        if(NOT OSS_SDK_V2_CURL_INCLUDE_DIR)
+            get_target_property(OSS_SDK_V2_CURL_INCLUDE_DIR CURL::libcurl
+                                INTERFACE_INCLUDE_DIRECTORIES)
+        endif()
+        if(NOT OSS_SDK_V2_CURL_LIBRARY)
+            foreach(CURL_CONFIG RELEASE RELWITHDEBINFO DEBUG NOCONFIG)
+                get_target_property(OSS_SDK_V2_CURL_LIBRARY CURL::libcurl
+                                    "IMPORTED_LOCATION_${CURL_CONFIG}")
+                if(OSS_SDK_V2_CURL_LIBRARY)
+                    break()
+                endif()
+            endforeach()
+        endif()
+        if(NOT OSS_SDK_V2_CURL_LIBRARY)
+            get_target_property(OSS_SDK_V2_CURL_LIBRARY CURL::libcurl IMPORTED_LOCATION)
+        endif()
+    endif()
+    if(OSS_SDK_V2_CURL_INCLUDE_DIR AND OSS_SDK_V2_CURL_LIBRARY)
+        list(APPEND
+             OSS_SDK_V2_CMAKE_ARGS
+             "-DCURL_INCLUDE_DIR=${OSS_SDK_V2_CURL_INCLUDE_DIR}"
+             "-DCURL_LIBRARY=${OSS_SDK_V2_CURL_LIBRARY}"
+             "-DCURL_LIBRARY_RELEASE=${OSS_SDK_V2_CURL_LIBRARY}")
+    endif()
+
+    externalproject_add(oss_sdk_v2_ep
+                        ${EP_COMMON_OPTIONS}
+                        URL ${OSS_SDK_V2_SOURCE_URL}
+                        URL_HASH "SHA256=${PAIMON_OSS_SDK_V2_BUILD_SHA256_CHECKSUM}"
+                        CMAKE_ARGS ${OSS_SDK_V2_CMAKE_ARGS} ${THIRDPARTY_LOG_OPTIONS}
+                        BUILD_BYPRODUCTS "${OSS_SDK_V2_STATIC_LIB}")
+
+    file(MAKE_DIRECTORY "${OSS_SDK_V2_INCLUDE_DIR}")
+    file(MAKE_DIRECTORY "${OSS_SDK_V2_LIB_DIR}")
+
+    add_library(alibabacloud_oss_v2::oss STATIC IMPORTED)
+    set_target_properties(alibabacloud_oss_v2::oss
+                          PROPERTIES IMPORTED_LOCATION "${OSS_SDK_V2_STATIC_LIB}"
+                                     INTERFACE_INCLUDE_DIRECTORIES
+                                     "${OSS_SDK_V2_INCLUDE_DIR}")
+    target_link_libraries(alibabacloud_oss_v2::oss INTERFACE CURL::libcurl
+                                                             Threads::Threads)
+    add_dependencies(alibabacloud_oss_v2::oss oss_sdk_v2_ep)
+endmacro()
+
 macro(build_protobuf)
     message(STATUS "Building protobuf from source")
     set(PROTOBUF_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/protobuf_ep-install")
@@ -2026,6 +2142,9 @@ if(PAIMON_ENABLE_JINDO)
     build_jindosdk_c()
     build_jindosdk_nextarch()
 endif()
+if(PAIMON_ENABLE_OSS)
+    resolve_dependency(OSS_SDK_V2)
+endif()
 if(PAIMON_ENABLE_S3)
     include(BuildAwsAuth)
     build_aws_auth()
diff --git a/docs/source/build_system.rst b/docs/source/build_system.rst
index a2a1a9e7b..40964f6f8 100644
--- a/docs/source/build_system.rst
+++ b/docs/source/build_system.rst
@@ -106,6 +106,8 @@ Paimon provides a set of built-in optional plugins that you can link to as neede
 
   - ``Paimon::paimon_local_file_system_shared``
   - ``Paimon::paimon_jindo_file_system_shared``
+  - ``Paimon::paimon_oss_file_system_shared``
+  - ``Paimon::paimon_s3_file_system_shared``
 
 - Index plugins:
 
diff --git a/docs/source/building.rst b/docs/source/building.rst
index 32c7e67cd..a057ff539 100644
--- a/docs/source/building.rst
+++ b/docs/source/building.rst
@@ -178,6 +178,8 @@ boolean flags to ``cmake``.
 * ``-DPAIMON_ENABLE_ORC=ON``: Paimon integration with Apache ORC
 * ``-DPAIMON_ENABLE_AVRO=ON``: Apache Avro libraries and Paimon integration
 * ``-DPAIMON_ENABLE_JINDO=ON``: Support for Alibaba Jindo filesystems
+* ``-DPAIMON_ENABLE_OSS=ON``: Support for Alibaba Cloud OSS through OSS SDK V2
+* ``-DPAIMON_ENABLE_S3=ON``: Support for Amazon S3-compatible filesystems
 * ``-DPAIMON_ENABLE_LUMINA=ON``: Support for the Lumina vector index. Requires
   Linux ``x86_64``; see :ref:`cpp-building-platforms`.
 * ``-DPAIMON_ENABLE_LUCENE=ON``: Support for Lucene full-text search indexes
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 3de2b667e..0ec23a0a9 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -201,7 +201,7 @@ set(PAIMON_REST_LINK_LIBS)
 if(PAIMON_ENABLE_REST)
     set(PAIMON_REST_LINK_LIBS OpenSSL::Crypto)
 endif()
-if(PAIMON_ENABLE_S3)
+if(PAIMON_ENABLE_OSS OR PAIMON_ENABLE_S3)
     list(APPEND PAIMON_COMMON_SRCS common/fs/object_store_file_system.cpp)
 endif()
 
@@ -948,7 +948,7 @@ if(PAIMON_BUILD_TESTS)
     endif()
 
     set(PAIMON_OBJECT_STORE_FS_TEST_SOURCES)
-    if(PAIMON_ENABLE_S3)
+    if(PAIMON_ENABLE_OSS OR PAIMON_ENABLE_S3)
         list(APPEND PAIMON_OBJECT_STORE_FS_TEST_SOURCES
              common/fs/object_store_file_system_test.cpp)
     endif()
diff --git a/src/paimon/common/fs/object_store_file_system.cpp b/src/paimon/common/fs/object_store_file_system.cpp
index ae44835e2..26cbe2aa4 100644
--- a/src/paimon/common/fs/object_store_file_system.cpp
+++ b/src/paimon/common/fs/object_store_file_system.cpp
@@ -30,6 +30,7 @@
 #include "paimon/common/utils/path_util.h"
 
 namespace paimon {
+
 namespace {
 
 constexpr int64_t kMaxReadAheadMemory = 64LL * 1024LL * 1024LL;
diff --git a/src/paimon/common/fs/object_store_file_system.h b/src/paimon/common/fs/object_store_file_system.h
index a1313b851..a40ef15c3 100644
--- a/src/paimon/common/fs/object_store_file_system.h
+++ b/src/paimon/common/fs/object_store_file_system.h
@@ -19,7 +19,12 @@
 
 #pragma once
 
+#include <curl/curl.h>
+
+#include <cctype>
+#include <cerrno>
 #include <cstdint>
+#include <ctime>
 #include <functional>
 #include <memory>
 #include <mutex>
@@ -31,6 +36,71 @@
 
 namespace paimon {
 
+class ObjectStoreFileSystemUtils {
+ public:
+    static inline int64_t ParseModificationTime(const std::string& value) {
+        std::tm parsed_time{};
+        const char* current = strptime(value.c_str(), "%Y-%m-%dT%H:%M:%S", &parsed_time);
+        if (current != nullptr) {
+            int32_t milliseconds = 0;
+            int32_t fraction_digits = 0;
+            if (*current == '.') {
+                ++current;
+                const char* fraction_begin = current;
+                while (std::isdigit(static_cast<unsigned char>(*current))) {
+                    if (fraction_digits < 3) {
+                        milliseconds = milliseconds * 10 + (*current - '0');
+                    }
+                    ++fraction_digits;
+                    ++current;
+                }
+                if (current == fraction_begin) {
+                    current = nullptr;
+                }
+                while (fraction_digits < 3) {
+                    milliseconds *= 10;
+                    ++fraction_digits;
+                }
+            }
+
+            int32_t timezone_offset_seconds = 0;
+            bool valid_timezone = false;
+            if (current != nullptr && *current == 'Z' && current[1] == '\0') {
+                valid_timezone = true;
+            } else if (current != nullptr && (*current == '+' || *current == '-') &&
+                       std::isdigit(static_cast<unsigned char>(current[1])) &&
+                       std::isdigit(static_cast<unsigned char>(current[2])) && current[3] == ':' &&
+                       std::isdigit(static_cast<unsigned char>(current[4])) &&
+                       std::isdigit(static_cast<unsigned char>(current[5])) && current[6] == '\0') {
+                int32_t hours = (current[1] - '0') * 10 + current[2] - '0';
+                int32_t minutes = (current[4] - '0') * 10 + current[5] - '0';
+                if (hours <= 23 && minutes <= 59) {
+                    timezone_offset_seconds = (hours * 60 + minutes) * 60;
+                    if (*current == '-') {
+                        timezone_offset_seconds = -timezone_offset_seconds;
+                    }
+                    valid_timezone = true;
+                }
+            }
+
+            if (valid_timezone) {
+                errno = 0;
+                time_t seconds = timegm(&parsed_time);
+                if (seconds != static_cast<time_t>(-1) || errno != EOVERFLOW) {
+                    int64_t utc_seconds = static_cast<int64_t>(seconds) - timezone_offset_seconds;
+                    return utc_seconds * 1000 + milliseconds;
+                }
+            }
+        }
+
+        time_t seconds = curl_getdate(value.c_str(), nullptr);
+        if (seconds == static_cast<time_t>(-1)) {
+            return FileStatus::kUnknownModificationTime;
+        }
+        return static_cast<int64_t>(seconds) * 1000;
+    }
+};
+
 struct ObjectStorePath {
     std::string bucket;
     std::string key;
@@ -39,7 +109,7 @@ struct ObjectStorePath {
 struct ObjectMetadata {
     std::string key;
     int64_t size = 0;
-    int64_t modification_time = 0;
+    int64_t modification_time = FileStatus::kUnknownModificationTime;
 };
 
 struct ListObjectsResult {
diff --git a/src/paimon/common/fs/object_store_file_system_test.cpp b/src/paimon/common/fs/object_store_file_system_test.cpp
index 824060f80..6b8b56503 100644
--- a/src/paimon/common/fs/object_store_file_system_test.cpp
+++ b/src/paimon/common/fs/object_store_file_system_test.cpp
@@ -33,6 +33,17 @@
 namespace paimon::test {
 namespace {
 
+TEST(ObjectStoreFileSystemTest, TestParseModificationTime) {
+    ASSERT_EQ(1704067200000,
+              ObjectStoreFileSystemUtils::ParseModificationTime("2024-01-01T00:00:00.000Z"));
+    ASSERT_EQ(1704067200123,
+              ObjectStoreFileSystemUtils::ParseModificationTime("2024-01-01T08:00:00.123+08:00"));
+    ASSERT_EQ(1704067200000,
+              ObjectStoreFileSystemUtils::ParseModificationTime("Mon, 01 Jan 2024 00:00:00 GMT"));
+    ASSERT_EQ(FileStatus::kUnknownModificationTime,
+              ObjectStoreFileSystemUtils::ParseModificationTime("not-a-timestamp"));
+}
+
 using Range = std::pair<int64_t, int64_t>;
 
 class MockObjectStoreClient : public ObjectStoreClient {
diff --git a/src/paimon/fs/oss/CMakeLists.txt b/src/paimon/fs/oss/CMakeLists.txt
new file mode 100644
index 000000000..fb0ea6744
--- /dev/null
+++ b/src/paimon/fs/oss/CMakeLists.txt
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if(PAIMON_ENABLE_OSS)
+    add_paimon_lib(paimon_oss_file_system
+                   SOURCES
+                   oss_file_system.cpp
+                   oss_file_system_factory.cpp
+                   EXTRA_INCLUDES
+                   ${OSS_SDK_V2_INCLUDE_DIR}
+                   DEPENDENCIES
+                   paimon_shared
+                   CURL::libcurl
+                   STATIC_LINK_LIBS
+                   alibabacloud_oss_v2::oss
+                   fmt
+                   SHARED_LINK_LIBS
+                   paimon_shared
+                   SHARED_LINK_FLAGS
+                   ${PAIMON_VERSION_SCRIPT_FLAGS})
+
+    add_dependencies(paimon_oss_file_system_objlib oss_sdk_v2_ep)
+
+    if(PAIMON_BUILD_TESTS)
+        add_paimon_test(oss_file_system_test
+                        SOURCES
+                        oss_file_system_test.cpp
+                        EXTRA_INCLUDES
+                        ${OSS_SDK_V2_INCLUDE_DIR}
+                        STATIC_LINK_LIBS
+                        paimon_shared
+                        test_utils_static
+                        ${PAIMON_OSS_FILE_SYSTEM_STATIC_LINK_LIBS}
+                        ${GTEST_LINK_TOOLCHAIN})
+    endif()
+endif()
diff --git a/src/paimon/fs/oss/oss_file_system.cpp b/src/paimon/fs/oss/oss_file_system.cpp
new file mode 100644
index 000000000..bc283cc8b
--- /dev/null
+++ b/src/paimon/fs/oss/oss_file_system.cpp
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/fs/oss/oss_file_system.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <memory>
+#include <utility>
+
+#include "alibabacloud/oss2/OSSClient.h"
+#include "alibabacloud/oss2/Operation.h"
+#include "alibabacloud/oss2/Types.h"
+#include "alibabacloud/oss2/io/ByteWriter.h"
+#include "alibabacloud/oss2/models/BucketBasic.h"
+#include "alibabacloud/oss2/models/ObjectBasic.h"
+#include "fmt/format.h"
+#include "paimon/executor.h"
+
+namespace paimon::oss {
+namespace {
+
+namespace oss2 = alibabacloud::oss2;
+
+constexpr int32_t kMaxKeysPerRequest = 1000;
+
+bool IsNotFoundError(const oss2::OperationError& error) {
+    return error.getStatusCode() == 404 || error.getCode() == "NoSuchKey" ||
+           error.getCode() == "NoSuchBucket" || error.getCode() == "NotFound";
+}
+
+Status ToPaimonStatus(const oss2::OperationError& error, const std::string& operation,
+                      const ObjectStorePath& path) {
+    std::string message = fmt::format("OSS {} 'oss://{}/{}' failed: code={}, status={}, message={}",
+                                      operation, path.bucket, path.key, error.getCode(),
+                                      error.getStatusCode(), error.getMessage());
+    if (!error.getRequestId().empty()) {
+        message += fmt::format(", request_id={}", error.getRequestId());
+    }
+    if (IsNotFoundError(error)) {
+        return Status::NotExist(message);
+    }
+    if (error.getCode() == "RequestCanceled") {
+        return Status::Cancelled(message);
+    }
+    return Status::IOError(message);
+}
+
+class OssObjectStoreClient : public ObjectStoreClient,
+                             public std::enable_shared_from_this<OssObjectStoreClient> {
+ public:
+    OssObjectStoreClient(std::string bucket, std::shared_ptr<oss2::OSSClient> client,
+                         std::unique_ptr<Executor> executor)
+        : bucket_(std::move(bucket)), client_(std::move(client)), executor_(std::move(executor)) {}
+
+    Result<ObjectMetadata> HeadObject(const ObjectStorePath& path) const override {
+        PAIMON_RETURN_NOT_OK(ValidateBucket(path));
+        oss2::models::HeadObjectRequest request;
+        request.setBucket(path.bucket).setKey(path.key);
+        oss2::HeadObjectOutcome outcome = client_->headObject(request);
+        if (!outcome.has_value()) {
+            return ToPaimonStatus(outcome.error(), "HeadObject", path);
+        }
+        const oss2::models::HeadObjectResult& result = outcome.value();
+        if (result.getContentLength() < 0) {
+            return Status::IOError("OSS HeadObject response is missing Content-Length");
+        }
+        return ObjectMetadata{
+            path.key, result.getContentLength(),
+            ObjectStoreFileSystemUtils::ParseModificationTime(result.getLastModified())};
+    }
+
+    Result<ListObjectsResult> ListObjects(const ObjectStorePath& path,
+                                          const std::string& continuation_token,
+                                          int32_t max_keys) const override {
+        PAIMON_RETURN_NOT_OK(ValidateBucket(path));
+        oss2::models::ListObjectsV2Request request;
+        request.setBucket(path.bucket).setPrefix(path.key).setDelimiter("/");
+        if (!continuation_token.empty()) {
+            request.setContinuationToken(continuation_token);
+        }
+        if (max_keys > 0) {
+            // OSS limits ListObjectsV2 requests to 1000 keys.
+            request.setMaxKeys(std::min(max_keys, kMaxKeysPerRequest));
+        }
+        oss2::ListObjectsV2Outcome outcome = client_->listObjectsV2(request);
+        if (!outcome.has_value()) {
+            return ToPaimonStatus(outcome.error(), "ListObjectsV2", path);
+        }
+        const oss2::models::ListObjectsV2Result& value = outcome.value();
+        ListObjectsResult result;
+        result.objects.reserve(value.getContents().size());
+        for (const oss2::models::ObjectSummary& object : value.getContents()) {
+            result.objects.push_back(ObjectMetadata{
+                object.key, object.size,
+                ObjectStoreFileSystemUtils::ParseModificationTime(object.lastModified)});
+        }
+        result.common_prefixes.reserve(value.getCommonPrefixes().size());
+        for (const oss2::models::CommonPrefix& prefix : value.getCommonPrefixes()) {
+            result.common_prefixes.push_back(prefix.prefix);
+        }
+        result.is_truncated = value.getIsTruncated();
+        result.continuation_token = value.getNextContinuationToken();
+        return result;
+    }
+
+    Result<int64_t> GetObjectRange(const ObjectStorePath& path, int64_t offset, int64_t size,
+                                   char* buffer) const override {
+        PAIMON_RETURN_NOT_OK(ValidateBucket(path));
+        if (size == 0) {
+            return 0;
+        }
+        auto writer = std::make_shared<std::shared_ptr<oss2::MemoryWriter>>();
+        oss2::SinkFactory sink;
+        sink.isOneShot = false;
+        sink.supplier = [buffer, size, writer](int64_t, const oss2::HeaderCollection&) {
+            auto memory_writer = std::make_shared<oss2::MemoryWriter>(
+                reinterpret_cast<uint8_t*>(buffer), static_cast<size_t>(size));
+            *writer = memory_writer;
+            return memory_writer;
+        };
+        oss2::models::GetObjectRequest request;
+        request.setBucket(path.bucket)
+            .setKey(path.key)
+            .setRange(fmt::format("bytes={}-{}", offset, offset + size - 1))
+            .setRangeBehavior("standard")
+            .setSinkFactory(std::move(sink));
+        oss2::GetObjectOutcome outcome = client_->getObject(request);
+        if (!outcome.has_value()) {
+            return ToPaimonStatus(outcome.error(), "GetObject", path);
+        }
+        int64_t written =
+            *writer ? static_cast<int64_t>((*writer)->written()) : static_cast<int64_t>(0);
+        if (written != size) {
+            return Status::IOError(
+                fmt::format("OSS GetObject read {} bytes for oss://{}/{}, expected {}", written,
+                            path.bucket, path.key, size));
+        }
+        return written;
+    }
+
+    void GetObjectRangeAsync(const ObjectStorePath& path, int64_t offset, int64_t size,
+                             char* buffer, std::function<void(Status)>&& callback) const override {
+        std::shared_ptr<const OssObjectStoreClient> self = shared_from_this();
+        executor_->Add([self = std::move(self), path, offset, size, buffer,
+                        callback = std::move(callback)]() mutable {
+            Result<int64_t> result = self->GetObjectRange(path, offset, size, buffer);
+            callback(result.ok() ? Status::OK() : result.status());
+        });
+    }
+
+ private:
+    Status ValidateBucket(const ObjectStorePath& path) const {
+        if (path.bucket != bucket_) {
+            return Status::Invalid(
+                fmt::format("OSS file system for bucket '{}' cannot access "
+                            "'oss://{}/{}'",
+                            bucket_, path.bucket, path.key));
+        }
+        return Status::OK();
+    }
+
+    std::string bucket_;
+    std::shared_ptr<oss2::OSSClient> client_;
+    std::unique_ptr<Executor> executor_;
+};
+
+}  // namespace
+
+OssFileSystem::OssFileSystem(std::string bucket, std::shared_ptr<oss2::OSSClient> client,
+                             std::unique_ptr<Executor> executor)
+    : ObjectStoreFileSystem("oss", std::make_shared<OssObjectStoreClient>(
+                                       std::move(bucket), std::move(client), std::move(executor))) {
+}
+
+}  // namespace paimon::oss
diff --git a/src/paimon/fs/oss/oss_file_system.h b/src/paimon/fs/oss/oss_file_system.h
new file mode 100644
index 000000000..0749718e9
--- /dev/null
+++ b/src/paimon/fs/oss/oss_file_system.h
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "paimon/common/fs/object_store_file_system.h"
+#include "paimon/executor.h"
+
+namespace alibabacloud::oss2 {
+class OSSClient;
+}
+
+namespace paimon::oss {
+
+inline constexpr char kOssAccessKeyIdOption[] = "fs.oss.accessKeyId";
+inline constexpr char kOssAccessKeySecretOption[] = "fs.oss.accessKeySecret";
+inline constexpr char kOssEndpointOption[] = "fs.oss.endpoint";
+inline constexpr char kOssRegionOption[] = "fs.oss.region";
+inline constexpr char kOssSignatureVersionOption[] = "fs.oss.signatureVersion";
+inline constexpr char kOssSecurityTokenOption[] = "fs.oss.securityToken";
+inline constexpr char kOssSessionTokenOption[] = "fs.oss.sessionToken";
+inline constexpr char kOssUsePathStyleOption[] = "fs.oss.usePathStyle";
+inline constexpr char kOssExecutorThreadCountOption[] = "fs.oss.executor.thread-count";
+
+class OssFileSystem : public ObjectStoreFileSystem {
+ public:
+    OssFileSystem(std::string bucket, std::shared_ptr<alibabacloud::oss2::OSSClient> client,
+                  std::unique_ptr<Executor> executor);
+    ~OssFileSystem() override = default;
+};
+
+}  // namespace paimon::oss
diff --git a/src/paimon/fs/oss/oss_file_system_factory.cpp b/src/paimon/fs/oss/oss_file_system_factory.cpp
new file mode 100644
index 000000000..327586d0e
--- /dev/null
+++ b/src/paimon/fs/oss/oss_file_system_factory.cpp
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/fs/oss/oss_file_system_factory.h"
+
+#include <cctype>
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <utility>
+
+#include "alibabacloud/oss2/ClientConfiguration.h"
+#include "alibabacloud/oss2/OSSClient.h"
+#include "alibabacloud/oss2/credentials/CredentialsProvider.h"
+#include "fmt/format.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/factories/factory.h"
+#include "paimon/fs/oss/oss_file_system.h"
+
+namespace paimon::oss {
+namespace {
+
+namespace oss2 = alibabacloud::oss2;
+
+constexpr std::string_view kOssOptionPrefix = "fs.oss.";
+constexpr char kEndpointPrefix[] = "oss-";
+constexpr char kEndpointSuffix[] = ".aliyuncs.com";
+constexpr char kDualStackEndpointSuffix[] = ".oss.aliyuncs.com";
+constexpr char kInternalEndpointSuffix[] = "-internal";
+
+bool IsValidRegion(const std::string& region) {
+    size_t first_separator = region.find('-');
+    if (first_separator < 2 || first_separator > 3 || first_separator + 1 == region.size()) {
+        return false;
+    }
+    for (size_t i = 0; i < first_separator; ++i) {
+        if (!std::islower(static_cast<unsigned char>(region[i]))) {
+            return false;
+        }
+    }
+    bool previous_was_separator = true;
+    for (size_t i = first_separator + 1; i < region.size(); ++i) {
+        char value = region[i];
+        if (value == '-') {
+            if (previous_was_separator || i + 1 == region.size()) {
+                return false;
+            }
+            previous_was_separator = true;
+        } else if (std::islower(static_cast<unsigned char>(value)) ||
+                   std::isdigit(static_cast<unsigned char>(value))) {
+            previous_was_separator = false;
+        } else {
+            return false;
+        }
+    }
+    return !StringUtils::EndsWith(region, "-dualstack") && !StringUtils::EndsWith(region, "-pub");
+}
+
+std::string GetBucketOptionKey(const std::string& bucket, std::string_view option) {
+    return fmt::format("fs.oss.bucket.{}.{}", bucket, option.substr(kOssOptionPrefix.size()));
+}
+
+const std::string* FindOption(const std::map<std::string, std::string>& options,
+                              const std::string& bucket, std::string_view option,
+                              std::string* option_key) {
+    std::string bucket_option_key = GetBucketOptionKey(bucket, option);
+    auto bucket_option = options.find(bucket_option_key);
+    if (bucket_option != options.end()) {
+        *option_key = std::move(bucket_option_key);
+        return &bucket_option->second;
+    }
+    option_key->assign(option);
+    auto global_option = options.find(*option_key);
+    return global_option == options.end() ? nullptr : &global_option->second;
+}
+
+std::string GetOption(const std::map<std::string, std::string>& options, const std::string& bucket,
+                      std::string_view option) {
+    std::string option_key;
+    const std::string* value = FindOption(options, bucket, option, &option_key);
+    return value == nullptr ? "" : *value;
+}
+
+Result<std::string> GetRequiredOption(const std::map<std::string, std::string>& options,
+                                      const std::string& bucket, std::string_view option) {
+    std::string option_key;
+    const std::string* value = FindOption(options, bucket, option, &option_key);
+    if (value == nullptr || value->empty()) {
+        return Status::Invalid(fmt::format("OSS option '{}' must not be empty", option_key));
+    }
+    return *value;
+}
+
+Result<std::unique_ptr<Executor>> CreateExecutor(const std::map<std::string, std::string>& options,
+                                                 const std::string& bucket) {
+    std::string option_key;
+    const std::string* value =
+        FindOption(options, bucket, kOssExecutorThreadCountOption, &option_key);
+    if (value == nullptr) {
+        return CreateDefaultExecutor();
+    }
+    std::optional<uint32_t> thread_count = StringUtils::StringToValue<uint32_t>(*value);
+    if (!thread_count.has_value() || *thread_count == 0) {
+        return Status::Invalid(fmt::format(
+            "OSS executor thread count for option '{}' must be greater than 0", option_key));
+    }
+    return CreateDefaultExecutor(*thread_count);
+}
+
+std::string NormalizeEndpoint(std::string endpoint) {
+    if (!endpoint.empty() && endpoint.find("://") == std::string::npos) {
+        endpoint = "https://" + endpoint;
+    }
+    return endpoint;
+}
+
+std::string InferRegion(std::string endpoint) {
+    size_t scheme = endpoint.find("://");
+    if (scheme != std::string::npos) {
+        endpoint.erase(0, scheme + 3);
+    }
+    size_t slash = endpoint.find('/');
+    if (slash != std::string::npos) {
+        endpoint.erase(slash);
+    }
+    size_t port_separator = endpoint.rfind(':');
+    if (port_separator != std::string::npos && endpoint.find(':') == port_separator) {
+        std::optional<uint16_t> port =
+            StringUtils::StringToValue<uint16_t>(endpoint.substr(port_separator + 1));
+        if (port.has_value()) {
+            endpoint.erase(port_separator);
+        }
+    }
+
+    std::string region;
+    if (StringUtils::StartsWith(endpoint, kEndpointPrefix) &&
+        StringUtils::EndsWith(endpoint, kEndpointSuffix)) {
+        region = endpoint.substr(
+            sizeof(kEndpointPrefix) - 1,
+            endpoint.size() - (sizeof(kEndpointPrefix) - 1) - (sizeof(kEndpointSuffix) - 1));
+        if (StringUtils::EndsWith(region, kInternalEndpointSuffix)) {
+            region.erase(region.size() - (sizeof(kInternalEndpointSuffix) - 1));
+        }
+    } else if (StringUtils::EndsWith(endpoint, kDualStackEndpointSuffix)) {
+        region = endpoint.substr(0, endpoint.size() - (sizeof(kDualStackEndpointSuffix) - 1));
+    }
+    return IsValidRegion(region) ? region : "";
+}
+
+}  // namespace
+
+const char OssFileSystemFactory::IDENTIFIER[] = "oss";
+
+Result<std::unique_ptr<FileSystem>> OssFileSystemFactory::Create(
+    const std::string& path, const std::map<std::string, std::string>& options) const {
+    PAIMON_ASSIGN_OR_RAISE(Path parsed_path, PathUtil::ToPath(path));
+    if (parsed_path.scheme != "oss" || parsed_path.authority.empty()) {
+        return Status::Invalid(fmt::format("invalid OSS path '{}'", path));
+    }
+    const std::string& bucket = parsed_path.authority;
+    PAIMON_ASSIGN_OR_RAISE(std::string access_key_id,
+                           GetRequiredOption(options, bucket, kOssAccessKeyIdOption));
+    PAIMON_ASSIGN_OR_RAISE(std::string access_key_secret,
+                           GetRequiredOption(options, bucket, kOssAccessKeySecretOption));
+    std::string endpoint = GetOption(options, bucket, kOssEndpointOption);
+    std::string region = GetOption(options, bucket, kOssRegionOption);
+    if (region.empty()) {
+        region = InferRegion(endpoint);
+    }
+    if (endpoint.empty() && region.empty()) {
+        return Status::Invalid("OSS endpoint or region must be configured");
+    }
+    std::string signature_version = GetOption(options, bucket, kOssSignatureVersionOption);
+    if (!signature_version.empty() && signature_version != "v1" && signature_version != "v4") {
+        return Status::Invalid(
+            fmt::format("invalid OSS signature version '{}'", signature_version));
+    }
+    if (region.empty() && signature_version != "v1") {
+        return Status::Invalid(
+            "OSS region must be configured when the endpoint does not identify a region");
+    }
+    std::string security_token = GetOption(options, bucket, kOssSecurityTokenOption);
+    if (security_token.empty()) {
+        security_token = GetOption(options, bucket, kOssSessionTokenOption);
+    }
+
+    oss2::ClientConfiguration config = oss2::ClientConfiguration::loadDefault();
+    if (!endpoint.empty()) {
+        config.endpoint = NormalizeEndpoint(endpoint);
+    }
+    if (!region.empty()) {
+        config.region = region;
+    }
+    if (!signature_version.empty()) {
+        config.signatureVersion = signature_version;
+    }
+    config.userAgent = "paimon-cpp";
+    config.credentialsProvider = std::make_shared<oss2::StaticCredentialsProvider>(
+        std::move(access_key_id), std::move(access_key_secret), std::move(security_token));
+    std::string path_style = GetOption(options, bucket, kOssUsePathStyleOption);
+    if (!path_style.empty()) {
+        std::optional<bool> value = StringUtils::StringToValue<bool>(path_style);
+        if (!value.has_value()) {
+            return Status::Invalid(fmt::format("invalid boolean value '{}' for OSS option '{}'",
+                                               path_style, kOssUsePathStyleOption));
+        }
+        config.usePathStyle = *value;
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<Executor> executor, CreateExecutor(options, bucket));
+    return std::make_unique<OssFileSystem>(bucket, std::make_shared<oss2::OSSClient>(config),
+                                           std::move(executor));
+}
+
+REGISTER_PAIMON_FACTORY(OssFileSystemFactory);
+
+}  // namespace paimon::oss
diff --git a/src/paimon/fs/oss/oss_file_system_factory.h b/src/paimon/fs/oss/oss_file_system_factory.h
new file mode 100644
index 000000000..fe92715e5
--- /dev/null
+++ b/src/paimon/fs/oss/oss_file_system_factory.h
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include "paimon/fs/file_system_factory.h"
+
+namespace paimon::oss {
+
+class OssFileSystemFactory : public FileSystemFactory {
+ public:
+    static const char IDENTIFIER[];
+
+    const char* Identifier() const override {
+        return IDENTIFIER;
+    }
+
+    Result<std::unique_ptr<FileSystem>> Create(
+        const std::string& path, const std::map<std::string, std::string>& options) const override;
+};
+
+}  // namespace paimon::oss
diff --git a/src/paimon/fs/oss/oss_file_system_test.cpp b/src/paimon/fs/oss/oss_file_system_test.cpp
new file mode 100644
index 000000000..fbd847f80
--- /dev/null
+++ b/src/paimon/fs/oss/oss_file_system_test.cpp
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/fs/oss/oss_file_system.h"
+
+#include <chrono>
+#include <cstdint>
+#include <future>
+#include <map>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <system_error>
+#include <utility>
+#include <vector>
+
+#include "alibabacloud/oss2/ClientConfiguration.h"
+#include "alibabacloud/oss2/OSSClient.h"
+#include "alibabacloud/oss2/credentials/CredentialsProvider.h"
+#include "alibabacloud/oss2/io/ByteWriter.h"
+#include "alibabacloud/oss2/transport/HttpTransport.h"
+#include "gtest/gtest.h"
+#include "paimon/fs/oss/oss_file_system_factory.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::oss {
+namespace {
+
+namespace oss2 = alibabacloud::oss2;
+
+class MockHttpTransport : public oss2::HttpTransport {
+ public:
+    oss2::ResponseResult send(std::unique_ptr<oss2::RequestMessage>& request,
+                              const oss2::RequestOptions& options) override {
+        requests_.emplace_back(std::make_unique<oss2::RequestMessage>(*request));
+        if (responses_.empty()) {
+            return oss2::TransportError{std::make_error_code(std::errc::no_message_available), "",
+                                        ""};
+        }
+        std::unique_ptr<oss2::ResponseMessage> response = std::move(responses_.front());
+        responses_.erase(responses_.begin());
+        if (response->statusCode / 100 == 2 && options.sinkFactory.has_value() &&
+            response->body != nullptr) {
+            int64_t content_length = -1;
+            auto content_length_header = response->headers.find("Content-Length");
+            if (content_length_header != response->headers.end()) {
+                content_length = std::stoll(content_length_header->second);
+            }
+            std::shared_ptr<oss2::ByteWriter> sink =
+                options.sinkFactory.value()(content_length, response->headers);
+            std::ostringstream body;
+            body << response->body->rdbuf();
+            const std::string data = body.str();
+            sink->write(reinterpret_cast<const uint8_t*>(data.data()), data.size());
+            response->body.reset();
+        }
+        return response;
+    }
+
+    std::string getName() const override {
+        return "MockHttpTransport";
+    }
+
+    void AddResponse(int status_code, oss2::HeaderCollection headers, std::string body = "") {
+        std::shared_ptr<std::iostream> response_body;
+        if (!body.empty()) {
+            response_body = std::make_shared<std::stringstream>(std::move(body));
+        }
+        responses_.emplace_back(std::make_unique<oss2::ResponseMessage>(oss2::ResponseMessage{
+            status_code, "", std::move(headers), std::move(response_body), nullptr}));
+    }
+
+    std::vector<std::unique_ptr<oss2::ResponseMessage>> responses_;
+    std::vector<std::unique_ptr<oss2::RequestMessage>> requests_;
+};
+
+std::unique_ptr<OssFileSystem> CreateFileSystem(
+    const std::shared_ptr<MockHttpTransport>& transport) {
+    oss2::ClientConfiguration config = oss2::ClientConfiguration::loadDefault();
+    config.region = "cn-hangzhou";
+    config.credentialsProvider =
+        std::make_shared<oss2::StaticCredentialsProvider>("access-key", "secret-key");
+    config.httpTransport = transport;
+    return std::make_unique<OssFileSystem>("bucket", std::make_shared<oss2::OSSClient>(config),
+                                           CreateDefaultExecutor());
+}
+
+}  // namespace
+
+TEST(OssFileSystemFactoryTest, TestOptionValidation) {
+    OssFileSystemFactory factory;
+    std::map<std::string, std::string> options;
+    ASSERT_NOK(factory.Create("s3://bucket/key", options));
+    ASSERT_NOK(factory.Create("oss://bucket/key", options));
+
+    options[kOssAccessKeyIdOption] = "access-key";
+    options[kOssAccessKeySecretOption] = "secret-key";
+    options[kOssEndpointOption] = "oss-cn-hangzhou.aliyuncs.com";
+    options[kOssUsePathStyleOption] = "treu";
+    ASSERT_NOK(factory.Create("oss://bucket/key", options));
+
+    options[kOssUsePathStyleOption] = "false";
+    options[kOssSignatureVersionOption] = "v2";
+    ASSERT_NOK(factory.Create("oss://bucket/key", options));
+
+    options[kOssSignatureVersionOption] = "v4";
+    options[kOssExecutorThreadCountOption] = "0";
+    ASSERT_NOK(factory.Create("oss://bucket/key", options));
+
+    options[kOssExecutorThreadCountOption] = "4";
+    ASSERT_OK(factory.Create("oss://bucket/key", options));
+
+    options[kOssEndpointOption] = "";
+    options[kOssRegionOption] = "cn-hangzhou";
+    ASSERT_OK(factory.Create("oss://bucket/key", options));
+}
+
+TEST(OssFileSystemFactoryTest, TestBucketOptionsOverrideGlobalOptions) {
+    OssFileSystemFactory factory;
+    std::map<std::string, std::string> options = {
+        {kOssAccessKeyIdOption, ""},
+        {kOssAccessKeySecretOption, ""},
+        {kOssEndpointOption, ""},
+        {"fs.oss.bucket.bucket.accessKeyId", "access-key"},
+        {"fs.oss.bucket.bucket.accessKeySecret", "secret-key"},
+        {"fs.oss.bucket.bucket.endpoint", "oss-cn-hangzhou.aliyuncs.com"},
+    };
+    ASSERT_OK(factory.Create("oss://bucket/key", options));
+}
+
+TEST(OssFileSystemFactoryTest, TestBucketOptionErrorReportsBucketKey) {
+    OssFileSystemFactory factory;
+    std::map<std::string, std::string> options = {
+        {kOssAccessKeyIdOption, "access-key"},
+        {kOssAccessKeySecretOption, "secret-key"},
+        {kOssEndpointOption, "oss-cn-hangzhou.aliyuncs.com"},
+        {"fs.oss.bucket.bucket.accessKeyId", ""},
+    };
+    ASSERT_NOK_WITH_MSG(factory.Create("oss://bucket/key", options),
+                        "fs.oss.bucket.bucket.accessKeyId");
+}
+
+TEST(OssFileSystemFactoryTest, TestEndpointRegionValidation) {
+    OssFileSystemFactory factory;
+    std::map<std::string, std::string> options = {
+        {kOssAccessKeyIdOption, "access-key"},
+        {kOssAccessKeySecretOption, "secret-key"},
+    };
+
+    options[kOssEndpointOption] = "oss-cn-hangzhou.aliyuncs.com";
+    ASSERT_OK(factory.Create("oss://bucket/key", options));
+
+    options[kOssEndpointOption] = "oss-cn-hangzhou-internal.aliyuncs.com";
+    ASSERT_OK(factory.Create("oss://bucket/key", options));
+
+    options[kOssEndpointOption] = "oss-ap-southeast-1.aliyuncs.com:443";
+    ASSERT_OK(factory.Create("oss://bucket/key", options));
+
+    options[kOssEndpointOption] = "cn-hangzhou.oss.aliyuncs.com";
+    ASSERT_OK(factory.Create("oss://bucket/key", options));
+
+    options[kOssEndpointOption] = "oss-accelerate.aliyuncs.com";
+    ASSERT_NOK_WITH_MSG(factory.Create("oss://bucket/key", options), "OSS region must be");
+
+    options[kOssEndpointOption] = "oss-accelerate-overseas.aliyuncs.com";
+    ASSERT_NOK_WITH_MSG(factory.Create("oss://bucket/key", options), "OSS region must be");
+
+    options[kOssEndpointOption] = "oss.example.com";
+    ASSERT_NOK_WITH_MSG(factory.Create("oss://bucket/key", options), "OSS region must be");
+
+    options[kOssRegionOption] = "cn-hangzhou";
+    ASSERT_OK(factory.Create("oss://bucket/key", options));
+
+    options.erase(kOssRegionOption);
+    options[kOssSignatureVersionOption] = "v1";
+    ASSERT_OK(factory.Create("oss://bucket/key", options));
+}
+
+TEST(OssFileSystemTest, TestHeadObjectParsesMetadata) {
+    std::shared_ptr<MockHttpTransport> transport = std::make_shared<MockHttpTransport>();
+    transport->AddResponse(200, {{"Content-Length", "3"},
+                                 {"Last-Modified", "not-a-timestamp"},
+                                 {"x-oss-request-id", "request-id"}});
+    std::unique_ptr<OssFileSystem> file_system = CreateFileSystem(transport);
+
+    ASSERT_OK_AND_ASSIGN(FileStatus status, file_system->GetFileStatus("oss://bucket/key"));
+    ASSERT_EQ(3, status.GetLen());
+    ASSERT_EQ(FileStatus::kUnknownModificationTime, status.GetModificationTime());
+    ASSERT_EQ(1U, transport->requests_.size());
+    ASSERT_EQ("HEAD", transport->requests_[0]->method);
+}
+
+TEST(OssFileSystemTest, TestHeadObjectNotFound) {
+    std::shared_ptr<MockHttpTransport> transport = std::make_shared<MockHttpTransport>();
+    transport->AddResponse(404, {{"x-oss-request-id", "request-id"}},
+                           "<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>");
+    transport->AddResponse(200, {},
+                           "<ListBucketResult><IsTruncated>false</IsTruncated></ListBucketResult>");
+    std::unique_ptr<OssFileSystem> file_system = CreateFileSystem(transport);
+
+    Result<FileStatus> status = file_system->GetFileStatus("oss://bucket/missing");
+    ASSERT_TRUE(status.status().IsNotExist()) << status.status().ToString();
+    ASSERT_NOK_WITH_MSG(status, "does not exist");
+}
+
+TEST(OssFileSystemTest, TestHeadObjectErrorMapping) {
+    std::shared_ptr<MockHttpTransport> transport = std::make_shared<MockHttpTransport>();
+    transport->AddResponse(403, {{"x-oss-request-id", "request-id"}},
+                           "<Error><Code>AccessDenied</Code><Message>denied</Message></Error>");
+    std::unique_ptr<OssFileSystem> file_system = CreateFileSystem(transport);
+
+    Result<FileStatus> status = file_system->GetFileStatus("oss://bucket/key");
+    ASSERT_TRUE(status.status().IsIOError()) << status.status().ToString();
+    ASSERT_NOK_WITH_MSG(status, "code=AccessDenied, status=403");
+}
+
+TEST(OssFileSystemTest, TestListObjects) {
+    std::shared_ptr<MockHttpTransport> transport = std::make_shared<MockHttpTransport>();
+    transport->AddResponse(404, {{"x-oss-request-id", "request-id"}},
+                           "<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>");
+    transport->AddResponse(200, {{"x-oss-request-id", "request-id"}}, R"(
+<ListBucketResult>
+  <IsTruncated>false</IsTruncated>
+  <Contents>
+    <Key>prefix/file</Key>
+    <Size>3</Size>
+    <LastModified>2024-01-01T00:00:00.000Z</LastModified>
+  </Contents>
+  <CommonPrefixes><Prefix>prefix/sub/</Prefix></CommonPrefixes>
+</ListBucketResult>)");
+    std::unique_ptr<OssFileSystem> file_system = CreateFileSystem(transport);
+    std::vector<FileStatus> statuses;
+
+    ASSERT_OK(file_system->ListFileStatus("oss://bucket/prefix", &statuses));
+    ASSERT_EQ(2U, statuses.size());
+    ASSERT_EQ(1704067200000, statuses[0].GetModificationTime());
+    ASSERT_EQ(2U, transport->requests_.size());
+    ASSERT_EQ("HEAD", transport->requests_[0]->method);
+    ASSERT_EQ("GET", transport->requests_[1]->method);
+    ASSERT_NE(std::string::npos, transport->requests_[1]->uri.find("list-type=2"));
+}
+
+TEST(OssFileSystemTest, TestListObjectsErrorMapping) {
+    std::shared_ptr<MockHttpTransport> transport = std::make_shared<MockHttpTransport>();
+    transport->AddResponse(403, {{"x-oss-request-id", "request-id"}},
+                           "<Error><Code>AccessDenied</Code><Message>denied</Message></Error>");
+    std::unique_ptr<OssFileSystem> file_system = CreateFileSystem(transport);
+    std::vector<BasicFileStatus> statuses;
+
+    Status status = file_system->ListDir("oss://bucket/prefix/", &statuses);
+    ASSERT_TRUE(status.IsIOError()) << status.ToString();
+    ASSERT_NOK_WITH_MSG(status, "code=AccessDenied, status=403");
+}
+
+TEST(OssFileSystemTest, TestGetObjectRangeAndShortRead) {
+    std::shared_ptr<MockHttpTransport> transport = std::make_shared<MockHttpTransport>();
+    transport->AddResponse(206, {{"Content-Length", "3"}}, "abc");
+    std::unique_ptr<OssFileSystem> file_system = CreateFileSystem(transport);
+    FileStatus file_status("oss://bucket/key", 3);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<InputStream> stream, file_system->Open(file_status));
+    char buffer[3];
+
+    ASSERT_OK_AND_ASSIGN(int64_t bytes_read, stream->Read(buffer, 3, 0));
+    ASSERT_EQ(3, bytes_read);
+    ASSERT_EQ("abc", std::string(buffer, sizeof(buffer)));
+    ASSERT_EQ(1U, transport->requests_.size());
+    ASSERT_EQ("bytes=0-2", transport->requests_[0]->headers.at("range"));
+
+    std::shared_ptr<MockHttpTransport> short_transport = std::make_shared<MockHttpTransport>();
+    short_transport->AddResponse(206, {{"Content-Length", "2"}}, "ab");
+    std::unique_ptr<OssFileSystem> short_file_system = CreateFileSystem(short_transport);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<InputStream> short_stream,
+                         short_file_system->Open(file_status));
+    ASSERT_NOK_WITH_MSG(short_stream->Read(buffer, 3, 0), "expected 3");
+}
+
+TEST(OssFileSystemTest, TestGetObjectRangeErrorMapping) {
+    std::shared_ptr<MockHttpTransport> transport = std::make_shared<MockHttpTransport>();
+    transport->AddResponse(403, {{"x-oss-request-id", "request-id"}},
+                           "<Error><Code>AccessDenied</Code><Message>denied</Message></Error>");
+    std::unique_ptr<OssFileSystem> file_system = CreateFileSystem(transport);
+    FileStatus file_status("oss://bucket/key", 3);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<InputStream> stream, file_system->Open(file_status));
+    char buffer[3];
+
+    Result<int64_t> result = stream->Read(buffer, 3, 0);
+    ASSERT_TRUE(result.status().IsIOError()) << result.status().ToString();
+    ASSERT_NOK_WITH_MSG(result, "code=AccessDenied, status=403");
+}
+
+TEST(OssFileSystemTest, TestGetObjectRangeAsync) {
+    std::shared_ptr<MockHttpTransport> transport = std::make_shared<MockHttpTransport>();
+    transport->AddResponse(206, {{"Content-Length", "3"}}, "abc");
+    std::unique_ptr<OssFileSystem> file_system = CreateFileSystem(transport);
+    FileStatus file_status("oss://bucket/key", 3);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<InputStream> stream, file_system->Open(file_status));
+    char buffer[3];
+    std::promise<Status> promise;
+    std::future<Status> future = promise.get_future();
+
+    stream->ReadAsync(buffer, 3, 0,
+                      [&promise](Status status) { promise.set_value(std::move(status)); });
+
+    ASSERT_EQ(std::future_status::ready, future.wait_for(std::chrono::seconds(5)));
+    ASSERT_OK(future.get());
+    ASSERT_EQ("abc", std::string(buffer, sizeof(buffer)));
+    ASSERT_EQ(1U, transport->requests_.size());
+    ASSERT_EQ("bytes=0-2", transport->requests_[0]->headers.at("range"));
+}
+
+}  // namespace paimon::oss
diff --git a/src/paimon/fs/s3/s3_file_system.cpp b/src/paimon/fs/s3/s3_file_system.cpp
index e31cc9ca3..b9b98f1e5 100644
--- a/src/paimon/fs/s3/s3_file_system.cpp
+++ b/src/paimon/fs/s3/s3_file_system.cpp
@@ -71,11 +71,6 @@ Result<int64_t> ParseNonNegativeInt64(const std::string& value, const std::strin
     return *result;
 }
 
-int64_t ParseModificationTime(const std::string& value) {
-    time_t seconds = curl_getdate(value.c_str(), nullptr);
-    return seconds == static_cast<time_t>(-1) ? 0 : static_cast<int64_t>(seconds) * 1000;
-}
-
 std::string XmlUnescape(const std::string& value) {
     const std::pair<const char*, const char*> entities[] = {
         {"&amp;", "&"}, {"&lt;", "<"}, {"&gt;", ">"}, {"&quot;", "\""}, {"&apos;", "'"}};
@@ -662,10 +657,10 @@ class S3ObjectStoreClient : public ObjectStoreClient,
         if (length == response.headers.end()) {
             return Status::IOError("HeadObject response is missing Content-Length");
         }
-        int64_t modification_time = 0;
+        int64_t modification_time = FileStatus::kUnknownModificationTime;
         auto modified = response.headers.find("last-modified");
         if (modified != response.headers.end()) {
-            modification_time = ParseModificationTime(modified->second);
+            modification_time = ObjectStoreFileSystemUtils::ParseModificationTime(modified->second);
         }
         PAIMON_ASSIGN_OR_RAISE(int64_t object_size,
                                ParseNonNegativeInt64(length->second, "Content-Length"));
@@ -712,10 +707,10 @@ class S3ObjectStoreClient : public ObjectStoreClient,
             PAIMON_ASSIGN_OR_RAISE(std::string decoded_key, PercentDecode(*key, "Key"));
             PAIMON_ASSIGN_OR_RAISE(int64_t object_size,
                                    ParseNonNegativeInt64(*size, "ListObjectsV2 Size"));
-            int64_t modified = 0;
+            int64_t modified = FileStatus::kUnknownModificationTime;
             auto last_modified = TagValue(block, "LastModified");
             if (last_modified) {
-                modified = ParseModificationTime(*last_modified);
+                modified = ObjectStoreFileSystemUtils::ParseModificationTime(*last_modified);
             }
             result.objects.push_back(ObjectMetadata{decoded_key, object_size, modified});
         }
diff --git a/src/paimon/fs/s3/s3_file_system_test.cpp b/src/paimon/fs/s3/s3_file_system_test.cpp
index 063a7dcb5..822e1aede 100644
--- a/src/paimon/fs/s3/s3_file_system_test.cpp
+++ b/src/paimon/fs/s3/s3_file_system_test.cpp
@@ -302,13 +302,13 @@ TEST(S3ObjectStoreClientTest, TestInvalidModificationTime) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<ObjectStoreClient> client,
                          MakeS3ObjectStoreClient(StaticOptions(), http));
     ASSERT_OK_AND_ASSIGN(auto metadata, client->HeadObject({"bucket", "file"}));
-    ASSERT_EQ(metadata.modification_time, 0);
+    ASSERT_EQ(metadata.modification_time, FileStatus::kUnknownModificationTime);
 
     http->body_ =
         "<ListBucketResult><IsTruncated>false</IsTruncated><Contents><Key>file</Key>"
         "<LastModified>invalid</LastModified><Size>12</Size></Contents></ListBucketResult>";
     ASSERT_OK_AND_ASSIGN(auto result, client->ListObjects({"bucket", ""}, "", 0));
-    ASSERT_EQ(result.objects[0].modification_time, 0);
+    ASSERT_EQ(result.objects[0].modification_time, FileStatus::kUnknownModificationTime);
 }
 
 TEST(S3ObjectStoreClientTest, TestRegionFromEnvironment) {
@@ -438,7 +438,9 @@ TEST(S3ObjectStoreClientTest, TestRangeAndListObjects) {
     ASSERT_TRUE(result.is_truncated);
     ASSERT_EQ(result.continuation_token, "next token");
     ASSERT_EQ(result.objects[0].key, "dir/a&b");
+    ASSERT_EQ(result.objects[0].modification_time, 1767225600000);
     ASSERT_EQ(result.objects[1].key, "dir/a&lt;b");
+    ASSERT_EQ(result.objects[1].modification_time, FileStatus::kUnknownModificationTime);
     ASSERT_EQ(result.common_prefixes[0], "dir/sub/");
     ASSERT_NE(http->request_.url.find("amazonaws.com/?list-type=2"), std::string::npos);
     ASSERT_NE(http->request_.url.find("encoding-type=url"), std::string::npos);
diff --git a/third_party/versions.txt b/third_party/versions.txt
index ffb1bbf3b..1afdf7a1c 100644
--- a/third_party/versions.txt
+++ b/third_party/versions.txt
@@ -101,6 +101,10 @@ PAIMON_AWS_S2N_BUILD_VERSION=v1.7.4
 PAIMON_AWS_S2N_BUILD_SHA256_CHECKSUM=af5ce0783fd9e05ed1899fda0c76e02fa5dd92128018e5bfe71634312d2ce8e7
 PAIMON_AWS_S2N_PKG_NAME=s2n-${PAIMON_AWS_S2N_BUILD_VERSION}.zip
 
+PAIMON_OSS_SDK_V2_BUILD_VERSION=0.1.2
+PAIMON_OSS_SDK_V2_BUILD_SHA256_CHECKSUM=6a6c00692c8fe8461594a359de8ba97f4468d90f92ffb6bcb92b82a9d1b9311b
+PAIMON_OSS_SDK_V2_PKG_NAME=alibabacloud-oss-cpp-sdk-v2-${PAIMON_OSS_SDK_V2_BUILD_VERSION}.tar.gz
+
 PAIMON_FMT_BUILD_VERSION=11.2.0
 PAIMON_FMT_BUILD_SHA256_CHECKSUM=bc23066d87ab3168f27cef3e97d545fa63314f5c79df5ea444d41d56f962c6af
 PAIMON_FMT_PKG_NAME=fmt-${PAIMON_FMT_BUILD_VERSION}.tar.gz
@@ -162,6 +166,7 @@ DEPENDENCIES=(
   "PAIMON_LZ4_URL ${PAIMON_LZ4_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/lz4/lz4/archive/${PAIMON_LZ4_BUILD_VERSION}.tar.gz"
   "PAIMON_PROTOBUF_URL ${PAIMON_PROTOBUF_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/protocolbuffers/protobuf/releases/download/v${PAIMON_PROTOBUF_BUILD_VERSION}/protobuf-all-${PAIMON_PROTOBUF_BUILD_VERSION}.tar.gz"
   "PAIMON_TBB_URL ${PAIMON_TBB_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/uxlfoundation/oneTBB/archive/refs/tags/${PAIMON_TBB_BUILD_VERSION}.tar.gz"
+  "PAIMON_OSS_SDK_V2_URL ${PAIMON_OSS_SDK_V2_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/aliyun/alibabacloud-oss-cpp-sdk-v2/archive/refs/tags/${PAIMON_OSS_SDK_V2_BUILD_VERSION}.tar.gz"
   "PAIMON_ORC_URL ${PAIMON_ORC_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/apache/orc/archive/refs/tags/${PAIMON_ORC_BUILD_VERSION}.tar.gz"
   "PAIMON_GTEST_URL ${PAIMON_GTEST_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/google/googletest/archive/release-${PAIMON_GTEST_BUILD_VERSION}.tar.gz"
   "PAIMON_BENCHMARK_URL ${PAIMON_BENCHMARK_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/google/benchmark/archive/refs/tags/v${PAIMON_BENCHMARK_BUILD_VERSION}.tar.gz"

From 53f9c86d45aabb0a6f1a379271da07d7a9f27a3d Mon Sep 17 00:00:00 2001
From: lszskye <57179283+lszskye@users.noreply.github.com>
Date: Sat, 29 Aug 2026 00:41:26 -0700
Subject: [PATCH 93/93] feat(core): support input and lookup changelog
 producers (#248)

---
 include/paimon/defs.h                         |  24 +-
 src/paimon/CMakeLists.txt                     |   2 +
 src/paimon/common/defs.cpp                    |   6 +
 src/paimon/common/utils/fields_comparator.cpp |  29 +-
 src/paimon/common/utils/fields_comparator.h   |   6 +-
 .../common/utils/fields_comparator_test.cpp   |  45 ++
 src/paimon/core/core_options.cpp              | 129 ++--
 src/paimon/core/core_options.h                |   5 +
 src/paimon/core/core_options_test.cpp         |  16 +
 .../async_key_value_producer_and_consumer.cpp | 171 +++--
 .../async_key_value_producer_and_consumer.h   | 103 ++-
 .../io/async_key_value_projection_reader.h    |   6 +-
 src/paimon/core/io/data_file_path_factory.h   |   5 +-
 .../core/io/data_file_path_factory_test.cpp   |   7 +
 .../key_value_data_file_writer_factories.cpp  |  51 ++
 .../io/key_value_data_file_writer_factories.h |  56 ++
 .../io/key_value_data_file_writer_factory.cpp |  54 +-
 .../io/key_value_data_file_writer_factory.h   |   8 +-
 ...ing_key_value_data_file_writer_factory.cpp |  31 +-
 ...dding_key_value_data_file_writer_factory.h |   2 +-
 src/paimon/core/manifest/manifest_list.h      |   3 +-
 .../core/manifest/manifest_list_test.cpp      |  28 +
 .../compact/changelog_merge_tree_rewriter.cpp | 208 +++++-
 .../compact/changelog_merge_tree_rewriter.h   |   9 +
 .../core/mergetree/compact/changelog_result.h |  35 +
 .../first_row_merge_function_wrapper.h        |  35 +-
 .../first_row_merge_function_wrapper_test.cpp |  31 +-
 .../compact/internal_row_equalizer.h          | 250 +++++++
 .../compact/internal_row_equalizer_test.cpp   | 204 ++++++
 .../lookup_changelog_merge_function_wrapper.h | 100 ++-
 ..._changelog_merge_function_wrapper_test.cpp | 198 +++++-
 .../mergetree/compact/lookup_merge_function.h |   5 +
 .../lookup_merge_tree_compact_rewriter.cpp    |  48 +-
 .../lookup_merge_tree_compact_rewriter.h      |  27 +-
 ...ookup_merge_tree_compact_rewriter_test.cpp | 283 ++++++--
 .../merge_tree_compact_manager_factory.cpp    | 217 +++---
 .../merge_tree_compact_manager_factory.h      |  21 +-
 ...erge_tree_compact_manager_factory_test.cpp |  16 +-
 .../compact/merge_tree_compact_rewriter.cpp   |  55 +-
 .../compact/merge_tree_compact_rewriter.h     |  10 +-
 .../core/mergetree/external_sort_buffer.cpp   |   5 +-
 .../core/mergetree/merge_tree_writer.cpp      | 130 +++-
 src/paimon/core/mergetree/merge_tree_writer.h |   5 +
 .../core/mergetree/merge_tree_writer_test.cpp | 214 +++++-
 src/paimon/core/mergetree/write_buffer.cpp    |   4 +
 src/paimon/core/mergetree/write_buffer.h      |   4 +
 src/paimon/core/operation/file_store_scan.cpp |   2 +
 .../operation/key_value_file_store_scan.cpp   |   7 +-
 .../operation/key_value_file_store_write.cpp  |   2 +-
 .../core/operation/merge_file_split_read.cpp  |  17 +
 .../core/operation/merge_file_split_read.h    |   6 +
 .../core/postpone/postpone_bucket_writer.cpp  |  22 +-
 src/paimon/core/schema/schema_validation.cpp  |  58 +-
 src/paimon/core/schema/schema_validation.h    |   2 +-
 .../core/schema/schema_validation_test.cpp    |  79 ++-
 .../table/source/data_table_stream_scan.cpp   |  30 +-
 src/paimon/core/table/source/scan_mode.h      |   4 +-
 .../snapshot/changelog_follow_up_scanner.h    |  53 ++
 .../source/snapshot/snapshot_reader_test.cpp  |  21 +
 test/inte/scan_inte_test.cpp                  |   4 +-
 test/inte/write_and_read_inte_test.cpp        | 652 +++++++++++++++++-
 61 files changed, 3328 insertions(+), 532 deletions(-)
 create mode 100644 src/paimon/core/io/key_value_data_file_writer_factories.cpp
 create mode 100644 src/paimon/core/io/key_value_data_file_writer_factories.h
 create mode 100644 src/paimon/core/mergetree/compact/changelog_result.h
 create mode 100644 src/paimon/core/mergetree/compact/internal_row_equalizer.h
 create mode 100644 src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp
 create mode 100644 src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h

diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index 0ca67e3a7..e05faefd8 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -386,14 +386,34 @@ struct PAIMON_EXPORT Options {
     /// @note: bitmap64 dv is not supported.
     static const char DELETION_VECTOR_BITMAP64[];
 
-    ///  @note `CHANGELOG_PRODUCER` currently only support `none`
-    ///
     /// "changelog-producer" - Whether to double write to a changelog file. This changelog file
     /// keeps the details of data changes, it can be read directly during stream reads. This can be
     /// applied to tables with primary keys. Values can be "none", "input", "lookup",
     /// "full-compaction". Default value is "none".
+    /// @note C++ Paimon currently supports "none", "input", and "lookup".
     static const char CHANGELOG_PRODUCER[];
 
+    /// "changelog-producer.row-deduplicate" - Whether to generate update-before and update-after
+    /// changelog records when the row has not changed. This option is only valid for "lookup" or
+    /// "full-compaction" changelog producers. Default value is "false".
+    static const char CHANGELOG_PRODUCER_ROW_DEDUPLICATE[];
+
+    /// "changelog-producer.row-deduplicate-ignore-fields" - Comma-separated fields to ignore when
+    /// comparing rows for changelog deduplication. This option is only valid when
+    /// "changelog-producer.row-deduplicate" is "true".
+    static const char CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS[];
+
+    /// "changelog-file.prefix" - Specify the file name prefix of changelog files. Default value is
+    /// "changelog-".
+    static const char CHANGELOG_FILE_PREFIX[];
+
+    /// "changelog-file.format" - Specify the file format of changelog files. No default value.
+    static const char CHANGELOG_FILE_FORMAT[];
+
+    /// "changelog-file.compression" - Specify the compression of changelog files. No default
+    /// value.
+    static const char CHANGELOG_FILE_COMPRESSION[];
+
     /// "force-lookup" - Whether to force the use of lookup for compaction. Default value is
     /// "false".
     static const char FORCE_LOOKUP[];
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 0ec23a0a9..1749634f4 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -288,6 +288,7 @@ set(PAIMON_CORE_SRCS
     core/io/vector_file_batch_reader.cpp
     core/io/file_index_evaluator.cpp
     core/io/key_value_data_file_record_reader.cpp
+    core/io/key_value_data_file_writer_factories.cpp
     core/io/key_value_data_file_writer_factory.cpp
     core/io/key_value_data_file_writer.cpp
     core/io/key_value_in_memory_record_reader.cpp
@@ -814,6 +815,7 @@ if(PAIMON_BUILD_TESTS)
                     core/mergetree/compact/deduplicate_merge_function_test.cpp
                     core/mergetree/compact/first_row_merge_function_test.cpp
                     core/mergetree/compact/first_row_merge_function_wrapper_test.cpp
+                    core/mergetree/compact/internal_row_equalizer_test.cpp
                     core/mergetree/compact/interval_partition_test.cpp
                     core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp
                     core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp
index 39e6d494c..f0b0f5611 100644
--- a/src/paimon/common/defs.cpp
+++ b/src/paimon/common/defs.cpp
@@ -96,6 +96,12 @@ const char Options::DELETION_VECTOR_INDEX_FILE_TARGET_SIZE[] =
     "deletion-vector.index-file.target-size";
 const char Options::DELETION_VECTOR_BITMAP64[] = "deletion-vectors.bitmap64";
 const char Options::CHANGELOG_PRODUCER[] = "changelog-producer";
+const char Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE[] = "changelog-producer.row-deduplicate";
+const char Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS[] =
+    "changelog-producer.row-deduplicate-ignore-fields";
+const char Options::CHANGELOG_FILE_PREFIX[] = "changelog-file.prefix";
+const char Options::CHANGELOG_FILE_FORMAT[] = "changelog-file.format";
+const char Options::CHANGELOG_FILE_COMPRESSION[] = "changelog-file.compression";
 const char Options::FORCE_LOOKUP[] = "force-lookup";
 const char Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE[] =
     "partial-update.remove-record-on-delete";
diff --git a/src/paimon/common/utils/fields_comparator.cpp b/src/paimon/common/utils/fields_comparator.cpp
index 224ec131f..2fee9aa15 100644
--- a/src/paimon/common/utils/fields_comparator.cpp
+++ b/src/paimon/common/utils/fields_comparator.cpp
@@ -85,28 +85,28 @@ Result<FieldsComparator::FieldComparatorFunc> FieldsComparator::CompareField(
     arrow::Type::type type = input_type->id();
     switch (type) {
         case arrow::Type::type::BOOL:
-            return FieldsComparator::FieldComparatorFunc(
+            return FieldComparatorFunc(
                 [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t {
                     bool lvalue = lhs.GetBoolean(field_idx);
                     bool rvalue = rhs.GetBoolean(field_idx);
                     return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1);
                 });
         case arrow::Type::type::INT8:
-            return FieldsComparator::FieldComparatorFunc(
+            return FieldComparatorFunc(
                 [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t {
                     int8_t lvalue = lhs.GetByte(field_idx);
                     int8_t rvalue = rhs.GetByte(field_idx);
                     return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1);
                 });
         case arrow::Type::type::INT16:
-            return FieldsComparator::FieldComparatorFunc(
+            return FieldComparatorFunc(
                 [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t {
                     int16_t lvalue = lhs.GetShort(field_idx);
                     int16_t rvalue = rhs.GetShort(field_idx);
                     return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1);
                 });
         case arrow::Type::type::DATE32:
-            return FieldsComparator::FieldComparatorFunc(
+            return FieldComparatorFunc(
                 [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t {
                     int32_t lvalue = lhs.GetDate(field_idx);
                     int32_t rvalue = rhs.GetDate(field_idx);
@@ -114,39 +114,36 @@ Result<FieldsComparator::FieldComparatorFunc> FieldsComparator::CompareField(
                 });
 
         case arrow::Type::type::INT32:
-            return FieldsComparator::FieldComparatorFunc(
+            return FieldComparatorFunc(
                 [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t {
                     int32_t lvalue = lhs.GetInt(field_idx);
                     int32_t rvalue = rhs.GetInt(field_idx);
                     return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1);
                 });
         case arrow::Type::type::INT64:
-            return FieldsComparator::FieldComparatorFunc(
+            return FieldComparatorFunc(
                 [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t {
                     int64_t lvalue = lhs.GetLong(field_idx);
                     int64_t rvalue = rhs.GetLong(field_idx);
                     return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1);
                 });
         case arrow::Type::type::FLOAT:
-            // TODO(xinyu.lxy):
-            // currently in java KeyComparatorSupplier: -inf < -0.0 == +0.0 < +inf = nan
-            // paimon-cpp: -inf < -0.0 == +0.0 < +inf and nan cannot be compared
-            return FieldsComparator::FieldComparatorFunc(
+            return FieldComparatorFunc(
                 [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t {
                     float lvalue = lhs.GetFloat(field_idx);
                     float rvalue = rhs.GetFloat(field_idx);
-                    return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1);
+                    return CompareFloatingPoint(lvalue, rvalue);
                 });
         case arrow::Type::type::DOUBLE:
-            return FieldsComparator::FieldComparatorFunc(
+            return FieldComparatorFunc(
                 [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t {
                     double lvalue = lhs.GetDouble(field_idx);
                     double rvalue = rhs.GetDouble(field_idx);
-                    return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1);
+                    return CompareFloatingPoint(lvalue, rvalue);
                 });
         case arrow::Type::type::STRING:
         case arrow::Type::type::BINARY: {
-            return FieldsComparator::FieldComparatorFunc(
+            return FieldComparatorFunc(
                 [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t {
                     auto lvalue = lhs.GetStringView(field_idx);
                     auto rvalue = rhs.GetStringView(field_idx);
@@ -157,7 +154,7 @@ Result<FieldsComparator::FieldComparatorFunc> FieldsComparator::CompareField(
         case arrow::Type::type::TIMESTAMP: {
             auto timestamp_type = checked_pointer_cast<arrow::TimestampType>(input_type);
             int32_t precision = DateTimeUtils::GetPrecisionFromType(timestamp_type);
-            return FieldsComparator::FieldComparatorFunc(
+            return FieldComparatorFunc(
                 [field_idx, precision](const InternalRow& lhs, const InternalRow& rhs) -> int32_t {
                     Timestamp lvalue = lhs.GetTimestamp(field_idx, precision);
                     Timestamp rvalue = rhs.GetTimestamp(field_idx, precision);
@@ -168,7 +165,7 @@ Result<FieldsComparator::FieldComparatorFunc> FieldsComparator::CompareField(
             auto* decimal_type = checked_cast<arrow::Decimal128Type*>(input_type.get());
             auto precision = decimal_type->precision();
             auto scale = decimal_type->scale();
-            return FieldsComparator::FieldComparatorFunc(
+            return FieldComparatorFunc(
                 [field_idx, precision, scale](const InternalRow& lhs,
                                               const InternalRow& rhs) -> int32_t {
                     Decimal lvalue = lhs.GetDecimal(field_idx, precision, scale);
diff --git a/src/paimon/common/utils/fields_comparator.h b/src/paimon/common/utils/fields_comparator.h
index 7dbeca15e..50e0260d9 100644
--- a/src/paimon/common/utils/fields_comparator.h
+++ b/src/paimon/common/utils/fields_comparator.h
@@ -41,6 +41,9 @@ class DataField;
 /// A `Comparator` that compares the file store key.
 class FieldsComparator {
  public:
+    using FieldComparatorFunc =
+        std::function<int32_t(const InternalRow& lhs, const InternalRow& rhs)>;
+
     static Result<std::unique_ptr<FieldsComparator>> Create(
         const std::vector<DataField>& input_data_field, bool is_ascending_order);
 
@@ -82,9 +85,6 @@ class FieldsComparator {
     }
 
  private:
-    using FieldComparatorFunc =
-        std::function<int32_t(const InternalRow& lhs, const InternalRow& rhs)>;
-
     FieldsComparator(bool is_ascending_order, const std::vector<int32_t>& sort_fields,
                      std::vector<FieldComparatorFunc>&& comparators)
         : is_ascending_order_(is_ascending_order),
diff --git a/src/paimon/common/utils/fields_comparator_test.cpp b/src/paimon/common/utils/fields_comparator_test.cpp
index 2f64f813c..111de404c 100644
--- a/src/paimon/common/utils/fields_comparator_test.cpp
+++ b/src/paimon/common/utils/fields_comparator_test.cpp
@@ -20,8 +20,10 @@
 #include "paimon/common/utils/fields_comparator.h"
 
 #include <cstddef>
+#include <limits>
 #include <string>
 #include <variant>
+#include <vector>
 
 #include "arrow/api.h"
 #include "gtest/gtest.h"
@@ -80,6 +82,44 @@ class FieldsComparatorTest : public ::testing::Test {
         }
         CheckResult(row1, row2, input_types, sort_fields, has_null);
     }
+
+    template <typename T>
+    void CheckFloatingPointOrder(const std::shared_ptr<arrow::DataType>& type) {
+        std::shared_ptr<MemoryPool> pool = GetDefaultPool();
+        std::vector<DataField> data_fields = {DataField(0, arrow::field("f0", type))};
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FieldsComparator> ascending_comparator,
+                             FieldsComparator::Create(data_fields,
+                                                      /*is_ascending_order=*/true));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FieldsComparator> descending_comparator,
+                             FieldsComparator::Create(data_fields,
+                                                      /*is_ascending_order=*/false));
+
+        const T nan = std::numeric_limits<T>::quiet_NaN();
+        const std::vector<T> values = {-std::numeric_limits<T>::infinity(),
+                                       static_cast<T>(-1),
+                                       static_cast<T>(-0.0),
+                                       static_cast<T>(0.0),
+                                       static_cast<T>(1),
+                                       std::numeric_limits<T>::infinity(),
+                                       nan};
+        std::vector<BinaryRow> rows;
+        rows.reserve(values.size());
+        for (T value : values) {
+            rows.emplace_back(BinaryRowGenerator::GenerateRow({value}, pool.get()));
+        }
+
+        for (size_t i = 0; i < rows.size(); ++i) {
+            for (size_t j = 0; j < rows.size(); ++j) {
+                int32_t expected = i == j ? 0 : (i < j ? -1 : 1);
+                ASSERT_EQ(expected, ascending_comparator->CompareTo(rows[i], rows[j]));
+                ASSERT_EQ(-expected, descending_comparator->CompareTo(rows[i], rows[j]));
+            }
+        }
+
+        BinaryRow negative_nan_row = BinaryRowGenerator::GenerateRow({-nan}, pool.get());
+        ASSERT_EQ(0, ascending_comparator->CompareTo(rows.back(), negative_nan_row));
+        ASSERT_EQ(0, ascending_comparator->CompareTo(negative_nan_row, rows.back()));
+    }
 };
 
 TEST_F(FieldsComparatorTest, TestSimple) {
@@ -202,6 +242,11 @@ TEST_F(FieldsComparatorTest, TestSimple) {
     }
 }
 
+TEST_F(FieldsComparatorTest, TestFloatingPointOrder) {
+    CheckFloatingPointOrder<float>(arrow::float32());
+    CheckFloatingPointOrder<double>(arrow::float64());
+}
+
 TEST_F(FieldsComparatorTest, TestTimestampType) {
     auto pool = GetDefaultPool();
     // test ts with different precision
diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp
index dfa0aa7ff..71ba73deb 100644
--- a/src/paimon/core/core_options.cpp
+++ b/src/paimon/core/core_options.cpp
@@ -330,9 +330,7 @@ class ConfigParser {
 // storing various configurable fields and their default values.
 struct CoreOptions::Impl {
     int64_t page_size = 64 * 1024;
-    std::optional<int64_t> target_file_size;
     int64_t target_file_row_num = std::numeric_limits<int64_t>::max();
-    std::optional<int64_t> blob_target_file_size;
     int64_t source_split_target_size = 128 * 1024 * 1024;
     int64_t source_split_open_file_cost = 4 * 1024 * 1024;
     int64_t manifest_target_file_size = 8 * 1024 * 1024;
@@ -343,20 +341,33 @@ struct CoreOptions::Impl {
     int64_t commit_timeout = std::numeric_limits<int64_t>::max();
     int64_t commit_min_retry_wait = 10;
     int64_t commit_max_retry_wait = 10 * 1000;
-    bool realtime_enabled = false;
     int64_t realtime_read_view_ttl_millis = 5 * 60 * 1000;
-    StatisticsMode realtime_store_statistics_mode = StatisticsMode::NONE;
+    int64_t write_buffer_spill_max_disk_size = std::numeric_limits<int64_t>::max();
+    double variant_shredding_min_field_cardinality_ratio = 0.1;
+    double variant_shredding_adaptive_retention_ratio = 0.05;
+    double lookup_cache_bloom_filter_fpp = 0.05;
+    int64_t cache_page_size = 64 * 1024;  // 64KB
+    int64_t lookup_cache_max_memory = 256 * 1024 * 1024;
+    double lookup_cache_high_prio_pool_ratio = 0.25;
+    int64_t lookup_cache_file_retention_ms = 1 * 3600 * 1000;  // 1 hour
+    int64_t lookup_cache_max_disk_size = INT64_MAX;
 
+    std::optional<int64_t> target_file_size;
+    std::optional<int64_t> blob_target_file_size;
     std::shared_ptr<FileFormat> file_format;
     std::shared_ptr<FileSystem> file_system;
     std::shared_ptr<FileFormat> manifest_file_format;
     std::shared_ptr<Cache> cache;
-
     std::optional<int64_t> scan_snapshot_id;
     std::optional<int64_t> scan_timestamp_millis;
+    std::optional<int64_t> optimized_compaction_interval;
+    std::optional<int64_t> compaction_total_size_threshold;
+    std::optional<int64_t> compaction_incremental_size_threshold;
+    std::shared_ptr<FileFormat> changelog_file_format;
     ExpireConfig expire_config;
     std::vector<std::string> sequence_field;
     std::vector<std::string> remove_record_on_sequence_group;
+    std::vector<std::string> changelog_row_deduplicate_ignore_fields;
     std::vector<std::string> blob_fields;
     std::vector<std::string> blob_descriptor_fields;
     std::vector<std::string> blob_view_fields;
@@ -367,20 +378,25 @@ struct CoreOptions::Impl {
     std::string manifest_compression = "zstd";
     std::string branch = BranchManager::DEFAULT_MAIN_BRANCH;
     std::string data_file_prefix = "data-";
+    std::string changelog_file_prefix = "changelog-";
     std::string file_system_scheme_to_identifier_map_str;
-
     std::optional<std::string> field_default_func;
     std::optional<std::string> scan_fallback_branch;
     std::optional<std::string> data_file_external_paths;
     std::optional<std::string> blob_view_upstream_warehouse;
-
+    std::optional<std::string> changelog_file_compression;
+    std::optional<std::string> global_index_external_path;
+    std::optional<std::string> scan_tag_name;
+    CompressOptions lookup_compress_options{"zstd", 1};
+    CompressOptions spill_compress_options{"zstd", 1};
     std::map<std::string, std::string> raw_options;
+    std::map<int32_t, std::shared_ptr<FileFormat>> file_format_per_level;
+    std::map<int32_t, std::string> file_compression_per_level;
 
+    StatisticsMode realtime_store_statistics_mode = StatisticsMode::NONE;
     int32_t bucket = -1;
-
     int32_t manifest_merge_min_count = 30;
     int32_t scan_manifest_entry_cache_max_snapshots = 0;
-    bool scan_manifest_entry_lazy_decode_enabled = true;
     int32_t read_batch_size = 1024;
     int32_t write_batch_size = 1024;
     int32_t local_sort_max_num_file_handles = 128;
@@ -389,32 +405,43 @@ struct CoreOptions::Impl {
     int32_t compaction_max_size_amplification_percent = 200;
     int32_t compaction_size_ratio = 1;
     int32_t num_sorted_runs_compaction_trigger = 5;
-    std::optional<int32_t> num_sorted_runs_stop_trigger;
-    std::optional<int32_t> num_levels;
-
     SortOrder sequence_field_sort_order = SortOrder::ASCENDING;
     MergeEngine merge_engine = MergeEngine::DEDUPLICATE;
     SortEngine sort_engine = SortEngine::LOSER_TREE;
     ChangelogProducer changelog_producer = ChangelogProducer::NONE;
     ExternalPathStrategy external_path_strategy = ExternalPathStrategy::NONE;
     LookupCompactMode lookup_compact_mode = LookupCompactMode::RADICAL;
-    std::optional<int32_t> lookup_compact_max_interval;
     BucketFunctionType bucket_function_type = BucketFunctionType::DEFAULT;
-
     int32_t file_compression_zstd_level = 1;
-    int64_t write_buffer_spill_max_disk_size = std::numeric_limits<int64_t>::max();
+    CoreOptions::SequenceNumberInitMode write_sequence_number_init_mode =
+        CoreOptions::SequenceNumberInitMode::SCAN;
+    VariantShreddingInferenceMode variant_shredding_inference_mode =
+        VariantShreddingInferenceMode::PER_FILE;
+    int32_t variant_shredding_max_schema_width = 300;
+    int32_t variant_shredding_max_schema_depth = 50;
+    int32_t variant_shredding_max_infer_buffer_row = 4096;
+    int32_t variant_shredding_adaptive_max_infer_buffer_row = 256;
+    int32_t compact_off_peak_start_hour = -1;
+    int32_t compact_off_peak_end_hour = -1;
+    int32_t compact_off_peak_ratio = 0;
+    int32_t lookup_remote_level_threshold = INT32_MIN;
+    std::optional<int32_t> num_sorted_runs_stop_trigger;
+    std::optional<int32_t> num_levels;
+    std::optional<int32_t> lookup_compact_max_interval;
+    std::optional<int32_t> global_index_thread_num;
 
+    bool realtime_enabled = false;
+    bool scan_manifest_entry_lazy_decode_enabled = true;
     bool ignore_delete = false;
     bool manifest_delete_file_drop_stats = false;
     bool write_buffer_spillable = true;
     bool write_only = false;
     bool bucket_append_ordered = false;
-    CoreOptions::SequenceNumberInitMode write_sequence_number_init_mode =
-        CoreOptions::SequenceNumberInitMode::SCAN;
     bool deletion_vectors_enabled = false;
     bool deletion_vectors_bitmap64 = false;
     bool force_lookup = false;
     bool lookup_wait = true;
+    bool changelog_row_deduplicate = false;
     bool partial_update_remove_record_on_delete = false;
     bool aggregation_remove_record_on_delete = false;
     bool table_read_sequence_number_enabled = false;
@@ -427,48 +454,19 @@ struct CoreOptions::Impl {
     bool row_tracking_partition_group_on_commit = true;
     bool data_evolution_enabled = false;
     bool variant_infer_shredding_schema = false;
-    VariantShreddingInferenceMode variant_shredding_inference_mode =
-        VariantShreddingInferenceMode::PER_FILE;
-    int32_t variant_shredding_max_schema_width = 300;
-    int32_t variant_shredding_max_schema_depth = 50;
-    double variant_shredding_min_field_cardinality_ratio = 0.1;
-    int32_t variant_shredding_max_infer_buffer_row = 4096;
-    int32_t variant_shredding_adaptive_max_infer_buffer_row = 256;
-    double variant_shredding_adaptive_retention_ratio = 0.05;
     bool blob_view_resolve_enabled = true;
     bool blob_as_descriptor = false;
-    std::optional<bool> blob_split_by_file_size;
     bool legacy_partition_name_enabled = true;
     bool global_index_enabled = true;
-    std::optional<int32_t> global_index_thread_num;
     bool commit_force_compact = false;
     bool commit_discard_duplicate_files = false;
     bool dynamic_partition_overwrite = true;
     bool overwrite_upgrade = true;
     bool compaction_force_rewrite_all_files = false;
     bool compaction_force_up_level_0 = false;
-    std::optional<std::string> global_index_external_path;
-
-    std::optional<std::string> scan_tag_name;
-    std::optional<int64_t> optimized_compaction_interval;
-    std::optional<int64_t> compaction_total_size_threshold;
-    std::optional<int64_t> compaction_incremental_size_threshold;
-    int32_t compact_off_peak_start_hour = -1;
-    int32_t compact_off_peak_end_hour = -1;
-    int32_t compact_off_peak_ratio = 0;
     bool lookup_cache_bloom_filter = true;
-    double lookup_cache_bloom_filter_fpp = 0.05;
     bool lookup_remote_file_enabled = false;
-    int32_t lookup_remote_level_threshold = INT32_MIN;
-    CompressOptions lookup_compress_options{"zstd", 1};
-    CompressOptions spill_compress_options{"zstd", 1};
-    int64_t cache_page_size = 64 * 1024;  // 64KB
-    std::map<int32_t, std::shared_ptr<FileFormat>> file_format_per_level;
-    std::map<int32_t, std::string> file_compression_per_level;
-    int64_t lookup_cache_max_memory = 256 * 1024 * 1024;
-    double lookup_cache_high_prio_pool_ratio = 0.25;
-    int64_t lookup_cache_file_retention_ms = 1 * 3600 * 1000;  // 1 hour
-    int64_t lookup_cache_max_disk_size = INT64_MAX;
+    std::optional<bool> blob_split_by_file_size;
 
     // Parse basic table options: bucket, partition, file sizes, batch sizes, file system, etc.
     Status ParseBasicOptions(
@@ -542,6 +540,8 @@ struct CoreOptions::Impl {
         PAIMON_RETURN_NOT_OK(parser.ParseExternalPathStrategy(&external_path_strategy));
         // Parse data-file.prefix - file name prefix of data files, default "data-"
         PAIMON_RETURN_NOT_OK(parser.Parse(Options::DATA_FILE_PREFIX, &data_file_prefix));
+        // Parse changelog-file.prefix - file name prefix of changelog files, default "changelog-"
+        PAIMON_RETURN_NOT_OK(parser.Parse(Options::CHANGELOG_FILE_PREFIX, &changelog_file_prefix));
         // Parse row-tracking.enabled - whether to enable unique row id for append table
         PAIMON_RETURN_NOT_OK(
             parser.Parse<bool>(Options::ROW_TRACKING_ENABLED, &row_tracking_enabled));
@@ -590,6 +590,14 @@ struct CoreOptions::Impl {
             Options::FILE_FORMAT, /*default_identifier=*/"parquet", &file_format));
         // Parse file.compression - default file compression, default "zstd"
         PAIMON_RETURN_NOT_OK(parser.Parse(Options::FILE_COMPRESSION, &file_compression));
+        // Parse changelog-file.format - no default value
+        if (parser.ContainsKey(Options::CHANGELOG_FILE_FORMAT)) {
+            PAIMON_RETURN_NOT_OK(parser.ParseObject<FileFormatFactory>(
+                Options::CHANGELOG_FILE_FORMAT, file_format->Identifier(), &changelog_file_format));
+        }
+        // Parse changelog-file.compression - no default value
+        PAIMON_RETURN_NOT_OK(
+            parser.Parse(Options::CHANGELOG_FILE_COMPRESSION, &changelog_file_compression));
         // Parse file.compression.zstd-level - zstd compression level, default 1
         PAIMON_RETURN_NOT_OK(
             parser.Parse(Options::FILE_COMPRESSION_ZSTD_LEVEL, &file_compression_zstd_level));
@@ -720,6 +728,13 @@ struct CoreOptions::Impl {
         PAIMON_RETURN_NOT_OK(parser.Parse(Options::FIELDS_DEFAULT_AGG_FUNC, &field_default_func));
         // Parse changelog-producer - whether to double write to a changelog file, default "none"
         PAIMON_RETURN_NOT_OK(parser.ParseChangelogProducer(&changelog_producer));
+        // Parse changelog-producer.row-deduplicate - skip unchanged row changelogs
+        PAIMON_RETURN_NOT_OK(parser.Parse<bool>(Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE,
+                                                &changelog_row_deduplicate));
+        // Parse changelog-producer.row-deduplicate-ignore-fields - ignored comparison fields
+        PAIMON_RETURN_NOT_OK(parser.ParseList<std::string>(
+            Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, Options::FIELDS_SEPARATOR,
+            &changelog_row_deduplicate_ignore_fields, /*need_trim=*/true));
         // Parse partial-update.remove-record-on-delete - remove whole row on delete
         PAIMON_RETURN_NOT_OK(parser.Parse<bool>(Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE,
                                                 &partial_update_remove_record_on_delete));
@@ -1578,6 +1593,26 @@ ChangelogProducer CoreOptions::GetChangelogProducer() const {
     return impl_->changelog_producer;
 }
 
+bool CoreOptions::ChangelogRowDeduplicate() const {
+    return impl_->changelog_row_deduplicate;
+}
+
+const std::vector<std::string>& CoreOptions::GetChangelogRowDeduplicateIgnoreFields() const {
+    return impl_->changelog_row_deduplicate_ignore_fields;
+}
+
+std::string CoreOptions::ChangelogFilePrefix() const {
+    return impl_->changelog_file_prefix;
+}
+
+std::shared_ptr<FileFormat> CoreOptions::GetChangelogFileFormat() const {
+    return impl_->changelog_file_format;
+}
+
+std::optional<std::string> CoreOptions::GetChangelogFileCompression() const {
+    return impl_->changelog_file_compression;
+}
+
 LookupStrategy CoreOptions::GetLookupStrategy() const {
     return LookupStrategy::From(
         /*is_first_row=*/GetMergeEngine() == MergeEngine::FIRST_ROW,
diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h
index f4e7964f7..345958e1a 100644
--- a/src/paimon/core/core_options.h
+++ b/src/paimon/core/core_options.h
@@ -207,6 +207,11 @@ class PAIMON_EXPORT CoreOptions {
     bool DeletionVectorsBitmap64() const;
     int64_t DeletionVectorTargetFileSize() const;
     ChangelogProducer GetChangelogProducer() const;
+    bool ChangelogRowDeduplicate() const;
+    const std::vector<std::string>& GetChangelogRowDeduplicateIgnoreFields() const;
+    std::string ChangelogFilePrefix() const;
+    std::shared_ptr<FileFormat> GetChangelogFileFormat() const;
+    std::optional<std::string> GetChangelogFileCompression() const;
     LookupStrategy GetLookupStrategy() const;
 
     bool NeedLookup() const;
diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp
index b0a3274ba..c424b9cfe 100644
--- a/src/paimon/core/core_options_test.cpp
+++ b/src/paimon/core/core_options_test.cpp
@@ -39,6 +39,7 @@ TEST(CoreOptionsTest, TestDefaultValue) {
     ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
     ASSERT_EQ(core_options.GetManifestFormat()->Identifier(), "avro");
     ASSERT_EQ(core_options.GetFileFormat()->Identifier(), "parquet");
+    ASSERT_EQ(nullptr, core_options.GetChangelogFileFormat());
     ASSERT_EQ(core_options.GetWriteFileFormat(0)->Identifier(), "parquet");
     ASSERT_EQ(core_options.GetWriteFileFormat(3)->Identifier(), "parquet");
     ASSERT_TRUE(core_options.GetFileSystem());
@@ -58,6 +59,7 @@ TEST(CoreOptionsTest, TestDefaultValue) {
     ASSERT_FALSE(core_options.RealtimeEnabled());
     ASSERT_EQ(StatisticsMode::NONE, core_options.GetRealtimeStoreStatisticsMode());
     ASSERT_EQ("zstd", core_options.GetFileCompression());
+    ASSERT_EQ(std::nullopt, core_options.GetChangelogFileCompression());
     ASSERT_EQ("zstd", core_options.GetWriteFileCompression(0));
     ASSERT_EQ("zstd", core_options.GetWriteFileCompression(3));
     ASSERT_EQ("zstd", core_options.GetManifestCompression());
@@ -125,6 +127,9 @@ TEST(CoreOptionsTest, TestDefaultValue) {
     ASSERT_FALSE(core_options.DeletionVectorsBitmap64());
     ASSERT_EQ(2 * 1024 * 1024, core_options.DeletionVectorTargetFileSize());
     ASSERT_EQ(ChangelogProducer::NONE, core_options.GetChangelogProducer());
+    ASSERT_FALSE(core_options.ChangelogRowDeduplicate());
+    ASSERT_TRUE(core_options.GetChangelogRowDeduplicateIgnoreFields().empty());
+    ASSERT_EQ("changelog-", core_options.ChangelogFilePrefix());
     ASSERT_FALSE(core_options.NeedLookup());
     ASSERT_FALSE(core_options.PrepareCommitWaitCompaction());
     LookupStrategy expected_lookup_strategy = {/*is_first_row=*/false,
@@ -192,6 +197,7 @@ TEST(CoreOptionsTest, TestFromMap) {
     std::map<std::string, std::string> options = {
         {Options::FILE_SYSTEM, "Local"},
         {Options::FILE_FORMAT, "ORC"},
+        {Options::CHANGELOG_FILE_FORMAT, "avro"},
         {Options::MANIFEST_FORMAT, "avRo"},
         {Options::BUCKET, "3"},
         {Options::PAGE_SIZE, "128 kb"},
@@ -248,6 +254,10 @@ TEST(CoreOptionsTest, TestFromMap) {
         {Options::DELETION_VECTOR_BITMAP64, "true"},
         {Options::DELETION_VECTOR_INDEX_FILE_TARGET_SIZE, "4MB"},
         {Options::CHANGELOG_PRODUCER, "full-compaction"},
+        {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"},
+        {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, "f0, f2"},
+        {Options::CHANGELOG_FILE_PREFIX, "test-changelog-"},
+        {Options::CHANGELOG_FILE_COMPRESSION, "lz4"},
         {Options::FORCE_LOOKUP, "true"},
         {"fields.g_1,g_3.sequence-group", "c,d"},
         {Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, "true"},
@@ -324,6 +334,7 @@ TEST(CoreOptionsTest, TestFromMap) {
     ASSERT_TRUE(fs);
 
     ASSERT_EQ(core_options.GetFileFormat()->Identifier(), "orc");
+    ASSERT_EQ(core_options.GetChangelogFileFormat()->Identifier(), "avro");
     ASSERT_EQ(core_options.GetWriteFileFormat(0)->Identifier(), "avro");
     ASSERT_EQ(core_options.GetWriteFileFormat(1)->Identifier(), "orc");
     ASSERT_EQ(core_options.GetWriteFileFormat(3)->Identifier(), "parquet");
@@ -393,6 +404,11 @@ TEST(CoreOptionsTest, TestFromMap) {
     ASSERT_TRUE(core_options.DeletionVectorsBitmap64());
     ASSERT_EQ(4 * 1024 * 1024, core_options.DeletionVectorTargetFileSize());
     ASSERT_EQ(ChangelogProducer::FULL_COMPACTION, core_options.GetChangelogProducer());
+    ASSERT_TRUE(core_options.ChangelogRowDeduplicate());
+    ASSERT_EQ(std::vector<std::string>({"f0", "f2"}),
+              core_options.GetChangelogRowDeduplicateIgnoreFields());
+    ASSERT_EQ("test-changelog-", core_options.ChangelogFilePrefix());
+    ASSERT_EQ(std::optional<std::string>("lz4"), core_options.GetChangelogFileCompression());
     ASSERT_TRUE(core_options.NeedLookup());
     ASSERT_TRUE(core_options.PrepareCommitWaitCompaction());
     LookupStrategy expected_lookup_strategy = {/*is_first_row=*/false,
diff --git a/src/paimon/core/io/async_key_value_producer_and_consumer.cpp b/src/paimon/core/io/async_key_value_producer_and_consumer.cpp
index 1792b43cd..b353aefc8 100644
--- a/src/paimon/core/io/async_key_value_producer_and_consumer.cpp
+++ b/src/paimon/core/io/async_key_value_producer_and_consumer.cpp
@@ -25,22 +25,96 @@
 
 #include "arrow/c/abi.h"
 #include "arrow/c/helpers.h"
+#include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/reader/reader_utils.h"
 #include "paimon/reader/batch_reader.h"
 
 namespace paimon {
 class MemoryPool;
 
+namespace {
+
+class AsyncKeyValueQueueBatchSink : public AsyncKeyValueBatchSink {
+ public:
+    AsyncKeyValueQueueBatchSink(std::atomic<bool>* consume_finished,
+                                tbb::concurrent_bounded_queue<AsyncKeyValueRowsBatch>* kv_queue)
+        : consume_finished_(consume_finished), kv_queue_(kv_queue) {}
+
+    Status Write(AsyncKeyValueBatchType type, std::vector<KeyValue>&& rows) override {
+        if (*consume_finished_) {
+            return Status::Cancelled("Key value conversion is cancelled");
+        }
+        kv_queue_->push(AsyncKeyValueRowsBatch{type, std::move(rows)});
+        return Status::OK();
+    }
+
+    bool IsCancelled() const override {
+        return *consume_finished_;
+    }
+
+ private:
+    std::atomic<bool>* consume_finished_;
+    tbb::concurrent_bounded_queue<AsyncKeyValueRowsBatch>* kv_queue_;
+};
+
+}  // namespace
+
+std::shared_ptr<Metrics> AsyncKeyValueBatchProducer::GetReaderMetrics() const {
+    return std::make_shared<MetricsImpl>();
+}
+
+SortMergeReaderBatchProducer::SortMergeReaderBatchProducer(
+    std::unique_ptr<SortMergeReader>&& sort_merge_reader, int32_t batch_size)
+    : sort_merge_reader_(std::move(sort_merge_reader)),
+      batch_size_(NormalizeProjectionBatchSize(batch_size)) {}
+
+Status SortMergeReaderBatchProducer::Produce(AsyncKeyValueBatchSink* sink) {
+    std::vector<KeyValue> batch;
+    batch.reserve(batch_size_);
+    while (!sink->IsCancelled()) {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader::Iterator> iterator,
+                               sort_merge_reader_->NextBatch());
+        if (iterator == nullptr) {
+            break;
+        }
+        while (!sink->IsCancelled()) {
+            PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext());
+            if (!has_next) {
+                break;
+            }
+            batch.push_back(std::move(iterator->Next()));
+            if (static_cast<int32_t>(batch.size()) >= batch_size_) {
+                PAIMON_RETURN_NOT_OK(sink->Write(AsyncKeyValueBatchType::DATA, std::move(batch)));
+                batch = std::vector<KeyValue>();
+                batch.reserve(batch_size_);
+            }
+        }
+    }
+    if (!batch.empty() && !sink->IsCancelled()) {
+        PAIMON_RETURN_NOT_OK(sink->Write(AsyncKeyValueBatchType::DATA, std::move(batch)));
+    }
+    return Status::OK();
+}
+
+std::shared_ptr<Metrics> SortMergeReaderBatchProducer::GetReaderMetrics() const {
+    return sort_merge_reader_->GetReaderMetrics();
+}
+
+void SortMergeReaderBatchProducer::Close() {
+    if (!closed_) {
+        sort_merge_reader_->Close();
+        closed_ = true;
+    }
+}
+
 template <typename T, typename R>
 AsyncKeyValueProducerAndConsumer<T, R>::AsyncKeyValueProducerAndConsumer(
-    std::unique_ptr<SortMergeReader>&& sort_merge_reader, ConsumerCreator create_consumer,
-    int32_t batch_size, int32_t consumer_thread_num, const std::shared_ptr<MemoryPool>& pool)
-    : batch_size_(std::min(batch_size, MAX_PROJECTION_BATCH_SIZE)),
-      consumer_thread_num_(consumer_thread_num),
-      pool_(pool),
-      sort_merge_reader_(std::move(sort_merge_reader)),
-      create_consumer_(std::move(create_consumer)) {
-    kv_queue_.set_capacity(consumer_thread_num * 2);
+    std::unique_ptr<AsyncKeyValueBatchProducer>&& producer, ConsumerCreator create_consumer,
+    int32_t consumer_thread_num)
+    : consumer_thread_num_(consumer_thread_num),
+      create_consumer_(std::move(create_consumer)),
+      producer_(std::move(producer)) {
+    kv_queue_.set_capacity(consumer_thread_num_ * 2);
     result_queue_.set_capacity(RESULT_BATCH_COUNT);
 }
 
@@ -70,6 +144,12 @@ Status AsyncKeyValueProducerAndConsumer<T, R>::CheckStatusAndCleanUp() {
 
 template <typename T, typename R>
 Result<R> AsyncKeyValueProducerAndConsumer<T, R>::NextBatch() {
+    PAIMON_ASSIGN_OR_RAISE(AsyncKeyValueResultBatch<R> result, NextBatchWithType());
+    return std::move(result.result);
+}
+
+template <typename T, typename R>
+Result<AsyncKeyValueResultBatch<R>> AsyncKeyValueProducerAndConsumer<T, R>::NextBatchWithType() {
     if (!producer_future_.valid()) {
         producer_future_ = std::async(std::launch::async,
                                       &AsyncKeyValueProducerAndConsumer<T, R>::ProduceLoop, this)
@@ -78,10 +158,10 @@ Result<R> AsyncKeyValueProducerAndConsumer<T, R>::NextBatch() {
     if (consumers_.empty()) {
         consumers_.reserve(consumer_thread_num_);
         for (int32_t i = 0; i < consumer_thread_num_; i++) {
-            Result<std::unique_ptr<RowToArrowArrayConverter<T, R>>> consumer = create_consumer_();
-            PAIMON_RETURN_NOT_OK(consumer.status());
+            std::unique_ptr<RowToArrowArrayConverter<T, R>> consumer;
+            PAIMON_ASSIGN_OR_RAISE(consumer, create_consumer_());
             auto async_consumer = std::make_unique<AsyncKeyValueConsumer<T, R>>(
-                std::move(consumer).value(), consume_finished_, consumer_finished_count_, kv_queue_,
+                std::move(consumer), consume_finished_, consumer_finished_count_, kv_queue_,
                 result_queue_);
             consumers_.push_back(std::move(async_consumer));
         }
@@ -90,16 +170,16 @@ Result<R> AsyncKeyValueProducerAndConsumer<T, R>::NextBatch() {
 
     if (next_batch_finished_) {
         // projection reader is eof
-        return R();
+        return AsyncKeyValueResultBatch<R>();
     }
 
-    R result;
+    AsyncKeyValueResultBatch<R> result;
     while (!result_queue_.try_pop(result)) {
         PAIMON_RETURN_NOT_OK(CheckStatusAndCleanUp());
         if (consumer_finished_count_ == consumer_thread_num_ && result_queue_.empty()) {
             // all consume thread finished
             next_batch_finished_ = true;
-            return R();
+            return AsyncKeyValueResultBatch<R>();
         }
         usleep(1000);
     }
@@ -109,33 +189,9 @@ Result<R> AsyncKeyValueProducerAndConsumer<T, R>::NextBatch() {
 
 template <typename T, typename R>
 Status AsyncKeyValueProducerAndConsumer<T, R>::ProduceLoop() {
-    std::vector<KeyValue> batch;
-    batch.reserve(batch_size_);
-    while (!consume_finished_) {
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader::Iterator> iterator,
-                               sort_merge_reader_->NextBatch());
-        if (iterator == nullptr) {
-            break;
-        }
-        while (!consume_finished_) {
-            PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext());
-            if (!has_next) {
-                break;
-            }
-            batch.push_back(std::move(iterator->Next()));
-            if (static_cast<int32_t>(batch.size()) >= batch_size_) {
-                kv_queue_.push(std::move(batch));
-                batch = std::vector<KeyValue>();
-                batch.reserve(batch_size_);
-            }
-        }
-    }
-    // Push remaining rows
-    if (!batch.empty()) {
-        kv_queue_.push(std::move(batch));
-    }
-    // Push empty batch as EOF signal
-    kv_queue_.push(std::vector<KeyValue>());
+    AsyncKeyValueQueueBatchSink sink(&consume_finished_, &kv_queue_);
+    PAIMON_RETURN_NOT_OK(producer_->Produce(&sink));
+    kv_queue_.push(AsyncKeyValueRowsBatch());
     return Status::OK();
 }
 
@@ -155,8 +211,9 @@ void AsyncKeyValueProducerAndConsumer<T, R>::CleanUp() {
 
 template <typename T, typename R>
 void AsyncKeyValueProducerAndConsumer<T, R>::CleanUpQueue() {
-    R read_batch;
-    while (result_queue_.try_pop(read_batch)) {
+    AsyncKeyValueResultBatch<R> tagged_batch;
+    while (result_queue_.try_pop(tagged_batch)) {
+        R& read_batch = tagged_batch.result;
         if constexpr (std::is_same_v<R, BatchReader::ReadBatch>) {
             if (!BatchReader::IsEofBatch(read_batch)) {
                 ReaderUtils::ReleaseReadBatch(std::move(read_batch));
@@ -168,7 +225,7 @@ void AsyncKeyValueProducerAndConsumer<T, R>::CleanUpQueue() {
         }
     }
 
-    std::vector<KeyValue> kv_batch;
+    AsyncKeyValueRowsBatch kv_batch;
     while (kv_queue_.try_pop(kv_batch)) {
     }
 }
@@ -178,11 +235,11 @@ template class AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>;
 
 template <typename T, typename R>
 AsyncKeyValueConsumer<T, R>::AsyncKeyValueConsumer(
-    std::unique_ptr<RowToArrowArrayConverter<T, R>>&& key_value_consumer,
-    std::atomic<bool>& consume_finished, std::atomic<int32_t>& consumer_finished_count,
-    tbb::concurrent_bounded_queue<std::vector<KeyValue>>& kv_queue,
-    tbb::concurrent_bounded_queue<R>& result_queue)
-    : key_value_consumer_(std::move(key_value_consumer)),
+    std::unique_ptr<RowToArrowArrayConverter<T, R>>&& consumer, std::atomic<bool>& consume_finished,
+    std::atomic<int32_t>& consumer_finished_count,
+    tbb::concurrent_bounded_queue<AsyncKeyValueRowsBatch>& kv_queue,
+    tbb::concurrent_bounded_queue<AsyncKeyValueResultBatch<R>>& result_queue)
+    : consumer_(std::move(consumer)),
       consume_finished_(consume_finished),
       consumer_finished_count_(consumer_finished_count),
       kv_queue_(kv_queue),
@@ -205,18 +262,18 @@ Status AsyncKeyValueConsumer<T, R>::GetStatus() const {
 template <typename T, typename R>
 Status AsyncKeyValueConsumer<T, R>::ConsumeLoop() {
     while (!consume_finished_) {
-        std::vector<KeyValue> key_value_vec;
-        if (!kv_queue_.try_pop(key_value_vec)) {
+        AsyncKeyValueRowsBatch rows_batch;
+        if (!kv_queue_.try_pop(rows_batch)) {
             usleep(100);
             continue;
         }
-        if (key_value_vec.empty()) {
+        if (rows_batch.rows.empty()) {
             // Empty batch is EOF signal; re-push for other consumers
-            kv_queue_.push(std::move(key_value_vec));
+            kv_queue_.push(std::move(rows_batch));
             break;
         }
-        PAIMON_ASSIGN_OR_RAISE(R result, key_value_consumer_->NextBatch(key_value_vec));
-        result_queue_.push(std::move(result));
+        PAIMON_ASSIGN_OR_RAISE(R result, consumer_->NextBatch(rows_batch.rows));
+        result_queue_.push(AsyncKeyValueResultBatch<R>{rows_batch.type, std::move(result)});
     }
     consumer_finished_count_++;
     return Status::OK();
@@ -227,7 +284,9 @@ void AsyncKeyValueConsumer<T, R>::CleanUp() {
     if (consumer_future_.valid()) {
         [[maybe_unused]] Status status = consumer_future_.get();
     }
-    key_value_consumer_->CleanUp();
+    if (consumer_) {
+        consumer_->CleanUp();
+    }
 }
 
 template class AsyncKeyValueConsumer<KeyValue, BatchReader::ReadBatch>;
diff --git a/src/paimon/core/io/async_key_value_producer_and_consumer.h b/src/paimon/core/io/async_key_value_producer_and_consumer.h
index af8bbed68..086ff1112 100644
--- a/src/paimon/core/io/async_key_value_producer_and_consumer.h
+++ b/src/paimon/core/io/async_key_value_producer_and_consumer.h
@@ -18,6 +18,7 @@
 
 #pragma once
 
+#include <algorithm>
 #include <atomic>
 #include <cstdint>
 #include <functional>
@@ -37,21 +38,80 @@
 namespace paimon {
 template <typename T, typename R>
 class AsyncKeyValueConsumer;
-class MemoryPool;
 class Metrics;
 
-// Asynchronous iterate SortMergeReader (producer) and row-to-array conversion (consumer), support
-// multi-threaded conversion, R can be BatchReader::ReadBatch, KeyValueBatch
+enum class AsyncKeyValueBatchType {
+    DATA,
+    CHANGELOG,
+};
+
+struct AsyncKeyValueRowsBatch {
+    AsyncKeyValueBatchType type = AsyncKeyValueBatchType::DATA;
+    std::vector<KeyValue> rows;
+};
+
+class AsyncKeyValueBatchSink {
+ public:
+    virtual ~AsyncKeyValueBatchSink() = default;
+
+    virtual Status Write(AsyncKeyValueBatchType type, std::vector<KeyValue>&& rows) = 0;
+
+    virtual bool IsCancelled() const = 0;
+};
+
+class AsyncKeyValueBatchProducer {
+ public:
+    virtual ~AsyncKeyValueBatchProducer() = default;
+
+    virtual Status Produce(AsyncKeyValueBatchSink* sink) = 0;
+
+    virtual std::shared_ptr<Metrics> GetReaderMetrics() const;
+
+    virtual void Close() {}
+
+ protected:
+    // Limits the number of rows sent to one Arrow projection call.
+    static int32_t NormalizeProjectionBatchSize(int32_t batch_size) {
+        return std::min(batch_size, MAX_PROJECTION_BATCH_SIZE);
+    }
+
+ private:
+    static constexpr int32_t MAX_PROJECTION_BATCH_SIZE = 100000;
+};
+
+class SortMergeReaderBatchProducer : public AsyncKeyValueBatchProducer {
+ public:
+    SortMergeReaderBatchProducer(std::unique_ptr<SortMergeReader>&& sort_merge_reader,
+                                 int32_t batch_size);
+
+    Status Produce(AsyncKeyValueBatchSink* sink) override;
+
+    std::shared_ptr<Metrics> GetReaderMetrics() const override;
+
+    void Close() override;
+
+ private:
+    std::unique_ptr<SortMergeReader> sort_merge_reader_;
+    int32_t batch_size_;
+    bool closed_ = false;
+};
+
+template <typename R>
+struct AsyncKeyValueResultBatch {
+    AsyncKeyValueBatchType type = AsyncKeyValueBatchType::DATA;
+    R result;
+};
+
+// Asynchronous iterates AsyncKeyValueBatchProducer(producer) and row-to-array conversion
+// (consumer), support multi-threaded conversion, R can be BatchReader::ReadBatch or KeyValueBatch.
 template <typename T, typename R>
 class AsyncKeyValueProducerAndConsumer {
  public:
     using ConsumerCreator =
         std::function<Result<std::unique_ptr<RowToArrowArrayConverter<T, R>>>()>;
 
-    AsyncKeyValueProducerAndConsumer(std::unique_ptr<SortMergeReader>&& sort_merge_reader,
-                                     ConsumerCreator create_consumer, int32_t batch_size,
-                                     int32_t consumer_thread_num,
-                                     const std::shared_ptr<MemoryPool>& pool);
+    AsyncKeyValueProducerAndConsumer(std::unique_ptr<AsyncKeyValueBatchProducer>&& producer,
+                                     ConsumerCreator create_consumer, int32_t consumer_thread_num);
 
     ~AsyncKeyValueProducerAndConsumer() {
         CleanUp();
@@ -59,21 +119,20 @@ class AsyncKeyValueProducerAndConsumer {
 
     Result<R> NextBatch();
 
+    Result<AsyncKeyValueResultBatch<R>> NextBatchWithType();
+
     std::shared_ptr<Metrics> GetReaderMetrics() const {
-        return sort_merge_reader_->GetReaderMetrics();
+        return producer_->GetReaderMetrics();
     }
 
     void Close() {
         CleanUp();
-        sort_merge_reader_->Close();
+        producer_->Close();
     }
 
  private:
     static constexpr int32_t RESULT_BATCH_COUNT = 3;
 
-    // in case write batch size is too large and overflow arrow array
-    static constexpr int32_t MAX_PROJECTION_BATCH_SIZE = 100000;
-
     void CleanUpQueue();
     Status ProduceLoop();
     void CleanUp();
@@ -81,11 +140,9 @@ class AsyncKeyValueProducerAndConsumer {
     Status CheckStatusAndCleanUp();
 
  private:
-    int32_t batch_size_;
     int32_t consumer_thread_num_;
-    std::shared_ptr<MemoryPool> pool_;
-    std::unique_ptr<SortMergeReader> sort_merge_reader_;
     ConsumerCreator create_consumer_;
+    std::unique_ptr<AsyncKeyValueBatchProducer> producer_;
 
     // produce: merge sort KeyValue and push result KeyValue to kv_queue_, consume: project KeyValue
     // to arrow array and push result array to result_queue_
@@ -94,18 +151,18 @@ class AsyncKeyValueProducerAndConsumer {
     std::shared_future<Status> producer_future_;
     std::vector<std::unique_ptr<AsyncKeyValueConsumer<T, R>>> consumers_;
     std::atomic<int32_t> consumer_finished_count_ = 0;
-    tbb::concurrent_bounded_queue<std::vector<KeyValue>> kv_queue_;
-    tbb::concurrent_bounded_queue<R> result_queue_;
+    tbb::concurrent_bounded_queue<AsyncKeyValueRowsBatch> kv_queue_;
+    tbb::concurrent_bounded_queue<AsyncKeyValueResultBatch<R>> result_queue_;
 };
 
 template <typename T, typename R>
 class AsyncKeyValueConsumer {
  public:
-    AsyncKeyValueConsumer(std::unique_ptr<RowToArrowArrayConverter<T, R>>&& key_value_consumer,
+    AsyncKeyValueConsumer(std::unique_ptr<RowToArrowArrayConverter<T, R>>&& consumer,
                           std::atomic<bool>& consume_finished,
                           std::atomic<int32_t>& consumer_finished_count,
-                          tbb::concurrent_bounded_queue<std::vector<KeyValue>>& kv_queue,
-                          tbb::concurrent_bounded_queue<R>& result_queue);
+                          tbb::concurrent_bounded_queue<AsyncKeyValueRowsBatch>& kv_queue,
+                          tbb::concurrent_bounded_queue<AsyncKeyValueResultBatch<R>>& result_queue);
 
     ~AsyncKeyValueConsumer() {
         CleanUp();
@@ -118,12 +175,12 @@ class AsyncKeyValueConsumer {
     Status ConsumeLoop();
 
  private:
-    std::unique_ptr<RowToArrowArrayConverter<T, R>> key_value_consumer_;
+    std::unique_ptr<RowToArrowArrayConverter<T, R>> consumer_;
     std::shared_future<Status> consumer_future_;
     std::atomic<bool>& consume_finished_;
     std::atomic<int32_t>& consumer_finished_count_;
-    tbb::concurrent_bounded_queue<std::vector<KeyValue>>& kv_queue_;
-    tbb::concurrent_bounded_queue<R>& result_queue_;
+    tbb::concurrent_bounded_queue<AsyncKeyValueRowsBatch>& kv_queue_;
+    tbb::concurrent_bounded_queue<AsyncKeyValueResultBatch<R>>& result_queue_;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/io/async_key_value_projection_reader.h b/src/paimon/core/io/async_key_value_projection_reader.h
index a272a0edf..79c7aa4de 100644
--- a/src/paimon/core/io/async_key_value_projection_reader.h
+++ b/src/paimon/core/io/async_key_value_projection_reader.h
@@ -38,10 +38,12 @@ class AsyncKeyValueProjectionReader : public BatchReader {
             -> Result<std::unique_ptr<RowToArrowArrayConverter<KeyValue, BatchReader::ReadBatch>>> {
             return KeyValueProjectionConsumer::Create(target_schema, target_to_src_mapping, pool);
         };
+        std::unique_ptr<AsyncKeyValueBatchProducer> producer =
+            std::make_unique<SortMergeReaderBatchProducer>(std::move(sort_merge_reader),
+                                                           batch_size);
         producer_and_consumer_ =
             std::make_unique<AsyncKeyValueProducerAndConsumer<KeyValue, BatchReader::ReadBatch>>(
-                std::move(sort_merge_reader), create_consumer, batch_size, projection_thread_num,
-                pool);
+                std::move(producer), create_consumer, projection_thread_num);
     }
 
     Result<BatchReader::ReadBatch> NextBatch() override {
diff --git a/src/paimon/core/io/data_file_path_factory.h b/src/paimon/core/io/data_file_path_factory.h
index b49154f1b..a9050eeee 100644
--- a/src/paimon/core/io/data_file_path_factory.h
+++ b/src/paimon/core/io/data_file_path_factory.h
@@ -56,8 +56,9 @@ class DataFilePathFactory : public PathFactory {
         return NewPath(data_file_prefix_);
     }
 
-    std::string NewChangelogPath() const {
-        return NewPath(std::string(CHANGELOG_FILE_PREFIX));
+    std::string NewChangelogPath(const std::string& changelog_file_prefix,
+                                 const std::string& format_identifier) const {
+        return NewPathFromName(NewFileName(changelog_file_prefix, "." + format_identifier));
     }
 
     std::string NewBlobPath() const {
diff --git a/src/paimon/core/io/data_file_path_factory_test.cpp b/src/paimon/core/io/data_file_path_factory_test.cpp
index 6283876df..fe4824c94 100644
--- a/src/paimon/core/io/data_file_path_factory_test.cpp
+++ b/src/paimon/core/io/data_file_path_factory_test.cpp
@@ -59,6 +59,13 @@ TEST_F(DataFilePathFactoryTest, TestNewPath) {
     ASSERT_EQ(factory_.NewPathFromName("index-file"), "/tmp/index-file");
 }
 
+TEST_F(DataFilePathFactoryTest, TestNewChangelogPath) {
+    std::string path = factory_.NewChangelogPath("changes-", "parquet");
+
+    ASSERT_TRUE(path.find("/tmp/changes-") != std::string::npos);
+    ASSERT_TRUE(StringUtils::EndsWith(path, ".parquet"));
+}
+
 TEST_F(DataFilePathFactoryTest, TestNewPathWithDataFilePrefixAndExternalPath) {
     DataFilePathFactory factory;
     ASSERT_OK_AND_ASSIGN(
diff --git a/src/paimon/core/io/key_value_data_file_writer_factories.cpp b/src/paimon/core/io/key_value_data_file_writer_factories.cpp
new file mode 100644
index 000000000..5aa5a8caa
--- /dev/null
+++ b/src/paimon/core/io/key_value_data_file_writer_factories.cpp
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/io/key_value_data_file_writer_factories.h"
+
+#include "paimon/common/data/shredding/shredding_write_plan_factories.h"
+#include "paimon/core/io/key_value_data_file_writer_factory.h"
+#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h"
+
+namespace paimon {
+
+Result<std::shared_ptr<KeyValueDataFileWriterFactories::WriterFactory>>
+KeyValueDataFileWriterFactories::Create(const CoreOptions& options, int64_t schema_id,
+                                        const std::shared_ptr<arrow::Schema>& write_schema,
+                                        int32_t level, FileSource file_source,
+                                        const std::vector<std::string>& primary_keys,
+                                        const std::shared_ptr<DataFilePathFactory>& path_factory,
+                                        bool create_stats_extractor, bool is_changelog,
+                                        const std::shared_ptr<MemoryPool>& pool) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ShreddingWritePlanFactory> plan_factory,
+                           ShreddingWritePlanFactories::SelectActive(options, write_schema, pool));
+    std::shared_ptr<WriterFactory> writer_factory;
+    if (plan_factory != nullptr) {
+        writer_factory = std::make_shared<ShreddingKeyValueDataFileWriterFactory>(
+            options, schema_id, write_schema, level, file_source, primary_keys, path_factory,
+            create_stats_extractor, plan_factory, is_changelog, pool);
+    } else {
+        writer_factory = std::make_shared<KeyValueDataFileWriterFactory>(
+            options, schema_id, write_schema, level, file_source, primary_keys, path_factory,
+            create_stats_extractor, is_changelog, pool);
+    }
+    return writer_factory;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/key_value_data_file_writer_factories.h b/src/paimon/core/io/key_value_data_file_writer_factories.h
new file mode 100644
index 000000000..335bb94d4
--- /dev/null
+++ b/src/paimon/core/io/key_value_data_file_writer_factories.h
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/single_file_writer_factory.h"
+#include "paimon/core/key_value.h"
+#include "paimon/core/manifest/file_source.h"
+#include "paimon/result.h"
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
+namespace paimon {
+
+class CoreOptions;
+class DataFilePathFactory;
+class MemoryPool;
+
+/// Creates the appropriate key-value data file writer factory for the configured write schema.
+class KeyValueDataFileWriterFactories {
+ public:
+    using WriterFactory = SingleFileWriterFactory<KeyValueBatch, std::shared_ptr<DataFileMeta>>;
+
+    static Result<std::shared_ptr<WriterFactory>> Create(
+        const CoreOptions& options, int64_t schema_id,
+        const std::shared_ptr<arrow::Schema>& write_schema, int32_t level, FileSource file_source,
+        const std::vector<std::string>& primary_keys,
+        const std::shared_ptr<DataFilePathFactory>& path_factory, bool create_stats_extractor,
+        bool is_changelog, const std::shared_ptr<MemoryPool>& pool);
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/key_value_data_file_writer_factory.cpp b/src/paimon/core/io/key_value_data_file_writer_factory.cpp
index 8f3885591..e0640ab61 100644
--- a/src/paimon/core/io/key_value_data_file_writer_factory.cpp
+++ b/src/paimon/core/io/key_value_data_file_writer_factory.cpp
@@ -20,6 +20,7 @@
 #include "paimon/core/io/key_value_data_file_writer_factory.h"
 
 #include <functional>
+#include <optional>
 #include <utility>
 
 #include "arrow/c/helpers.h"
@@ -37,14 +38,15 @@ KeyValueDataFileWriterFactory::KeyValueDataFileWriterFactory(
     const std::shared_ptr<arrow::Schema>& write_schema, int32_t level, FileSource file_source,
     const std::vector<std::string>& primary_keys,
     const std::shared_ptr<DataFilePathFactory>& path_factory, bool create_stats_extractor,
-    const std::shared_ptr<MemoryPool>& pool)
+    bool is_changelog, const std::shared_ptr<MemoryPool>& pool)
     : DataFileWriterFactory(options, schema_id, pool),
       write_schema_(write_schema),
       level_(level),
       file_source_(file_source),
       primary_keys_(primary_keys),
       path_factory_(path_factory),
-      create_stats_extractor_(create_stats_extractor) {}
+      create_stats_extractor_(create_stats_extractor),
+      is_changelog_(is_changelog) {}
 
 Result<std::unique_ptr<SingleFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>>
 KeyValueDataFileWriterFactory::CreateWriter() const {
@@ -54,22 +56,50 @@ KeyValueDataFileWriterFactory::CreateWriter() const {
         return Status::OK();
     };
 
-    auto format = options_.GetWriteFileFormat(level_);
+    std::shared_ptr<FileFormat> format = GetFileFormat();
     PAIMON_ASSIGN_OR_RAISE(WriterResources resources,
                            CreateWriterResources(*format, write_schema_, create_stats_extractor_));
     auto writer = std::make_unique<KeyValueDataFileWriter>(
-        options_.GetWriteFileCompression(level_), std::move(converter), schema_id_, level_,
-        file_source_, primary_keys_, resources.stats_extractor, write_schema_,
-        path_factory_->IsExternalPath(), pool_);
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<DataFileIndexWriter> file_index_writer,
-                           CreateFileIndexWriter(write_schema_, path_factory_));
-    if (file_index_writer) {
-        writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_);
+        GetFileCompression(), std::move(converter), schema_id_, level_, file_source_, primary_keys_,
+        resources.stats_extractor, write_schema_, path_factory_->IsExternalPath(), pool_);
+    if (!is_changelog_) {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<DataFileIndexWriter> file_index_writer,
+                               CreateFileIndexWriter(write_schema_, path_factory_));
+        if (file_index_writer) {
+            writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_);
+        }
     }
-    PAIMON_RETURN_NOT_OK(
-        writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder));
+    PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), NewFilePath(format->Identifier()),
+                                      resources.writer_builder));
     return std::unique_ptr<SingleFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>(
         std::move(writer));
 }
 
+std::shared_ptr<FileFormat> KeyValueDataFileWriterFactory::GetFileFormat() const {
+    if (is_changelog_) {
+        std::shared_ptr<FileFormat> changelog_format = options_.GetChangelogFileFormat();
+        if (changelog_format) {
+            return changelog_format;
+        }
+    }
+    return options_.GetWriteFileFormat(level_);
+}
+
+std::string KeyValueDataFileWriterFactory::GetFileCompression() const {
+    if (is_changelog_) {
+        std::optional<std::string> changelog_compression = options_.GetChangelogFileCompression();
+        if (changelog_compression) {
+            return changelog_compression.value();
+        }
+    }
+    return options_.GetWriteFileCompression(level_);
+}
+
+std::string KeyValueDataFileWriterFactory::NewFilePath(const std::string& format_identifier) const {
+    if (is_changelog_) {
+        return path_factory_->NewChangelogPath(options_.ChangelogFilePrefix(), format_identifier);
+    }
+    return path_factory_->NewPath();
+}
+
 }  // namespace paimon
diff --git a/src/paimon/core/io/key_value_data_file_writer_factory.h b/src/paimon/core/io/key_value_data_file_writer_factory.h
index 6ac50aba3..533b84c6e 100644
--- a/src/paimon/core/io/key_value_data_file_writer_factory.h
+++ b/src/paimon/core/io/key_value_data_file_writer_factory.h
@@ -38,6 +38,7 @@ namespace paimon {
 
 class CoreOptions;
 class DataFilePathFactory;
+class FileFormat;
 class MemoryPool;
 
 class KeyValueDataFileWriterFactory
@@ -49,19 +50,24 @@ class KeyValueDataFileWriterFactory
                                   FileSource file_source,
                                   const std::vector<std::string>& primary_keys,
                                   const std::shared_ptr<DataFilePathFactory>& path_factory,
-                                  bool create_stats_extractor,
+                                  bool create_stats_extractor, bool is_changelog,
                                   const std::shared_ptr<MemoryPool>& pool);
 
     Result<std::unique_ptr<SingleFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>>
     CreateWriter() const override;
 
  protected:
+    std::shared_ptr<FileFormat> GetFileFormat() const;
+    std::string GetFileCompression() const;
+    std::string NewFilePath(const std::string& format_identifier) const;
+
     std::shared_ptr<arrow::Schema> write_schema_;
     int32_t level_;
     FileSource file_source_;
     std::vector<std::string> primary_keys_;
     std::shared_ptr<DataFilePathFactory> path_factory_;
     bool create_stats_extractor_;
+    bool is_changelog_;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp
index 30d4c9fce..f56ba8d14 100644
--- a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp
+++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp
@@ -37,10 +37,11 @@ ShreddingKeyValueDataFileWriterFactory::ShreddingKeyValueDataFileWriterFactory(
     const std::shared_ptr<arrow::Schema>& write_schema, int32_t level, FileSource file_source,
     const std::vector<std::string>& primary_keys,
     const std::shared_ptr<DataFilePathFactory>& path_factory, bool create_stats_extractor,
-    const std::shared_ptr<ShreddingWritePlanFactory>& plan_factory,
+    const std::shared_ptr<ShreddingWritePlanFactory>& plan_factory, bool is_changelog,
     const std::shared_ptr<MemoryPool>& pool)
     : KeyValueDataFileWriterFactory(options, schema_id, write_schema, level, file_source,
-                                    primary_keys, path_factory, create_stats_extractor, pool),
+                                    primary_keys, path_factory, create_stats_extractor,
+                                    is_changelog, pool),
       plan_factory_(plan_factory) {}
 
 Result<std::unique_ptr<SingleFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>>
@@ -48,7 +49,7 @@ ShreddingKeyValueDataFileWriterFactory::CreateWriter() const {
     if (!plan_factory_) {
         return Status::Invalid("Shredding key-value writer requires a write-plan factory.");
     }
-    const std::string format_identifier = options_.GetWriteFileFormat(level_)->Identifier();
+    const std::string format_identifier = GetFileFormat()->Identifier();
     if (plan_factory_->ShouldInferWritePlan()) {
         auto create_inner = [this](const std::shared_ptr<ShreddingBatchConverter>& converter) {
             return CreateShreddedWriter(converter);
@@ -74,7 +75,8 @@ ShreddingKeyValueDataFileWriterFactory::CreateShreddedWriter(
             [factory = plan_factory_, converter]() { return factory->OnFileCompleted(converter); });
         return writer;
     }
-    auto format = options_.GetWriteFileFormat(level_);
+    std::shared_ptr<FileFormat> format = GetFileFormat();
+    std::string compression = GetFileCompression();
     std::shared_ptr<arrow::Schema> file_schema = converter->GetPhysicalSchema();
     std::function<Status(KeyValueBatch&&, ::ArrowArray*)> batch_converter =
         [converter](KeyValueBatch key_value_batch, ::ArrowArray* array) -> Status {
@@ -86,18 +88,19 @@ ShreddingKeyValueDataFileWriterFactory::CreateShreddedWriter(
     PAIMON_ASSIGN_OR_RAISE(WriterResources resources,
                            CreateWriterResources(*format, file_schema, create_stats_extractor_));
     auto writer = std::make_unique<KeyValueDataFileWriter>(
-        options_.GetWriteFileCompression(level_), std::move(batch_converter), schema_id_, level_,
-        file_source_, primary_keys_, resources.stats_extractor, file_schema,
-        path_factory_->IsExternalPath(), pool_);
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<DataFileIndexWriter> file_index_writer,
-                           CreateFileIndexWriter(write_schema_, path_factory_));
-    if (file_index_writer) {
-        writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_);
+        compression, std::move(batch_converter), schema_id_, level_, file_source_, primary_keys_,
+        resources.stats_extractor, file_schema, path_factory_->IsExternalPath(), pool_);
+    if (!is_changelog_) {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<DataFileIndexWriter> file_index_writer,
+                               CreateFileIndexWriter(write_schema_, path_factory_));
+        if (file_index_writer) {
+            writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_);
+        }
     }
-    PAIMON_RETURN_NOT_OK(
-        writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder));
+    PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), NewFilePath(format->Identifier()),
+                                      resources.writer_builder));
     ShreddingWritePlanFactory::MetadataFinalizer finalizer =
-        plan_factory_->CreateMetadataFinalizer(converter, options_.GetWriteFileCompression(level_));
+        plan_factory_->CreateMetadataFinalizer(converter, compression);
     if (finalizer) {
         writer->SetMetadataFinalizer(std::move(finalizer));
     }
diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h
index fb967588e..e32dc8a04 100644
--- a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h
+++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h
@@ -41,7 +41,7 @@ class ShreddingKeyValueDataFileWriterFactory : public KeyValueDataFileWriterFact
         const std::shared_ptr<arrow::Schema>& write_schema, int32_t level, FileSource file_source,
         const std::vector<std::string>& primary_keys,
         const std::shared_ptr<DataFilePathFactory>& path_factory, bool create_stats_extractor,
-        const std::shared_ptr<ShreddingWritePlanFactory>& plan_factory,
+        const std::shared_ptr<ShreddingWritePlanFactory>& plan_factory, bool is_changelog,
         const std::shared_ptr<MemoryPool>& pool);
 
     Result<std::unique_ptr<SingleFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>>
diff --git a/src/paimon/core/manifest/manifest_list.h b/src/paimon/core/manifest/manifest_list.h
index 31959a5c9..ff67e8080 100644
--- a/src/paimon/core/manifest/manifest_list.h
+++ b/src/paimon/core/manifest/manifest_list.h
@@ -113,8 +113,7 @@ class ManifestList : public ObjectsFile<ManifestFileMeta> {
         const std::optional<std::string>& changelog_manifest_list =
             snapshot.ChangelogManifestList();
         if (changelog_manifest_list) {
-            return Status::NotImplemented("do not support read changelog manifest list");
-            // return Read(changelog_manifest_list.value(), /*filter=*/nullptr, manifests);
+            return Read(changelog_manifest_list.value(), /*filter=*/nullptr, manifests);
         } else {
             return Status::OK();
         }
diff --git a/src/paimon/core/manifest/manifest_list_test.cpp b/src/paimon/core/manifest/manifest_list_test.cpp
index 889f18f2b..23949bfdd 100644
--- a/src/paimon/core/manifest/manifest_list_test.cpp
+++ b/src/paimon/core/manifest/manifest_list_test.cpp
@@ -25,6 +25,7 @@
 #include "gtest/gtest.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/manifest/manifest_file_meta.h"
+#include "paimon/core/snapshot.h"
 #include "paimon/core/stats/simple_stats.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/format/file_format.h"
@@ -141,6 +142,33 @@ TEST_F(ManifestListTest, TestEmptyManifestList) {
     ASSERT_EQ(manifest_file_metas.size(), 0);
 }
 
+TEST_F(ManifestListTest, TestReadChangelogManifests) {
+    auto pool = GetDefaultPool();
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto manifest_list = CreateManifestList("orc", dir->Str(), pool);
+    ManifestFileMeta expected_meta(
+        "changelog-manifest", /*file_size=*/100, /*num_added_files=*/1,
+        /*num_deleted_files=*/0, SimpleStats::EmptyStats(), /*schema_id=*/0,
+        /*min_bucket=*/0, /*max_bucket=*/0, /*min_level=*/0, /*max_level=*/0,
+        /*min_row_id=*/std::nullopt, /*max_row_id=*/std::nullopt);
+    ASSERT_OK_AND_ASSIGN(auto changelog_manifest_list, manifest_list->Write({expected_meta}));
+    Snapshot snapshot(
+        /*id=*/1, /*schema_id=*/0, /*base_manifest_list=*/"",
+        /*base_manifest_list_size=*/std::nullopt, /*delta_manifest_list=*/"",
+        /*delta_manifest_list_size=*/std::nullopt,
+        /*changelog_manifest_list=*/changelog_manifest_list.first,
+        /*changelog_manifest_list_size=*/changelog_manifest_list.second,
+        /*index_manifest=*/std::nullopt, /*commit_user=*/"user", /*commit_identifier=*/1,
+        Snapshot::CommitKind::Append(), /*time_millis=*/0, /*total_record_count=*/1,
+        /*delta_record_count=*/1, /*changelog_record_count=*/1, /*watermark=*/std::nullopt,
+        /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/std::nullopt);
+
+    std::vector<ManifestFileMeta> actual_metas;
+    ASSERT_OK(manifest_list->ReadChangelogManifests(snapshot, &actual_metas));
+    ASSERT_EQ(std::vector<ManifestFileMeta>({expected_meta}), actual_metas);
+}
+
 TEST_F(ManifestListTest, TestManifestListCompatibleWithJavaPaimon09) {
     auto pool = GetDefaultPool();
     auto manifest_file_metas = ReadManifestFileMeta("avro", paimon::test::GetDataDir() + "/avro",
diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp
index 3efd40f64..4f632425b 100644
--- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp
+++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp
@@ -17,7 +17,142 @@
  */
 
 #include "paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h"
+
+#include <utility>
+
+#include "paimon/common/utils/fields_comparator.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/io/key_value_meta_projection_consumer.h"
+#include "paimon/core/io/row_to_arrow_array_converter.h"
+#include "paimon/format/file_format.h"
 namespace paimon {
+
+namespace {
+
+using CancellationChecker = std::function<bool()>;
+
+class ChangelogCompactionBatchProducer : public AsyncKeyValueBatchProducer {
+ public:
+    ChangelogCompactionBatchProducer(
+        std::unique_ptr<SortMergeReader>&& sort_merge_reader, int32_t write_batch_size,
+        std::shared_ptr<MergeFunctionWrapper<ChangelogResult>>&& merge_function_wrapper,
+        const FieldsComparator::FieldComparatorFunc& key_comparator,
+        const CancellationChecker& cancellation_checker, bool drop_delete, bool produce_data,
+        bool produce_changelog)
+        : sort_merge_reader_(std::move(sort_merge_reader)),
+          write_batch_size_(NormalizeProjectionBatchSize(write_batch_size)),
+          merge_function_wrapper_(std::move(merge_function_wrapper)),
+          key_comparator_(key_comparator),
+          cancellation_checker_(cancellation_checker),
+          drop_delete_(drop_delete),
+          produce_data_(produce_data),
+          produce_changelog_(produce_changelog) {}
+
+    Status Produce(AsyncKeyValueBatchSink* sink) override {
+        std::vector<KeyValue> compact_buffer;
+        std::vector<KeyValue> changelog_buffer;
+        compact_buffer.reserve(write_batch_size_);
+        changelog_buffer.reserve(write_batch_size_);
+
+        auto flush = [&](AsyncKeyValueBatchType type, std::vector<KeyValue>* buffer) -> Status {
+            if (buffer->empty()) {
+                return Status::OK();
+            }
+            std::vector<KeyValue> rows = std::move(*buffer);
+            buffer->clear();
+            buffer->reserve(write_batch_size_);
+            return sink->Write(type, std::move(rows));
+        };
+
+        auto emit_result = [&](ChangelogResult&& result) -> Status {
+            if (produce_data_ && result.result &&
+                (!drop_delete_ || result.result->value_kind->IsAdd())) {
+                compact_buffer.emplace_back(std::move(result.result).value());
+                if (static_cast<int32_t>(compact_buffer.size()) >= write_batch_size_) {
+                    PAIMON_RETURN_NOT_OK(flush(AsyncKeyValueBatchType::DATA, &compact_buffer));
+                }
+            }
+            if (produce_changelog_) {
+                for (auto& changelog : result.changelogs) {
+                    changelog_buffer.emplace_back(std::move(changelog));
+                    if (static_cast<int32_t>(changelog_buffer.size()) >= write_batch_size_) {
+                        PAIMON_RETURN_NOT_OK(
+                            flush(AsyncKeyValueBatchType::CHANGELOG, &changelog_buffer));
+                    }
+                }
+            }
+            return Status::OK();
+        };
+
+        std::shared_ptr<InternalRow> current_key;
+        auto finish_group = [&]() -> Status {
+            if (!current_key) {
+                return Status::OK();
+            }
+            PAIMON_ASSIGN_OR_RAISE(std::optional<ChangelogResult> result,
+                                   merge_function_wrapper_->GetResult());
+            current_key.reset();
+            if (result) {
+                PAIMON_RETURN_NOT_OK(emit_result(std::move(result).value()));
+            }
+            return Status::OK();
+        };
+
+        while (true) {
+            if (cancellation_checker_()) {
+                return Status::Cancelled("Compaction is cancelled");
+            }
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader::Iterator> iterator,
+                                   sort_merge_reader_->NextBatch());
+            if (!iterator) {
+                break;
+            }
+            while (true) {
+                PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext());
+                if (!has_next) {
+                    break;
+                }
+                KeyValue key_value = iterator->Next();
+                if (current_key && key_comparator_(*current_key, *key_value.key) != 0) {
+                    PAIMON_RETURN_NOT_OK(finish_group());
+                }
+                if (!current_key) {
+                    merge_function_wrapper_->Reset();
+                    current_key = key_value.key;
+                }
+                PAIMON_RETURN_NOT_OK(merge_function_wrapper_->Add(std::move(key_value)));
+            }
+        }
+        PAIMON_RETURN_NOT_OK(finish_group());
+        sort_merge_reader_->Close();
+        closed_ = true;
+        PAIMON_RETURN_NOT_OK(flush(AsyncKeyValueBatchType::DATA, &compact_buffer));
+        PAIMON_RETURN_NOT_OK(flush(AsyncKeyValueBatchType::CHANGELOG, &changelog_buffer));
+        return Status::OK();
+    }
+
+    void Close() override {
+        if (closed_) {
+            return;
+        }
+        sort_merge_reader_->Close();
+        closed_ = true;
+    }
+
+ private:
+    std::unique_ptr<SortMergeReader> sort_merge_reader_;
+    int32_t write_batch_size_;
+    std::shared_ptr<MergeFunctionWrapper<ChangelogResult>> merge_function_wrapper_;
+    FieldsComparator::FieldComparatorFunc key_comparator_;
+    CancellationChecker cancellation_checker_;
+    bool drop_delete_;
+    bool produce_data_;
+    bool produce_changelog_;
+    bool closed_ = false;
+};
+
+}  // namespace
+
 ChangelogMergeTreeRewriter::ChangelogMergeTreeRewriter(
     int32_t max_level, bool force_drop_delete, const BinaryRow& partition, int32_t bucket,
     int64_t schema_id, const std::vector<std::string>& trimmed_primary_keys,
@@ -26,14 +161,18 @@ ChangelogMergeTreeRewriter::ChangelogMergeTreeRewriter(
     const std::shared_ptr<FileStorePathFactoryCache>& path_factory_cache,
     std::unique_ptr<MergeFileSplitRead>&& merge_file_split_read,
     MergeFunctionWrapperFactory merge_function_wrapper_factory,
-    const std::shared_ptr<CancellationController>& cancellation_controller,
+    ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory,
+    bool produce_changelog, const std::shared_ptr<CancellationController>& cancellation_controller,
     const std::shared_ptr<MemoryPool>& pool)
     : MergeTreeCompactRewriter(
           partition, bucket, schema_id, trimmed_primary_keys, options, data_schema, write_schema,
           std::move(dv_factory), path_factory_cache, std::move(merge_file_split_read),
           std::move(merge_function_wrapper_factory), cancellation_controller, pool),
       max_level_(max_level),
-      force_drop_delete_(force_drop_delete) {}
+      force_drop_delete_(force_drop_delete),
+      changelog_merge_function_wrapper_factory_(
+          std::move(changelog_merge_function_wrapper_factory)),
+      produce_changelog_(produce_changelog) {}
 
 Result<CompactResult> ChangelogMergeTreeRewriter::Rewrite(
     int32_t output_level, bool drop_delete, const std::vector<std::vector<SortedRun>>& sections) {
@@ -78,31 +217,78 @@ Result<CompactResult> ChangelogMergeTreeRewriter::RewriteOrProduceChangelog(
     bool rewrite_compact_file) {
     PAIMON_ASSIGN_OR_RAISE(MergeTreeCompactRewriter::KeyValueConsumerCreator create_consumer,
                            GenerateKeyValueConsumer());
-    std::vector<std::shared_ptr<MergeTreeCompactRewriter::KeyValueMergeReader>> reader_holders;
-
     auto before = ExtractFilesFromSections(sections);
     std::unique_ptr<MergeTreeCompactRewriter::KeyValueRollingFileWriter> compact_file_writer;
     if (rewrite_compact_file) {
         PAIMON_ASSIGN_OR_RAISE(compact_file_writer, CreateRollingRowWriter(output_level));
     }
-    // TODO(xinyu.lxy): produce changelog
+    std::unique_ptr<MergeTreeCompactRewriter::KeyValueRollingFileWriter> changelog_file_writer;
+    if (produce_changelog_) {
+        PAIMON_ASSIGN_OR_RAISE(changelog_file_writer, CreateRollingChangelogWriter(output_level));
+    }
+
+    std::vector<std::shared_ptr<MergeTreeCompactRewriter::KeyValueMergeReader>> reader_holders;
     ScopeGuard write_guard([&]() -> void {
         if (compact_file_writer) {
             compact_file_writer->Abort();
+            compact_file_writer.reset();
+        }
+        if (changelog_file_writer) {
+            changelog_file_writer->Abort();
+            changelog_file_writer.reset();
         }
-        merge_file_split_read_.reset();
         for (const auto& reader : reader_holders) {
             reader->Close();
         }
+        merge_file_split_read_.reset();
     });
 
+    bool produce_data = compact_file_writer != nullptr;
+    bool produce_changelog = changelog_file_writer != nullptr;
+    FieldsComparator::FieldComparatorFunc key_comparator = [this](const InternalRow& lhs,
+                                                                  const InternalRow& rhs) {
+        return merge_file_split_read_->GetKeyComparator()->CompareTo(lhs, rhs);
+    };
+    CancellationChecker cancellation_checker = [this]() { return IsCancelled(); };
+
     for (const auto& section : sections) {
-        PAIMON_RETURN_NOT_OK(MergeReadAndWrite(output_level, drop_delete, section, create_consumer,
-                                               compact_file_writer.get(), &reader_holders));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortMergeReader> sort_merge_reader,
+                               CreateRawSortMergeReaderForSection(section));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr<MergeFunctionWrapper<ChangelogResult>> merge_function_wrapper,
+            changelog_merge_function_wrapper_factory_(output_level));
+        std::unique_ptr<AsyncKeyValueBatchProducer> producer =
+            std::make_unique<ChangelogCompactionBatchProducer>(
+                std::move(sort_merge_reader), options_.GetWriteBatchSize(),
+                std::move(merge_function_wrapper), key_comparator, cancellation_checker,
+                drop_delete, produce_data, produce_changelog);
+        auto producer_and_consumer =
+            std::make_shared<AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>>(
+                std::move(producer), create_consumer, /*consumer_thread_num=*/1);
+        reader_holders.emplace_back(producer_and_consumer);
+
+        while (true) {
+            if (IsCancelled()) {
+                return Status::Cancelled("Compaction is cancelled");
+            }
+            PAIMON_ASSIGN_OR_RAISE(AsyncKeyValueResultBatch<KeyValueBatch> output,
+                                   producer_and_consumer->NextBatchWithType());
+            if (output.result.batch == nullptr) {
+                break;
+            }
+            if (output.type == AsyncKeyValueBatchType::DATA) {
+                PAIMON_RETURN_NOT_OK(compact_file_writer->Write(std::move(output.result)));
+            } else {
+                PAIMON_RETURN_NOT_OK(changelog_file_writer->Write(std::move(output.result)));
+            }
+        }
     }
     if (compact_file_writer) {
         PAIMON_RETURN_NOT_OK(compact_file_writer->Close());
     }
+    if (changelog_file_writer) {
+        PAIMON_RETURN_NOT_OK(changelog_file_writer->Close());
+    }
     std::vector<std::shared_ptr<DataFileMeta>> after;
     if (compact_file_writer) {
         PAIMON_ASSIGN_OR_RAISE(after, compact_file_writer->GetResult());
@@ -118,8 +304,12 @@ Result<CompactResult> ChangelogMergeTreeRewriter::RewriteOrProduceChangelog(
         NotifyRewriteCompactBefore(before);
     }
     PAIMON_ASSIGN_OR_RAISE(after, NotifyRewriteCompactAfter(after));
+    std::vector<std::shared_ptr<DataFileMeta>> changelog_files;
+    if (changelog_file_writer) {
+        PAIMON_ASSIGN_OR_RAISE(changelog_files, changelog_file_writer->GetResult());
+    }
     write_guard.Release();
-    return CompactResult(before, after);
+    return CompactResult(before, after, changelog_files);
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h
index c5d8e8914..c080b30a7 100644
--- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h
+++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h
@@ -22,11 +22,15 @@
 #include "paimon/core/core_options.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/key_value.h"
+#include "paimon/core/mergetree/compact/changelog_result.h"
 #include "paimon/core/mergetree/compact/merge_tree_compact_rewriter.h"
 namespace paimon {
 /// A `MergeTreeCompactRewriter` which produces changelog files while performing compaction.
 class ChangelogMergeTreeRewriter : public MergeTreeCompactRewriter {
  public:
+    using ChangelogMergeFunctionWrapperFactory =
+        std::function<Result<std::shared_ptr<MergeFunctionWrapper<ChangelogResult>>>(int32_t)>;
+
     Result<CompactResult> Rewrite(int32_t output_level, bool drop_delete,
                                   const std::vector<std::vector<SortedRun>>& sections) override;
 
@@ -42,6 +46,8 @@ class ChangelogMergeTreeRewriter : public MergeTreeCompactRewriter {
         const std::shared_ptr<FileStorePathFactoryCache>& path_factory_cache,
         std::unique_ptr<MergeFileSplitRead>&& merge_file_split_read,
         MergeFunctionWrapperFactory merge_function_wrapper_factory,
+        ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory,
+        bool produce_changelog,
         const std::shared_ptr<CancellationController>& cancellation_controller,
         const std::shared_ptr<MemoryPool>& pool);
 
@@ -85,5 +91,8 @@ class ChangelogMergeTreeRewriter : public MergeTreeCompactRewriter {
     Result<CompactResult> RewriteOrProduceChangelog(
         int32_t output_level, const std::vector<std::vector<SortedRun>>& sections, bool drop_delete,
         bool rewrite_compact_file);
+
+    ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory_;
+    bool produce_changelog_;
 };
 }  // namespace paimon
diff --git a/src/paimon/core/mergetree/compact/changelog_result.h b/src/paimon/core/mergetree/compact/changelog_result.h
new file mode 100644
index 000000000..778183ed7
--- /dev/null
+++ b/src/paimon/core/mergetree/compact/changelog_result.h
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <optional>
+#include <vector>
+
+#include "paimon/core/key_value.h"
+
+namespace paimon {
+
+/// The result of merging all records with the same primary key while producing changelog.
+struct ChangelogResult {
+    std::optional<KeyValue> result;
+    std::vector<KeyValue> changelogs;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h b/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h
index 319ed5ab6..e0d7e6d7f 100644
--- a/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h
+++ b/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h
@@ -24,7 +24,9 @@
 #include <optional>
 #include <utility>
 
+#include "paimon/common/data/serializer/row_compacted_serializer.h"
 #include "paimon/core/key_value.h"
+#include "paimon/core/mergetree/compact/changelog_result.h"
 #include "paimon/core/mergetree/compact/first_row_merge_function.h"
 #include "paimon/core/mergetree/compact/merge_function_wrapper.h"
 #include "paimon/result.h"
@@ -32,12 +34,15 @@
 
 namespace paimon {
 /// Wrapper for `MergeFunction`s to produce changelog by lookup for first row.
-class FirstRowMergeFunctionWrapper : public MergeFunctionWrapper<KeyValue> {
+class FirstRowMergeFunctionWrapper : public MergeFunctionWrapper<ChangelogResult> {
  public:
     FirstRowMergeFunctionWrapper(
         std::unique_ptr<FirstRowMergeFunction>&& merge_function,
-        std::function<Result<bool>(const std::shared_ptr<InternalRow>&)> contains)
-        : merge_function_(std::move(merge_function)), contains_(std::move(contains)) {}
+        std::function<Result<bool>(const std::shared_ptr<InternalRow>&)> contains,
+        std::unique_ptr<RowCompactedSerializer>&& value_serializer)
+        : merge_function_(std::move(merge_function)),
+          contains_(std::move(contains)),
+          value_serializer_(std::move(value_serializer)) {}
 
     void Reset() override {
         merge_function_->Reset();
@@ -47,11 +52,13 @@ class FirstRowMergeFunctionWrapper : public MergeFunctionWrapper<KeyValue> {
         return merge_function_->Add(std::move(kv));
     }
 
-    Result<std::optional<KeyValue>> GetResult() override {
+    Result<std::optional<ChangelogResult>> GetResult() override {
         PAIMON_ASSIGN_OR_RAISE(std::optional<KeyValue> result, merge_function_->GetResult());
+        ChangelogResult changelog_result;
         if (merge_function_->ContainsHighLevel()) {
+            changelog_result.result = std::move(result);
             Reset();
-            return result;
+            return std::optional<ChangelogResult>(std::move(changelog_result));
         }
         if (!result) {
             Reset();
@@ -62,17 +69,27 @@ class FirstRowMergeFunctionWrapper : public MergeFunctionWrapper<KeyValue> {
         if (contains) {
             // empty
             Reset();
-            return std::optional<KeyValue>();
+            return std::optional<ChangelogResult>(std::move(changelog_result));
         }
-        // new record, output changelog
-        // TODO(xinyu.lxy) support changelog
+        // TODO(lisizhuo.lsz): avoid serialize & deserialize here.
+        if (value_serializer_) {
+            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> bytes,
+                                   value_serializer_->SerializeToBytes(*result->value));
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<InternalRow> changelog_value,
+                                   value_serializer_->Deserialize(bytes));
+            changelog_result.changelogs.emplace_back(result->value_kind, result->sequence_number,
+                                                     result->level, result->key,
+                                                     std::move(changelog_value));
+        }
+        changelog_result.result = std::move(result);
         Reset();
-        return result;
+        return std::optional<ChangelogResult>(std::move(changelog_result));
     }
 
  private:
     std::unique_ptr<FirstRowMergeFunction> merge_function_;
     std::function<Result<bool>(const std::shared_ptr<InternalRow>&)> contains_;
+    std::unique_ptr<RowCompactedSerializer> value_serializer_;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper_test.cpp b/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper_test.cpp
index 36a2ef01e..47a2d38ed 100644
--- a/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper_test.cpp
+++ b/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper_test.cpp
@@ -29,6 +29,15 @@
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
+namespace {
+std::unique_ptr<RowCompactedSerializer> CreateValueSerializer(
+    const std::shared_ptr<MemoryPool>& pool) {
+    return RowCompactedSerializer::Create(arrow::schema({arrow::field("value", arrow::int32())}),
+                                          pool)
+        .value();
+}
+}  // namespace
+
 TEST(FirstRowMergeFunctionWrapperTest, TestSimple) {
     auto pool = GetDefaultPool();
     KeyValue kv1(RowKind::Insert(), /*sequence_number=*/0, /*level=*/0, /*key=*/
@@ -45,14 +54,17 @@ TEST(FirstRowMergeFunctionWrapperTest, TestSimple) {
 
     auto contains = [](const std::shared_ptr<InternalRow>& row) { return true; };
 
-    FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains));
+    FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains),
+                                         CreateValueSerializer(pool));
     wrapper.Reset();
     ASSERT_OK(wrapper.Add(std::move(kv1)));
     ASSERT_OK(wrapper.Add(std::move(kv2)));
     ASSERT_OK(wrapper.Add(std::move(kv3)));
     ASSERT_OK_AND_ASSIGN(auto result, wrapper.GetResult());
     ASSERT_TRUE(result);
-    ASSERT_EQ(result.value().sequence_number, 0);
+    ASSERT_TRUE(result->result);
+    ASSERT_EQ(result->result->sequence_number, 0);
+    ASSERT_TRUE(result->changelogs.empty());
 }
 
 TEST(FirstRowMergeFunctionWrapperTest, TestAllLevel0WithContain) {
@@ -71,13 +83,16 @@ TEST(FirstRowMergeFunctionWrapperTest, TestAllLevel0WithContain) {
 
     auto contains = [](const std::shared_ptr<InternalRow>& row) { return true; };
 
-    FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains));
+    FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains),
+                                         CreateValueSerializer(pool));
     wrapper.Reset();
     ASSERT_OK(wrapper.Add(std::move(kv1)));
     ASSERT_OK(wrapper.Add(std::move(kv2)));
     ASSERT_OK(wrapper.Add(std::move(kv3)));
     ASSERT_OK_AND_ASSIGN(auto result, wrapper.GetResult());
-    ASSERT_FALSE(result);
+    ASSERT_TRUE(result);
+    ASSERT_FALSE(result->result);
+    ASSERT_TRUE(result->changelogs.empty());
 }
 
 TEST(FirstRowMergeFunctionWrapperTest, TestAllLevel0WithoutContain) {
@@ -96,14 +111,18 @@ TEST(FirstRowMergeFunctionWrapperTest, TestAllLevel0WithoutContain) {
 
     auto contains = [](const std::shared_ptr<InternalRow>& row) { return false; };
 
-    FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains));
+    FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains),
+                                         CreateValueSerializer(pool));
     wrapper.Reset();
     ASSERT_OK(wrapper.Add(std::move(kv1)));
     ASSERT_OK(wrapper.Add(std::move(kv2)));
     ASSERT_OK(wrapper.Add(std::move(kv3)));
     ASSERT_OK_AND_ASSIGN(auto result, wrapper.GetResult());
     ASSERT_TRUE(result);
-    ASSERT_EQ(result.value().sequence_number, 0);
+    ASSERT_TRUE(result->result);
+    ASSERT_EQ(result->result->sequence_number, 0);
+    ASSERT_EQ(result->changelogs.size(), 1);
+    ASSERT_EQ(result->changelogs[0].sequence_number, 0);
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/core/mergetree/compact/internal_row_equalizer.h b/src/paimon/core/mergetree/compact/internal_row_equalizer.h
new file mode 100644
index 000000000..22f15d8bd
--- /dev/null
+++ b/src/paimon/core/mergetree/compact/internal_row_equalizer.h
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "fmt/format.h"
+#include "paimon/common/data/data_getters.h"
+#include "paimon/common/data/internal_array.h"
+#include "paimon/common/data/internal_map.h"
+#include "paimon/common/data/internal_row.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/date_time_utils.h"
+#include "paimon/common/utils/fields_comparator.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+/// Creates equality functions for internal rows, including nested values.
+/// Java's RecordEqualiser also compares the RowKind embedded in InternalRow. This comparator
+/// currently compares field values only. This does not affect the current lookup changelog results
+/// because changelog kinds are tracked separately by KeyValue::value_kind.
+class InternalRowEqualizer {
+ public:
+    static Result<FieldsComparator::FieldComparatorFunc> Create(
+        const std::shared_ptr<arrow::Schema>& schema,
+        const std::vector<std::string>& ignore_fields) {
+        std::set<std::string> ignored(ignore_fields.begin(), ignore_fields.end());
+        std::vector<std::pair<int32_t, ValueEqualizer>> equalizers;
+        for (int32_t i = 0; i < schema->num_fields(); ++i) {
+            if (ignored.find(schema->field(i)->name()) != ignored.end()) {
+                continue;
+            }
+            PAIMON_ASSIGN_OR_RAISE(ValueEqualizer equalizer,
+                                   CreateValueEqualizer(schema->field(i)->type()));
+            equalizers.emplace_back(i, std::move(equalizer));
+        }
+        return FieldsComparator::FieldComparatorFunc(
+            [equalizers = std::move(equalizers)](const InternalRow& lhs, const InternalRow& rhs) {
+                for (const auto& [field_idx, equalizer] : equalizers) {
+                    if (!EqualAt(lhs, field_idx, rhs, field_idx, equalizer)) {
+                        return 1;
+                    }
+                }
+                return 0;
+            });
+    }
+
+ private:
+    using ValueEqualizer =
+        std::function<bool(const DataGetters&, int32_t, const DataGetters&, int32_t)>;
+
+    static bool EqualAt(const DataGetters& lhs, int32_t lhs_pos, const DataGetters& rhs,
+                        int32_t rhs_pos, const ValueEqualizer& equalizer) {
+        bool lhs_null = lhs.IsNullAt(lhs_pos);
+        bool rhs_null = rhs.IsNullAt(rhs_pos);
+        if (lhs_null || rhs_null) {
+            return lhs_null == rhs_null;
+        }
+        return equalizer(lhs, lhs_pos, rhs, rhs_pos);
+    }
+
+    static Result<ValueEqualizer> CreateValueEqualizer(
+        const std::shared_ptr<arrow::DataType>& type) {
+        switch (type->id()) {
+            case arrow::Type::BOOL:
+                return PrimitiveEqualizer<bool>(
+                    [](const DataGetters& row, int32_t pos) { return row.GetBoolean(pos); });
+            case arrow::Type::INT8:
+                return PrimitiveEqualizer<char>(
+                    [](const DataGetters& row, int32_t pos) { return row.GetByte(pos); });
+            case arrow::Type::INT16:
+                return PrimitiveEqualizer<int16_t>(
+                    [](const DataGetters& row, int32_t pos) { return row.GetShort(pos); });
+            case arrow::Type::INT32:
+                return PrimitiveEqualizer<int32_t>(
+                    [](const DataGetters& row, int32_t pos) { return row.GetInt(pos); });
+            case arrow::Type::DATE32:
+                return PrimitiveEqualizer<int32_t>(
+                    [](const DataGetters& row, int32_t pos) { return row.GetDate(pos); });
+            case arrow::Type::INT64:
+                return PrimitiveEqualizer<int64_t>(
+                    [](const DataGetters& row, int32_t pos) { return row.GetLong(pos); });
+            case arrow::Type::FLOAT:
+                return ValueEqualizer([](const DataGetters& lhs, int32_t lhs_pos,
+                                         const DataGetters& rhs, int32_t rhs_pos) {
+                    return FieldsComparator::CompareFloatingPoint(lhs.GetFloat(lhs_pos),
+                                                                  rhs.GetFloat(rhs_pos)) == 0;
+                });
+            case arrow::Type::DOUBLE:
+                return ValueEqualizer([](const DataGetters& lhs, int32_t lhs_pos,
+                                         const DataGetters& rhs, int32_t rhs_pos) {
+                    return FieldsComparator::CompareFloatingPoint(lhs.GetDouble(lhs_pos),
+                                                                  rhs.GetDouble(rhs_pos)) == 0;
+                });
+            case arrow::Type::STRING:
+            case arrow::Type::BINARY:
+                return ValueEqualizer([](const DataGetters& lhs, int32_t lhs_pos,
+                                         const DataGetters& rhs, int32_t rhs_pos) {
+                    return lhs.GetStringView(lhs_pos) == rhs.GetStringView(rhs_pos);
+                });
+            case arrow::Type::TIMESTAMP: {
+                std::shared_ptr<arrow::TimestampType> timestamp_type =
+                    checked_pointer_cast<arrow::TimestampType>(type);
+                int32_t precision = DateTimeUtils::GetPrecisionFromType(timestamp_type);
+                return ValueEqualizer([precision](const DataGetters& lhs, int32_t lhs_pos,
+                                                  const DataGetters& rhs, int32_t rhs_pos) {
+                    return lhs.GetTimestamp(lhs_pos, precision) ==
+                           rhs.GetTimestamp(rhs_pos, precision);
+                });
+            }
+            case arrow::Type::DECIMAL128: {
+                std::shared_ptr<arrow::Decimal128Type> decimal_type =
+                    checked_pointer_cast<arrow::Decimal128Type>(type);
+                int32_t precision = decimal_type->precision();
+                int32_t scale = decimal_type->scale();
+                return ValueEqualizer([precision, scale](const DataGetters& lhs, int32_t lhs_pos,
+                                                         const DataGetters& rhs, int32_t rhs_pos) {
+                    return lhs.GetDecimal(lhs_pos, precision, scale)
+                               .CompareTo(rhs.GetDecimal(rhs_pos, precision, scale)) == 0;
+                });
+            }
+            case arrow::Type::LIST: {
+                std::shared_ptr<arrow::ListType> list_type =
+                    checked_pointer_cast<arrow::ListType>(type);
+                PAIMON_ASSIGN_OR_RAISE(ValueEqualizer element_equalizer,
+                                       CreateValueEqualizer(list_type->value_type()));
+                return ValueEqualizer([element_equalizer = std::move(element_equalizer)](
+                                          const DataGetters& lhs, int32_t lhs_pos,
+                                          const DataGetters& rhs, int32_t rhs_pos) {
+                    std::shared_ptr<InternalArray> lhs_array = lhs.GetArray(lhs_pos);
+                    std::shared_ptr<InternalArray> rhs_array = rhs.GetArray(rhs_pos);
+                    if (lhs_array->Size() != rhs_array->Size()) {
+                        return false;
+                    }
+                    for (int32_t i = 0; i < lhs_array->Size(); ++i) {
+                        if (!EqualAt(*lhs_array, i, *rhs_array, i, element_equalizer)) {
+                            return false;
+                        }
+                    }
+                    return true;
+                });
+            }
+            case arrow::Type::MAP: {
+                std::shared_ptr<arrow::MapType> map_type =
+                    checked_pointer_cast<arrow::MapType>(type);
+                PAIMON_ASSIGN_OR_RAISE(ValueEqualizer key_equalizer,
+                                       CreateValueEqualizer(map_type->key_type()));
+                PAIMON_ASSIGN_OR_RAISE(ValueEqualizer item_equalizer,
+                                       CreateValueEqualizer(map_type->item_type()));
+                return ValueEqualizer([key_equalizer = std::move(key_equalizer),
+                                       item_equalizer = std::move(item_equalizer)](
+                                          const DataGetters& lhs, int32_t lhs_pos,
+                                          const DataGetters& rhs, int32_t rhs_pos) {
+                    std::shared_ptr<InternalMap> lhs_map = lhs.GetMap(lhs_pos);
+                    std::shared_ptr<InternalMap> rhs_map = rhs.GetMap(rhs_pos);
+                    if (lhs_map->Size() != rhs_map->Size()) {
+                        return false;
+                    }
+                    std::shared_ptr<InternalArray> lhs_keys = lhs_map->KeyArray();
+                    std::shared_ptr<InternalArray> rhs_keys = rhs_map->KeyArray();
+                    std::shared_ptr<InternalArray> lhs_values = lhs_map->ValueArray();
+                    std::shared_ptr<InternalArray> rhs_values = rhs_map->ValueArray();
+                    std::vector<bool> matched(rhs_map->Size(), false);
+                    for (int32_t lhs_index = 0; lhs_index < lhs_map->Size(); ++lhs_index) {
+                        bool found = false;
+                        for (int32_t rhs_index = 0; rhs_index < rhs_map->Size(); ++rhs_index) {
+                            if (matched[rhs_index]) {
+                                continue;
+                            }
+                            if (EqualAt(*lhs_keys, lhs_index, *rhs_keys, rhs_index,
+                                        key_equalizer) &&
+                                EqualAt(*lhs_values, lhs_index, *rhs_values, rhs_index,
+                                        item_equalizer)) {
+                                matched[rhs_index] = true;
+                                found = true;
+                                break;
+                            }
+                        }
+                        if (!found) {
+                            return false;
+                        }
+                    }
+                    return true;
+                });
+            }
+            case arrow::Type::STRUCT: {
+                std::shared_ptr<arrow::StructType> struct_type =
+                    checked_pointer_cast<arrow::StructType>(type);
+                std::vector<ValueEqualizer> field_equalizers;
+                field_equalizers.reserve(struct_type->num_fields());
+                for (const auto& field : struct_type->fields()) {
+                    PAIMON_ASSIGN_OR_RAISE(ValueEqualizer field_equalizer,
+                                           CreateValueEqualizer(field->type()));
+                    field_equalizers.emplace_back(std::move(field_equalizer));
+                }
+                int32_t field_count = struct_type->num_fields();
+                return ValueEqualizer([field_equalizers = std::move(field_equalizers), field_count](
+                                          const DataGetters& lhs, int32_t lhs_pos,
+                                          const DataGetters& rhs, int32_t rhs_pos) {
+                    std::shared_ptr<InternalRow> lhs_row = lhs.GetRow(lhs_pos, field_count);
+                    std::shared_ptr<InternalRow> rhs_row = rhs.GetRow(rhs_pos, field_count);
+                    for (int32_t i = 0; i < field_count; ++i) {
+                        if (!EqualAt(*lhs_row, i, *rhs_row, i, field_equalizers[i])) {
+                            return false;
+                        }
+                    }
+                    return true;
+                });
+            }
+            default:
+                return Status::NotImplemented(
+                    fmt::format("Do not support equality for type {}", type->ToString()));
+        }
+    }
+
+    template <typename T, typename Getter>
+    static ValueEqualizer PrimitiveEqualizer(Getter getter) {
+        return [getter = std::move(getter)](const DataGetters& lhs, int32_t lhs_pos,
+                                            const DataGetters& rhs, int32_t rhs_pos) {
+            return static_cast<T>(getter(lhs, lhs_pos)) == static_cast<T>(getter(rhs, rhs_pos));
+        };
+    }
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp b/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp
new file mode 100644
index 000000000..80bc37b55
--- /dev/null
+++ b/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/mergetree/compact/internal_row_equalizer.h"
+
+#include <cstdint>
+#include <limits>
+#include <memory>
+#include <string_view>
+#include <vector>
+
+#include "arrow/api.h"
+#include "gtest/gtest.h"
+#include "paimon/common/data/binary_array.h"
+#include "paimon/common/data/binary_array_writer.h"
+#include "paimon/common/data/binary_map.h"
+#include "paimon/common/data/generic_row.h"
+#include "paimon/common/data/internal_array.h"
+#include "paimon/common/data/internal_map.h"
+#include "paimon/common/data/internal_row.h"
+#include "paimon/common/utils/decimal_utils.h"
+#include "paimon/data/decimal.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+namespace {
+
+std::shared_ptr<InternalArray> CreateIntArray(const std::vector<int32_t>& values,
+                                              MemoryPool* pool) {
+    return std::make_shared<BinaryArray>(BinaryArray::FromIntArray(values, pool));
+}
+
+std::shared_ptr<InternalArray> CreateNullableIntArray(const std::vector<int32_t>& values,
+                                                      int32_t null_pos, MemoryPool* pool) {
+    auto array = std::make_shared<BinaryArray>();
+    BinaryArrayWriter writer(array.get(), static_cast<int32_t>(values.size()), sizeof(int32_t),
+                             pool);
+    for (int32_t i = 0; i < static_cast<int32_t>(values.size()); ++i) {
+        if (i == null_pos) {
+            writer.SetNullValue<int32_t>(i);
+        } else {
+            writer.WriteInt(i, values[i]);
+        }
+    }
+    writer.Complete();
+    return array;
+}
+
+std::shared_ptr<InternalMap> CreateIntMap(const std::vector<int32_t>& keys,
+                                          const std::vector<int32_t>& values, MemoryPool* pool) {
+    BinaryArray key_array = BinaryArray::FromIntArray(keys, pool);
+    BinaryArray value_array = BinaryArray::FromIntArray(values, pool);
+    return BinaryMap::ValueOf(key_array, value_array, pool);
+}
+
+std::shared_ptr<InternalRow> CreateNestedRow(int32_t value, double floating_point) {
+    return GenericRow::Of({value, floating_point});
+}
+
+}  // namespace
+
+TEST(InternalRowEqualizerTest, PrimitiveTypesAndIgnoreFields) {
+    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(
+        {arrow::field("boolean", arrow::boolean()), arrow::field("tinyint", arrow::int8()),
+         arrow::field("smallint", arrow::int16()), arrow::field("int", arrow::int32()),
+         arrow::field("date", arrow::date32()), arrow::field("bigint", arrow::int64()),
+         arrow::field("float", arrow::float32()), arrow::field("double", arrow::float64()),
+         arrow::field("string", arrow::utf8()), arrow::field("binary", arrow::binary()),
+         arrow::field("timestamp", arrow::timestamp(arrow::TimeUnit::MICRO)),
+         arrow::field("decimal", arrow::decimal128(10, 2)),
+         arrow::field("ignored", arrow::int32())});
+
+    const float float_nan = std::numeric_limits<float>::quiet_NaN();
+    const double double_nan = std::numeric_limits<double>::quiet_NaN();
+    auto binary = std::make_shared<Bytes>("binary", pool.get());
+    Decimal decimal(10, 2, DecimalUtils::StrToInt128("12345").value());
+    std::vector<VariantType> left_values = {true,
+                                            static_cast<char>(1),
+                                            static_cast<int16_t>(2),
+                                            static_cast<int32_t>(3),
+                                            int32_t{4},
+                                            int64_t{5},
+                                            float_nan,
+                                            double_nan,
+                                            std::string_view("string"),
+                                            binary,
+                                            Timestamp(1234, 567000),
+                                            decimal,
+                                            int32_t{10}};
+    std::vector<VariantType> right_values = left_values;
+    right_values.back() = int32_t{20};
+
+    std::unique_ptr<GenericRow> left = GenericRow::Of(left_values);
+    std::unique_ptr<GenericRow> right = GenericRow::Of(right_values);
+    ASSERT_OK_AND_ASSIGN(FieldsComparator::FieldComparatorFunc equalizer,
+                         InternalRowEqualizer::Create(schema, {"ignored"}));
+    ASSERT_EQ(0, equalizer(*left, *right));
+
+    right->SetField(/*pos=*/3, int32_t{30});
+    ASSERT_NE(0, equalizer(*left, *right));
+}
+
+TEST(InternalRowEqualizerTest, NullAndFloatingPointSemantics) {
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(
+        {arrow::field("value", arrow::float64()), arrow::field("nullable", arrow::int32())});
+    ASSERT_OK_AND_ASSIGN(FieldsComparator::FieldComparatorFunc equalizer,
+                         InternalRowEqualizer::Create(schema, {}));
+
+    std::unique_ptr<GenericRow> negative_zero =
+        GenericRow::Of({static_cast<double>(-0.0), NullType()});
+    std::unique_ptr<GenericRow> positive_zero =
+        GenericRow::Of({static_cast<double>(0.0), NullType()});
+    ASSERT_NE(0, equalizer(*negative_zero, *positive_zero));
+
+    double nan1 = std::numeric_limits<double>::quiet_NaN();
+    double nan2 = -std::numeric_limits<double>::quiet_NaN();
+    std::unique_ptr<GenericRow> left_nan = GenericRow::Of({nan1, NullType()});
+    std::unique_ptr<GenericRow> right_nan = GenericRow::Of({nan2, NullType()});
+    ASSERT_EQ(0, equalizer(*left_nan, *right_nan));
+
+    right_nan->SetField(/*pos=*/1, int32_t{1});
+    ASSERT_NE(0, equalizer(*left_nan, *right_nan));
+}
+
+TEST(InternalRowEqualizerTest, NestedTypes) {
+    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
+    std::shared_ptr<arrow::Schema> schema = arrow::schema(
+        {arrow::field("array", arrow::list(arrow::int32())),
+         arrow::field("map", arrow::map(arrow::int32(), arrow::int32())),
+         arrow::field("row", arrow::struct_({arrow::field("value", arrow::int32()),
+                                             arrow::field("floating", arrow::float64())}))});
+    ASSERT_OK_AND_ASSIGN(FieldsComparator::FieldComparatorFunc equalizer,
+                         InternalRowEqualizer::Create(schema, {}));
+
+    std::unique_ptr<GenericRow> left =
+        GenericRow::Of({CreateNullableIntArray({1, 0, 3}, /*null_pos=*/1, pool.get()),
+                        CreateIntMap({1, 2}, {10, 20}, pool.get()),
+                        CreateNestedRow(100, std::numeric_limits<double>::quiet_NaN())});
+    std::unique_ptr<GenericRow> right =
+        GenericRow::Of({CreateNullableIntArray({1, 9, 3}, /*null_pos=*/1, pool.get()),
+                        CreateIntMap({1, 2}, {10, 20}, pool.get()),
+                        CreateNestedRow(100, -std::numeric_limits<double>::quiet_NaN())});
+    ASSERT_EQ(0, equalizer(*left, *right));
+
+    right->SetField(/*pos=*/0, CreateIntArray({1, 2}, pool.get()));
+    ASSERT_NE(0, equalizer(*left, *right));
+
+    right->SetField(/*pos=*/0, CreateNullableIntArray({1, 0, 3}, /*null_pos=*/1, pool.get()));
+    right->SetField(/*pos=*/0, CreateNullableIntArray({1, 0, 4}, /*null_pos=*/1, pool.get()));
+    ASSERT_NE(0, equalizer(*left, *right));
+
+    right->SetField(/*pos=*/0, CreateNullableIntArray({1, 0, 3}, /*null_pos=*/1, pool.get()));
+    right->SetField(/*pos=*/1, CreateIntMap({1, 3}, {10, 20}, pool.get()));
+    ASSERT_NE(0, equalizer(*left, *right));
+
+    right->SetField(/*pos=*/1, CreateIntMap({1, 2}, {10, 20}, pool.get()));
+    right->SetField(/*pos=*/2, CreateNestedRow(101, 1.0));
+    ASSERT_NE(0, equalizer(*left, *right));
+}
+
+TEST(InternalRowEqualizerTest, MapEqualityDoesNotDependOnEntryOrder) {
+    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
+    std::shared_ptr<arrow::Schema> schema =
+        arrow::schema({arrow::field("map", arrow::map(arrow::int32(), arrow::int32()))});
+    ASSERT_OK_AND_ASSIGN(FieldsComparator::FieldComparatorFunc equalizer,
+                         InternalRowEqualizer::Create(schema, {}));
+
+    std::unique_ptr<GenericRow> left =
+        GenericRow::Of({CreateIntMap({1, 2, 3}, {10, 20, 30}, pool.get())});
+    std::unique_ptr<GenericRow> reordered =
+        GenericRow::Of({CreateIntMap({3, 1, 2}, {30, 10, 20}, pool.get())});
+    ASSERT_EQ(0, equalizer(*left, *reordered));
+
+    reordered->SetField(/*pos=*/0, CreateIntMap({3, 1, 2}, {30, 10, 21}, pool.get()));
+    ASSERT_NE(0, equalizer(*left, *reordered));
+}
+
+TEST(InternalRowEqualizerTest, UnsupportedType) {
+    ASSERT_NOK_WITH_MSG(InternalRowEqualizer::Create(
+                            arrow::schema({arrow::field("unsupported", arrow::null())}), {}),
+                        "Do not support equality for type null");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h
index d2b21b2c0..34f7cf970 100644
--- a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h
+++ b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h
@@ -25,9 +25,11 @@
 #include <string>
 #include <utility>
 
+#include "paimon/common/data/serializer/row_compacted_serializer.h"
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/deletionvectors/bucketed_dv_maintainer.h"
 #include "paimon/core/key_value.h"
+#include "paimon/core/mergetree/compact/changelog_result.h"
 #include "paimon/core/mergetree/compact/lookup_merge_function.h"
 #include "paimon/core/mergetree/compact/merge_function_wrapper.h"
 #include "paimon/core/mergetree/lookup/file_position.h"
@@ -47,23 +49,28 @@ namespace paimon {
 ///       should be AFTER.
 ///  With level-0 record, without level-x record, need to lookup the history value of the upper
 ///       level as BEFORE.
-/// TODO(xinyu.lxy) : add changelog
 template <typename T>
-class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper<KeyValue> {
+class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper<ChangelogResult> {
  public:
     static Result<std::unique_ptr<LookupChangelogMergeFunctionWrapper>> Create(
         std::unique_ptr<LookupMergeFunction>&& merge_function,
         std::function<Result<std::optional<T>>(const std::shared_ptr<InternalRow>&)> lookup,
-        const LookupStrategy& lookup_strategy,
+        const LookupStrategy& lookup_strategy, bool should_produce_changelog,
         const std::shared_ptr<BucketedDvMaintainer>& deletion_vectors_maintainer,
-        const std::shared_ptr<FieldsComparator>& comparator) {
+        const std::shared_ptr<FieldsComparator>& comparator,
+        std::unique_ptr<RowCompactedSerializer>&& value_serializer,
+        FieldsComparator::FieldComparatorFunc value_equalizer) {
         if (lookup_strategy.deletion_vector && !deletion_vectors_maintainer) {
             return Status::Invalid("deletionVectorsMaintainer should not be null, there is a bug.");
         }
+        if (should_produce_changelog && !value_serializer) {
+            return Status::Invalid("valueSerializer is required when producing changelog.");
+        }
         return std::unique_ptr<LookupChangelogMergeFunctionWrapper>(
-            new LookupChangelogMergeFunctionWrapper(std::move(merge_function), std::move(lookup),
-                                                    lookup_strategy, deletion_vectors_maintainer,
-                                                    comparator));
+            new LookupChangelogMergeFunctionWrapper(
+                std::move(merge_function), std::move(lookup), lookup_strategy,
+                should_produce_changelog, deletion_vectors_maintainer, comparator,
+                std::move(value_serializer), std::move(value_equalizer)));
     }
     void Reset() override {
         merge_function_->Reset();
@@ -73,12 +80,17 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper<KeyValue
         return merge_function_->Add(std::move(kv));
     }
 
-    Result<std::optional<KeyValue>> GetResult() override {
+    Result<std::optional<ChangelogResult>> GetResult() override {
         // 1. Find the latest high level record and compute containLevel0
-        std::optional<int32_t> high_level_idx = merge_function_->PickHighLevelIdx();
+        const KeyValue* high_level = merge_function_->PickHighLevel();
+        bool contain_level0 = merge_function_->ContainLevel0();
+        std::optional<KeyValue> before;
+        if (contain_level0 && should_produce_changelog_ && high_level != nullptr) {
+            PAIMON_ASSIGN_OR_RAISE(before, CloneKeyValue(*high_level, high_level->value_kind));
+        }
 
         // 2. Lookup if latest high level record is absent
-        if (high_level_idx == std::nullopt) {
+        if (high_level == nullptr) {
             std::optional<KeyValue> lookup_high_level;
             PAIMON_ASSIGN_OR_RAISE(std::optional<T> lookup_result,
                                    lookup_(merge_function_->GetKey()));
@@ -105,30 +117,83 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper<KeyValue
                 }
             }
             if (lookup_high_level) {
+                if (contain_level0 && should_produce_changelog_) {
+                    PAIMON_ASSIGN_OR_RAISE(before,
+                                           CloneKeyValue(lookup_high_level.value(),
+                                                         lookup_high_level.value().value_kind));
+                }
                 merge_function_->InsertInto(std::move(lookup_high_level), comparator_);
             }
         }
 
         // 3. Calculate result
         PAIMON_ASSIGN_OR_RAISE(std::optional<KeyValue> result, merge_function_->GetResult());
-        Reset();
+
         // 4. Set changelog when there's level-0 records
-        // TODO(liancheng.lsz): setChangelog
-        return result;
+        ChangelogResult changelog_result;
+        if (contain_level0 && should_produce_changelog_) {
+            PAIMON_RETURN_NOT_OK(
+                SetChangelog(std::move(before), result, &changelog_result.changelogs));
+        }
+        changelog_result.result = std::move(result);
+        Reset();
+        return std::optional<ChangelogResult>(std::move(changelog_result));
     }
 
  private:
     LookupChangelogMergeFunctionWrapper(
         std::unique_ptr<LookupMergeFunction>&& merge_function,
         std::function<Result<std::optional<T>>(const std::shared_ptr<InternalRow>&)> lookup,
-        const LookupStrategy& lookup_strategy,
+        const LookupStrategy& lookup_strategy, bool should_produce_changelog,
         const std::shared_ptr<BucketedDvMaintainer>& deletion_vectors_maintainer,
-        const std::shared_ptr<FieldsComparator>& user_defined_seq_comparator)
+        const std::shared_ptr<FieldsComparator>& user_defined_seq_comparator,
+        std::unique_ptr<RowCompactedSerializer>&& value_serializer,
+        FieldsComparator::FieldComparatorFunc value_equalizer)
         : merge_function_(std::move(merge_function)),
           lookup_(std::move(lookup)),
           lookup_strategy_(lookup_strategy),
+          should_produce_changelog_(should_produce_changelog),
           deletion_vectors_maintainer_(deletion_vectors_maintainer),
-          comparator_(CreateSequenceComparator(user_defined_seq_comparator)) {}
+          comparator_(CreateSequenceComparator(user_defined_seq_comparator)),
+          value_serializer_(std::move(value_serializer)),
+          value_equalizer_(std::move(value_equalizer)) {}
+
+    Result<KeyValue> CloneKeyValue(const KeyValue& from, const RowKind* value_kind) {
+        // TODO(lisizhuo.lsz): avoid serialize & deserialize here.
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> bytes,
+                               value_serializer_->SerializeToBytes(*from.value));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<InternalRow> value,
+                               value_serializer_->Deserialize(bytes));
+        return KeyValue(value_kind, from.sequence_number, KeyValue::UNKNOWN_LEVEL, from.key,
+                        std::move(value));
+    }
+
+    Status SetChangelog(std::optional<KeyValue>&& before, const std::optional<KeyValue>& after,
+                        std::vector<KeyValue>* changelogs) {
+        if (!before || !before->value_kind->IsAdd()) {
+            if (after && after->value_kind->IsAdd()) {
+                PAIMON_ASSIGN_OR_RAISE(KeyValue insert,
+                                       CloneKeyValue(after.value(), RowKind::Insert()));
+                changelogs->emplace_back(std::move(insert));
+            }
+            return Status::OK();
+        }
+
+        if (!after || !after->value_kind->IsAdd()) {
+            before->value_kind = RowKind::Delete();
+            changelogs->emplace_back(std::move(before.value()));
+            return Status::OK();
+        }
+
+        if (!value_equalizer_ || value_equalizer_(*before->value, *after->value) != 0) {
+            before->value_kind = RowKind::UpdateBefore();
+            PAIMON_ASSIGN_OR_RAISE(KeyValue update_after,
+                                   CloneKeyValue(after.value(), RowKind::UpdateAfter()));
+            changelogs->emplace_back(std::move(before.value()));
+            changelogs->emplace_back(std::move(update_after));
+        }
+        return Status::OK();
+    }
 
     static std::function<bool(const KeyValue& o1, const KeyValue& o2)> CreateSequenceComparator(
         const std::shared_ptr<FieldsComparator>& user_defined_seq_comparator) {
@@ -150,8 +215,11 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper<KeyValue
     std::unique_ptr<LookupMergeFunction> merge_function_;
     std::function<Result<std::optional<T>>(const std::shared_ptr<InternalRow>&)> lookup_;
     LookupStrategy lookup_strategy_;
+    bool should_produce_changelog_;
     std::shared_ptr<BucketedDvMaintainer> deletion_vectors_maintainer_;
     std::function<bool(const KeyValue& o1, const KeyValue& o2)> comparator_;
+    std::unique_ptr<RowCompactedSerializer> value_serializer_;
+    FieldsComparator::FieldComparatorFunc value_equalizer_;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp
index 446beb091..c4d255d9a 100644
--- a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp
+++ b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp
@@ -26,12 +26,22 @@
 #include "paimon/core/deletionvectors/bucketed_dv_maintainer.h"
 #include "paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
+#include "paimon/core/mergetree/compact/internal_row_equalizer.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/testing/utils/binary_row_generator.h"
 #include "paimon/testing/utils/key_value_checker.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
+namespace {
+std::unique_ptr<RowCompactedSerializer> CreateValueSerializer(
+    const std::shared_ptr<MemoryPool>& pool) {
+    return RowCompactedSerializer::Create(arrow::schema({arrow::field("value", arrow::int32())}),
+                                          pool)
+        .value();
+}
+}  // namespace
+
 TEST(LookupChangelogMergeFunctionWrapperTest, TestCreateInvalid) {
     auto pool = GetDefaultPool();
     auto mfunc = std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/true);
@@ -45,8 +55,10 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestCreateInvalid) {
                                    /*deletion_vector=*/true, /*force_lookup=*/true);
     ASSERT_NOK_WITH_MSG(LookupChangelogMergeFunctionWrapper<KeyValue>::Create(
                             std::move(lookup_mfunc), lookup, lookup_strategy,
+                            /*should_produce_changelog=*/false,
                             /*deletion_vectors_maintainer=*/nullptr,
-                            /*comparator=*/nullptr),
+                            /*comparator=*/nullptr, CreateValueSerializer(pool),
+                            /*value_equalizer=*/{}),
                         "deletionVectorsMaintainer should not be null, there is a bug.");
 }
 
@@ -69,12 +81,15 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestSimple) {
             KeyValue(RowKind::Insert(), /*sequence_number=*/1000, /*level=*/3, key,
                      BinaryRowGenerator::GenerateRowPtr({1001}, pool.get())));
     };
-    LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/false,
+    LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true,
                                    /*deletion_vector=*/false, /*force_lookup=*/true);
     ASSERT_OK_AND_ASSIGN(auto wrapper, LookupChangelogMergeFunctionWrapper<KeyValue>::Create(
                                            std::move(lookup_mfunc), lookup, lookup_strategy,
+                                           /*should_produce_changelog=*/false,
                                            /*deletion_vectors_maintainer=*/nullptr,
-                                           /*comparator=*/nullptr));
+                                           /*comparator=*/nullptr,
+                                           /*value_serializer=*/nullptr,
+                                           /*value_equalizer=*/{}));
 
     wrapper->Reset();
     ASSERT_OK(wrapper->Add(std::move(kv1)));
@@ -82,7 +97,9 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestSimple) {
     ASSERT_OK(wrapper->Add(std::move(kv3)));
     ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult());
     ASSERT_TRUE(result);
-    ASSERT_EQ(result.value().sequence_number, 2);
+    ASSERT_TRUE(result->result);
+    ASSERT_EQ(result->result->sequence_number, 2);
+    ASSERT_TRUE(result->changelogs.empty());
 }
 
 TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookup) {
@@ -104,12 +121,14 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookup) {
             KeyValue(RowKind::Insert(), /*sequence_number=*/0, /*level=*/3, key,
                      BinaryRowGenerator::GenerateRowPtr({1001}, pool.get())));
     };
-    LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/false,
+    LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true,
                                    /*deletion_vector=*/false, /*force_lookup=*/true);
     ASSERT_OK_AND_ASSIGN(auto wrapper, LookupChangelogMergeFunctionWrapper<KeyValue>::Create(
                                            std::move(lookup_mfunc), lookup, lookup_strategy,
+                                           /*should_produce_changelog=*/true,
                                            /*deletion_vectors_maintainer=*/nullptr,
-                                           /*comparator=*/nullptr));
+                                           /*comparator=*/nullptr, CreateValueSerializer(pool),
+                                           /*value_equalizer=*/{}));
 
     wrapper->Reset();
     ASSERT_OK(wrapper->Add(std::move(kv1)));
@@ -117,7 +136,98 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookup) {
     ASSERT_OK(wrapper->Add(std::move(kv3)));
     ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult());
     ASSERT_TRUE(result);
-    ASSERT_EQ(result.value().sequence_number, 3);
+    ASSERT_TRUE(result->result);
+    ASSERT_EQ(result->result->sequence_number, 3);
+    ASSERT_EQ(result->changelogs.size(), 2);
+    ASSERT_EQ(result->changelogs[0].value_kind, RowKind::UpdateBefore());
+    ASSERT_EQ(result->changelogs[0].level, KeyValue::UNKNOWN_LEVEL);
+    ASSERT_EQ(result->changelogs[0].value->GetInt(0), 1001);
+    ASSERT_EQ(result->changelogs[1].value_kind, RowKind::UpdateAfter());
+    ASSERT_EQ(result->changelogs[1].level, KeyValue::UNKNOWN_LEVEL);
+    ASSERT_EQ(result->changelogs[1].value->GetInt(0), 300);
+}
+
+TEST(LookupChangelogMergeFunctionWrapperTest, TestRowDeduplicate) {
+    auto pool = GetDefaultPool();
+    KeyValue kv(RowKind::Insert(), /*sequence_number=*/1, /*level=*/0,
+                BinaryRowGenerator::GenerateRowPtr({10}, pool.get()),
+                BinaryRowGenerator::GenerateRowPtr({300}, pool.get()));
+
+    auto merge_function = std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/true);
+    auto lookup_merge_function = std::make_unique<LookupMergeFunction>(std::move(merge_function));
+    auto lookup = [&](const std::shared_ptr<InternalRow>& key) -> Result<std::optional<KeyValue>> {
+        return std::optional<KeyValue>(
+            KeyValue(RowKind::Insert(), /*sequence_number=*/0, /*level=*/3, key,
+                     BinaryRowGenerator::GenerateRowPtr({300}, pool.get())));
+    };
+    LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true,
+                                   /*deletion_vector=*/false, /*force_lookup=*/true);
+    auto value_equalizer = [](const InternalRow& lhs, const InternalRow& rhs) {
+        return lhs.GetInt(0) == rhs.GetInt(0) ? 0 : 1;
+    };
+    ASSERT_OK_AND_ASSIGN(auto wrapper,
+                         LookupChangelogMergeFunctionWrapper<KeyValue>::Create(
+                             std::move(lookup_merge_function), lookup, lookup_strategy,
+                             /*should_produce_changelog=*/true,
+                             /*deletion_vectors_maintainer=*/nullptr,
+                             /*comparator=*/nullptr, CreateValueSerializer(pool), value_equalizer));
+
+    wrapper->Reset();
+    ASSERT_OK(wrapper->Add(std::move(kv)));
+    ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult());
+    ASSERT_TRUE(result);
+    ASSERT_TRUE(result->result);
+    ASSERT_TRUE(result->changelogs.empty());
+}
+
+TEST(LookupChangelogMergeFunctionWrapperTest, TestRowDeduplicateWithIgnoreFields) {
+    auto pool = GetDefaultPool();
+    auto value_schema = arrow::schema(
+        {arrow::field("value", arrow::int32()), arrow::field("ignored", arrow::int32())});
+    auto merge_function = std::make_unique<DeduplicateMergeFunction>(/*ignore_delete=*/true);
+    auto lookup_merge_function = std::make_unique<LookupMergeFunction>(std::move(merge_function));
+    auto lookup = [&](const std::shared_ptr<InternalRow>& key) -> Result<std::optional<KeyValue>> {
+        return std::optional<KeyValue>(
+            KeyValue(RowKind::Insert(), /*sequence_number=*/0, /*level=*/3, key,
+                     BinaryRowGenerator::GenerateRowPtr({300, 1}, pool.get())));
+    };
+    LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true,
+                                   /*deletion_vector=*/false, /*force_lookup=*/true);
+    ASSERT_OK_AND_ASSIGN(auto value_equalizer,
+                         InternalRowEqualizer::Create(value_schema, {"ignored"}));
+    ASSERT_OK_AND_ASSIGN(auto value_serializer, RowCompactedSerializer::Create(value_schema, pool));
+    ASSERT_OK_AND_ASSIGN(auto wrapper,
+                         LookupChangelogMergeFunctionWrapper<KeyValue>::Create(
+                             std::move(lookup_merge_function), lookup, lookup_strategy,
+                             /*should_produce_changelog=*/true,
+                             /*deletion_vectors_maintainer=*/nullptr,
+                             /*comparator=*/nullptr, std::move(value_serializer), value_equalizer));
+
+    // A change to an ignored field does not produce an update changelog.
+    wrapper->Reset();
+    ASSERT_OK(wrapper->Add(KeyValue(RowKind::Insert(), /*sequence_number=*/1, /*level=*/0,
+                                    BinaryRowGenerator::GenerateRowPtr({10}, pool.get()),
+                                    BinaryRowGenerator::GenerateRowPtr({300, 2}, pool.get()))));
+    ASSERT_OK_AND_ASSIGN(auto ignored_field_result, wrapper->GetResult());
+    ASSERT_TRUE(ignored_field_result);
+    ASSERT_TRUE(ignored_field_result->result);
+    ASSERT_TRUE(ignored_field_result->changelogs.empty());
+
+    // A change to a non-ignored field still produces update-before and update-after.
+    wrapper->Reset();
+    ASSERT_OK(wrapper->Add(KeyValue(RowKind::Insert(), /*sequence_number=*/2, /*level=*/0,
+                                    BinaryRowGenerator::GenerateRowPtr({20}, pool.get()),
+                                    BinaryRowGenerator::GenerateRowPtr({301, 2}, pool.get()))));
+    ASSERT_OK_AND_ASSIGN(auto value_field_result, wrapper->GetResult());
+    ASSERT_TRUE(value_field_result);
+    ASSERT_TRUE(value_field_result->result);
+    ASSERT_EQ(value_field_result->changelogs.size(), 2);
+    ASSERT_EQ(value_field_result->changelogs[0].value_kind, RowKind::UpdateBefore());
+    ASSERT_EQ(value_field_result->changelogs[0].value->GetInt(0), 300);
+    ASSERT_EQ(value_field_result->changelogs[0].value->GetInt(1), 1);
+    ASSERT_EQ(value_field_result->changelogs[1].value_kind, RowKind::UpdateAfter());
+    ASSERT_EQ(value_field_result->changelogs[1].value->GetInt(0), 301);
+    ASSERT_EQ(value_field_result->changelogs[1].value->GetInt(1), 2);
 }
 
 TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookupWithDv) {
@@ -154,8 +264,66 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookupWithDv) {
                                    /*deletion_vector=*/true, /*force_lookup=*/false);
     ASSERT_OK_AND_ASSIGN(auto wrapper,
                          LookupChangelogMergeFunctionWrapper<PositionedKeyValue>::Create(
-                             std::move(lookup_mfunc), lookup, lookup_strategy, dv_maintainer,
-                             /*comparator=*/nullptr));
+                             std::move(lookup_mfunc), lookup, lookup_strategy,
+                             /*should_produce_changelog=*/false, dv_maintainer,
+                             /*comparator=*/nullptr,
+                             /*value_serializer=*/nullptr,
+                             /*value_equalizer=*/{}));
+
+    wrapper->Reset();
+    ASSERT_OK(wrapper->Add(std::move(kv1)));
+    ASSERT_OK(wrapper->Add(std::move(kv2)));
+    ASSERT_OK(wrapper->Add(std::move(kv3)));
+    ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult());
+    ASSERT_TRUE(result);
+    ASSERT_TRUE(result->result);
+    ASSERT_EQ(result->result->sequence_number, 3);
+    ASSERT_EQ(result->result->value->GetInt(0), 100 + 200 + 300 + 1001);
+
+    auto dv = dv_maintainer->DeletionVectorOf("data.file");
+    ASSERT_TRUE(dv);
+    ASSERT_FALSE(dv.value()->IsDeleted(0).value());
+    ASSERT_TRUE(dv.value()->IsDeleted(10).value());
+}
+
+TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookupWithDvAndChangelog) {
+    auto pool = GetDefaultPool();
+    KeyValue kv1(RowKind::Insert(), /*sequence_number=*/1, /*level=*/0, /*key=*/
+                 BinaryRowGenerator::GenerateRowPtr({10}, pool.get()),
+                 /*value=*/BinaryRowGenerator::GenerateRowPtr({100}, pool.get()));
+    KeyValue kv2(RowKind::Insert(), /*sequence_number=*/2, /*level=*/0,
+                 /*key=*/BinaryRowGenerator::GenerateRowPtr({10}, pool.get()),
+                 /*value=*/BinaryRowGenerator::GenerateRowPtr({200}, pool.get()));
+    KeyValue kv3(RowKind::Insert(), /*sequence_number=*/3, /*level=*/0,
+                 /*key=*/BinaryRowGenerator::GenerateRowPtr({10}, pool.get()),
+                 /*value=*/BinaryRowGenerator::GenerateRowPtr({300}, pool.get()));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                         CoreOptions::FromMap({{Options::FIELDS_DEFAULT_AGG_FUNC, "sum"}}));
+    ASSERT_OK_AND_ASSIGN(auto mfunc, AggregateMergeFunction::Create(
+                                         arrow::schema({arrow::field("value", arrow::int32())}),
+                                         {"key"}, core_options, pool));
+    auto lookup_mfunc = std::make_unique<LookupMergeFunction>(std::move(mfunc));
+    auto lookup =
+        [&](const std::shared_ptr<InternalRow>& key) -> Result<std::optional<PositionedKeyValue>> {
+        return std::optional<PositionedKeyValue>(
+            {KeyValue(RowKind::Insert(), /*sequence_number=*/0, /*level=*/3, key,
+                      BinaryRowGenerator::GenerateRowPtr({1001}, pool.get())),
+             "data.file", /*row_position=*/10});
+    };
+    auto dv_index_file =
+        std::make_shared<DeletionVectorsIndexFile>(/*fs=*/nullptr, /*path_factory=*/nullptr,
+                                                   /*bitmap64=*/false, pool);
+    std::map<std::string, std::shared_ptr<DeletionVector>> deletion_vectors;
+    auto dv_maintainer = std::make_shared<BucketedDvMaintainer>(dv_index_file, deletion_vectors);
+
+    LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true,
+                                   /*deletion_vector=*/true, /*force_lookup=*/false);
+    ASSERT_OK_AND_ASSIGN(auto wrapper,
+                         LookupChangelogMergeFunctionWrapper<PositionedKeyValue>::Create(
+                             std::move(lookup_mfunc), lookup, lookup_strategy,
+                             /*should_produce_changelog=*/true, dv_maintainer,
+                             /*comparator=*/nullptr, CreateValueSerializer(pool),
+                             /*value_equalizer=*/{}));
 
     wrapper->Reset();
     ASSERT_OK(wrapper->Add(std::move(kv1)));
@@ -163,8 +331,16 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookupWithDv) {
     ASSERT_OK(wrapper->Add(std::move(kv3)));
     ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult());
     ASSERT_TRUE(result);
-    ASSERT_EQ(result.value().sequence_number, 3);
-    ASSERT_EQ(result.value().value->GetInt(0), 100 + 200 + 300 + 1001);
+    ASSERT_TRUE(result->result);
+    ASSERT_EQ(result->result->sequence_number, 3);
+    ASSERT_EQ(result->result->value->GetInt(0), 100 + 200 + 300 + 1001);
+    ASSERT_EQ(result->changelogs.size(), 2);
+    ASSERT_EQ(result->changelogs[0].value_kind, RowKind::UpdateBefore());
+    ASSERT_EQ(result->changelogs[0].sequence_number, 0);
+    ASSERT_EQ(result->changelogs[0].value->GetInt(0), 1001);
+    ASSERT_EQ(result->changelogs[1].value_kind, RowKind::UpdateAfter());
+    ASSERT_EQ(result->changelogs[1].sequence_number, 3);
+    ASSERT_EQ(result->changelogs[1].value->GetInt(0), 100 + 200 + 300 + 1001);
 
     auto dv = dv_maintainer->DeletionVectorOf("data.file");
     ASSERT_TRUE(dv);
diff --git a/src/paimon/core/mergetree/compact/lookup_merge_function.h b/src/paimon/core/mergetree/compact/lookup_merge_function.h
index e50d85eb2..7d8ee1a02 100644
--- a/src/paimon/core/mergetree/compact/lookup_merge_function.h
+++ b/src/paimon/core/mergetree/compact/lookup_merge_function.h
@@ -100,6 +100,11 @@ class LookupMergeFunction : public MergeFunction {
         return high_level_idx;
     }
 
+    const KeyValue* PickHighLevel() const {
+        std::optional<int32_t> high_level_idx = PickHighLevelIdx();
+        return high_level_idx ? &candidates_[high_level_idx.value()] : nullptr;
+    }
+
  private:
     std::unique_ptr<MergeFunction> merge_function_;
     std::vector<KeyValue> candidates_;
diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp
index 071456e48..1a818dd01 100644
--- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp
+++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp
@@ -23,8 +23,10 @@
 #include "paimon/common/table/special_fields.h"
 #include "paimon/core/mergetree/compact/first_row_merge_function_wrapper.h"
 #include "paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h"
+#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/mergetree/lookup/file_position.h"
 #include "paimon/core/mergetree/lookup/positioned_key_value.h"
+#include "paimon/core/utils/primary_key_table_utils.h"
 
 namespace paimon {
 template <typename T>
@@ -38,7 +40,8 @@ LookupMergeTreeCompactRewriter<T>::LookupMergeTreeCompactRewriter(
     const std::shared_ptr<FileStorePathFactoryCache>& path_factory_cache,
     std::unique_ptr<MergeFileSplitRead>&& merge_file_split_read,
     MergeFunctionWrapperFactory merge_function_wrapper_factory,
-    const std::shared_ptr<CancellationController>& cancellation_controller,
+    ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory,
+    bool produce_changelog, const std::shared_ptr<CancellationController>& cancellation_controller,
     const std::shared_ptr<RemoteLookupFileManager>& remote_lookup_file_manager,
     const std::shared_ptr<MemoryPool>& pool)
     : ChangelogMergeTreeRewriter(
@@ -46,6 +49,7 @@ LookupMergeTreeCompactRewriter<T>::LookupMergeTreeCompactRewriter(
           trimmed_primary_keys, options, data_schema, write_schema,
           DeletionVector::CreateFactory(dv_maintainer), path_factory_cache,
           std::move(merge_file_split_read), std::move(merge_function_wrapper_factory),
+          std::move(changelog_merge_function_wrapper_factory), produce_changelog,
           cancellation_controller, pool),
       lookup_levels_(std::move(lookup_levels)),
       dv_maintainer_(dv_maintainer),
@@ -56,10 +60,10 @@ Result<std::unique_ptr<LookupMergeTreeCompactRewriter<T>>>
 LookupMergeTreeCompactRewriter<T>::Create(
     int32_t max_level, std::unique_ptr<LookupLevels<T>>&& lookup_levels,
     const std::shared_ptr<BucketedDvMaintainer>& dv_maintainer,
-    MergeFunctionWrapperFactory merge_function_wrapper_factory, int32_t bucket,
+    ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory, int32_t bucket,
     const BinaryRow& partition, const std::shared_ptr<TableSchema>& table_schema,
     const std::shared_ptr<FileStorePathFactoryCache>& path_factory_cache,
-    const CoreOptions& options,
+    const CoreOptions& options, bool produce_changelog,
     const std::shared_ptr<CancellationController>& cancellation_controller,
     const std::shared_ptr<RemoteLookupFileManager>& remote_lookup_file_manager,
     const std::shared_ptr<MemoryPool>& pool) {
@@ -88,44 +92,60 @@ LookupMergeTreeCompactRewriter<T>::Create(
         std::unique_ptr<MergeFileSplitRead> merge_file_split_read,
         MergeFileSplitRead::Create(path_factory, internal_context, pool, CreateDefaultExecutor()));
 
+    MergeFunctionWrapperFactory merge_function_wrapper_factory =
+        [data_schema, options, trimmed_primary_keys, pool](
+            int32_t /*output_level*/) -> Result<std::shared_ptr<MergeFunctionWrapper<KeyValue>>> {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<MergeFunction> merge_function,
+                               PrimaryKeyTableUtils::CreateMergeFunction(
+                                   data_schema, trimmed_primary_keys, options, pool));
+        if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) {
+            merge_function = std::make_unique<LookupMergeFunction>(std::move(merge_function));
+        }
+        return std::make_shared<ReducerMergeFunctionWrapper>(std::move(merge_function));
+    };
+
     return std::unique_ptr<LookupMergeTreeCompactRewriter>(new LookupMergeTreeCompactRewriter(
         std::move(lookup_levels), dv_maintainer, max_level, partition, bucket, table_schema->Id(),
         trimmed_primary_keys, options, data_schema, write_schema, path_factory_cache,
         std::move(merge_file_split_read), std::move(merge_function_wrapper_factory),
+        std::move(changelog_merge_function_wrapper_factory), produce_changelog,
         cancellation_controller, remote_lookup_file_manager, pool));
 }
 
 template <typename T>
-std::shared_ptr<MergeFunctionWrapper<KeyValue>>
+std::shared_ptr<MergeFunctionWrapper<ChangelogResult>>
 LookupMergeTreeCompactRewriter<T>::CreateFirstRowMergeFunctionWrapper(
     std::unique_ptr<FirstRowMergeFunction>&& merge_func, int32_t output_level,
-    LookupLevels<bool>* lookup_levels) {
+    std::unique_ptr<RowCompactedSerializer>&& value_serializer, LookupLevels<bool>* lookup_levels) {
     auto contains = [output_level,
                      lookup_levels](const std::shared_ptr<InternalRow>& key) -> Result<bool> {
         PAIMON_ASSIGN_OR_RAISE(std::optional<bool> contain,
                                lookup_levels->Lookup(key, output_level + 1));
         return contain != std::nullopt;
     };
-    return std::make_shared<FirstRowMergeFunctionWrapper>(std::move(merge_func),
-                                                          std::move(contains));
+    return std::make_shared<FirstRowMergeFunctionWrapper>(
+        std::move(merge_func), std::move(contains), std::move(value_serializer));
 }
 
 template <typename T>
-Result<std::shared_ptr<MergeFunctionWrapper<KeyValue>>>
+Result<std::shared_ptr<MergeFunctionWrapper<ChangelogResult>>>
 LookupMergeTreeCompactRewriter<T>::CreateLookupMergeFunctionWrapper(
     std::unique_ptr<LookupMergeFunction>&& merge_func, int32_t output_level,
     const std::shared_ptr<BucketedDvMaintainer>& deletion_vectors_maintainer,
-    const LookupStrategy& lookup_strategy,
+    const LookupStrategy& lookup_strategy, bool should_produce_changelog,
     const std::shared_ptr<FieldsComparator>& user_defined_seq_comparator,
-    LookupLevels<T>* lookup_levels) {
+    std::unique_ptr<RowCompactedSerializer>&& value_serializer,
+    FieldsComparator::FieldComparatorFunc value_equalizer, LookupLevels<T>* lookup_levels) {
     auto lookup = [output_level, lookup_levels](
                       const std::shared_ptr<InternalRow>& key) -> Result<std::optional<T>> {
         return lookup_levels->Lookup(key, output_level + 1);
     };
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<MergeFunctionWrapper<KeyValue>> wrapper,
-                           LookupChangelogMergeFunctionWrapper<T>::Create(
-                               std::move(merge_func), std::move(lookup), lookup_strategy,
-                               deletion_vectors_maintainer, user_defined_seq_comparator));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<MergeFunctionWrapper<ChangelogResult>> wrapper,
+        LookupChangelogMergeFunctionWrapper<T>::Create(
+            std::move(merge_func), std::move(lookup), lookup_strategy, should_produce_changelog,
+            deletion_vectors_maintainer, user_defined_seq_comparator, std::move(value_serializer),
+            std::move(value_equalizer)));
     return wrapper;
 }
 
diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h
index 06c5f47f4..a461177ba 100644
--- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h
+++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h
@@ -19,10 +19,12 @@
 #pragma once
 
 #include "arrow/api.h"
+#include "paimon/common/data/serializer/row_compacted_serializer.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h"
 #include "paimon/core/mergetree/compact/first_row_merge_function.h"
+#include "paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h"
 #include "paimon/core/mergetree/compact/lookup_merge_function.h"
 #include "paimon/core/mergetree/lookup/remote_lookup_file_manager.h"
 #include "paimon/core/mergetree/lookup_levels.h"
@@ -38,10 +40,11 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter {
     static Result<std::unique_ptr<LookupMergeTreeCompactRewriter>> Create(
         int32_t max_level, std::unique_ptr<LookupLevels<T>>&& lookup_levels,
         const std::shared_ptr<BucketedDvMaintainer>& dv_maintainer,
-        MergeFunctionWrapperFactory merge_function_wrapper_factory, int32_t bucket,
-        const BinaryRow& partition, const std::shared_ptr<TableSchema>& table_schema,
+        ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory,
+        int32_t bucket, const BinaryRow& partition,
+        const std::shared_ptr<TableSchema>& table_schema,
         const std::shared_ptr<FileStorePathFactoryCache>& path_factory_cache,
-        const CoreOptions& options,
+        const CoreOptions& options, bool produce_changelog,
         const std::shared_ptr<CancellationController>& cancellation_controller,
         const std::shared_ptr<RemoteLookupFileManager>& remote_lookup_file_manager,
         const std::shared_ptr<MemoryPool>& pool);
@@ -50,16 +53,20 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter {
         return lookup_levels_->Close();
     }
 
-    static std::shared_ptr<MergeFunctionWrapper<KeyValue>> CreateFirstRowMergeFunctionWrapper(
-        std::unique_ptr<FirstRowMergeFunction>&& merge_func, int32_t output_level,
-        LookupLevels<bool>* lookup_levels);
+    static std::shared_ptr<MergeFunctionWrapper<ChangelogResult>>
+    CreateFirstRowMergeFunctionWrapper(std::unique_ptr<FirstRowMergeFunction>&& merge_func,
+                                       int32_t output_level,
+                                       std::unique_ptr<RowCompactedSerializer>&& value_serializer,
+                                       LookupLevels<bool>* lookup_levels);
 
-    static Result<std::shared_ptr<MergeFunctionWrapper<KeyValue>>> CreateLookupMergeFunctionWrapper(
+    static Result<std::shared_ptr<MergeFunctionWrapper<ChangelogResult>>>
+    CreateLookupMergeFunctionWrapper(
         std::unique_ptr<LookupMergeFunction>&& merge_func, int32_t output_level,
         const std::shared_ptr<BucketedDvMaintainer>& deletion_vectors_maintainer,
-        const LookupStrategy& lookup_strategy,
+        const LookupStrategy& lookup_strategy, bool should_produce_changelog,
         const std::shared_ptr<FieldsComparator>& user_defined_seq_comparator,
-        LookupLevels<T>* lookup_levels);
+        std::unique_ptr<RowCompactedSerializer>&& value_serializer,
+        FieldsComparator::FieldComparatorFunc value_equalizer, LookupLevels<T>* lookup_levels);
 
  private:
     LookupMergeTreeCompactRewriter(
@@ -72,6 +79,8 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter {
         const std::shared_ptr<FileStorePathFactoryCache>& path_factory_cache,
         std::unique_ptr<MergeFileSplitRead>&& merge_file_split_read,
         MergeFunctionWrapperFactory merge_function_wrapper_factory,
+        ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory,
+        bool produce_changelog,
         const std::shared_ptr<CancellationController>& cancellation_controller,
         const std::shared_ptr<RemoteLookupFileManager>& remote_lookup_file_manager,
         const std::shared_ptr<MemoryPool>& pool);
diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
index 169170da1..c930c5ca7 100644
--- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
+++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
@@ -24,6 +24,7 @@
 #include "arrow/ipc/json_simple.h"
 #include "gtest/gtest.h"
 #include "paimon/catalog/catalog.h"
+#include "paimon/common/data/shredding/map_shared_shredding_utils.h"
 #include "paimon/common/factories/io_hook.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/utils/fields_comparator.h"
@@ -35,6 +36,7 @@
 #include "paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h"
 #include "paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h"
 #include "paimon/core/mergetree/compact/deduplicate_merge_function.h"
+#include "paimon/core/mergetree/compact/internal_row_equalizer.h"
 #include "paimon/core/mergetree/compact/interval_partition.h"
 #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
 #include "paimon/core/mergetree/lookup/default_lookup_serializer_factory.h"
@@ -148,16 +150,25 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam<std::
 
     Result<std::unique_ptr<LookupMergeTreeCompactRewriter<bool>>> CreateCompactRewriterForFirstRow(
         const std::string& table_path, const std::shared_ptr<TableSchema>& table_schema,
-        const CoreOptions& options, std::unique_ptr<LookupLevels<bool>>&& lookup_levels) const {
+        const CoreOptions& options, std::unique_ptr<LookupLevels<bool>>&& lookup_levels,
+        std::optional<bool> produce_changelog = std::nullopt) const {
         auto path_factory_cache =
             std::make_shared<FileStorePathFactoryCache>(table_path, table_schema, options, pool_);
+        bool should_produce_changelog =
+            produce_changelog.value_or(options.GetLookupStrategy().produce_changelog);
         auto merge_function_wrapper_factory =
-            [lookup_levels_ptr = lookup_levels.get()](
-                int32_t output_level) -> Result<std::shared_ptr<MergeFunctionWrapper<KeyValue>>> {
-            std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper =
+            [lookup_levels_ptr = lookup_levels.get(), data_schema = arrow_schema_,
+             should_produce_changelog, pool = pool_](int32_t output_level)
+            -> Result<std::shared_ptr<MergeFunctionWrapper<ChangelogResult>>> {
+            std::unique_ptr<RowCompactedSerializer> value_serializer;
+            if (should_produce_changelog) {
+                PAIMON_ASSIGN_OR_RAISE(value_serializer,
+                                       RowCompactedSerializer::Create(data_schema, pool));
+            }
+            std::shared_ptr<MergeFunctionWrapper<ChangelogResult>> merge_function_wrapper =
                 LookupMergeTreeCompactRewriter<bool>::CreateFirstRowMergeFunctionWrapper(
                     std::make_unique<FirstRowMergeFunction>(/*ignore_delete=*/true), output_level,
-                    lookup_levels_ptr);
+                    std::move(value_serializer), lookup_levels_ptr);
             return merge_function_wrapper;
         };
         auto cancellation_controller = std::make_shared<CancellationController>();
@@ -166,7 +177,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam<std::
             std::move(merge_function_wrapper_factory),
             /*bucket=*/0,
             /*partition=*/BinaryRow::EmptyRow(), table_schema, path_factory_cache, options,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
+            should_produce_changelog, cancellation_controller,
+            /*remote_lookup_file_manager=*/nullptr, pool_);
     }
 
     Result<std::unique_ptr<LookupMergeTreeCompactRewriter<KeyValue>>>
@@ -177,18 +189,29 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam<std::
             std::make_shared<FileStorePathFactoryCache>(table_path, table_schema, options, pool_);
         auto merge_function_wrapper_factory =
             [this, table_schema, options, lookup_levels_ptr = lookup_levels.get(),
-             lookup_strategy = options.GetLookupStrategy()](
-                int32_t output_level) -> Result<std::shared_ptr<MergeFunctionWrapper<KeyValue>>> {
-            PAIMON_ASSIGN_OR_RAISE(auto merge_func,
-                                   AggregateMergeFunction::Create(
-                                       arrow_schema_, table_schema->TrimmedPrimaryKeys().value(),
-                                       options, GetDefaultPool()));
+             lookup_strategy = options.GetLookupStrategy()](int32_t output_level)
+            -> Result<std::shared_ptr<MergeFunctionWrapper<ChangelogResult>>> {
             PAIMON_ASSIGN_OR_RAISE(
-                std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper,
+                auto merge_func,
+                AggregateMergeFunction::Create(
+                    arrow_schema_, table_schema->TrimmedPrimaryKeys().value(), options, pool_));
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RowCompactedSerializer> value_serializer,
+                                   RowCompactedSerializer::Create(arrow_schema_, pool_));
+            FieldsComparator::FieldComparatorFunc value_equalizer;
+            if (options.ChangelogRowDeduplicate()) {
+                PAIMON_ASSIGN_OR_RAISE(
+                    value_equalizer,
+                    InternalRowEqualizer::Create(arrow_schema_,
+                                                 options.GetChangelogRowDeduplicateIgnoreFields()));
+            }
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr<MergeFunctionWrapper<ChangelogResult>> merge_function_wrapper,
                 LookupMergeTreeCompactRewriter<KeyValue>::CreateLookupMergeFunctionWrapper(
                     std::make_unique<LookupMergeFunction>(std::move(merge_func)), output_level,
                     /*deletion_vectors_maintainer=*/nullptr, lookup_strategy,
-                    /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr));
+                    /*should_produce_changelog=*/lookup_strategy.produce_changelog,
+                    /*user_defined_seq_comparator=*/nullptr, std::move(value_serializer),
+                    std::move(value_equalizer), lookup_levels_ptr));
             return merge_function_wrapper;
         };
         auto cancellation_controller = std::make_shared<CancellationController>();
@@ -196,7 +219,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam<std::
             /*max_level=*/5, std::move(lookup_levels), /*dv_maintainer=*/nullptr,
             std::move(merge_function_wrapper_factory), /*bucket=*/0,
             /*partition=*/BinaryRow::EmptyRow(), table_schema, path_factory_cache, options,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
+            options.GetLookupStrategy().produce_changelog, cancellation_controller,
+            /*remote_lookup_file_manager=*/nullptr, pool_);
     }
 
     Result<std::unique_ptr<LookupMergeTreeCompactRewriter<FilePosition>>>
@@ -213,17 +237,22 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam<std::
         auto dv_maintainer =
             std::make_shared<BucketedDvMaintainer>(dv_index_file, deletion_vectors);
 
-        auto merge_function_wrapper_factory =
-            [lookup_levels_ptr = lookup_levels.get(), lookup_strategy = options.GetLookupStrategy(),
-             dv_maintainer_ptr = dv_maintainer](
-                int32_t output_level) -> Result<std::shared_ptr<MergeFunctionWrapper<KeyValue>>> {
+        auto merge_function_wrapper_factory = [this, lookup_levels_ptr = lookup_levels.get(),
+                                               lookup_strategy = options.GetLookupStrategy(),
+                                               dv_maintainer_ptr =
+                                                   dv_maintainer](int32_t output_level)
+            -> Result<std::shared_ptr<MergeFunctionWrapper<ChangelogResult>>> {
             auto merge_func = std::make_unique<DeduplicateMergeFunction>(false);
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RowCompactedSerializer> value_serializer,
+                                   RowCompactedSerializer::Create(arrow_schema_, pool_));
             PAIMON_ASSIGN_OR_RAISE(
-                std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper,
+                std::shared_ptr<MergeFunctionWrapper<ChangelogResult>> merge_function_wrapper,
                 LookupMergeTreeCompactRewriter<FilePosition>::CreateLookupMergeFunctionWrapper(
                     std::make_unique<LookupMergeFunction>(std::move(merge_func)), output_level,
                     dv_maintainer_ptr, lookup_strategy,
-                    /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr));
+                    /*should_produce_changelog=*/lookup_strategy.produce_changelog,
+                    /*user_defined_seq_comparator=*/nullptr, std::move(value_serializer),
+                    /*value_equalizer=*/{}, lookup_levels_ptr));
             return merge_function_wrapper;
         };
         auto cancellation_controller = std::make_shared<CancellationController>();
@@ -231,7 +260,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam<std::
             /*max_level=*/5, std::move(lookup_levels), dv_maintainer,
             std::move(merge_function_wrapper_factory), /*bucket=*/0,
             /*partition=*/BinaryRow::EmptyRow(), table_schema, path_factory_cache, options,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
+            options.GetLookupStrategy().produce_changelog, cancellation_controller,
+            /*remote_lookup_file_manager=*/nullptr, pool_);
     }
 
     Result<std::unique_ptr<LookupMergeTreeCompactRewriter<PositionedKeyValue>>>
@@ -253,19 +283,31 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam<std::
 
         auto merge_function_wrapper_factory =
             [this, table_schema, options, lookup_levels_ptr = lookup_levels.get(),
-             lookup_strategy = options.GetLookupStrategy(), dv_maintainer_ptr = dv_maintainer](
-                int32_t output_level) -> Result<std::shared_ptr<MergeFunctionWrapper<KeyValue>>> {
-            PAIMON_ASSIGN_OR_RAISE(auto merge_func,
-                                   AggregateMergeFunction::Create(
-                                       arrow_schema_, table_schema->TrimmedPrimaryKeys().value(),
-                                       options, GetDefaultPool()));
+             lookup_strategy = options.GetLookupStrategy(),
+             dv_maintainer_ptr = dv_maintainer](int32_t output_level)
+            -> Result<std::shared_ptr<MergeFunctionWrapper<ChangelogResult>>> {
             PAIMON_ASSIGN_OR_RAISE(
-                std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper,
+                auto merge_func,
+                AggregateMergeFunction::Create(
+                    arrow_schema_, table_schema->TrimmedPrimaryKeys().value(), options, pool_));
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RowCompactedSerializer> value_serializer,
+                                   RowCompactedSerializer::Create(arrow_schema_, pool_));
+            FieldsComparator::FieldComparatorFunc value_equalizer;
+            if (options.ChangelogRowDeduplicate()) {
+                PAIMON_ASSIGN_OR_RAISE(
+                    value_equalizer,
+                    InternalRowEqualizer::Create(arrow_schema_,
+                                                 options.GetChangelogRowDeduplicateIgnoreFields()));
+            }
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr<MergeFunctionWrapper<ChangelogResult>> merge_function_wrapper,
                 LookupMergeTreeCompactRewriter<PositionedKeyValue>::
                     CreateLookupMergeFunctionWrapper(
                         std::make_unique<LookupMergeFunction>(std::move(merge_func)), output_level,
                         dv_maintainer_ptr, lookup_strategy,
-                        /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr));
+                        /*should_produce_changelog=*/lookup_strategy.produce_changelog,
+                        /*user_defined_seq_comparator=*/nullptr, std::move(value_serializer),
+                        std::move(value_equalizer), lookup_levels_ptr));
             return merge_function_wrapper;
         };
         auto cancellation_controller = std::make_shared<CancellationController>();
@@ -273,7 +315,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam<std::
             /*max_level=*/5, std::move(lookup_levels), dv_maintainer,
             std::move(merge_function_wrapper_factory), /*bucket=*/0,
             /*partition=*/BinaryRow::EmptyRow(), table_schema, path_factory_cache, options,
-            cancellation_controller, remote_lookup_file_manager, pool_);
+            options.GetLookupStrategy().produce_changelog, cancellation_controller,
+            remote_lookup_file_manager, pool_);
     }
 
     void CheckResult(const std::string& compact_file_name,
@@ -298,6 +341,34 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam<std::
         ASSERT_TRUE(expected_array->Equals(*result_array)) << result_array->ToString();
     }
 
+    void CheckShreddingFileSchema(const std::string& file_name,
+                                  const std::shared_ptr<TableSchema>& table_schema,
+                                  const std::string& file_format_name,
+                                  const std::shared_ptr<arrow::Schema>& expected_physical_schema,
+                                  int32_t field_index,
+                                  const MapSharedShreddingFieldMeta& expected_meta) const {
+        ASSERT_OK_AND_ASSIGN(auto file_format,
+                             FileFormatFactory::Get(file_format_name, table_schema->Options()));
+        ASSERT_OK_AND_ASSIGN(auto reader_builder,
+                             file_format->CreateReaderBuilder(/*batch_size=*/10));
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input_stream, fs_->Open(file_name));
+        ASSERT_OK_AND_ASSIGN(auto file_batch_reader, reader_builder->Build(input_stream));
+        ASSERT_OK_AND_ASSIGN(auto c_file_schema, file_batch_reader->GetFileSchema());
+        std::shared_ptr<arrow::Schema> file_schema =
+            arrow::ImportSchema(c_file_schema.get()).ValueOrDie();
+
+        ASSERT_TRUE(file_schema->Equals(*expected_physical_schema, /*check_metadata=*/false))
+            << "Expected schema:\n"
+            << expected_physical_schema->ToString() << "\nActual schema:\n"
+            << file_schema->ToString();
+        std::shared_ptr<const arrow::KeyValueMetadata> metadata =
+            file_schema->field(field_index)->metadata();
+        ASSERT_NE(nullptr, metadata);
+        ASSERT_OK_AND_ASSIGN(MapSharedShreddingFieldMeta actual_meta,
+                             MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy()));
+        ASSERT_EQ(expected_meta, actual_meta);
+    }
+
     Result<std::shared_ptr<FileStorePathFactory>> CreateFileStorePathFactory(
         const std::string& table_path, const CoreOptions& options) const {
         PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> external_paths,
@@ -422,7 +493,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam<std::
 
 TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowRewrite) {
     std::map<std::string, std::string> options = {{Options::MERGE_ENGINE, "first-row"},
-                                                  {Options::FILE_FORMAT, "orc"}};
+                                                  {Options::FILE_FORMAT, "orc"},
+                                                  {Options::CHANGELOG_PRODUCER, "lookup"}};
     ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options));
     ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options));
     auto schema_manager = std::make_shared<SchemaManager>(fs_, table_path);
@@ -447,6 +519,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowRewrite) {
                                                   /*output_level=*/5, /*drop_delete=*/true, runs));
     ASSERT_EQ(2, compact_result.Before().size());
     ASSERT_EQ(1, compact_result.After().size());
+    ASSERT_EQ(1, compact_result.Changelog().size());
 
     // check compact result
     const auto& compact_file_meta = compact_result.After()[0];
@@ -483,11 +556,15 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowRewrite) {
                                                          &expected_array);
     ASSERT_TRUE(array_status.ok());
     CheckResult(compact_file_name, table_schema, "orc", expected_array);
+    std::string changelog_file_name =
+        table_path + "/bucket-0/" + compact_result.Changelog()[0]->file_name;
+    CheckResult(changelog_file_name, table_schema, "orc", expected_array);
 }
 
 TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowLooksUpExistingKeys) {
     std::map<std::string, std::string> options = {{Options::MERGE_ENGINE, "first-row"},
-                                                  {Options::FILE_FORMAT, "orc"}};
+                                                  {Options::FILE_FORMAT, "orc"},
+                                                  {Options::CHANGELOG_PRODUCER, "lookup"}};
     ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options));
     ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options));
     auto schema_manager = std::make_shared<SchemaManager>(fs_, table_path);
@@ -513,6 +590,8 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowLooksUpExistingKeys) {
 
     ASSERT_EQ(1, compact_result.After().size());
     ASSERT_EQ(1, compact_result.After()[0]->row_count);
+    ASSERT_EQ(1, compact_result.Changelog().size());
+    ASSERT_EQ(1, compact_result.Changelog()[0]->row_count);
 
     auto type_with_special_fields =
         arrow::struct_(SpecialFields::CompleteSequenceAndValueKindField(arrow_schema_)->fields());
@@ -522,6 +601,103 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowLooksUpExistingKeys) {
                     .ok());
     CheckResult(table_path + "/bucket-0/" + compact_result.After()[0]->file_name, table_schema,
                 "orc", expected);
+    CheckResult(table_path + "/bucket-0/" + compact_result.Changelog()[0]->file_name, table_schema,
+                "orc", expected);
+}
+
+TEST_F(LookupMergeTreeCompactRewriterTest, TestLookupChangelogCanBeDisabled) {
+    std::map<std::string, std::string> options = {{Options::MERGE_ENGINE, "first-row"},
+                                                  {Options::FILE_FORMAT, "orc"},
+                                                  {Options::CHANGELOG_PRODUCER, "lookup"}};
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options));
+    ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options));
+    auto schema_manager = std::make_shared<SchemaManager>(fs_, table_path);
+    ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager->ReadSchema(0));
+    ASSERT_OK_AND_ASSIGN(auto file, NewFiles(/*level=*/0, /*last_sequence_number=*/-1, table_path,
+                                             core_options, "[[1, 11], [2, 22]]"));
+    auto processor_factory = std::make_shared<PersistEmptyProcessor::Factory>();
+    ASSERT_OK_AND_ASSIGN(auto lookup_levels, CreateLookupLevels<bool>(table_path, table_schema,
+                                                                      processor_factory, {file}));
+    ASSERT_OK_AND_ASSIGN(auto rewriter,
+                         CreateCompactRewriterForFirstRow(table_path, table_schema, core_options,
+                                                          std::move(lookup_levels),
+                                                          /*produce_changelog=*/false));
+    ASSERT_OK_AND_ASSIGN(auto runs, GenerateSortedRuns({file}));
+    ASSERT_OK_AND_ASSIGN(auto compact_result, rewriter->Rewrite(
+                                                  /*output_level=*/1, /*drop_delete=*/true, runs));
+
+    ASSERT_EQ(1, compact_result.After().size());
+    ASSERT_TRUE(compact_result.Changelog().empty());
+}
+
+TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowRewriteWithSharedShreddingChangelog) {
+    arrow::FieldVector fields = {
+        arrow::field("key", arrow::int32()),
+        arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
+    };
+    arrow_schema_ = arrow::schema(fields);
+    key_schema_ = arrow::schema({fields[0]});
+    std::map<std::string, std::string> options = {
+        {Options::MERGE_ENGINE, "first-row"},
+        {Options::FILE_FORMAT, "orc"},
+        {Options::CHANGELOG_PRODUCER, "lookup"},
+        {Options::CHANGELOG_FILE_FORMAT, "parquet"},
+        {"fields.tags.map.storage-layout", "shared-shredding"},
+        {"fields.tags.map.shared-shredding.max-columns", "3"},
+        {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
+    };
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options));
+    ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options));
+    auto schema_manager = std::make_shared<SchemaManager>(fs_, table_path);
+    ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager->ReadSchema(0));
+
+    ASSERT_OK_AND_ASSIGN(
+        auto file0, NewFiles(/*level=*/0, /*last_sequence_number=*/-1, table_path, core_options,
+                             R"([[1, [["a", 11]]], [3, [["c", 33]]], [5, [["e", 55]]]])"));
+    ASSERT_OK_AND_ASSIGN(auto file1,
+                         NewFiles(/*level=*/0, /*last_sequence_number=*/2, table_path, core_options,
+                                  R"([[2, [["b", 22]]], [5, [["f", 555]]]])"));
+    std::vector<std::shared_ptr<DataFileMeta>> files = {file0, file1};
+    auto processor_factory = std::make_shared<PersistEmptyProcessor::Factory>();
+    ASSERT_OK_AND_ASSIGN(auto lookup_levels, CreateLookupLevels<bool>(table_path, table_schema,
+                                                                      processor_factory, files));
+    ASSERT_OK_AND_ASSIGN(auto rewriter,
+                         CreateCompactRewriterForFirstRow(table_path, table_schema, core_options,
+                                                          std::move(lookup_levels)));
+    ASSERT_OK_AND_ASSIGN(auto runs, GenerateSortedRuns(files));
+    ASSERT_OK_AND_ASSIGN(auto compact_result, rewriter->Rewrite(
+                                                  /*output_level=*/5, /*drop_delete=*/true, runs));
+
+    ASSERT_EQ(1, compact_result.Changelog().size());
+    const std::shared_ptr<DataFileMeta>& changelog_file = compact_result.Changelog()[0];
+    ASSERT_EQ(4, changelog_file->row_count);
+    ASSERT_EQ(FileSource::Append(), changelog_file->file_source);
+    ASSERT_TRUE(StringUtils::EndsWith(changelog_file->file_name, ".parquet"));
+    std::string changelog_file_name = table_path + "/bucket-0/" + changelog_file->file_name;
+
+    std::shared_ptr<arrow::Schema> write_schema = SpecialFields::CompleteSequenceAndValueKindField(
+        DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()));
+    ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                                                   write_schema, {{"tags", 3}}));
+    std::shared_ptr<arrow::ChunkedArray> expected_changelog;
+    ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(
+                    arrow::struct_(physical_schema->fields()), {R"([
+      [0, 0, 1, [[0, -1, -1], 11, null, null, null]],
+      [3, 0, 2, [[1, -1, -1], 22, null, null, null]],
+      [1, 0, 3, [[2, -1, -1], 33, null, null, null]],
+      [2, 0, 5, [[3, -1, -1], 55, null, null, null]]
+    ])"},
+                    &expected_changelog)
+                    .ok());
+    CheckResult(changelog_file_name, table_schema, "parquet", expected_changelog);
+
+    MapSharedShreddingFieldMeta expected_meta;
+    expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"e", 3}};
+    expected_meta.field_to_columns = {{0, {0}}, {1, {0}}, {2, {0}}, {3, {0}}};
+    expected_meta.num_columns = 3;
+    expected_meta.max_row_width = 1;
+    CheckShreddingFileSchema(changelog_file_name, table_schema, "parquet", physical_schema,
+                             /*field_index=*/3, expected_meta);
 }
 
 TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowUpgrade) {
@@ -680,7 +856,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithAllHighLevel) {
 TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithForceLookupAndSumAgg) {
     std::map<std::string, std::string> options = {{Options::MERGE_ENGINE, "aggregation"},
                                                   {Options::FILE_FORMAT, "orc"},
-                                                  {Options::FORCE_LOOKUP, "true"},
+                                                  {Options::CHANGELOG_PRODUCER, "lookup"},
                                                   {Options::FIELDS_DEFAULT_AGG_FUNC, "sum"}};
     ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options));
     ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options));
@@ -709,6 +885,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithForceLookupAndSumAgg)
                                                   /*output_level=*/4, /*drop_delete=*/true, runs));
     ASSERT_EQ(2, compact_result.Before().size());
     ASSERT_EQ(1, compact_result.After().size());
+    ASSERT_EQ(1, compact_result.Changelog().size());
 
     const auto& compact_file_meta = compact_result.After()[0];
     // check compact file exist
@@ -729,6 +906,20 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithForceLookupAndSumAgg)
                                                          &expected_array);
     ASSERT_TRUE(array_status.ok());
     CheckResult(compact_file_name, table_schema, "orc", expected_array);
+
+    const auto& changelog_file_meta = compact_result.Changelog()[0];
+    std::string changelog_file_name = table_path + "/bucket-0/" + changelog_file_meta->file_name;
+    std::shared_ptr<arrow::ChunkedArray> expected_changelog;
+    auto changelog_status =
+        arrow::ipc::internal::json::ChunkedArrayFromJSON(type_with_special_fields, {R"([
+[6,  0,  2, 244],
+[4,  0,  4, 44],
+[2,  1,  5, 55],
+[7,  2,  5, 615]
+])"},
+                                                         &expected_changelog);
+    ASSERT_TRUE(changelog_status.ok());
+    CheckResult(changelog_file_name, table_schema, "orc", expected_changelog);
 }
 
 TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithDvAndDeduplicate) {
@@ -1122,7 +1313,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) {
             /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr,
             /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr,
             /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
+            /*changelog_merge_function_wrapper_factory=*/nullptr,
+            /*produce_changelog=*/false, cancellation_controller,
+            /*remote_lookup_file_manager=*/nullptr, pool_);
         auto file = create_meta(/*level=*/1, /*delete_row_count=*/std::nullopt);
         ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::NoChangelogNoRewrite(),
                   rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file));
@@ -1137,7 +1330,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) {
             /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr,
             /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr,
             /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
+            /*changelog_merge_function_wrapper_factory=*/nullptr,
+            /*produce_changelog=*/false, cancellation_controller,
+            /*remote_lookup_file_manager=*/nullptr, pool_);
         auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt);
         ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(),
                   rewriter.GenerateUpgradeStrategy(/*output_level=*/5, file));
@@ -1156,7 +1351,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) {
             /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr,
             /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr,
             /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
+            /*changelog_merge_function_wrapper_factory=*/nullptr,
+            /*produce_changelog=*/false, cancellation_controller,
+            /*remote_lookup_file_manager=*/nullptr, pool_);
         auto file = create_meta(/*level=*/0, /*delete_row_count=*/1);
         ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(),
                   rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file));
@@ -1171,7 +1368,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) {
             /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr,
             /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr,
             /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
+            /*changelog_merge_function_wrapper_factory=*/nullptr,
+            /*produce_changelog=*/false, cancellation_controller,
+            /*remote_lookup_file_manager=*/nullptr, pool_);
         auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt);
         ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogNoRewrite(),
                   rewriter.GenerateUpgradeStrategy(/*output_level=*/5, file));
@@ -1186,7 +1385,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) {
             /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr,
             /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr,
             /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
+            /*changelog_merge_function_wrapper_factory=*/nullptr,
+            /*produce_changelog=*/false, cancellation_controller,
+            /*remote_lookup_file_manager=*/nullptr, pool_);
         auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt);
         ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogNoRewrite(),
                   rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file));
@@ -1202,7 +1403,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) {
             /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr,
             /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr,
             /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
+            /*changelog_merge_function_wrapper_factory=*/nullptr,
+            /*produce_changelog=*/false, cancellation_controller,
+            /*remote_lookup_file_manager=*/nullptr, pool_);
         auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt);
         ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(),
                   rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file));
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp
index e5cb3abf6..258336698 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp
@@ -25,6 +25,7 @@
 #include "paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h"
 #include "paimon/core/mergetree/compact/early_full_compaction.h"
 #include "paimon/core/mergetree/compact/force_up_level0_compaction.h"
+#include "paimon/core/mergetree/compact/internal_row_equalizer.h"
 #include "paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h"
 #include "paimon/core/mergetree/compact/merge_tree_compact_manager.h"
 #include "paimon/core/mergetree/compact/merge_tree_compact_rewriter.h"
@@ -50,6 +51,24 @@ namespace paimon {
 
 namespace {
 
+Result<FieldsComparator::FieldComparatorFunc> CreateChangelogValueEqualizer(
+    const std::shared_ptr<arrow::Schema>& schema, const CoreOptions& options,
+    bool produce_changelog) {
+    if (!produce_changelog || !options.ChangelogRowDeduplicate()) {
+        return FieldsComparator::FieldComparatorFunc();
+    }
+    return InternalRowEqualizer::Create(schema, options.GetChangelogRowDeduplicateIgnoreFields());
+}
+
+Result<std::unique_ptr<RowCompactedSerializer>> CreateChangelogValueSerializer(
+    const std::shared_ptr<arrow::Schema>& schema, bool produce_changelog,
+    const std::shared_ptr<MemoryPool>& pool) {
+    if (!produce_changelog) {
+        return std::unique_ptr<RowCompactedSerializer>();
+    }
+    return RowCompactedSerializer::Create(schema, pool);
+}
+
 template <typename T>
 Result<std::unique_ptr<LookupLevels<T>>> CreateLookupLevelsInternal(
     const CoreOptions& options, const std::shared_ptr<SchemaManager>& schema_manager,
@@ -174,6 +193,8 @@ Result<std::shared_ptr<CompactRewriter>> MergeTreeCompactManagerFactory::CreateL
     const LookupStrategy& lookup_strategy,
     const std::shared_ptr<FileStorePathFactoryCache>& path_factory_cache,
     const std::shared_ptr<CancellationController>& cancellation_controller) const {
+    const bool should_produce_changelog =
+        lookup_strategy.produce_changelog && !ignore_previous_files_;
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RemoteLookupFileManager> remote_lookup_file_manager,
                            CreateRemoteLookupFileManager(partition, bucket));
     if (lookup_strategy.is_first_row) {
@@ -190,157 +211,133 @@ Result<std::shared_ptr<CompactRewriter>> MergeTreeCompactManagerFactory::CreateL
                 options_, schema_manager_, io_manager_, cache_manager_, file_store_path_factory_,
                 table_schema_, partition, bucket, levels, processor_factory, dv_maintainer,
                 lookup_file_cache_, remote_lookup_file_manager, pool_));
-        auto merge_function_wrapper_factory =
-            [lookup_levels_ptr = lookup_levels.get(), ignore_delete = options_.IgnoreDelete()](
-                int32_t output_level) -> Result<std::shared_ptr<MergeFunctionWrapper<KeyValue>>> {
-            std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper =
+        auto merge_function_wrapper_factory = [lookup_levels_ptr = lookup_levels.get(),
+                                               data_schema = schema_, should_produce_changelog,
+                                               ignore_delete = options_.IgnoreDelete(),
+                                               pool = pool_](int32_t output_level)
+            -> Result<std::shared_ptr<MergeFunctionWrapper<ChangelogResult>>> {
+            PAIMON_ASSIGN_OR_RAISE(
+                std::unique_ptr<RowCompactedSerializer> value_serializer,
+                CreateChangelogValueSerializer(data_schema, should_produce_changelog, pool));
+            std::shared_ptr<MergeFunctionWrapper<ChangelogResult>> merge_function_wrapper =
                 LookupMergeTreeCompactRewriter<bool>::CreateFirstRowMergeFunctionWrapper(
                     std::make_unique<FirstRowMergeFunction>(ignore_delete), output_level,
-                    lookup_levels_ptr);
+                    std::move(value_serializer), lookup_levels_ptr);
             return merge_function_wrapper;
         };
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<LookupMergeTreeCompactRewriter<bool>> rewriter,
-                               LookupMergeTreeCompactRewriter<bool>::Create(
-                                   max_level, std::move(lookup_levels), dv_maintainer,
-                                   std::move(merge_function_wrapper_factory), bucket, partition,
-                                   table_schema_, path_factory_cache, options_,
-                                   cancellation_controller, remote_lookup_file_manager, pool_));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<LookupMergeTreeCompactRewriter<bool>> rewriter,
+            LookupMergeTreeCompactRewriter<bool>::Create(
+                max_level, std::move(lookup_levels), dv_maintainer,
+                std::move(merge_function_wrapper_factory), bucket, partition, table_schema_,
+                path_factory_cache, options_, should_produce_changelog, cancellation_controller,
+                remote_lookup_file_manager, pool_));
         return std::shared_ptr<CompactRewriter>(std::move(rewriter));
     }
 
     if (lookup_strategy.deletion_vector) {
         return CreateLookupRewriterWithDeletionVector(
             partition, bucket, levels, dv_maintainer, max_level, lookup_strategy,
-            path_factory_cache, cancellation_controller, remote_lookup_file_manager);
+            should_produce_changelog, path_factory_cache, cancellation_controller,
+            remote_lookup_file_manager);
     }
 
     return CreateLookupRewriterWithoutDeletionVector(
-        partition, bucket, levels, dv_maintainer, max_level, lookup_strategy, path_factory_cache,
-        cancellation_controller, remote_lookup_file_manager);
+        partition, bucket, levels, dv_maintainer, max_level, lookup_strategy,
+        should_produce_changelog, path_factory_cache, cancellation_controller,
+        remote_lookup_file_manager);
 }
 
+template <typename T>
 Result<std::shared_ptr<CompactRewriter>>
-MergeTreeCompactManagerFactory::CreateLookupRewriterWithDeletionVector(
+MergeTreeCompactManagerFactory::CreateLookupRewriterInternal(
     const BinaryRow& partition, int32_t bucket, const std::shared_ptr<Levels>& levels,
     const std::shared_ptr<BucketedDvMaintainer>& dv_maintainer, int32_t max_level,
-    const LookupStrategy& lookup_strategy,
+    const LookupStrategy& lookup_strategy, bool should_produce_changelog,
+    const std::shared_ptr<typename PersistProcessor<T>::Factory>& processor_factory,
     const std::shared_ptr<FileStorePathFactoryCache>& path_factory_cache,
     const std::shared_ptr<CancellationController>& cancellation_controller,
     const std::shared_ptr<RemoteLookupFileManager>& remote_lookup_file_manager) const {
-    auto merge_engine = options_.GetMergeEngine();
     PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> trimmed_primary_keys,
                            table_schema_->TrimmedPrimaryKeys());
-    if (lookup_strategy.produce_changelog || merge_engine != MergeEngine::DEDUPLICATE ||
-        !options_.GetSequenceField().empty()) {
-        auto processor_factory = std::make_shared<PersistValueAndPosProcessor::Factory>(schema_);
-        PAIMON_ASSIGN_OR_RAISE(
-            std::unique_ptr<LookupLevels<PositionedKeyValue>> lookup_levels,
-            CreateLookupLevelsInternal<PositionedKeyValue>(
-                options_, schema_manager_, io_manager_, cache_manager_, file_store_path_factory_,
-                table_schema_, partition, bucket, levels, processor_factory, dv_maintainer,
-                lookup_file_cache_, remote_lookup_file_manager, pool_));
-        auto merge_function_wrapper_factory =
-            [data_schema = schema_, options = options_, trimmed_primary_keys,
-             lookup_levels_ptr = lookup_levels.get(), lookup_strategy,
-             dv_maintainer_ptr = dv_maintainer, pool = pool_,
-             user_defined_seq_comparator = user_defined_seq_comparator_](
-                int32_t output_level) -> Result<std::shared_ptr<MergeFunctionWrapper<KeyValue>>> {
-            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<MergeFunction> merge_func,
-                                   PrimaryKeyTableUtils::CreateMergeFunction(
-                                       data_schema, trimmed_primary_keys, options, pool));
-            PAIMON_ASSIGN_OR_RAISE(
-                std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper,
-                LookupMergeTreeCompactRewriter<PositionedKeyValue>::
-                    CreateLookupMergeFunctionWrapper(
-                        std::make_unique<LookupMergeFunction>(std::move(merge_func)), output_level,
-                        dv_maintainer_ptr, lookup_strategy, user_defined_seq_comparator,
-                        lookup_levels_ptr));
-            return merge_function_wrapper;
-        };
-        PAIMON_ASSIGN_OR_RAISE(
-            std::unique_ptr<LookupMergeTreeCompactRewriter<PositionedKeyValue>> rewriter,
-            LookupMergeTreeCompactRewriter<PositionedKeyValue>::Create(
-                max_level, std::move(lookup_levels), dv_maintainer,
-                std::move(merge_function_wrapper_factory), bucket, partition, table_schema_,
-                path_factory_cache, options_, cancellation_controller, remote_lookup_file_manager,
-                pool_));
-        return std::shared_ptr<CompactRewriter>(std::move(rewriter));
-    }
-    auto processor_factory = std::make_shared<PersistPositionProcessor::Factory>();
     PAIMON_ASSIGN_OR_RAISE(
-        std::unique_ptr<LookupLevels<FilePosition>> lookup_levels,
-        CreateLookupLevelsInternal<FilePosition>(
-            options_, schema_manager_, io_manager_, cache_manager_, file_store_path_factory_,
-            table_schema_, partition, bucket, levels, processor_factory, dv_maintainer,
-            lookup_file_cache_, remote_lookup_file_manager, pool_));
+        std::unique_ptr<LookupLevels<T>> lookup_levels,
+        CreateLookupLevelsInternal<T>(options_, schema_manager_, io_manager_, cache_manager_,
+                                      file_store_path_factory_, table_schema_, partition, bucket,
+                                      levels, processor_factory, dv_maintainer, lookup_file_cache_,
+                                      remote_lookup_file_manager, pool_));
     auto merge_function_wrapper_factory =
         [data_schema = schema_, options = options_, trimmed_primary_keys,
-         lookup_levels_ptr = lookup_levels.get(), lookup_strategy,
-         dv_maintainer_ptr = dv_maintainer, pool = pool_,
-         user_defined_seq_comparator = user_defined_seq_comparator_](
-            int32_t output_level) -> Result<std::shared_ptr<MergeFunctionWrapper<KeyValue>>> {
+         lookup_levels_ptr = lookup_levels.get(), lookup_strategy, should_produce_changelog,
+         dv_maintainer_ptr = dv_maintainer,
+         user_defined_seq_comparator = user_defined_seq_comparator_,
+         pool = pool_](int32_t output_level)
+        -> Result<std::shared_ptr<MergeFunctionWrapper<ChangelogResult>>> {
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<MergeFunction> merge_func,
                                PrimaryKeyTableUtils::CreateMergeFunction(
                                    data_schema, trimmed_primary_keys, options, pool));
         PAIMON_ASSIGN_OR_RAISE(
-            std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper,
-            LookupMergeTreeCompactRewriter<FilePosition>::CreateLookupMergeFunctionWrapper(
-                std::make_unique<LookupMergeFunction>(std::move(merge_func)), output_level,
-                dv_maintainer_ptr, lookup_strategy, user_defined_seq_comparator,
-                lookup_levels_ptr));
-        return merge_function_wrapper;
+            std::unique_ptr<RowCompactedSerializer> value_serializer,
+            CreateChangelogValueSerializer(data_schema, should_produce_changelog, pool));
+        PAIMON_ASSIGN_OR_RAISE(
+            FieldsComparator::FieldComparatorFunc value_equalizer,
+            CreateChangelogValueEqualizer(data_schema, options, should_produce_changelog));
+        return LookupMergeTreeCompactRewriter<T>::CreateLookupMergeFunctionWrapper(
+            std::make_unique<LookupMergeFunction>(std::move(merge_func)), output_level,
+            dv_maintainer_ptr, lookup_strategy, should_produce_changelog,
+            user_defined_seq_comparator, std::move(value_serializer), std::move(value_equalizer),
+            lookup_levels_ptr);
     };
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<LookupMergeTreeCompactRewriter<FilePosition>> rewriter,
-                           LookupMergeTreeCompactRewriter<FilePosition>::Create(
-                               max_level, std::move(lookup_levels), dv_maintainer,
-                               std::move(merge_function_wrapper_factory), bucket, partition,
-                               table_schema_, path_factory_cache, options_, cancellation_controller,
-                               remote_lookup_file_manager, pool_));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<LookupMergeTreeCompactRewriter<T>> rewriter,
+        LookupMergeTreeCompactRewriter<T>::Create(
+            max_level, std::move(lookup_levels), dv_maintainer,
+            std::move(merge_function_wrapper_factory), bucket, partition, table_schema_,
+            path_factory_cache, options_, should_produce_changelog, cancellation_controller,
+            remote_lookup_file_manager, pool_));
     return std::shared_ptr<CompactRewriter>(std::move(rewriter));
 }
 
 Result<std::shared_ptr<CompactRewriter>>
-MergeTreeCompactManagerFactory::CreateLookupRewriterWithoutDeletionVector(
+MergeTreeCompactManagerFactory::CreateLookupRewriterWithDeletionVector(
     const BinaryRow& partition, int32_t bucket, const std::shared_ptr<Levels>& levels,
     const std::shared_ptr<BucketedDvMaintainer>& dv_maintainer, int32_t max_level,
-    const LookupStrategy& lookup_strategy,
+    const LookupStrategy& lookup_strategy, bool should_produce_changelog,
     const std::shared_ptr<FileStorePathFactoryCache>& path_factory_cache,
     const std::shared_ptr<CancellationController>& cancellation_controller,
     const std::shared_ptr<RemoteLookupFileManager>& remote_lookup_file_manager) const {
-    PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> trimmed_primary_keys,
-                           table_schema_->TrimmedPrimaryKeys());
-    auto processor_factory = std::make_shared<PersistValueProcessor::Factory>(schema_);
-    PAIMON_ASSIGN_OR_RAISE(
-        std::unique_ptr<LookupLevels<KeyValue>> lookup_levels,
-        CreateLookupLevelsInternal<KeyValue>(
-            options_, schema_manager_, io_manager_, cache_manager_, file_store_path_factory_,
-            table_schema_, partition, bucket, levels, processor_factory, dv_maintainer,
-            lookup_file_cache_, remote_lookup_file_manager, pool_));
-    auto merge_function_wrapper_factory =
-        [data_schema = schema_, options = options_, trimmed_primary_keys,
-         lookup_levels_ptr = lookup_levels.get(), lookup_strategy,
-         dv_maintainer_ptr = dv_maintainer, pool = pool_,
-         user_defined_seq_comparator = user_defined_seq_comparator_](
-            int32_t output_level) -> Result<std::shared_ptr<MergeFunctionWrapper<KeyValue>>> {
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<MergeFunction> merge_func,
-                               PrimaryKeyTableUtils::CreateMergeFunction(
-                                   data_schema, trimmed_primary_keys, options, pool));
-        PAIMON_ASSIGN_OR_RAISE(
-            std::shared_ptr<MergeFunctionWrapper<KeyValue>> merge_function_wrapper,
-            LookupMergeTreeCompactRewriter<KeyValue>::CreateLookupMergeFunctionWrapper(
-                std::make_unique<LookupMergeFunction>(std::move(merge_func)), output_level,
-                dv_maintainer_ptr, lookup_strategy, user_defined_seq_comparator,
-                lookup_levels_ptr));
-        return merge_function_wrapper;
-    };
+    auto merge_engine = options_.GetMergeEngine();
+    if (lookup_strategy.produce_changelog || merge_engine != MergeEngine::DEDUPLICATE ||
+        !options_.GetSequenceField().empty()) {
+        std::shared_ptr<PersistProcessor<PositionedKeyValue>::Factory> processor_factory =
+            std::make_shared<PersistValueAndPosProcessor::Factory>(schema_);
+        return CreateLookupRewriterInternal<PositionedKeyValue>(
+            partition, bucket, levels, dv_maintainer, max_level, lookup_strategy,
+            should_produce_changelog, processor_factory, path_factory_cache,
+            cancellation_controller, remote_lookup_file_manager);
+    }
+    std::shared_ptr<PersistProcessor<FilePosition>::Factory> processor_factory =
+        std::make_shared<PersistPositionProcessor::Factory>();
+    return CreateLookupRewriterInternal<FilePosition>(
+        partition, bucket, levels, dv_maintainer, max_level, lookup_strategy,
+        should_produce_changelog, processor_factory, path_factory_cache, cancellation_controller,
+        remote_lookup_file_manager);
+}
 
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<LookupMergeTreeCompactRewriter<KeyValue>> rewriter,
-                           LookupMergeTreeCompactRewriter<KeyValue>::Create(
-                               max_level, std::move(lookup_levels), dv_maintainer,
-                               std::move(merge_function_wrapper_factory), bucket, partition,
-                               table_schema_, path_factory_cache, options_, cancellation_controller,
-                               remote_lookup_file_manager, pool_));
-    return std::shared_ptr<CompactRewriter>(std::move(rewriter));
+Result<std::shared_ptr<CompactRewriter>>
+MergeTreeCompactManagerFactory::CreateLookupRewriterWithoutDeletionVector(
+    const BinaryRow& partition, int32_t bucket, const std::shared_ptr<Levels>& levels,
+    const std::shared_ptr<BucketedDvMaintainer>& dv_maintainer, int32_t max_level,
+    const LookupStrategy& lookup_strategy, bool should_produce_changelog,
+    const std::shared_ptr<FileStorePathFactoryCache>& path_factory_cache,
+    const std::shared_ptr<CancellationController>& cancellation_controller,
+    const std::shared_ptr<RemoteLookupFileManager>& remote_lookup_file_manager) const {
+    std::shared_ptr<PersistProcessor<KeyValue>::Factory> processor_factory =
+        std::make_shared<PersistValueProcessor::Factory>(schema_);
+    return CreateLookupRewriterInternal<KeyValue>(
+        partition, bucket, levels, dv_maintainer, max_level, lookup_strategy,
+        should_produce_changelog, processor_factory, path_factory_cache, cancellation_controller,
+        remote_lookup_file_manager);
 }
 
 Result<std::shared_ptr<RemoteLookupFileManager>>
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h
index 13a059620..2eaa6ffd4 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h
@@ -71,7 +71,8 @@ class MergeTreeCompactManagerFactory {
         const std::shared_ptr<IOManager>& io_manager,
         const std::shared_ptr<CacheManager>& cache_manager,
         const std::shared_ptr<FileStorePathFactory>& file_store_path_factory,
-        const std::string& root_path, const std::shared_ptr<MemoryPool>& pool)
+        const std::string& root_path, bool ignore_previous_files,
+        const std::shared_ptr<MemoryPool>& pool)
         : options_(options),
           pool_(pool),
           key_comparator_(key_comparator),
@@ -83,7 +84,8 @@ class MergeTreeCompactManagerFactory {
           io_manager_(io_manager),
           cache_manager_(cache_manager),
           file_store_path_factory_(file_store_path_factory),
-          root_path_(root_path) {}
+          root_path_(root_path),
+          ignore_previous_files_(ignore_previous_files) {}
 
     std::shared_ptr<CompactStrategy> CreateCompactStrategy() const;
 
@@ -115,10 +117,20 @@ class MergeTreeCompactManagerFactory {
         const std::shared_ptr<FileStorePathFactoryCache>& path_factory_cache,
         const std::shared_ptr<CancellationController>& cancellation_controller) const;
 
+    template <typename T>
+    Result<std::shared_ptr<CompactRewriter>> CreateLookupRewriterInternal(
+        const BinaryRow& partition, int32_t bucket, const std::shared_ptr<Levels>& levels,
+        const std::shared_ptr<BucketedDvMaintainer>& dv_maintainer, int32_t max_level,
+        const LookupStrategy& lookup_strategy, bool should_produce_changelog,
+        const std::shared_ptr<typename PersistProcessor<T>::Factory>& processor_factory,
+        const std::shared_ptr<FileStorePathFactoryCache>& path_factory_cache,
+        const std::shared_ptr<CancellationController>& cancellation_controller,
+        const std::shared_ptr<RemoteLookupFileManager>& remote_lookup_file_manager) const;
+
     Result<std::shared_ptr<CompactRewriter>> CreateLookupRewriterWithDeletionVector(
         const BinaryRow& partition, int32_t bucket, const std::shared_ptr<Levels>& levels,
         const std::shared_ptr<BucketedDvMaintainer>& dv_maintainer, int32_t max_level,
-        const LookupStrategy& lookup_strategy,
+        const LookupStrategy& lookup_strategy, bool should_produce_changelog,
         const std::shared_ptr<FileStorePathFactoryCache>& path_factory_cache,
         const std::shared_ptr<CancellationController>& cancellation_controller,
         const std::shared_ptr<RemoteLookupFileManager>& remote_lookup_file_manager) const;
@@ -126,7 +138,7 @@ class MergeTreeCompactManagerFactory {
     Result<std::shared_ptr<CompactRewriter>> CreateLookupRewriterWithoutDeletionVector(
         const BinaryRow& partition, int32_t bucket, const std::shared_ptr<Levels>& levels,
         const std::shared_ptr<BucketedDvMaintainer>& dv_maintainer, int32_t max_level,
-        const LookupStrategy& lookup_strategy,
+        const LookupStrategy& lookup_strategy, bool should_produce_changelog,
         const std::shared_ptr<FileStorePathFactoryCache>& path_factory_cache,
         const std::shared_ptr<CancellationController>& cancellation_controller,
         const std::shared_ptr<RemoteLookupFileManager>& remote_lookup_file_manager) const;
@@ -146,6 +158,7 @@ class MergeTreeCompactManagerFactory {
     std::shared_ptr<CacheManager> cache_manager_;
     std::shared_ptr<FileStorePathFactory> file_store_path_factory_;
     std::string root_path_;
+    bool ignore_previous_files_;
     std::shared_ptr<LookupFile::LookupFileCache> lookup_file_cache_;
 };
 
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp
index 91e3bb1cc..06a4ec197 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp
@@ -88,6 +88,7 @@ class MergeTreeCompactManagerFactoryStrategyTest : public ::testing::Test {
                                               /*cache_manager=*/nullptr,
                                               /*file_store_path_factory=*/nullptr,
                                               /*root_path=*/"",
+                                              /*ignore_previous_files=*/false,
                                               /*pool=*/nullptr);
     }
 };
@@ -333,7 +334,8 @@ TEST_F(MergeTreeCompactManagerFactoryWriteTest,
     ASSERT_NOK_WITH_MSG(CreateSingleStringFileStoreWrite(
                             {{"bucket", "1"}, {Options::CHANGELOG_PRODUCER, "full-compaction"}},
                             /*with_io_manager=*/false),
-                        "C++ Paimon does not support changelog-producer yet");
+                        "C++ Paimon only supports 'none', 'input' and 'lookup' "
+                        "changelog-producer now");
 }
 
 TEST_F(MergeTreeCompactManagerFactoryWriteTest,
@@ -378,13 +380,11 @@ TEST_F(MergeTreeCompactManagerFactoryWriteTest,
 }
 
 TEST_F(MergeTreeCompactManagerFactoryWriteTest,
-       TestCreateFileStoreWriteShouldFailWhenLookupChangelogConfigured) {
-    ASSERT_NOK_WITH_MSG(
-        CreateSingleStringFileStoreWrite({{"bucket", "1"},
-                                          {Options::DELETION_VECTORS_ENABLED, "true"},
-                                          {Options::CHANGELOG_PRODUCER, "lookup"}},
-                                         /*with_io_manager=*/true),
-        "C++ Paimon does not support changelog-producer yet");
+       TestCreateFileStoreWriteShouldSucceedWhenLookupChangelogConfigured) {
+    ASSERT_OK(CreateSingleStringFileStoreWrite({{"bucket", "1"},
+                                                {Options::DELETION_VECTORS_ENABLED, "true"},
+                                                {Options::CHANGELOG_PRODUCER, "lookup"}},
+                                               /*with_io_manager=*/true));
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp
index fd7c7cbd2..37c1cfa04 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp
@@ -23,14 +23,12 @@
 
 #include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
-#include "paimon/common/data/shredding/shredding_write_plan_factories.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/utils/scope_guard.h"
-#include "paimon/core/io/key_value_data_file_writer_factory.h"
+#include "paimon/core/io/key_value_data_file_writer_factories.h"
 #include "paimon/core/io/key_value_meta_projection_consumer.h"
 #include "paimon/core/io/key_value_record_reader.h"
 #include "paimon/core/io/row_to_arrow_array_converter.h"
-#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/operation/internal_read_context.h"
 #include "paimon/format/file_format.h"
@@ -133,20 +131,31 @@ MergeTreeCompactRewriter::CreateRollingRowWriter(int32_t level) {
     auto format = options_.GetWriteFileFormat(level);
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_file_path_factory,
                            CreateDataFilePathFactory(format->Identifier()));
-    std::shared_ptr<SingleFileWriterFactory<KeyValueBatch, std::shared_ptr<DataFileMeta>>> factory;
     PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<ShreddingWritePlanFactory> plan_factory,
-        ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_));
-    if (plan_factory != nullptr) {
-        factory = std::make_shared<ShreddingKeyValueDataFileWriterFactory>(
+        std::shared_ptr<KeyValueDataFileWriterFactories::WriterFactory> factory,
+        KeyValueDataFileWriterFactories::Create(
             options_, schema_id_, write_schema_, level, FileSource::Compact(),
             trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true,
-            plan_factory, pool_);
-    } else {
-        factory = std::make_shared<KeyValueDataFileWriterFactory>(
-            options_, schema_id_, write_schema_, level, FileSource::Compact(),
-            trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true, pool_);
+            /*is_changelog=*/false, pool_));
+    return std::make_unique<MergeTreeCompactRewriter::KeyValueRollingFileWriter>(
+        options_.GetTargetFileSize(/*has_primary_key=*/true),
+        /*target_file_row_num=*/std::numeric_limits<int64_t>::max(), factory);
+}
+
+Result<std::unique_ptr<MergeTreeCompactRewriter::KeyValueRollingFileWriter>>
+MergeTreeCompactRewriter::CreateRollingChangelogWriter(int32_t level) {
+    std::shared_ptr<FileFormat> format = options_.GetChangelogFileFormat();
+    if (!format) {
+        format = options_.GetWriteFileFormat(level);
     }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_file_path_factory,
+                           CreateDataFilePathFactory(format->Identifier()));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<KeyValueDataFileWriterFactories::WriterFactory> factory,
+        KeyValueDataFileWriterFactories::Create(
+            options_, schema_id_, write_schema_, level, FileSource::Append(), trimmed_primary_keys_,
+            data_file_path_factory, /*create_stats_extractor=*/true,
+            /*is_changelog=*/true, pool_));
     return std::make_unique<MergeTreeCompactRewriter::KeyValueRollingFileWriter>(
         options_.GetTargetFileSize(/*has_primary_key=*/true),
         /*target_file_row_num=*/std::numeric_limits<int64_t>::max(), factory);
@@ -178,6 +187,19 @@ Result<std::shared_ptr<DataFilePathFactory>> MergeTreeCompactRewriter::CreateDat
     return path_factory->CreateDataFilePathFactory(partition_, bucket_);
 }
 
+Result<std::unique_ptr<SortMergeReader>>
+MergeTreeCompactRewriter::CreateRawSortMergeReaderForSection(
+    const std::vector<SortedRun>& section) {
+    if (!merge_file_split_read_) {
+        return Status::Invalid(
+            "merge_file_split_read in MergeTreeCompactRewriter cannot be nullptr");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_file_path_factory,
+                           CreateDataFilePathFactory(options_.GetFileFormat()->Identifier()));
+    return merge_file_split_read_->CreateRawSortMergeReaderForSection(
+        section, partition_, dv_factory_, /*predicate=*/nullptr, data_file_path_factory);
+}
+
 Status MergeTreeCompactRewriter::MergeReadAndWrite(
     int32_t output_level, bool drop_delete, const std::vector<SortedRun>& section,
     const MergeTreeCompactRewriter::KeyValueConsumerCreator& create_consumer,
@@ -222,11 +244,12 @@ Status MergeTreeCompactRewriter::MergeReadAndWrite(
         return Status::OK();
     }
 
-    // consumer batch size is WriteBatchSize
+    std::unique_ptr<AsyncKeyValueBatchProducer> producer =
+        std::make_unique<SortMergeReaderBatchProducer>(std::move(sort_merge_reader),
+                                                       options_.GetWriteBatchSize());
     auto async_key_value_producer_consumer =
         std::make_shared<AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>>(
-            std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(),
-            /*projection_thread_num=*/1, pool_);
+            std::move(producer), create_consumer, /*projection_thread_num=*/1);
     reader_holders.push_back(async_key_value_producer_consumer);
     // read KeyValueBatch from SortMergeReader and write to RollingWriter
     while (true) {
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h
index c9c624704..c15e16fc5 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h
@@ -92,6 +92,8 @@ class MergeTreeCompactRewriter : public CompactRewriter {
 
     Result<std::unique_ptr<KeyValueRollingFileWriter>> CreateRollingRowWriter(int32_t level);
 
+    Result<std::unique_ptr<KeyValueRollingFileWriter>> CreateRollingChangelogWriter(int32_t level);
+
     Result<KeyValueConsumerCreator> GenerateKeyValueConsumer() const;
 
     Status MergeReadAndWrite(int32_t output_level, bool drop_delete,
@@ -100,6 +102,13 @@ class MergeTreeCompactRewriter : public CompactRewriter {
                              KeyValueRollingFileWriter* rolling_writer,
                              std::vector<std::shared_ptr<KeyValueMergeReader>>* reader_holders_ptr);
 
+    Result<std::unique_ptr<SortMergeReader>> CreateRawSortMergeReaderForSection(
+        const std::vector<SortedRun>& section);
+
+    bool IsCancelled() const {
+        return cancellation_controller_->IsCancelled();
+    }
+
  protected:
     CoreOptions options_;
     std::unique_ptr<MergeFileSplitRead> merge_file_split_read_;
@@ -108,7 +117,6 @@ class MergeTreeCompactRewriter : public CompactRewriter {
     Result<std::shared_ptr<DataFilePathFactory>> CreateDataFilePathFactory(
         const std::string& format);
 
- private:
     std::shared_ptr<MemoryPool> pool_;
     BinaryRow partition_;
     int32_t bucket_;
diff --git a/src/paimon/core/mergetree/external_sort_buffer.cpp b/src/paimon/core/mergetree/external_sort_buffer.cpp
index 9bfeec899..718a2de62 100644
--- a/src/paimon/core/mergetree/external_sort_buffer.cpp
+++ b/src/paimon/core/mergetree/external_sort_buffer.cpp
@@ -222,10 +222,11 @@ Result<FileChannelInfo> ExternalSortBuffer::SpillToDisk(
         -> Result<std::unique_ptr<RowToArrowArrayConverter<KeyValue, KeyValueBatch>>> {
         return KeyValueMetaProjectionConsumer::Create(target_schema, pool);
     };
+    std::unique_ptr<AsyncKeyValueBatchProducer> producer =
+        std::make_unique<SortMergeReaderBatchProducer>(std::move(sorted_reader), write_batch_size);
     auto async_key_value_producer_consumer =
         std::make_unique<AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>>(
-            std::move(sorted_reader), create_consumer, write_batch_size,
-            /*projection_thread_num=*/1, pool_);
+            std::move(producer), create_consumer, /*projection_thread_num=*/1);
     auto close_guard = ScopeGuard([&]() { async_key_value_producer_consumer->Close(); });
 
     while (true) {
diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp
index 3b6806c73..bb7a01054 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer.cpp
@@ -20,6 +20,7 @@
 
 #include <algorithm>
 #include <cassert>
+#include <limits>
 #include <map>
 #include <unordered_set>
 #include <utility>
@@ -27,7 +28,6 @@
 #include "arrow/api.h"
 #include "arrow/c/abi.h"
 #include "arrow/c/helpers.h"
-#include "paimon/common/data/shredding/shredding_write_plan_factories.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/utils/arrow/status_utils.h"
@@ -37,13 +37,13 @@
 #include "paimon/core/io/compact_increment.h"
 #include "paimon/core/io/data_file_path_factory.h"
 #include "paimon/core/io/data_increment.h"
-#include "paimon/core/io/key_value_data_file_writer_factory.h"
+#include "paimon/core/io/key_value_data_file_writer_factories.h"
 #include "paimon/core/io/key_value_meta_projection_consumer.h"
 #include "paimon/core/io/key_value_record_reader.h"
 #include "paimon/core/io/row_to_arrow_array_converter.h"
-#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/mergetree/compact/sort_merge_reader_with_loser_tree.h"
+#include "paimon/core/mergetree/compact/sort_merge_reader_with_min_heap.h"
 #include "paimon/core/mergetree/write_buffer.h"
 #include "paimon/core/utils/commit_increment.h"
 
@@ -104,8 +104,13 @@ Status MergeTreeWriter::DoClose() {
 
     // delete temporary files
     std::vector<std::shared_ptr<DataFileMeta>> delete_files;
-    delete_files.reserve(new_files_.size() + compact_after_.size());
+    delete_files.reserve(new_files_.size() + new_changelog_files_.size() + compact_after_.size() +
+                         compact_changelog_files_.size());
     delete_files.insert(delete_files.end(), new_files_.begin(), new_files_.end());
+    delete_files.insert(delete_files.end(), new_changelog_files_.begin(),
+                        new_changelog_files_.end());
+    delete_files.insert(delete_files.end(), compact_changelog_files_.begin(),
+                        compact_changelog_files_.end());
     for (const auto& file : compact_after_) {
         // Upgrade file is required by previous snapshot, so we should ensure that this file is
         // not the output of upgraded.
@@ -125,9 +130,11 @@ Status MergeTreeWriter::DoClose() {
 
     write_buffer_->Clear();
     new_files_.clear();
+    new_changelog_files_.clear();
     deleted_files_.clear();
     compact_before_.clear();
     compact_after_.clear();
+    compact_changelog_files_.clear();
 
     if (compact_deletion_file_) {
         compact_deletion_file_->Clean();
@@ -210,7 +217,9 @@ Status MergeTreeWriter::UpdateCompactResult(const std::shared_ptr<CompactResult>
 
     compact_after_.insert(compact_after_.end(), compact_result->After().begin(),
                           compact_result->After().end());
-    // TODO(yonghao.fyh): support compact changelog
+    compact_changelog_files_.insert(compact_changelog_files_.end(),
+                                    compact_result->Changelog().begin(),
+                                    compact_result->Changelog().end());
     return UpdateCompactDeletionFile(compact_result->DeletionFile());
 }
 
@@ -256,23 +265,62 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction,
             wait_for_latest_compaction = true;
         }
         auto cleanup_guard = ScopeGuard([&]() { write_buffer_->Clear(); });
-        // 1. flush write buffer to get sorted readers
+
+        auto create_consumer = [target_schema = write_schema_, pool = pool_]()
+            -> Result<std::unique_ptr<RowToArrowArrayConverter<KeyValue, KeyValueBatch>>> {
+            return KeyValueMetaProjectionConsumer::Create(target_schema, pool);
+        };
+
+        std::unique_ptr<AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>>
+            async_changelog_producer_consumer;
+        std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>
+            changelog_writer;
+        std::vector<std::shared_ptr<DataFileMeta>> flushed_changelog_files;
+        ScopeGuard changelog_write_guard([&]() -> void {
+            if (changelog_writer) {
+                changelog_writer->Abort();
+            }
+            if (async_changelog_producer_consumer) {
+                async_changelog_producer_consumer->Close();
+            }
+        });
+        if (options_.GetChangelogProducer() == ChangelogProducer::INPUT) {
+            PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> raw_readers,
+                                   write_buffer_->CreateRawReaders());
+            auto raw_sort_merge_reader = std::make_unique<SortMergeReaderWithMinHeap>(
+                std::move(raw_readers), key_comparator_, user_defined_seq_comparator_,
+                /*merge_function_wrapper=*/nullptr);
+            std::unique_ptr<AsyncKeyValueBatchProducer> producer =
+                std::make_unique<SortMergeReaderBatchProducer>(std::move(raw_sort_merge_reader),
+                                                               options_.GetWriteBatchSize());
+            async_changelog_producer_consumer =
+                std::make_unique<AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>>(
+                    std::move(producer), create_consumer, /*projection_thread_num=*/1);
+            PAIMON_ASSIGN_OR_RAISE(changelog_writer, CreateRollingChangelogWriter());
+            while (true) {
+                PAIMON_ASSIGN_OR_RAISE(KeyValueBatch key_value_batch,
+                                       async_changelog_producer_consumer->NextBatch());
+                if (key_value_batch.batch == nullptr) {
+                    break;
+                }
+                PAIMON_RETURN_NOT_OK(changelog_writer->Write(std::move(key_value_batch)));
+            }
+            PAIMON_RETURN_NOT_OK(changelog_writer->Close());
+            PAIMON_ASSIGN_OR_RAISE(flushed_changelog_files, changelog_writer->GetResult());
+        }
+
+        // Flush write buffer to get sorted and merged data readers.
         PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<KeyValueRecordReader>> readers,
                                write_buffer_->CreateReaders());
-        // 2. prepare loser tree sort merge reader
         auto sort_merge_reader = std::make_unique<SortMergeReaderWithLoserTree>(
             std::move(readers), key_comparator_, user_defined_seq_comparator_,
             merge_function_wrapper_);
-        // 3. project key value to arrow array
-        auto create_consumer = [target_schema = write_schema_, pool = pool_]()
-            -> Result<std::unique_ptr<RowToArrowArrayConverter<KeyValue, KeyValueBatch>>> {
-            return KeyValueMetaProjectionConsumer::Create(target_schema, pool);
-        };
-        // consumer batch size is WriteBatchSize
+        std::unique_ptr<AsyncKeyValueBatchProducer> producer =
+            std::make_unique<SortMergeReaderBatchProducer>(std::move(sort_merge_reader),
+                                                           options_.GetWriteBatchSize());
         auto async_key_value_producer_consumer =
             std::make_unique<AsyncKeyValueProducerAndConsumer<KeyValue, KeyValueBatch>>(
-                std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(),
-                /*projection_thread_num=*/1, pool_);
+                std::move(producer), create_consumer, /*projection_thread_num=*/1);
         std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>
             rolling_writer;
         PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter());
@@ -292,13 +340,24 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction,
         PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<DataFileMeta>> flushed_files,
                                rolling_writer->GetResult());
         async_key_value_producer_consumer->Close();
+        if (async_changelog_producer_consumer) {
+            async_changelog_producer_consumer->Close();
+        }
+
+        new_changelog_files_.insert(new_changelog_files_.end(), flushed_changelog_files.begin(),
+                                    flushed_changelog_files.end());
+        new_files_.insert(new_files_.end(), flushed_files.begin(), flushed_files.end());
+
         write_guard.Release();
+        changelog_write_guard.Release();
 
         for (const auto& flushed_file : flushed_files) {
-            new_files_.emplace_back(flushed_file);
             PAIMON_RETURN_NOT_OK(compact_manager_->AddNewFile(flushed_file));
         }
         metrics_->Merge(rolling_writer->GetMetrics());
+        if (changelog_writer) {
+            metrics_->Merge(changelog_writer->GetMetrics());
+        }
     }
     PAIMON_RETURN_NOT_OK(TrySyncLatestCompaction(wait_for_latest_compaction));
     PAIMON_RETURN_NOT_OK(compact_manager_->TriggerCompaction(forced_full_compaction));
@@ -306,14 +365,18 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction,
 }
 
 Result<CommitIncrement> MergeTreeWriter::DrainIncrement() {
-    DataIncrement data_increment(std::move(new_files_), std::move(deleted_files_), {});
-    CompactIncrement compact_increment(std::move(compact_before_), std::move(compact_after_), {});
+    DataIncrement data_increment(std::move(new_files_), std::move(deleted_files_),
+                                 std::move(new_changelog_files_));
+    CompactIncrement compact_increment(std::move(compact_before_), std::move(compact_after_),
+                                       std::move(compact_changelog_files_));
     auto drain_deletion_file = compact_deletion_file_;
 
     new_files_.clear();
+    new_changelog_files_.clear();
     deleted_files_.clear();
     compact_before_.clear();
     compact_after_.clear();
+    compact_changelog_files_.clear();
     compact_deletion_file_ = nullptr;
 
     return CommitIncrement(data_increment, compact_increment, drain_deletion_file);
@@ -321,23 +384,28 @@ Result<CommitIncrement> MergeTreeWriter::DrainIncrement() {
 
 Result<std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>>
 MergeTreeWriter::CreateRollingRowWriter() const {
-    std::shared_ptr<SingleFileWriterFactory<KeyValueBatch, std::shared_ptr<DataFileMeta>>> factory;
     PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<ShreddingWritePlanFactory> plan_factory,
-        ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_));
-    if (plan_factory != nullptr) {
-        factory = std::make_shared<ShreddingKeyValueDataFileWriterFactory>(
-            options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(),
-            trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, plan_factory,
-            pool_);
-    } else {
-        factory = std::make_shared<KeyValueDataFileWriterFactory>(
-            options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(),
-            trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, pool_);
-    }
+        std::shared_ptr<KeyValueDataFileWriterFactories::WriterFactory> factory,
+        KeyValueDataFileWriterFactories::Create(options_, schema_id_, write_schema_, /*level=*/0,
+                                                FileSource::Append(), trimmed_primary_keys_,
+                                                path_factory_, /*create_stats_extractor=*/true,
+                                                /*is_changelog=*/false, pool_));
     return std::make_unique<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>(
         options_.GetTargetFileSize(/*has_primary_key=*/true), options_.GetTargetFileRowNum(),
         factory);
 }
 
+Result<std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>>
+MergeTreeWriter::CreateRollingChangelogWriter() const {
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<KeyValueDataFileWriterFactories::WriterFactory> factory,
+        KeyValueDataFileWriterFactories::Create(options_, schema_id_, write_schema_, /*level=*/0,
+                                                FileSource::Append(), trimmed_primary_keys_,
+                                                path_factory_, /*create_stats_extractor=*/true,
+                                                /*is_changelog=*/true, pool_));
+    return std::make_unique<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>(
+        options_.GetTargetFileSize(/*has_primary_key=*/true),
+        /*target_file_row_num=*/std::numeric_limits<int64_t>::max(), factory);
+}
+
 }  // namespace paimon
diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h
index febce2afb..10f35fbbe 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.h
+++ b/src/paimon/core/mergetree/merge_tree_writer.h
@@ -100,6 +100,9 @@ class MergeTreeWriter : public BatchWriter {
     Result<std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>>
     CreateRollingRowWriter() const;
 
+    Result<std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>>
+    CreateRollingChangelogWriter() const;
+
     Status TrySyncLatestCompaction(bool blocking);
     Status UpdateCompactResult(const std::shared_ptr<CompactResult>& compact_result);
     Status UpdateCompactDeletionFile(const std::shared_ptr<CompactDeletionFile>& new_deletion_file);
@@ -134,9 +137,11 @@ class MergeTreeWriter : public BatchWriter {
     std::shared_ptr<Metrics> metrics_;
 
     std::vector<std::shared_ptr<DataFileMeta>> new_files_;
+    std::vector<std::shared_ptr<DataFileMeta>> new_changelog_files_;
     std::vector<std::shared_ptr<DataFileMeta>> deleted_files_;
     std::vector<std::shared_ptr<DataFileMeta>> compact_before_;
     std::vector<std::shared_ptr<DataFileMeta>> compact_after_;
+    std::vector<std::shared_ptr<DataFileMeta>> compact_changelog_files_;
 
     std::shared_ptr<CompactDeletionFile> compact_deletion_file_;
 };
diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index c5a114f36..b3f53f487 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -37,6 +37,7 @@
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/common/utils/scope_guard.h"
+#include "paimon/common/utils/string_utils.h"
 #include "paimon/core/compact/noop_compact_manager.h"
 #include "paimon/core/disk/io_manager.h"
 #include "paimon/core/io/compact_increment.h"
@@ -144,11 +145,13 @@ class MergeTreeWriterTest : public ::testing::TestWithParam<bool> {
     }
 
     void CheckFileContent(const std::string& data_file_name,
-                          const std::shared_ptr<arrow::ChunkedArray>& expected_array) const {
+                          const std::shared_ptr<arrow::ChunkedArray>& expected_array,
+                          const std::string& file_format_name = "orc") const {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input_stream,
                              file_system_->Open(data_file_name));
         ASSERT_TRUE(input_stream);
-        ASSERT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get("orc", /*options=*/{}));
+        ASSERT_OK_AND_ASSIGN(auto file_format,
+                             FileFormatFactory::Get(file_format_name, /*options=*/{}));
         ASSERT_OK_AND_ASSIGN(auto reader_builder,
                              file_format->CreateReaderBuilder(/*batch_size=*/10));
         ASSERT_OK_AND_ASSIGN(auto orc_batch_reader, reader_builder->Build(input_stream));
@@ -160,11 +163,13 @@ class MergeTreeWriterTest : public ::testing::TestWithParam<bool> {
     void CheckShreddingFileSchema(const std::string& data_file_name,
                                   const std::shared_ptr<arrow::Schema>& expected_physical_schema,
                                   int32_t field_index,
-                                  const MapSharedShreddingFieldMeta& expected_meta) const {
+                                  const MapSharedShreddingFieldMeta& expected_meta,
+                                  const std::string& file_format_name = "orc") const {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input_stream,
                              file_system_->Open(data_file_name));
         ASSERT_TRUE(input_stream);
-        ASSERT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get("orc", /*options=*/{}));
+        ASSERT_OK_AND_ASSIGN(auto file_format,
+                             FileFormatFactory::Get(file_format_name, /*options=*/{}));
         ASSERT_OK_AND_ASSIGN(auto reader_builder,
                              file_format->CreateReaderBuilder(/*batch_size=*/10));
         ASSERT_OK_AND_ASSIGN(auto orc_batch_reader, reader_builder->Build(input_stream));
@@ -296,6 +301,182 @@ TEST_P(MergeTreeWriterTest, TestSimple) {
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
 }
 
+TEST_P(MergeTreeWriterTest, TestInputChangelog) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"},
+                                               {Options::CHANGELOG_PRODUCER, "input"},
+                                               {Options::CHANGELOG_FILE_PREFIX, "changes-"},
+                                               {Options::CHANGELOG_FILE_FORMAT, "parquet"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+    std::string uuid = path_factory->uuid_;
+
+    ASSERT_OK_AND_ASSIGN(auto merge_writer,
+                         CreateMergeWriter(/*last_sequence_number=*/-1, dir->Str(), path_factory,
+                                           /*schema_id=*/1, options));
+
+    std::shared_ptr<arrow::Array> array =
+        arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([
+      ["Bob", 20, 0, 20.0],
+      ["Alice", 10, 0, 10.0],
+      ["Alice", 11, 0, 11.0],
+      ["Bob", 21, 0, 21.0]
+    ])")
+            .ValueOrDie();
+    WriteBatch(array,
+               {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_BEFORE,
+                RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE},
+               merge_writer.get());
+    if (GetParam()) {
+        ASSERT_OK(merge_writer->FlushMemory());
+    }
+
+    ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment,
+                         merge_writer->PrepareCommit(/*wait_compaction=*/false));
+    ASSERT_OK(merge_writer->Close());
+
+    const DataIncrement& data_increment = commit_increment.GetNewFilesIncrement();
+    ASSERT_EQ(1, data_increment.NewFiles().size());
+    ASSERT_EQ(1, data_increment.ChangelogFiles().size());
+    ASSERT_EQ("data-" + uuid + "-1.orc", data_increment.NewFiles()[0]->file_name);
+    ASSERT_EQ("changes-" + uuid + "-0.parquet", data_increment.ChangelogFiles()[0]->file_name);
+    ASSERT_EQ(2, data_increment.NewFiles()[0]->row_count);
+    ASSERT_EQ(4, data_increment.ChangelogFiles()[0]->row_count);
+
+    std::shared_ptr<arrow::ChunkedArray> expected_data;
+    auto data_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(write_type_, {R"([
+      [2, 2, "Alice", 11, 0, 11.0],
+      [3, 3, "Bob", 21, 0, 21.0]
+    ])"},
+                                                                        &expected_data);
+    ASSERT_TRUE(data_status.ok());
+    CheckFileContent(dir->Str() + "/" + data_increment.NewFiles()[0]->file_name, expected_data);
+
+    std::shared_ptr<arrow::ChunkedArray> expected_changelog;
+    auto changelog_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(write_type_, {R"([
+      [1, 1, "Alice", 10, 0, 10.0],
+      [2, 2, "Alice", 11, 0, 11.0],
+      [0, 0, "Bob", 20, 0, 20.0],
+      [3, 3, "Bob", 21, 0, 21.0]
+    ])"},
+                                                                             &expected_changelog);
+    ASSERT_TRUE(changelog_status.ok());
+    CheckFileContent(dir->Str() + "/" + data_increment.ChangelogFiles()[0]->file_name,
+                     expected_changelog, "parquet");
+}
+
+TEST_P(MergeTreeWriterTest, TestInputChangelogIgnoresTargetFileRowNum) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"},
+                                               {Options::CHANGELOG_PRODUCER, "input"},
+                                               {Options::TARGET_FILE_ROW_NUM, "1"},
+                                               {Options::WRITE_BATCH_SIZE, "1"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+    ASSERT_OK_AND_ASSIGN(auto merge_writer,
+                         CreateMergeWriter(/*last_sequence_number=*/-1, dir->Str(), path_factory,
+                                           /*schema_id=*/1, options));
+
+    std::shared_ptr<arrow::Array> array =
+        arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([
+      ["Bob", 20, 0, 20.0],
+      ["Alice", 10, 0, 10.0],
+      ["Alice", 11, 0, 11.0],
+      ["Bob", 21, 0, 21.0]
+    ])")
+            .ValueOrDie();
+    WriteBatch(array, /*row_kinds=*/{}, merge_writer.get());
+    if (GetParam()) {
+        ASSERT_OK(merge_writer->FlushMemory());
+    }
+
+    ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment,
+                         merge_writer->PrepareCommit(/*wait_compaction=*/false));
+    ASSERT_OK(merge_writer->Close());
+
+    const std::vector<std::shared_ptr<DataFileMeta>>& changelog_files =
+        commit_increment.GetNewFilesIncrement().ChangelogFiles();
+    ASSERT_EQ(1, changelog_files.size());
+    ASSERT_EQ(4, changelog_files[0]->row_count);
+}
+
+TEST_P(MergeTreeWriterTest, TestInputChangelogWithSharedShredding) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({
+                             {Options::FILE_FORMAT, "orc"},
+                             {Options::CHANGELOG_PRODUCER, "input"},
+                             {Options::CHANGELOG_FILE_PREFIX, "changes-"},
+                             {Options::CHANGELOG_FILE_FORMAT, "parquet"},
+                             {"fields.tags.map.storage-layout", "shared-shredding"},
+                             {"fields.tags.map.shared-shredding.max-columns", "3"},
+                             {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
+                             {Options::WRITE_ONLY, "true"},
+                         }));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+
+    std::vector<DataField> value_fields = {
+        DataField(0, arrow::field("id", arrow::int32())),
+        DataField(1, arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64()))),
+    };
+    auto value_schema = DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    auto value_type = DataField::ConvertDataFieldsToArrowStructType(value_fields);
+    auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FieldsComparator> key_comparator,
+                         FieldsComparator::Create({value_fields[0]},
+                                                  /*is_ascending_order=*/true));
+    ASSERT_OK_AND_ASSIGN(
+        auto merge_writer,
+        MergeTreeWriter::Create(
+            /*last_sequence_number=*/-1, /*trimmed_primary_keys=*/{"id"}, path_factory,
+            key_comparator,
+            /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/5,
+            value_schema, options, noop_compact_manager_,
+            GetParam() ? std::make_shared<IOManager>(dir->Str() + "/tmp", file_system_) : nullptr,
+            /*enable_multi_thread_spill=*/false, pool_));
+
+    auto array = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([
+      [1, [["a", 10], ["b", 20]]],
+      [2, [["c", 30]]],
+      [1, [["a", 11], ["c", 31]]]
+    ])")
+                     .ValueOrDie();
+    WriteBatch(array, /*row_kinds=*/{}, merge_writer.get());
+    if (GetParam()) {
+        ASSERT_OK(merge_writer->FlushMemory());
+    }
+
+    ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment,
+                         merge_writer->PrepareCommit(/*wait_compaction=*/false));
+    ASSERT_OK(merge_writer->Close());
+
+    const std::vector<std::shared_ptr<DataFileMeta>>& changelog_files =
+        commit_increment.GetNewFilesIncrement().ChangelogFiles();
+    ASSERT_EQ(1, changelog_files.size());
+    ASSERT_EQ(3, changelog_files[0]->row_count);
+    ASSERT_TRUE(StringUtils::EndsWith(changelog_files[0]->file_name, ".parquet"));
+
+    std::map<std::string, int32_t> column_to_k = {{"tags", 3}};
+    ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                                                   write_schema, column_to_k));
+    MapSharedShreddingFieldMeta expected_shredding_meta;
+    expected_shredding_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}};
+    expected_shredding_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0, 1}}};
+    expected_shredding_meta.num_columns = 3;
+    expected_shredding_meta.max_row_width = 2;
+    CheckShreddingFileSchema(path_factory->ToPath(changelog_files[0]->file_name), physical_schema,
+                             /*field_index=*/3, expected_shredding_meta, "parquet");
+}
+
 TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
                          CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
@@ -1195,6 +1376,31 @@ TEST_P(MergeTreeWriterTest, TestUpdateCompactResultDeleteIntermediateFile) {
     ASSERT_EQ(merge_writer->compact_after_, std::vector<std::shared_ptr<DataFileMeta>>({file_y}));
 }
 
+TEST_P(MergeTreeWriterTest, TestUpdateCompactResultPropagatesChangelog) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared<DataFilePathFactory>();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+
+    auto fake_compact_manager = std::make_shared<FakeCompactManager>();
+    ASSERT_OK_AND_ASSIGN(
+        auto merge_writer,
+        CreateMergeWriter(/*last_sequence_number=*/-1, dir->Str(), path_factory, /*schema_id=*/0,
+                          options, /*user_defined_seq_comparator=*/nullptr, fake_compact_manager));
+
+    auto changelog = CreateMeta("changelog", /*level=*/0);
+    auto compact_result = std::make_shared<CompactResult>(
+        std::vector<std::shared_ptr<DataFileMeta>>(), std::vector<std::shared_ptr<DataFileMeta>>(),
+        std::vector<std::shared_ptr<DataFileMeta>>({changelog}));
+    ASSERT_OK(merge_writer->UpdateCompactResult(compact_result));
+
+    ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, merge_writer->DrainIncrement());
+    ASSERT_EQ(commit_increment.GetCompactIncrement().ChangelogFiles(),
+              std::vector<std::shared_ptr<DataFileMeta>>({changelog}));
+}
+
 TEST_P(MergeTreeWriterTest, TestUpdateCompactResultWithFileInCompactAfter) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
                          CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
diff --git a/src/paimon/core/mergetree/write_buffer.cpp b/src/paimon/core/mergetree/write_buffer.cpp
index 549975a33..d6c09a1cb 100644
--- a/src/paimon/core/mergetree/write_buffer.cpp
+++ b/src/paimon/core/mergetree/write_buffer.cpp
@@ -82,6 +82,10 @@ Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> WriteBuffer::CreateRe
     return merged_readers;
 }
 
+Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> WriteBuffer::CreateRawReaders() {
+    return sort_buffer_->CreateReaders();
+}
+
 Result<bool> WriteBuffer::FlushMemory() {
     return sort_buffer_->FlushMemory();
 }
diff --git a/src/paimon/core/mergetree/write_buffer.h b/src/paimon/core/mergetree/write_buffer.h
index f4f231000..1c71f7faf 100644
--- a/src/paimon/core/mergetree/write_buffer.h
+++ b/src/paimon/core/mergetree/write_buffer.h
@@ -72,6 +72,10 @@ class WriteBuffer {
     /// @return list of KeyValueRecordReaders built from buffered data
     Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateReaders();
 
+    /// Create KeyValueRecordReaders containing the raw input records without merging duplicate
+    /// keys. The caller should invoke Clear() after consuming the readers.
+    Result<std::vector<std::unique_ptr<KeyValueRecordReader>>> CreateRawReaders();
+
     /// Try to spill current buffered data. Return false when the call completed normally but the
     /// caller should fall back to FlushWriteBuffer before buffering more data.
     Result<bool> FlushMemory();
diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp
index 467ba6b27..befe7eb97 100644
--- a/src/paimon/core/operation/file_store_scan.cpp
+++ b/src/paimon/core/operation/file_store_scan.cpp
@@ -275,6 +275,8 @@ Status FileStoreScan::ReadManifestsWithSnapshot(const Snapshot& snapshot,
             return manifest_list_->ReadDataManifests(snapshot, manifests);
         case ScanMode::DELTA:
             return manifest_list_->ReadDeltaManifests(snapshot, manifests);
+        case ScanMode::CHANGELOG:
+            return manifest_list_->ReadChangelogManifests(snapshot, manifests);
         default:
             return Status::NotImplemented("Unknown scan mode ",
                                           std::to_string(static_cast<int32_t>(scan_mode_)));
diff --git a/src/paimon/core/operation/key_value_file_store_scan.cpp b/src/paimon/core/operation/key_value_file_store_scan.cpp
index 54197f871..54af98a0c 100644
--- a/src/paimon/core/operation/key_value_file_store_scan.cpp
+++ b/src/paimon/core/operation/key_value_file_store_scan.cpp
@@ -177,8 +177,13 @@ Result<bool> KeyValueFileStoreScan::IsValueFilterEnabled() const {
             return value_filter_force_enabled_;
         case ScanMode::DELTA:
             return false;
+        case ScanMode::CHANGELOG: {
+            ChangelogProducer producer = core_options_.GetChangelogProducer();
+            return producer == ChangelogProducer::LOOKUP ||
+                   producer == ChangelogProducer::FULL_COMPACTION;
+        }
         default:
-            return Status::NotImplemented("only support ALL and DELTA scan mode");
+            return Status::NotImplemented("unknown scan mode");
     }
 }
 
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 08c5ea0c3..09755984d 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -73,7 +73,7 @@ KeyValueFileStoreWrite::KeyValueFileStoreWrite(
       compact_manager_factory_(std::make_unique<MergeTreeCompactManagerFactory>(
           options_, key_comparator_, user_defined_seq_comparator_, compaction_metrics_,
           table_schema_, schema_, schema_manager_, io_manager_, cache_manager_,
-          file_store_path_factory_, root_path_, pool_)),
+          file_store_path_factory_, root_path_, ignore_previous_files, pool_)),
       logger_(Logger::GetLogger("KeyValueFileStoreWrite")) {}
 
 Result<std::unique_ptr<FileStoreScan>> KeyValueFileStoreWrite::CreateFileStoreScan(
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index b753ea431..2e517cdff 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -491,6 +491,23 @@ Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateSortMergeRead
     return sort_merge_reader;
 }
 
+Result<std::unique_ptr<SortMergeReader>> MergeFileSplitRead::CreateRawSortMergeReaderForSection(
+    const std::vector<SortedRun>& section, const BinaryRow& partition,
+    DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
+    const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) {
+    std::vector<std::unique_ptr<KeyValueRecordReader>> record_readers;
+    record_readers.reserve(section.size());
+    for (const auto& run : section) {
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<KeyValueRecordReader> run_reader,
+            CreateReaderForRun(partition, run, dv_factory, predicate, data_file_path_factory));
+        record_readers.emplace_back(std::move(run_reader));
+    }
+    return std::make_unique<SortMergeReaderWithMinHeap>(std::move(record_readers), key_comparator_,
+                                                        user_defined_seq_comparator_,
+                                                        /*merge_function_wrapper=*/nullptr);
+}
+
 Result<std::unique_ptr<KeyValueRecordReader>> MergeFileSplitRead::CreateReaderForRun(
     const BinaryRow& partition, const SortedRun& sorted_run, DeletionVector::Factory dv_factory,
     const std::shared_ptr<Predicate>& predicate,
diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h
index 11dcd0b37..bda6e2401 100644
--- a/src/paimon/core/operation/merge_file_split_read.h
+++ b/src/paimon/core/operation/merge_file_split_read.h
@@ -104,6 +104,12 @@ class MergeFileSplitRead : public AbstractSplitRead {
         DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
         const std::shared_ptr<DataFilePathFactory>& data_file_path_factory, bool drop_delete);
 
+    /// Creates a min-heap reader which only sorts records and preserves duplicate keys.
+    Result<std::unique_ptr<SortMergeReader>> CreateRawSortMergeReaderForSection(
+        const std::vector<SortedRun>& section, const BinaryRow& partition,
+        DeletionVector::Factory dv_factory, const std::shared_ptr<Predicate>& predicate,
+        const std::shared_ptr<DataFilePathFactory>& data_file_path_factory);
+
     std::shared_ptr<FileStorePathFactory> GetPathFactory() const {
         return path_factory_;
     }
diff --git a/src/paimon/core/postpone/postpone_bucket_writer.cpp b/src/paimon/core/postpone/postpone_bucket_writer.cpp
index 45bcfd364..71dfbc074 100644
--- a/src/paimon/core/postpone/postpone_bucket_writer.cpp
+++ b/src/paimon/core/postpone/postpone_bucket_writer.cpp
@@ -31,7 +31,6 @@
 #include "arrow/c/helpers.h"
 #include "arrow/scalar.h"
 #include "fmt/format.h"
-#include "paimon/common/data/shredding/shredding_write_plan_factories.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/row_kind.h"
@@ -42,8 +41,7 @@
 #include "paimon/core/io/compact_increment.h"
 #include "paimon/core/io/data_file_path_factory.h"
 #include "paimon/core/io/data_increment.h"
-#include "paimon/core/io/key_value_data_file_writer_factory.h"
-#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h"
+#include "paimon/core/io/key_value_data_file_writer_factories.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/utils/commit_increment.h"
 #include "paimon/format/file_format.h"
@@ -242,20 +240,12 @@ PostponeBucketWriter::PrepareMinMaxKey(
 
 Result<std::unique_ptr<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>>
 PostponeBucketWriter::CreateRollingRowWriter() const {
-    std::shared_ptr<SingleFileWriterFactory<KeyValueBatch, std::shared_ptr<DataFileMeta>>> factory;
     PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr<ShreddingWritePlanFactory> plan_factory,
-        ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_));
-    if (plan_factory != nullptr) {
-        factory = std::make_shared<ShreddingKeyValueDataFileWriterFactory>(
-            options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(),
-            trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, plan_factory,
-            pool_);
-    } else {
-        factory = std::make_shared<KeyValueDataFileWriterFactory>(
-            options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(),
-            trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, pool_);
-    }
+        std::shared_ptr<KeyValueDataFileWriterFactories::WriterFactory> factory,
+        KeyValueDataFileWriterFactories::Create(options_, schema_id_, write_schema_, /*level=*/0,
+                                                FileSource::Append(), trimmed_primary_keys_,
+                                                path_factory_, /*create_stats_extractor=*/false,
+                                                /*is_changelog=*/false, pool_));
     return std::make_unique<RollingFileWriter<KeyValueBatch, std::shared_ptr<DataFileMeta>>>(
         options_.GetTargetFileSize(/*has_primary_key=*/true), options_.GetTargetFileRowNum(),
         factory);
diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp
index a6844d9e6..90f508b47 100644
--- a/src/paimon/core/schema/schema_validation.cpp
+++ b/src/paimon/core/schema/schema_validation.cpp
@@ -52,6 +52,7 @@
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/table/bucket_mode.h"
 #include "paimon/defs.h"
+#include "paimon/format/file_format.h"
 #include "paimon/result.h"
 
 namespace paimon {
@@ -163,14 +164,8 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) {
     PAIMON_RETURN_NOT_OK(ValidateFieldsPrefix(schema, options));
     PAIMON_RETURN_NOT_OK(ValidateSequenceField(schema, options));
     PAIMON_RETURN_NOT_OK(ValidateSequenceGroup(schema, options));
+    PAIMON_RETURN_NOT_OK(ValidateChangelogProducer(schema, options));
 
-    ChangelogProducer changelog_producer = options.GetChangelogProducer();
-    if (schema.PrimaryKeys().empty() && changelog_producer != ChangelogProducer::NONE) {
-        return Status::Invalid(
-            fmt::format("Can not set {} on table without primary keys, please define primary keys.",
-                        Options::CHANGELOG_PRODUCER));
-    }
-    PAIMON_RETURN_NOT_OK(ValidateChangelogProducer(options));
     PAIMON_RETURN_NOT_OK(Preconditions::CheckState(
         options.GetExpireConfig().GetSnapshotRetainMin() > 0,
         std::string(Options::SNAPSHOT_NUM_RETAINED_MIN) + " should be at least 1"));
@@ -324,10 +319,41 @@ Status SchemaValidation::ValidateBucket(const TableSchema& schema, const CoreOpt
     return Status::OK();
 }
 
-Status SchemaValidation::ValidateChangelogProducer(const CoreOptions& options) {
-    return Preconditions::CheckState(options.GetChangelogProducer() == ChangelogProducer::NONE,
-                                     "C++ Paimon does not support changelog-producer yet. Please "
-                                     "keep changelog-producer as 'none'.");
+Status SchemaValidation::ValidateChangelogProducer(const TableSchema& schema,
+                                                   const CoreOptions& options) {
+    ChangelogProducer changelog_producer = options.GetChangelogProducer();
+    if (schema.PrimaryKeys().empty() && changelog_producer != ChangelogProducer::NONE) {
+        return Status::Invalid(
+            fmt::format("Can not set {} on table without primary keys, please define primary keys.",
+                        Options::CHANGELOG_PRODUCER));
+    }
+
+    bool row_deduplicate = options.ChangelogRowDeduplicate();
+    const std::vector<std::string>& ignore_fields =
+        options.GetChangelogRowDeduplicateIgnoreFields();
+    PAIMON_RETURN_NOT_OK(Preconditions::CheckState(
+        ignore_fields.empty() || row_deduplicate, "'{}' is only valid when '{}' is true.",
+        Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS,
+        Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE));
+    PAIMON_RETURN_NOT_OK(Preconditions::CheckState(
+        ObjectUtils::ContainsAll(schema.FieldNames(), ignore_fields),
+        "Fields {} configured in '{}' can not be found in table schema.", ignore_fields,
+        Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS));
+    PAIMON_RETURN_NOT_OK(Preconditions::CheckState(
+        !row_deduplicate || changelog_producer == ChangelogProducer::LOOKUP ||
+            changelog_producer == ChangelogProducer::FULL_COMPACTION,
+        "'{}' is only valid for 'lookup' or 'full-compaction' changelog producer.",
+        Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE));
+    PAIMON_RETURN_NOT_OK(Preconditions::CheckState(
+        changelog_producer == ChangelogProducer::NONE ||
+            changelog_producer == ChangelogProducer::INPUT ||
+            changelog_producer == ChangelogProducer::LOOKUP,
+        "C++ Paimon only supports 'none', 'input' and 'lookup' changelog-producer now."));
+    return Preconditions::CheckState(
+        options.GetMergeEngine() != MergeEngine::FIRST_ROW ||
+            changelog_producer == ChangelogProducer::NONE ||
+            changelog_producer == ChangelogProducer::LOOKUP,
+        "Only support 'none' and 'lookup' changelog-producer on FIRST_ROW merge engine");
 }
 
 Status SchemaValidation::ValidateForDeletionVectors(const CoreOptions& options) {
@@ -725,10 +751,20 @@ Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema,
                                                            options.GetFileFormat()->Identifier()));
     PAIMON_RETURN_NOT_OK(ValidatePerLevelOption(options_map, Options::FILE_FORMAT_PER_LEVEL,
                                                 ValidateSharedShreddingFileFormat));
+    std::shared_ptr<FileFormat> changelog_format = options.GetChangelogFileFormat();
+    if (changelog_format) {
+        PAIMON_RETURN_NOT_OK(ValidateSharedShreddingFileFormat(Options::CHANGELOG_FILE_FORMAT,
+                                                               changelog_format->Identifier()));
+    }
     PAIMON_RETURN_NOT_OK(ValidateSharedShreddingCompression(Options::FILE_COMPRESSION,
                                                             options.GetFileCompression()));
     PAIMON_RETURN_NOT_OK(ValidatePerLevelOption(options_map, Options::FILE_COMPRESSION_PER_LEVEL,
                                                 ValidateSharedShreddingCompression));
+    std::optional<std::string> changelog_compression = options.GetChangelogFileCompression();
+    if (changelog_compression) {
+        PAIMON_RETURN_NOT_OK(ValidateSharedShreddingCompression(Options::CHANGELOG_FILE_COMPRESSION,
+                                                                changelog_compression.value()));
+    }
 
     return Status::OK();
 }
diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h
index d331bf10f..388ab42e2 100644
--- a/src/paimon/core/schema/schema_validation.h
+++ b/src/paimon/core/schema/schema_validation.h
@@ -66,7 +66,7 @@ class SchemaValidation {
     static Status ValidateFieldsPrefix(const TableSchema& schema, const CoreOptions& options);
     static Status ValidateSequenceField(const TableSchema& schema, const CoreOptions& options);
     static Status ValidateSequenceGroup(const TableSchema& schema, const CoreOptions& options);
-    static Status ValidateChangelogProducer(const CoreOptions& options);
+    static Status ValidateChangelogProducer(const TableSchema& schema, const CoreOptions& options);
     static Status ValidateForDeletionVectors(const CoreOptions& options);
 
     static Status ValidateRowTracking(const TableSchema& table_schema, const CoreOptions& options);
diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp
index 65ab7f7d1..53d0c3c8c 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -714,7 +714,18 @@ TEST(SchemaValidationTest, ValidateDeletionVector) {
             std::shared_ptr<TableSchema> table_schema,
             TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options));
         ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
-                            "C++ Paimon does not support changelog-producer yet");
+                            "C++ Paimon only supports 'none', 'input' and 'lookup' "
+                            "changelog-producer now");
+    }
+    {
+        std::map<std::string, std::string> options = {{Options::BUCKET, "2"},
+                                                      {Options::BUCKET_KEY, "f0"},
+                                                      {Options::DELETION_VECTORS_ENABLED, "true"},
+                                                      {Options::CHANGELOG_PRODUCER, "input"}};
+        ASSERT_OK_AND_ASSIGN(
+            std::shared_ptr<TableSchema> table_schema,
+            TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options));
+        ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
     }
     {
         std::map<std::string, std::string> options = {{Options::BUCKET, "2"},
@@ -866,6 +877,41 @@ TEST(SchemaValidationTest, ValidateInvalidConfiguration) {
                             "Can not set changelog-producer on table without primary keys, please "
                             "define primary keys.");
     }
+    {
+        std::map<std::string, std::string> options = {
+            {Options::CHANGELOG_PRODUCER, "input"},
+            {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"}};
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{"f0"}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "'changelog-producer.row-deduplicate' is only valid for 'lookup' or "
+                            "'full-compaction' changelog producer");
+    }
+    {
+        std::map<std::string, std::string> options = {
+            {Options::CHANGELOG_PRODUCER, "input"},
+            {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, "f1"}};
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{"f0"}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "'changelog-producer.row-deduplicate-ignore-fields' is only valid when "
+                            "'changelog-producer.row-deduplicate' is true");
+    }
+    {
+        std::map<std::string, std::string> options = {
+            {Options::CHANGELOG_PRODUCER, "input"},
+            {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"},
+            {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, "missing"}};
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{"f0"}, options));
+        ASSERT_NOK_WITH_MSG(
+            SchemaValidation::ValidateTableSchema(*table_schema),
+            "Fields [\"missing\"] configured in "
+            "'changelog-producer.row-deduplicate-ignore-fields' can not be found in table schema");
+    }
     {
         auto invalid_field = arrow::field("_SEQUENCE_NUMBER", arrow::int64());
         arrow::FieldVector invalid_fields = fields;
@@ -897,15 +943,18 @@ TEST(SchemaValidationTest, ValidateInvalidConfiguration) {
                              TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
                                                  /*primary_keys=*/{"f0"}, options));
         ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
-                            "C++ Paimon does not support changelog-producer yet");
+                            "Only support 'none' and 'lookup' changelog-producer on FIRST_ROW "
+                            "merge engine");
     }
     {
-        std::map<std::string, std::string> options = {{Options::CHANGELOG_PRODUCER, "lookup"}};
+        std::map<std::string, std::string> options = {
+            {Options::CHANGELOG_PRODUCER, "full-compaction"}};
         ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
                              TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
                                                  /*primary_keys=*/{"f0"}, options));
-        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
-                            "C++ Paimon does not support changelog-producer yet");
+        ASSERT_NOK_WITH_MSG(
+            SchemaValidation::ValidateTableSchema(*table_schema),
+            "C++ Paimon only supports 'none', 'input' and 'lookup' changelog-producer now.");
     }
     // test for row tracking
     {
@@ -1182,6 +1231,16 @@ TEST(SchemaValidationTest, TestMapSharedShreddingCompression) {
                             "MAP shared-shredding only supports none/lz4/zstd compression, but "
                             "file.compression.per.level.1 is snappy.");
     }
+    {
+        auto options = base_options;
+        options[Options::CHANGELOG_FILE_COMPRESSION] = "snappy";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "MAP shared-shredding only supports none/lz4/zstd compression, but "
+                            "changelog-file.compression is snappy.");
+    }
     {
         auto options = base_options;
         options.erase("fields.f1.map.storage-layout");
@@ -1233,6 +1292,16 @@ TEST(SchemaValidationTest, TestMapSharedShreddingFileFormat) {
                             "MAP shared-shredding only supports parquet/orc file formats, but "
                             "file.format.per.level.1 is avro.");
     }
+    {
+        auto options = base_options;
+        options[Options::CHANGELOG_FILE_FORMAT] = "avro";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "MAP shared-shredding only supports parquet/orc file formats, but "
+                            "changelog-file.format is avro.");
+    }
 }
 
 TEST(SchemaValidationTest, TestMapSharedShreddingRejectsPostponeBucketMode) {
diff --git a/src/paimon/core/table/source/data_table_stream_scan.cpp b/src/paimon/core/table/source/data_table_stream_scan.cpp
index 4c6d4df11..1657c9c7b 100644
--- a/src/paimon/core/table/source/data_table_stream_scan.cpp
+++ b/src/paimon/core/table/source/data_table_stream_scan.cpp
@@ -26,6 +26,7 @@
 #include "paimon/core/options/changelog_producer.h"
 #include "paimon/core/table/bucket_mode.h"
 #include "paimon/core/table/source/plan_impl.h"
+#include "paimon/core/table/source/snapshot/changelog_follow_up_scanner.h"
 #include "paimon/core/table/source/snapshot/delta_follow_up_scanner.h"
 #include "paimon/core/table/source/snapshot/follow_up_scanner.h"
 #include "paimon/core/table/source/snapshot/snapshot_reader.h"
@@ -55,10 +56,15 @@ Result<std::shared_ptr<Plan>> DataTableStreamScan::CreatePlan() {
 
 Result<std::shared_ptr<Plan>> DataTableStreamScan::TryFirstPlan() {
     std::shared_ptr<StartingScanner::ScanResult> scan_result;
-    if (core_options_.GetChangelogProducer() == ChangelogProducer::LOOKUP) {
-        return Status::NotImplemented("do not support lookup changelog producer");
-    } else if (core_options_.GetChangelogProducer() == ChangelogProducer::FULL_COMPACTION) {
+    if (core_options_.GetChangelogProducer() == ChangelogProducer::FULL_COMPACTION) {
         return Status::NotImplemented("do not support full compaction changelog producer");
+    } else if (core_options_.GetChangelogProducer() == ChangelogProducer::LOOKUP) {
+        // Level-0 files will be compacted later to produce changelog records. Exclude them from
+        // the initial full scan so that the same changes are not emitted both in the full phase
+        // and again in the incremental changelog phase.
+        snapshot_reader_->WithLevelFilter([](int32_t level) -> bool { return level > 0; });
+        PAIMON_ASSIGN_OR_RAISE(scan_result, starting_scanner_->Scan(snapshot_reader_));
+        snapshot_reader_->WithLevelFilter([](int32_t) -> bool { return true; });
     } else {
         PAIMON_ASSIGN_OR_RAISE(scan_result, starting_scanner_->Scan(snapshot_reader_));
     }
@@ -85,6 +91,9 @@ Result<std::shared_ptr<Plan>> DataTableStreamScan::NextPlan() {
             PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
                                    follow_up_scanner_->Scan(snapshot.value(), snapshot_reader_));
             next_snapshot_id_.value()++;
+            if (plan->Splits().empty()) {
+                continue;
+            }
             return plan;
         } else {
             next_snapshot_id_.value()++;
@@ -122,8 +131,19 @@ Result<std::optional<Snapshot>> DataTableStreamScan::GetNextSnapshot(
 
 Status DataTableStreamScan::InitScanner() {
     PAIMON_ASSIGN_OR_RAISE(starting_scanner_, CreateStartingScanner(/*is_streaming=*/true));
-    follow_up_scanner_ = std::make_shared<DeltaFollowUpScanner>();
-    return Status::OK();
+    switch (core_options_.GetChangelogProducer()) {
+        case ChangelogProducer::NONE:
+            follow_up_scanner_ = std::make_shared<DeltaFollowUpScanner>();
+            return Status::OK();
+        case ChangelogProducer::INPUT:
+        case ChangelogProducer::LOOKUP:
+            follow_up_scanner_ = std::make_shared<ChangelogFollowUpScanner>();
+            return Status::OK();
+        case ChangelogProducer::FULL_COMPACTION:
+            return Status::NotImplemented("do not support full compaction changelog producer");
+        default:
+            return Status::NotImplemented("unknown changelog producer");
+    }
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/table/source/scan_mode.h b/src/paimon/core/table/source/scan_mode.h
index c236aad69..a01fde506 100644
--- a/src/paimon/core/table/source/scan_mode.h
+++ b/src/paimon/core/table/source/scan_mode.h
@@ -26,10 +26,10 @@ enum class ScanMode {
     ALL = 0,
 
     /// Only scan newly changed files of a snapshot.
-    DELTA = 1
+    DELTA = 1,
 
     /// Only scan changelog files of a snapshot.
-    /* CHANGELOG = 2 */
+    CHANGELOG = 2
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h b/src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h
new file mode 100644
index 000000000..ee2b73dfd
--- /dev/null
+++ b/src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "paimon/core/table/source/snapshot/follow_up_scanner.h"
+#include "paimon/logging.h"
+
+namespace paimon {
+
+/// Follow-up scanner for snapshots containing changelog manifests.
+class ChangelogFollowUpScanner : public FollowUpScanner {
+ public:
+    ChangelogFollowUpScanner() : logger_(Logger::GetLogger("ChangelogFollowUpScanner")) {}
+
+    bool NeedScanSnapshot(const Snapshot& snapshot) const override {
+        if (snapshot.ChangelogManifestList()) {
+            return true;
+        }
+        PAIMON_LOG_DEBUG(logger_, "Snapshot #%ld has no changelog, check the next snapshot.",
+                         snapshot.Id());
+        return false;
+    }
+
+    Result<std::shared_ptr<Plan>> Scan(
+        const Snapshot& snapshot,
+        const std::shared_ptr<SnapshotReader>& snapshot_reader) const override {
+        return snapshot_reader->WithMode(ScanMode::CHANGELOG)->WithSnapshot(snapshot)->Read();
+    }
+
+ private:
+    std::unique_ptr<Logger> logger_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp
index f3174fc6e..427d68fe2 100644
--- a/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp
+++ b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp
@@ -30,6 +30,8 @@
 #include "paimon/core/index/index_file_handler.h"
 #include "paimon/core/index/index_file_meta.h"
 #include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/table/source/snapshot/changelog_follow_up_scanner.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/index_file_path_factories.h"
 #include "paimon/fs/local/local_file_system.h"
@@ -112,4 +114,23 @@ TEST_F(SnapshotReaderTest, GetDeletionFilesOverwritesDuplicateDataFileName) {
     EXPECT_EQ(deletion_files[0]->cardinality, std::optional<int64_t>(4));
 }
 
+TEST_F(SnapshotReaderTest, ChangelogFollowUpScannerSkipsSnapshotsWithoutChangelog) {
+    auto create_snapshot = [](const std::optional<std::string>& changelog_manifest_list) {
+        return Snapshot(
+            /*id=*/1, /*schema_id=*/0, /*base_manifest_list=*/"",
+            /*base_manifest_list_size=*/std::nullopt, /*delta_manifest_list=*/"",
+            /*delta_manifest_list_size=*/std::nullopt, changelog_manifest_list,
+            /*changelog_manifest_list_size=*/std::nullopt,
+            /*index_manifest=*/std::nullopt, /*commit_user=*/"user", /*commit_identifier=*/1,
+            Snapshot::CommitKind::Append(), /*time_millis=*/0, /*total_record_count=*/0,
+            /*delta_record_count=*/0, /*changelog_record_count=*/std::nullopt,
+            /*watermark=*/std::nullopt, /*statistics=*/std::nullopt,
+            /*properties=*/std::nullopt, /*next_row_id=*/std::nullopt);
+    };
+
+    ChangelogFollowUpScanner scanner;
+    ASSERT_FALSE(scanner.NeedScanSnapshot(create_snapshot(std::nullopt)));
+    ASSERT_TRUE(scanner.NeedScanSnapshot(create_snapshot("changelog-manifest-list")));
+}
+
 }  // namespace paimon::test
diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp
index b33fcf36b..f626dba6b 100644
--- a/test/inte/scan_inte_test.cpp
+++ b/test/inte/scan_inte_test.cpp
@@ -1273,9 +1273,9 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) {
                                                      .value());
 
     std::vector<std::vector<std::shared_ptr<DataSplitImpl>>> expected_data_splits = {
-        {}, {expected_data_split1_2}, {expected_data_split2_1}, {}, {expected_data_split4_1}};
+        {}, {expected_data_split1_2}, {expected_data_split2_1}, {expected_data_split4_1}};
 
-    std::vector<std::optional<int64_t>> expected_snapshot_ids = {std::nullopt, 1, 2, 3, 4};
+    std::vector<std::optional<int64_t>> expected_snapshot_ids = {std::nullopt, 1, 2, 4};
     CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, expected_data_splits);
 }
 
diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp
index 397566d03..43ed0d1fb 100644
--- a/test/inte/write_and_read_inte_test.cpp
+++ b/test/inte/write_and_read_inte_test.cpp
@@ -90,6 +90,25 @@ class WriteAndReadInteTest
         return new_options;
     }
 
+    std::string LookupTempDirectory() const {
+        return PathUtil::JoinPath(test_dir_, "tmp");
+    }
+
+    Result<std::unique_ptr<TestHelper>> CreateLookupTestHelper(
+        const std::shared_ptr<arrow::Schema>& schema,
+        const std::map<std::string, std::string>& options) const {
+        return TestHelper::Create(test_dir_, schema, /*partition_keys=*/{},
+                                  /*primary_keys=*/{"pk"}, options,
+                                  /*is_streaming_mode=*/true, /*ignore_if_exists=*/false,
+                                  LookupTempDirectory());
+    }
+
+    Result<std::unique_ptr<TestHelper>> CreateLookupTestHelper(
+        const std::string& table_path, const std::map<std::string, std::string>& options) const {
+        return TestHelper::Create(table_path, options, /*is_streaming_mode=*/true,
+                                  LookupTempDirectory());
+    }
+
     Status WriteNextSchema(const std::vector<DataField>& fields, int32_t highest_field_id,
                            const std::map<std::string, std::string>& options) const {
         return TestHelper::WriteNextSchema(dir_->GetFileSystem(),
@@ -129,6 +148,33 @@ class WriteAndReadInteTest
         return file_system->AtomicStore(schema_path, std::string(buffer.GetString()));
     }
 
+    Status CompactAndCommit(const std::string& table_path,
+                            const std::map<std::string, std::string>& options,
+                            int64_t commit_identifier) const {
+        WriteContextBuilder write_context_builder(table_path, "commit_user");
+        write_context_builder.WithTempDirectory(LookupTempDirectory());
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<WriteContext> write_context,
+            write_context_builder.SetOptions(options).WithStreamingMode(true).Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreWrite> file_store_write,
+                               FileStoreWrite::Create(std::move(write_context)));
+        PAIMON_RETURN_NOT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0,
+                                                       /*full_compaction=*/true));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::vector<std::shared_ptr<CommitMessage>> compact_messages,
+            file_store_write->PrepareCommit(/*wait_compaction=*/true, commit_identifier));
+        PAIMON_RETURN_NOT_OK(file_store_write->Close());
+        if (compact_messages.empty()) {
+            return Status::Invalid("expected compaction commit messages");
+        }
+        CommitContextBuilder commit_context_builder(table_path, "commit_user");
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> commit_context,
+                               commit_context_builder.SetOptions(options).Finish());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> file_store_commit,
+                               FileStoreCommit::Create(std::move(commit_context)));
+        return file_store_commit->Commit(compact_messages, commit_identifier);
+    }
+
     Result<bool> ReadAndCheckProjectedResult(const std::map<std::string, std::string>& options,
                                              const std::vector<std::string>& read_fields,
                                              const std::shared_ptr<arrow::DataType>& expected_type,
@@ -630,6 +676,594 @@ TEST_P(WriteAndReadInteTest, TestPKSimple) {
     ASSERT_TRUE(success);
 }
 
+TEST_P(WriteAndReadInteTest, TestInputChangelogStreamRead) {
+    arrow::FieldVector fields = {
+        arrow::field("pk", arrow::utf8()),
+        arrow::field("value", arrow::int32()),
+    };
+    auto [file_format, file_system] = GetParam();
+    std::map<std::string, std::string> options = {
+        {Options::MANIFEST_FORMAT, "avro"},  {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
+        {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "input"},
+    };
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+    ASSERT_OK_AND_ASSIGN(
+        auto helper,
+        TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{},
+                           /*primary_keys=*/{"pk"}, options, /*is_streaming_mode=*/true));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<Split>> initial_splits,
+                         helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt));
+    ASSERT_TRUE(initial_splits.empty());
+
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr<RecordBatch> batch,
+        TestHelper::MakeRecordBatch(
+            arrow::struct_(fields), R"([["Alice", 10], ["Bob", 20], ["Alice", 11], ["Bob", 21]])",
+            /*partition_map=*/{}, /*bucket=*/0,
+            {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_BEFORE,
+             RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<Split>> changelog_splits, helper->Scan());
+    ASSERT_TRUE(changelog_splits.empty());
+    ASSERT_OK_AND_ASSIGN(changelog_splits, helper->Scan());
+    ASSERT_FALSE(changelog_splits.empty());
+    auto expected_type = arrow::struct_({
+        arrow::field("_VALUE_KIND", arrow::int8()),
+        fields[0],
+        fields[1],
+    });
+    ASSERT_OK_AND_ASSIGN(bool success,
+                         helper->ReadAndCheckResult(expected_type, changelog_splits,
+                                                    R"([[0, "Alice", 10], [2, "Alice", 11],
+                                       [1, "Bob", 20], [3, "Bob", 21]])"));
+    ASSERT_TRUE(success);
+}
+
+TEST_P(WriteAndReadInteTest, TestLookupChangelogStreamRead) {
+    auto [file_format, file_system] = GetParam();
+    arrow::FieldVector fields = {
+        arrow::field("pk", arrow::utf8()),
+        arrow::field("value", arrow::int32()),
+    };
+    std::map<std::string, std::string> options = {
+        {Options::MANIFEST_FORMAT, "avro"},  {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
+        {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"},
+    };
+    ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> initial_batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar");
+    // Move the initial value to a high level so the next compaction must look it up.
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1));
+    helper.reset();
+    ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<Split>> initial_splits,
+                         helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt));
+    ASSERT_TRUE(initial_splits.empty());
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> update_batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2,
+                                     /*expected_commit_messages=*/std::nullopt));
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<Split>> changelog_splits, helper->Scan());
+    ASSERT_FALSE(changelog_splits.empty());
+    auto expected_type = arrow::struct_({
+        arrow::field("_VALUE_KIND", arrow::int8()),
+        fields[0],
+        fields[1],
+    });
+    ASSERT_OK_AND_ASSIGN(bool success,
+                         helper->ReadAndCheckResult(expected_type, changelog_splits,
+                                                    R"([[1, "Alice", 10], [2, "Alice", 20]])"));
+    ASSERT_TRUE(success);
+}
+
+TEST_P(WriteAndReadInteTest, TestLookupChangelogInitialFullScanExcludesLevelZero) {
+    auto [file_format, file_system] = GetParam();
+    arrow::FieldVector fields = {
+        arrow::field("pk", arrow::utf8()),
+        arrow::field("value", arrow::int32()),
+    };
+    std::map<std::string, std::string> options = {
+        {Options::MANIFEST_FORMAT, "avro"},  {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
+        {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"},
+    };
+    ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options));
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> initial_batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar");
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1));
+
+    helper.reset();
+    ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> update_batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<Split>> initial_full_splits,
+                         helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt));
+    ASSERT_FALSE(initial_full_splits.empty());
+    for (const auto& split : initial_full_splits) {
+        auto data_split = std::dynamic_pointer_cast<DataSplitImpl>(split);
+        ASSERT_TRUE(data_split);
+        for (const auto& file : data_split->DataFiles()) {
+            ASSERT_GT(file->level, 0);
+        }
+    }
+
+    auto expected_type = arrow::struct_({
+        arrow::field("_VALUE_KIND", arrow::int8()),
+        fields[0],
+        fields[1],
+    });
+    ASSERT_OK_AND_ASSIGN(
+        bool success,
+        helper->ReadAndCheckResult(expected_type, initial_full_splits, R"([[0, "Alice", 10]])"));
+    ASSERT_TRUE(success);
+
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3));
+
+    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<Split>> changelog_splits, helper->Scan());
+    ASSERT_FALSE(changelog_splits.empty());
+    ASSERT_OK_AND_ASSIGN(bool changelog_success,
+                         helper->ReadAndCheckResult(expected_type, changelog_splits,
+                                                    R"([[1, "Alice", 10], [2, "Alice", 20]])"));
+    ASSERT_TRUE(changelog_success);
+}
+
+TEST_P(WriteAndReadInteTest, TestLookupChangelogInsertUpdateDelete) {
+    auto [file_format, file_system] = GetParam();
+    arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()),
+                                 arrow::field("value", arrow::int32())};
+    std::map<std::string, std::string> options = {
+        {Options::MANIFEST_FORMAT, "avro"},  {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
+        {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}};
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+    ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options));
+    ASSERT_OK_AND_ASSIGN(
+        auto initial_batch,
+        TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10], ["Bob", 20]])",
+                                    /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+    std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar");
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1));
+
+    helper.reset();
+    ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(auto initial_splits,
+                         helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt));
+    ASSERT_TRUE(initial_splits.empty());
+    ASSERT_OK_AND_ASSIGN(
+        auto change_batch,
+        TestHelper::MakeRecordBatch(arrow::struct_(fields),
+                                    R"([["Alice", 11], ["Bob", 0], ["Carol", 30], ["Dave", 0]])",
+                                    /*partition_map=*/{}, /*bucket=*/0,
+                                    {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE,
+                                     RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(change_batch), /*commit_identifier=*/2,
+                                     /*expected_commit_messages=*/std::nullopt));
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3));
+
+    ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan());
+    ASSERT_FALSE(changelog_splits.empty());
+    auto expected_type =
+        arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]});
+    ASSERT_OK_AND_ASSIGN(bool success,
+                         helper->ReadAndCheckResult(expected_type, changelog_splits,
+                                                    R"([[1, "Alice", 10], [2, "Alice", 11],
+                                       [3, "Bob", 20], [0, "Carol", 30]])"));
+    ASSERT_TRUE(success);
+}
+
+TEST_P(WriteAndReadInteTest, TestLookupChangelogWithFirstRow) {
+    auto [file_format, file_system] = GetParam();
+    arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()),
+                                 arrow::field("value", arrow::int32())};
+    std::map<std::string, std::string> options = {
+        {Options::MANIFEST_FORMAT, "avro"},  {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
+        {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"},
+        {Options::MERGE_ENGINE, "first-row"}};
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+    ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options));
+    ASSERT_OK_AND_ASSIGN(auto initial_batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+    std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar");
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1));
+
+    helper.reset();
+    ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(auto initial_splits,
+                         helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt));
+    ASSERT_TRUE(initial_splits.empty());
+    ASSERT_OK_AND_ASSIGN(
+        auto change_batch,
+        TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20], ["Bob", 30]])",
+                                    /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(change_batch), /*commit_identifier=*/2,
+                                     /*expected_commit_messages=*/std::nullopt));
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3));
+
+    ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan());
+    ASSERT_FALSE(changelog_splits.empty());
+    auto expected_type =
+        arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]});
+    ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(expected_type, changelog_splits,
+                                                                  R"([[0, "Bob", 30]])"));
+    ASSERT_TRUE(success);
+}
+
+TEST_P(WriteAndReadInteTest, TestLookupChangelogWithDeletionVector) {
+    auto [file_format, file_system] = GetParam();
+    arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()),
+                                 arrow::field("value", arrow::int32())};
+    std::map<std::string, std::string> options = {
+        {Options::MANIFEST_FORMAT, "avro"},         {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"},        {Options::BUCKET, "1"},
+        {Options::FILE_SYSTEM, file_system},        {Options::CHANGELOG_PRODUCER, "lookup"},
+        {Options::DELETION_VECTORS_ENABLED, "true"}};
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+    ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options));
+    ASSERT_OK_AND_ASSIGN(auto initial_batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+    std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar");
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1));
+
+    helper.reset();
+    ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(auto initial_splits,
+                         helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt));
+    ASSERT_TRUE(initial_splits.empty());
+    ASSERT_OK_AND_ASSIGN(auto update_batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2,
+                                     /*expected_commit_messages=*/std::nullopt));
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3));
+
+    ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan());
+    ASSERT_FALSE(changelog_splits.empty());
+    auto expected_type =
+        arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]});
+    ASSERT_OK_AND_ASSIGN(bool changelog_success,
+                         helper->ReadAndCheckResult(expected_type, changelog_splits,
+                                                    R"([[1, "Alice", 10], [2, "Alice", 20]])"));
+    ASSERT_TRUE(changelog_success);
+
+    ASSERT_OK_AND_ASSIGN(auto batch_splits,
+                         helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt,
+                                         /*is_streaming=*/false));
+    ASSERT_OK_AND_ASSIGN(bool batch_success, helper->ReadAndCheckResult(expected_type, batch_splits,
+                                                                        R"([[0, "Alice", 20]])"));
+    ASSERT_TRUE(batch_success);
+}
+
+TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicate) {
+    auto [file_format, file_system] = GetParam();
+    arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()),
+                                 arrow::field("value", arrow::int32())};
+    std::map<std::string, std::string> options = {
+        {Options::MANIFEST_FORMAT, "avro"},
+        {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"},
+        {Options::BUCKET, "1"},
+        {Options::FILE_SYSTEM, file_system},
+        {Options::CHANGELOG_PRODUCER, "lookup"},
+        {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"}};
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+    ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options));
+    ASSERT_OK_AND_ASSIGN(auto initial_batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+    std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar");
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1));
+
+    helper.reset();
+    ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(auto initial_splits,
+                         helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt));
+    ASSERT_TRUE(initial_splits.empty());
+    ASSERT_OK_AND_ASSIGN(auto write_helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(auto unchanged_batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(write_helper->WriteAndCommit(std::move(unchanged_batch), /*commit_identifier=*/2,
+                                           /*expected_commit_messages=*/std::nullopt));
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3));
+    ASSERT_OK_AND_ASSIGN(auto empty_splits, helper->Scan());
+    ASSERT_TRUE(empty_splits.empty());
+
+    write_helper.reset();
+    ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(auto changed_batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(write_helper->WriteAndCommit(std::move(changed_batch), /*commit_identifier=*/4,
+                                           /*expected_commit_messages=*/std::nullopt));
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/5));
+    ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan());
+    ASSERT_FALSE(changelog_splits.empty());
+    auto expected_type =
+        arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]});
+    ASSERT_OK_AND_ASSIGN(bool success,
+                         helper->ReadAndCheckResult(expected_type, changelog_splits,
+                                                    R"([[1, "Alice", 10], [2, "Alice", 20]])"));
+    ASSERT_TRUE(success);
+}
+
+TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicateIgnoreFields) {
+    auto [file_format, file_system] = GetParam();
+    arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()),
+                                 arrow::field("value", arrow::int32()),
+                                 arrow::field("ignored", arrow::int32())};
+    std::map<std::string, std::string> options = {
+        {Options::MANIFEST_FORMAT, "avro"},
+        {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"},
+        {Options::BUCKET, "1"},
+        {Options::FILE_SYSTEM, file_system},
+        {Options::CHANGELOG_PRODUCER, "lookup"},
+        {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"},
+        {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, "ignored"}};
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+    ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options));
+    ASSERT_OK_AND_ASSIGN(auto initial_batch, TestHelper::MakeRecordBatch(
+                                                 arrow::struct_(fields), R"([["Alice", 10, 100]])",
+                                                 /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+    std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar");
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1));
+
+    helper.reset();
+    ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(auto initial_splits,
+                         helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt));
+    ASSERT_TRUE(initial_splits.empty());
+    ASSERT_OK_AND_ASSIGN(auto write_helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(
+        auto ignored_change_batch,
+        TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10, 200]])",
+                                    /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(write_helper->WriteAndCommit(std::move(ignored_change_batch),
+                                           /*commit_identifier=*/2,
+                                           /*expected_commit_messages=*/std::nullopt));
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3));
+    ASSERT_OK_AND_ASSIGN(auto empty_splits, helper->Scan());
+    ASSERT_TRUE(empty_splits.empty());
+
+    write_helper.reset();
+    ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(
+        auto real_change_batch,
+        TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20, 300]])",
+                                    /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(write_helper->WriteAndCommit(std::move(real_change_batch), /*commit_identifier=*/4,
+                                           /*expected_commit_messages=*/std::nullopt));
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/5));
+    ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan());
+    ASSERT_FALSE(changelog_splits.empty());
+    auto expected_type = arrow::struct_(
+        {arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1], fields[2]});
+    ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(
+                                           expected_type, changelog_splits,
+                                           R"([[1, "Alice", 10, 200], [2, "Alice", 20, 300]])"));
+    ASSERT_TRUE(success);
+}
+
+TEST_P(WriteAndReadInteTest, TestChangelogWithSchemaEvolution) {
+    auto [file_format, file_system] = GetParam();
+    arrow::FieldVector fields_v0 = {arrow::field("pk", arrow::utf8()),
+                                    arrow::field("value", arrow::int32())};
+    std::map<std::string, std::string> options = {
+        {Options::MANIFEST_FORMAT, "avro"},  {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
+        {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}};
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+    ASSERT_OK_AND_ASSIGN(auto write_helper,
+                         CreateLookupTestHelper(arrow::schema(fields_v0), options));
+    ASSERT_OK_AND_ASSIGN(auto initial_batch, TestHelper::MakeRecordBatch(
+                                                 arrow::struct_(fields_v0), R"([["Alice", 10]])",
+                                                 /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(write_helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0,
+                                           /*expected_commit_messages=*/std::nullopt));
+    std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar");
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1));
+
+    ASSERT_OK_AND_ASSIGN(auto scan_helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(auto initial_splits,
+                         scan_helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt));
+    ASSERT_TRUE(initial_splits.empty());
+
+    write_helper.reset();
+    ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(
+        auto batch_v0, TestHelper::MakeRecordBatch(arrow::struct_(fields_v0), R"([["Alice", 11]])",
+                                                   /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/2,
+                                           /*expected_commit_messages=*/std::nullopt));
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3));
+
+    arrow::FieldVector fields_v1 = {arrow::field("pk", arrow::utf8()),
+                                    arrow::field("value", arrow::int32()),
+                                    arrow::field("extra", arrow::utf8())};
+    ASSERT_OK(WriteNextSchema(
+        {DataField(0, fields_v1[0]), DataField(1, fields_v1[1]), DataField(2, fields_v1[2])},
+        /*highest_field_id=*/2, options));
+    write_helper.reset();
+    ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(auto batch_v1, TestHelper::MakeRecordBatch(
+                                            arrow::struct_(fields_v1), R"([["Alice", 12, "v1"]])",
+                                            /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/4,
+                                           /*expected_commit_messages=*/std::nullopt));
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/5));
+
+    arrow::FieldVector fields_v2 = {arrow::field("pk", arrow::utf8()),
+                                    arrow::field("value", arrow::int32()),
+                                    arrow::field("renamed_extra", arrow::utf8())};
+    ASSERT_OK(WriteNextSchema(
+        {DataField(0, fields_v2[0]), DataField(1, fields_v2[1]), DataField(2, fields_v2[2])},
+        /*highest_field_id=*/2, options));
+    write_helper.reset();
+    ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(auto batch_v2, TestHelper::MakeRecordBatch(
+                                            arrow::struct_(fields_v2), R"([["Alice", 13, "v2"]])",
+                                            /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v2), /*commit_identifier=*/6,
+                                           /*expected_commit_messages=*/std::nullopt));
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/7));
+
+    arrow::FieldVector fields_v3 = {arrow::field("pk", arrow::utf8()),
+                                    arrow::field("value", arrow::int64()),
+                                    arrow::field("renamed_extra", arrow::utf8())};
+    ASSERT_OK(WriteNextSchema(
+        {DataField(0, fields_v3[0]), DataField(1, fields_v3[1]), DataField(2, fields_v3[2])},
+        /*highest_field_id=*/2, options));
+    write_helper.reset();
+    ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(auto batch_v3, TestHelper::MakeRecordBatch(
+                                            arrow::struct_(fields_v3), R"([["Alice", 14, "v3"]])",
+                                            /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v3), /*commit_identifier=*/8,
+                                           /*expected_commit_messages=*/std::nullopt));
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/9));
+
+    auto expected_type = arrow::struct_(
+        {arrow::field("_VALUE_KIND", arrow::int8()), fields_v3[0], fields_v3[1], fields_v3[2]});
+    std::vector<std::string> expected_data = {
+        R"([[1, "Alice", 10, null], [2, "Alice", 11, null]])",
+        R"([[1, "Alice", 11, null], [2, "Alice", 12, "v1"]])",
+        R"([[1, "Alice", 12, "v1"], [2, "Alice", 13, "v2"]])",
+        R"([[1, "Alice", 13, "v2"], [2, "Alice", 14, "v3"]])"};
+    for (int64_t schema_id = 0; schema_id < static_cast<int64_t>(expected_data.size());
+         schema_id++) {
+        ASSERT_OK_AND_ASSIGN(auto changelog_splits, scan_helper->Scan());
+        ASSERT_FALSE(changelog_splits.empty());
+        for (const auto& split : changelog_splits) {
+            auto data_split = std::dynamic_pointer_cast<DataSplitImpl>(split);
+            ASSERT_TRUE(data_split);
+            for (const auto& file : data_split->DataFiles()) {
+                ASSERT_EQ(file->schema_id, schema_id);
+            }
+        }
+        ASSERT_OK_AND_ASSIGN(bool success, scan_helper->ReadAndCheckResult(
+                                               expected_type, changelog_splits,
+                                               expected_data[static_cast<size_t>(schema_id)]));
+        ASSERT_TRUE(success);
+    }
+}
+
+TEST_P(WriteAndReadInteTest, TestLookupChangelogWithExternalPath) {
+    auto [file_format, file_system] = GetParam();
+    if (file_system == "jindo") {
+        return;
+    }
+    std::unique_ptr<UniqueTestDirectory> external_dir = UniqueTestDirectory::Create(file_system);
+    ASSERT_TRUE(external_dir);
+    arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()),
+                                 arrow::field("value", arrow::int32())};
+    std::map<std::string, std::string> options = {
+        {Options::MANIFEST_FORMAT, "avro"},
+        {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"},
+        {Options::BUCKET, "1"},
+        {Options::FILE_SYSTEM, file_system},
+        {Options::CHANGELOG_PRODUCER, "lookup"},
+        {Options::DATA_FILE_EXTERNAL_PATHS, "FILE://" + external_dir->Str()},
+        {Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY, "round-robin"}};
+    ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options));
+    ASSERT_OK_AND_ASSIGN(auto initial_batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+    std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar");
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1));
+
+    helper.reset();
+    ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options));
+    ASSERT_OK_AND_ASSIGN(auto initial_splits,
+                         helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt));
+    ASSERT_TRUE(initial_splits.empty());
+    ASSERT_OK_AND_ASSIGN(auto update_batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2,
+                                     /*expected_commit_messages=*/std::nullopt));
+    ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3));
+
+    ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan());
+    ASSERT_FALSE(changelog_splits.empty());
+    bool found_external_changelog = false;
+    for (const auto& split : changelog_splits) {
+        auto data_split = std::dynamic_pointer_cast<DataSplitImpl>(split);
+        ASSERT_TRUE(data_split);
+        for (const auto& file : data_split->DataFiles()) {
+            ASSERT_TRUE(file->external_path.has_value());
+            ASSERT_TRUE(StringUtils::StartsWith(PathUtil::GetName(file->file_name), "changelog-"));
+            ASSERT_OK_AND_ASSIGN(
+                bool exists, external_dir->GetFileSystem()->Exists(file->external_path.value()));
+            ASSERT_TRUE(exists);
+            found_external_changelog = true;
+        }
+    }
+    ASSERT_TRUE(found_external_changelog);
+
+    auto expected_type =
+        arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]});
+    ASSERT_OK_AND_ASSIGN(bool success,
+                         helper->ReadAndCheckResult(expected_type, changelog_splits,
+                                                    R"([[1, "Alice", 10], [2, "Alice", 20]])"));
+    ASSERT_TRUE(success);
+}
+
 TEST_P(WriteAndReadInteTest, TestNestedType) {
     arrow::FieldVector fields = {
         arrow::field("f1", arrow::map(arrow::int8(), arrow::int16())),
@@ -2963,23 +3597,7 @@ TEST_P(WriteAndReadInteTest, TestAppendMapStorageLayoutSharedShreddingToDefaultC
     ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1_file3), commit_identifier++,
                                      /*expected_commit_messages=*/std::nullopt));
 
-    WriteContextBuilder write_context_builder(table_path, "commit_user");
-    ASSERT_OK_AND_ASSIGN(
-        auto write_context,
-        write_context_builder.SetOptions(options_v1).WithStreamingMode(true).Finish());
-    ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context)));
-    ASSERT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0,
-                                        /*full_compaction=*/true));
-    ASSERT_OK_AND_ASSIGN(auto compact_messages, file_store_write->PrepareCommit(
-                                                    /*wait_compaction=*/true, commit_identifier));
-    ASSERT_FALSE(compact_messages.empty());
-
-    CommitContextBuilder commit_context_builder(table_path, "commit_user");
-    ASSERT_OK_AND_ASSIGN(auto commit_context,
-                         commit_context_builder.SetOptions(options_v1).Finish());
-    ASSERT_OK_AND_ASSIGN(auto file_store_commit,
-                         FileStoreCommit::Create(std::move(commit_context)));
-    ASSERT_OK(file_store_commit->Commit(compact_messages, commit_identifier));
+    ASSERT_OK(CompactAndCommit(table_path, options_v1, commit_identifier));
 
     arrow::FieldVector expected_fields = fields;
     expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8()));