diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 8fb6c17d1..3de2b667e 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -275,6 +275,7 @@ set(PAIMON_CORE_SRCS core/io/data_file_meta_first_row_id_legacy_serializer.cpp core/io/data_file_meta.cpp core/io/data_file_meta_serializer.cpp + core/io/data_file_meta_write_cols_legacy_serializer.cpp core/io/data_file_path_factory.cpp core/io/data_file_index_writer.cpp core/io/file_index_options.cpp diff --git a/src/paimon/core/append/bucketed_append_compact_manager_test.cpp b/src/paimon/core/append/bucketed_append_compact_manager_test.cpp index ee56825b5..712dd3667 100644 --- a/src/paimon/core/append/bucketed_append_compact_manager_test.cpp +++ b/src/paimon/core/append/bucketed_append_compact_manager_test.cpp @@ -115,7 +115,7 @@ class BucketedAppendCompactManagerTest : public testing::Test { /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } std::shared_ptr NewNamedFile(const std::string& file_name, int64_t file_size, @@ -135,7 +135,7 @@ class BucketedAppendCompactManagerTest : public testing::Test { /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } std::shared_ptr CreateTestDvMaintainer( diff --git a/src/paimon/core/deletionvectors/deletion_vector_test.cpp b/src/paimon/core/deletionvectors/deletion_vector_test.cpp index daf2ecf1c..bff4b46a3 100644 --- a/src/paimon/core/deletionvectors/deletion_vector_test.cpp +++ b/src/paimon/core/deletionvectors/deletion_vector_test.cpp @@ -54,7 +54,7 @@ std::shared_ptr CreateDataFileMeta(const std::string& file_name) { /*min_sequence_number=*/0, /*max_sequence_number=*/0, /*schema_id=*/0, DataFileMeta::DUMMY_LEVEL, std::vector>{}, Timestamp(0, 0), std::nullopt, nullptr, FileSource::Append(), std::nullopt, std::nullopt, std::nullopt, - std::nullopt); + std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } } // namespace diff --git a/src/paimon/core/global_index/indexed_split_test.cpp b/src/paimon/core/global_index/indexed_split_test.cpp index 857f50bfa..7cd121257 100644 --- a/src/paimon/core/global_index/indexed_split_test.cpp +++ b/src/paimon/core/global_index/indexed_split_test.cpp @@ -56,17 +56,20 @@ TEST(IndexedSplitTest, TestSimple) { "file1.orc", 100l, 200l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 50l, 249l, 0, 0, std::vector>(), Timestamp(1765535214349l, 0), 0, nullptr, - FileSource::Append(), std::nullopt, std::nullopt, 50l, std::nullopt); + FileSource::Append(), std::nullopt, std::nullopt, 50l, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto meta2 = std::make_shared( "file2.orc", 101l, 100l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 250l, 349l, 0, 0, std::vector>(), Timestamp(1765535214349l, 0), 0, nullptr, - FileSource::Append(), std::nullopt, std::nullopt, 250l, std::nullopt); + FileSource::Append(), std::nullopt, std::nullopt, 250l, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto meta3 = std::make_shared( "file3.orc", 102l, 200l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 1000l, 1199l, 0, 0, std::vector>(), Timestamp(1765535214349l, 0), 0, nullptr, - FileSource::Append(), std::nullopt, std::nullopt, 1000l, std::nullopt); + FileSource::Append(), std::nullopt, std::nullopt, 1000l, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRow::EmptyRow(), @@ -82,8 +85,11 @@ TEST(IndexedSplitTest, TestSimple) { ASSERT_EQ(*result_indexed_split, *expected_indexed_split) << result_indexed_split->ToString(); ASSERT_OK_AND_ASSIGN(std::string serialize_bytes, Split::Serialize(result_indexed_split, pool)); - ASSERT_EQ(serialize_bytes, - std::string(reinterpret_cast(split_bytes.data()), split_bytes.size())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr roundtrip, + Split::Deserialize(serialize_bytes.data(), serialize_bytes.size(), pool)); + auto roundtrip_indexed_split = std::dynamic_pointer_cast(roundtrip); + ASSERT_EQ(*roundtrip_indexed_split, *expected_indexed_split) + << roundtrip_indexed_split->ToString(); } TEST(IndexedSplitTest, TestIndexedSplitWithScore) { @@ -107,17 +113,20 @@ TEST(IndexedSplitTest, TestIndexedSplitWithScore) { "file1.orc", 100l, 200l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 50l, 249l, 0, 0, std::vector>(), Timestamp(1765549435648l, 0), 0, nullptr, - FileSource::Append(), std::nullopt, std::nullopt, 50l, std::nullopt); + FileSource::Append(), std::nullopt, std::nullopt, 50l, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto meta2 = std::make_shared( "file2.orc", 101l, 100l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 250l, 349l, 0, 0, std::vector>(), Timestamp(1765549435649l, 0), 0, nullptr, - FileSource::Append(), std::nullopt, std::nullopt, 250l, std::nullopt); + FileSource::Append(), std::nullopt, std::nullopt, 250l, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto meta3 = std::make_shared( "file3.orc", 102l, 200l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 1000l, 1199l, 0, 0, std::vector>(), Timestamp(1765549435649l, 0), 0, nullptr, - FileSource::Append(), std::nullopt, std::nullopt, 1000l, std::nullopt); + FileSource::Append(), std::nullopt, std::nullopt, 1000l, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRow::EmptyRow(), @@ -139,8 +148,11 @@ TEST(IndexedSplitTest, TestIndexedSplitWithScore) { "rowRanges=[[55, 56],[270, 270],[1001, 1002]], scores=[1.01,2.1,-1.32,4.23,50.74]") != std::string::npos); ASSERT_OK_AND_ASSIGN(std::string serialize_bytes, Split::Serialize(result_indexed_split, pool)); - ASSERT_EQ(serialize_bytes, - std::string(reinterpret_cast(split_bytes.data()), split_bytes.size())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr roundtrip, + Split::Deserialize(serialize_bytes.data(), serialize_bytes.size(), pool)); + auto roundtrip_indexed_split = std::dynamic_pointer_cast(roundtrip); + ASSERT_EQ(*roundtrip_indexed_split, *expected_indexed_split) + << roundtrip_indexed_split->ToString(); } TEST(IndexedSplitTest, TestValidate) { @@ -148,7 +160,8 @@ TEST(IndexedSplitTest, TestValidate) { "file.orc", 1l, 200l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 1000l, 1199l, 0, 0, std::vector>(), Timestamp(0l, 0), 0, nullptr, - FileSource::Append(), std::nullopt, std::nullopt, 1000l, std::nullopt); + FileSource::Append(), std::nullopt, std::nullopt, 1000l, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRow::EmptyRow(), diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp index d01f18c44..ec65f4ff1 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp @@ -53,7 +53,7 @@ class PkSortedBucketIndexStateTest : public ::testing::Test { /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, file_source, /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } /// Builds a payload whose source metadata lists the given sources in the given order. diff --git a/src/paimon/core/io/data_file_meta.cpp b/src/paimon/core/io/data_file_meta.cpp index 7ce767f4b..cd8666098 100644 --- a/src/paimon/core/io/data_file_meta.cpp +++ b/src/paimon/core/io/data_file_meta.cpp @@ -72,7 +72,8 @@ Result> DataFileMeta::ForAppend( file_name, file_size, row_count, EmptyMinKey(), EmptyMaxKey(), SimpleStats::EmptyStats(), row_stats, min_sequence_number, max_sequence_number, schema_id, DUMMY_LEVEL, extra_files, Timestamp(/*millisecond=*/local_micro / 1000, /*nano_of_millisecond=*/0), 0ll, - embedded_index, file_source, value_stats_cols, external_path, first_row_id, write_cols); + embedded_index, file_source, value_stats_cols, external_path, first_row_id, write_cols, + /*column_max_sequence_numbers=*/std::nullopt); } Result> DataFileMeta::Upgrade(int32_t new_level) const { @@ -84,7 +85,7 @@ Result> DataFileMeta::Upgrade(int32_t new_level) c file_name, file_size, row_count, min_key, max_key, key_stats, value_stats, min_sequence_number, max_sequence_number, schema_id, new_level, extra_files, creation_time, delete_row_count, embedded_index, file_source, value_stats_cols, external_path, - first_row_id, write_cols); + first_row_id, write_cols, column_max_sequence_numbers); } std::shared_ptr DataFileMeta::CopyWithExtraFiles( @@ -93,7 +94,16 @@ std::shared_ptr DataFileMeta::CopyWithExtraFiles( file_name, file_size, row_count, min_key, max_key, key_stats, value_stats, min_sequence_number, max_sequence_number, schema_id, level, new_extra_files, creation_time, delete_row_count, embedded_index, file_source, value_stats_cols, external_path, - first_row_id, write_cols); + first_row_id, write_cols, column_max_sequence_numbers); +} + +std::shared_ptr DataFileMeta::CopyWithColumnMaxSequenceNumbers( + const std::optional>& new_column_max_sequence_numbers) const { + return std::make_shared( + file_name, file_size, row_count, min_key, max_key, key_stats, value_stats, + min_sequence_number, max_sequence_number, schema_id, level, extra_files, creation_time, + delete_row_count, embedded_index, file_source, value_stats_cols, external_path, + first_row_id, write_cols, new_column_max_sequence_numbers); } std::shared_ptr DataFileMeta::CopyWithoutStats() const { @@ -101,7 +111,7 @@ std::shared_ptr DataFileMeta::CopyWithoutStats() const { file_name, file_size, row_count, min_key, max_key, key_stats, SimpleStats::EmptyStats(), min_sequence_number, max_sequence_number, schema_id, level, extra_files, creation_time, delete_row_count, embedded_index, file_source, std::vector(), external_path, - first_row_id, write_cols); + first_row_id, write_cols, column_max_sequence_numbers); } DataFileMeta::DataFileMeta( @@ -113,7 +123,8 @@ DataFileMeta::DataFileMeta( const std::shared_ptr& _embedded_index, const std::optional& _file_source, const std::optional>& _value_stats_cols, const std::optional& _external_path, const std::optional& _first_row_id, - const std::optional>& _write_cols) + const std::optional>& _write_cols, + const std::optional>& _column_max_sequence_numbers) : file_name(_file_name), file_size(_file_size), row_count(_row_count), @@ -133,7 +144,8 @@ DataFileMeta::DataFileMeta( value_stats_cols(_value_stats_cols), external_path(_external_path), first_row_id(_first_row_id), - write_cols(_write_cols) {} + write_cols(_write_cols), + column_max_sequence_numbers(_column_max_sequence_numbers) {} Result DataFileMeta::FileFormat() const { size_t last_dot_index = file_name.find_last_of("."); @@ -198,7 +210,8 @@ bool DataFileMeta::operator==(const DataFileMeta& other) const { creation_time == other.creation_time && delete_row_count == other.delete_row_count && file_source == other.file_source && value_stats_cols == other.value_stats_cols && external_path == other.external_path && first_row_id == other.first_row_id && - write_cols == other.write_cols; + write_cols == other.write_cols && + column_max_sequence_numbers == other.column_max_sequence_numbers; } bool DataFileMeta::operator!=(const DataFileMeta& other) const { @@ -243,7 +256,8 @@ bool DataFileMeta::TEST_Equal(const DataFileMeta& other) const { level == other.level && delete_row_count == other.delete_row_count && file_source == other.file_source && value_stats_cols == other.value_stats_cols && compare_optional_ignore_name(external_path, other.external_path) && - first_row_id == other.first_row_id && write_cols == other.write_cols; + first_row_id == other.first_row_id && write_cols == other.write_cols && + column_max_sequence_numbers == other.column_max_sequence_numbers; } std::string DataFileMeta::ToString() const { @@ -261,7 +275,8 @@ std::string DataFileMeta::ToString() const { "{}, " "keyStats: {}, valueStats: {}, minSequenceNumber: {}, maxSequenceNumber: {}, schemaId: " "{}, level: {}, extraFiles: {}, creationTime: {}, deleteRowCount: {}, fileSource: {}, " - "valueStatsCols: {}, externalPath: {}, firstRowId: {}, writeCols: {}}}", + "valueStatsCols: {}, externalPath: {}, firstRowId: {}, writeCols: {}, " + "columnMaxSequenceNumbers: {}}}", file_name, file_size, row_count, embedded_index == nullptr ? "null" : std::string(embedded_index->data(), embedded_index->size()), @@ -275,7 +290,10 @@ std::string DataFileMeta::ToString() const { : fmt::format("{}", fmt::join(value_stats_cols.value(), ", ")), external_path == std::nullopt ? "null" : external_path.value(), first_row_id == std::nullopt ? "null" : std::to_string(first_row_id.value()), - write_cols == std::nullopt ? "null" : fmt::format("{}", write_cols.value())); + write_cols == std::nullopt ? "null" : fmt::format("{}", write_cols.value()), + column_max_sequence_numbers == std::nullopt + ? "null" + : fmt::format("{}", column_max_sequence_numbers.value())); } int64_t DataFileMeta::GetMaxSequenceNumber( @@ -315,6 +333,9 @@ const std::shared_ptr& DataFileMeta::DataType() { arrow::field("_FIRST_ROW_ID", arrow::int64(), /*nullable=*/true), arrow::field("_WRITE_COLS", arrow::list(arrow::field("item", arrow::utf8(), /*nullable=*/false)), + /*nullable=*/true), + arrow::field("_WRITE_COLS_SEQUENCES", + arrow::list(arrow::field("item", arrow::int64(), /*nullable=*/false)), /*nullable=*/true)}); return schema; } diff --git a/src/paimon/core/io/data_file_meta.h b/src/paimon/core/io/data_file_meta.h index 98aaa0ee7..436d73785 100644 --- a/src/paimon/core/io/data_file_meta.h +++ b/src/paimon/core/io/data_file_meta.h @@ -58,7 +58,8 @@ struct DataFileMeta { const std::optional>& _value_stats_cols, const std::optional& _external_path, const std::optional& _first_row_id, - const std::optional>& _write_cols); + const std::optional>& _write_cols, + const std::optional>& _column_max_sequence_numbers); static Result> ForAppend( const std::string& file_name, int64_t file_size, int64_t row_count, @@ -83,6 +84,9 @@ struct DataFileMeta { std::shared_ptr CopyWithExtraFiles( const std::vector>& new_extra_files) const; + std::shared_ptr CopyWithColumnMaxSequenceNumbers( + const std::optional>& new_column_max_sequence_numbers) const; + /// Create a copy without value statistics. All other metadata is preserved. /// /// @return A new metadata object with empty value statistics and value-stat columns. @@ -167,5 +171,12 @@ struct DataFileMeta { std::optional first_row_id; std::optional> write_cols; + + /// Maximum sequence number per physical table field after data-evolution compaction. + /// + /// Values follow the table-field order selected by `write_cols` when it is non-null (system + /// fields are ignored), or the file schema field order otherwise. A null value means that only + /// the file-level sequence range is available. + std::optional> column_max_sequence_numbers; }; } // namespace paimon diff --git a/src/paimon/core/io/data_file_meta_09_serializer.cpp b/src/paimon/core/io/data_file_meta_09_serializer.cpp index 7eaffc911..7eae41d13 100644 --- a/src/paimon/core/io/data_file_meta_09_serializer.cpp +++ b/src/paimon/core/io/data_file_meta_09_serializer.cpp @@ -114,7 +114,7 @@ Result> DataFileMeta09Serializer::FromRow( embedded_file_index, file_source, /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } } // namespace paimon diff --git a/src/paimon/core/io/data_file_meta_10_serializer.cpp b/src/paimon/core/io/data_file_meta_10_serializer.cpp index fea60eeb5..bf99f99d8 100644 --- a/src/paimon/core/io/data_file_meta_10_serializer.cpp +++ b/src/paimon/core/io/data_file_meta_10_serializer.cpp @@ -123,7 +123,8 @@ Result> DataFileMeta10Serializer::FromRow( min_sequence_number, max_sequence_number, schema_id, level, InternalRowUtils::FromStringArrayData(extra_files.get()), creation_time, delete_row_count, embedded_file_index, file_source, std::optional>(value_stats_cols), - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); } } // namespace paimon diff --git a/src/paimon/core/io/data_file_meta_12_serializer.cpp b/src/paimon/core/io/data_file_meta_12_serializer.cpp index fe46a85c8..07fa1353a 100644 --- a/src/paimon/core/io/data_file_meta_12_serializer.cpp +++ b/src/paimon/core/io/data_file_meta_12_serializer.cpp @@ -128,7 +128,8 @@ Result> DataFileMeta12Serializer::FromRow( min_sequence_number, max_sequence_number, schema_id, level, InternalRowUtils::FromStringArrayData(extra_files.get()), creation_time, delete_row_count, embedded_file_index, file_source, std::optional>(value_stats_cols), - external_path, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + external_path, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); } } // namespace paimon diff --git a/src/paimon/core/io/data_file_meta_first_row_id_legacy_serializer.cpp b/src/paimon/core/io/data_file_meta_first_row_id_legacy_serializer.cpp index a834e47f0..3e82290d6 100644 --- a/src/paimon/core/io/data_file_meta_first_row_id_legacy_serializer.cpp +++ b/src/paimon/core/io/data_file_meta_first_row_id_legacy_serializer.cpp @@ -134,7 +134,8 @@ Result> DataFileMetaFirstRowIdLegacySerializer::Fr min_sequence_number, max_sequence_number, schema_id, level, InternalRowUtils::FromStringArrayData(extra_files.get()), creation_time, delete_row_count, embedded_file_index, file_source, std::optional>(value_stats_cols), - external_path, first_row_id, /*write_cols=*/std::nullopt); + external_path, first_row_id, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); } } // namespace paimon diff --git a/src/paimon/core/io/data_file_meta_serializer.cpp b/src/paimon/core/io/data_file_meta_serializer.cpp index 560299684..9f88ebec0 100644 --- a/src/paimon/core/io/data_file_meta_serializer.cpp +++ b/src/paimon/core/io/data_file_meta_serializer.cpp @@ -24,6 +24,7 @@ #include #include +#include "paimon/common/data/binary_array.h" #include "paimon/common/data/binary_row_writer.h" #include "paimon/common/data/binary_string.h" #include "paimon/common/data/internal_row.h" @@ -41,7 +42,7 @@ class InternalArray; class MemoryPool; Result DataFileMetaSerializer::ToRow(const std::shared_ptr& meta) const { - BinaryRow row(20); + BinaryRow row(NumFields()); BinaryRowWriter writer(&row, 32 * 1024, pool_.get()); writer.WriteString(0, BinaryString::FromString(meta->file_name, pool_.get())); writer.WriteLong(1, meta->file_size); @@ -95,6 +96,12 @@ Result DataFileMetaSerializer::ToRow(const std::shared_ptrwrite_cols.value(), pool_)); } + if (meta->column_max_sequence_numbers == std::nullopt) { + writer.SetNullAt(20); + } else { + writer.WriteArray( + 20, BinaryArray::FromLongArray(meta->column_max_sequence_numbers.value(), pool_.get())); + } writer.Complete(); return row; } @@ -160,6 +167,15 @@ Result> DataFileMetaSerializer::FromRow( } write_cols = InternalRowUtils::FromNotNullStringArrayData(array.get()); } + + std::optional> column_max_sequence_numbers; + if (!row.IsNullAt(20)) { + std::shared_ptr array = row.GetArray(20); + if (array == nullptr) { + return Status::Invalid("invalid column max sequence numbers"); + } + PAIMON_ASSIGN_OR_RAISE(column_max_sequence_numbers, array->ToLongArray()); + } PAIMON_ASSIGN_OR_RAISE(BinaryRow min_values, SerializationUtils::DeserializeBinaryRow(min_key)); PAIMON_ASSIGN_OR_RAISE(BinaryRow max_values, SerializationUtils::DeserializeBinaryRow(max_key)); PAIMON_ASSIGN_OR_RAISE(SimpleStats key_stats, @@ -171,7 +187,7 @@ Result> DataFileMetaSerializer::FromRow( min_sequence_number, max_sequence_number, schema_id, level, InternalRowUtils::FromStringArrayData(extra_files.get()), creation_time, delete_row_count, embedded_file_index, file_source, std::optional>(value_stats_cols), - external_path, first_row_id, write_cols); + external_path, first_row_id, write_cols, column_max_sequence_numbers); } } // namespace paimon diff --git a/src/paimon/core/io/data_file_meta_serializer_test.cpp b/src/paimon/core/io/data_file_meta_serializer_test.cpp index 18ba25788..5fdc452b3 100644 --- a/src/paimon/core/io/data_file_meta_serializer_test.cpp +++ b/src/paimon/core/io/data_file_meta_serializer_test.cpp @@ -21,13 +21,16 @@ #include #include #include +#include #include "arrow/api.h" #include "arrow/array/builder_base.h" #include "gtest/gtest.h" #include "paimon/common/io/memory_segment_output_stream.h" #include "paimon/common/memory/memory_segment_utils.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/data_file_meta_write_cols_legacy_serializer.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/data/timestamp.h" #include "paimon/io/byte_array_input_stream.h" @@ -49,7 +52,9 @@ class DataFileMetaSerializerTest : public testing::Test { } private: - std::shared_ptr GetDataFileMeta() { + std::shared_ptr GetDataFileMeta( + const std::optional>& column_max_sequence_numbers = + std::vector{16, 32}) { return std::make_shared( "some_file_name", 1024, 8, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16, @@ -58,7 +63,8 @@ class DataFileMetaSerializerTest : public testing::Test { /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/3, /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*external_path=*/std::optional(), - /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + column_max_sequence_numbers); } const int32_t TRIES = 100; @@ -68,6 +74,7 @@ class DataFileMetaSerializerTest : public testing::Test { TEST_F(DataFileMetaSerializerTest, TestToFromRow) { DataFileMetaSerializer serializer(memory_pool_); + ASSERT_EQ(serializer.NumFields(), 21); auto expected = GetDataFileMeta(); for (int32_t i = 0; i < TRIES; i++) { ASSERT_OK_AND_ASSIGN(auto row, serializer.ToRow(expected)); @@ -76,6 +83,15 @@ TEST_F(DataFileMetaSerializerTest, TestToFromRow) { } } +TEST_F(DataFileMetaSerializerTest, TestLegacySerializerSchema) { + DataFileMetaWriteColsLegacySerializer serializer(memory_pool_); + ASSERT_EQ(serializer.NumFields(), 20); + auto legacy_type = + checked_pointer_cast(DataFileMetaWriteColsLegacySerializer::DataType()); + ASSERT_EQ(legacy_type->num_fields(), 20); + ASSERT_EQ(legacy_type->field(19)->name(), "_WRITE_COLS"); +} + TEST_F(DataFileMetaSerializerTest, TestSerialize) { DataFileMetaSerializer serializer(memory_pool_); auto expected = GetDataFileMeta(); diff --git a/src/paimon/core/io/data_file_meta_test.cpp b/src/paimon/core/io/data_file_meta_test.cpp index f7283f158..f76bc0ae8 100644 --- a/src/paimon/core/io/data_file_meta_test.cpp +++ b/src/paimon/core/io/data_file_meta_test.cpp @@ -37,7 +37,8 @@ TEST(DataFileMetaTest, TestCopyWithoutStats) { /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::vector({"f0", "f1"}), /*external_path=*/"file:/tmp/bucket-0/data-0.orc", /*first_row_id=*/100, - /*write_cols=*/std::vector({"f0"})); + /*write_cols=*/std::vector({"f0"}), + /*column_max_sequence_numbers=*/std::nullopt); std::shared_ptr result = file_meta->CopyWithoutStats(); @@ -68,7 +69,8 @@ TEST(DataFileMetaTest, TestAddRowCount) { /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); ASSERT_EQ(3, file_meta.AddRowCount().value()); // test null delete row count file_meta.delete_row_count = std::nullopt; @@ -85,7 +87,8 @@ TEST(DataFileMetaTest, TestFileFormat) { /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); ASSERT_OK_AND_ASSIGN(auto file_format, file_meta.FileFormat()); ASSERT_EQ("orc", file_format); file_meta.file_name = "data-80110e15-97b5-4bcf-ac09-6ca2659a4950-0.parquet"; @@ -107,7 +110,8 @@ TEST(DataFileMetaTest, TestExternalPathDir) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/"file:/tmp/bucket-0/data-80110e15-97b5-4bcf-ac09-6ca2659a4950-0.orc", - /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); ASSERT_EQ("file:/tmp/bucket-0", file_meta.ExternalPathDir().value()); file_meta.external_path = std::nullopt; ASSERT_EQ(std::nullopt, file_meta.ExternalPathDir()); @@ -123,7 +127,8 @@ TEST(DataFileMetaTest, TestGetMaxSequenceNumber) { /*creation_time=*/Timestamp(1737111915429ll, 0), /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-80110e15-97b5-4bcf-ac09-6ca2659a4950-1.orc", /*file_size=*/645, /*row_count=*/5, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(), SimpleStats::EmptyStats(), @@ -133,7 +138,8 @@ TEST(DataFileMetaTest, TestGetMaxSequenceNumber) { /*creation_time=*/Timestamp(1737111915429ll, 0), /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); ASSERT_EQ(4, DataFileMeta::GetMaxSequenceNumber({file_meta1})); ASSERT_EQ(10, DataFileMeta::GetMaxSequenceNumber({file_meta1, file_meta2})); ASSERT_EQ(-1, DataFileMeta::GetMaxSequenceNumber({})); @@ -152,7 +158,8 @@ TEST(DataFileMetaTest, TestNonNullFirstRowId) { /*creation_time=*/Timestamp(1737111915429ll, 0), /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); ASSERT_OK_AND_ASSIGN(int64_t first_row_id, file_meta->NonNullFirstRowId()); ASSERT_EQ(100, first_row_id); } @@ -167,7 +174,7 @@ TEST(DataFileMetaTest, TestNonNullFirstRowId) { /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); ASSERT_NOK_WITH_MSG(file_meta->NonNullFirstRowId(), "First row id of data-1.orc should not be null."); } @@ -184,7 +191,8 @@ TEST(DataFileMetaTest, TestToFileSelection) { /*creation_time=*/Timestamp(1737111915429ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); { ASSERT_OK_AND_ASSIGN(std::optional result, @@ -223,7 +231,8 @@ TEST(DataFileMetaTest, TestUpgrade) { /*creation_time=*/Timestamp(1737111915429ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); // test normal upgrade ASSERT_OK_AND_ASSIGN(auto new_file_meta, file_meta->Upgrade(10)); ASSERT_EQ(new_file_meta->level, 10); diff --git a/src/paimon/core/io/data_file_meta_write_cols_legacy_serializer.cpp b/src/paimon/core/io/data_file_meta_write_cols_legacy_serializer.cpp new file mode 100644 index 000000000..ac2d480b9 --- /dev/null +++ b/src/paimon/core/io/data_file_meta_write_cols_legacy_serializer.cpp @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/data_file_meta_write_cols_legacy_serializer.h" + +#include +#include +#include +#include + +#include "arrow/type.h" +#include "paimon/common/data/internal_row.h" +#include "paimon/common/utils/internal_row_utils.h" +#include "paimon/common/utils/serialization_utils.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/data/timestamp.h" +#include "paimon/status.h" + +namespace paimon { + +class Bytes; +class InternalArray; + +const std::shared_ptr& DataFileMetaWriteColsLegacySerializer::DataType() { + static std::shared_ptr schema = arrow::struct_( + {arrow::field("_FILE_NAME", arrow::utf8(), /*nullable=*/false), + arrow::field("_FILE_SIZE", arrow::int64(), /*nullable=*/false), + arrow::field("_ROW_COUNT", arrow::int64(), /*nullable=*/false), + arrow::field("_MIN_KEY", arrow::binary(), /*nullable=*/false), + arrow::field("_MAX_KEY", arrow::binary(), /*nullable=*/false), + arrow::field("_KEY_STATS", SimpleStats::DataType(), /*nullable=*/false), + arrow::field("_VALUE_STATS", SimpleStats::DataType(), /*nullable=*/false), + arrow::field("_MIN_SEQUENCE_NUMBER", arrow::int64(), /*nullable=*/false), + arrow::field("_MAX_SEQUENCE_NUMBER", arrow::int64(), /*nullable=*/false), + arrow::field("_SCHEMA_ID", arrow::int64(), /*nullable=*/false), + arrow::field("_LEVEL", arrow::int32(), /*nullable=*/false), + arrow::field("_EXTRA_FILES", + arrow::list(arrow::field("item", arrow::utf8(), /*nullable=*/false)), + /*nullable=*/false), + arrow::field("_CREATION_TIME", arrow::timestamp(arrow::TimeUnit::MILLI), + /*nullable=*/true), + arrow::field("_DELETE_ROW_COUNT", arrow::int64(), /*nullable=*/true), + arrow::field("_EMBEDDED_FILE_INDEX", arrow::binary(), /*nullable=*/true), + arrow::field("_FILE_SOURCE", arrow::int8(), /*nullable=*/true), + arrow::field("_VALUE_STATS_COLS", + arrow::list(arrow::field("item", arrow::utf8(), /*nullable=*/false)), + /*nullable=*/true), + arrow::field("_EXTERNAL_PATH", arrow::utf8(), /*nullable=*/true), + arrow::field("_FIRST_ROW_ID", arrow::int64(), /*nullable=*/true), + arrow::field("_WRITE_COLS", + arrow::list(arrow::field("item", arrow::utf8(), /*nullable=*/false)), + /*nullable=*/true)}); + return schema; +} + +Result DataFileMetaWriteColsLegacySerializer::ToRow( + const std::shared_ptr&) const { + assert(false); + return Status::Invalid("to row for DataFileMetaWriteColsLegacySerializer is invalid"); +} + +Result> DataFileMetaWriteColsLegacySerializer::FromRow( + const InternalRow& row) const { + auto file_name = row.GetString(0); + auto file_size = row.GetLong(1); + auto row_count = row.GetLong(2); + auto min_key = row.GetBinary(3); + auto max_key = row.GetBinary(4); + auto key_stats_row = row.GetRow(5, 3); + auto value_stats_row = row.GetRow(6, 3); + auto min_sequence_number = row.GetLong(7); + auto max_sequence_number = row.GetLong(8); + auto schema_id = row.GetLong(9); + auto level = row.GetInt(10); + std::shared_ptr extra_files = row.GetArray(11); + auto creation_time = row.GetTimestamp(12, 3); + + assert(min_key && max_key && key_stats_row && value_stats_row); + if (extra_files == nullptr) { + return Status::Invalid("extra files is empty"); + } + + std::optional delete_row_count; + if (!row.IsNullAt(13)) { + delete_row_count = row.GetLong(13); + } + std::shared_ptr embedded_file_index; + if (!row.IsNullAt(14)) { + embedded_file_index = row.GetBinary(14); + } + + std::optional file_source; + if (!row.IsNullAt(15)) { + PAIMON_ASSIGN_OR_RAISE(file_source, FileSource::FromByteValue(row.GetByte(15))); + } + + std::optional> value_stats_cols; + if (!row.IsNullAt(16)) { + std::shared_ptr array = row.GetArray(16); + if (array == nullptr) { + return Status::Invalid("invalid value stats cols"); + } + value_stats_cols = InternalRowUtils::FromNotNullStringArrayData(array.get()); + } + + std::optional external_path; + if (!row.IsNullAt(17)) { + external_path = row.GetString(17).ToString(); + } + std::optional first_row_id; + if (!row.IsNullAt(18)) { + first_row_id = row.GetLong(18); + } + + std::optional> write_cols; + if (!row.IsNullAt(19)) { + std::shared_ptr array = row.GetArray(19); + if (array == nullptr) { + return Status::Invalid("invalid write cols"); + } + write_cols = InternalRowUtils::FromNotNullStringArrayData(array.get()); + } + + PAIMON_ASSIGN_OR_RAISE(BinaryRow min_values, SerializationUtils::DeserializeBinaryRow(min_key)); + PAIMON_ASSIGN_OR_RAISE(BinaryRow max_values, SerializationUtils::DeserializeBinaryRow(max_key)); + PAIMON_ASSIGN_OR_RAISE(SimpleStats key_stats, + SimpleStats::FromRow(key_stats_row.get(), pool_.get())); + PAIMON_ASSIGN_OR_RAISE(SimpleStats value_stats, + SimpleStats::FromRow(value_stats_row.get(), pool_.get())); + return std::make_shared( + file_name.ToString(), file_size, row_count, min_values, max_values, key_stats, value_stats, + min_sequence_number, max_sequence_number, schema_id, level, + InternalRowUtils::FromStringArrayData(extra_files.get()), creation_time, delete_row_count, + embedded_file_index, file_source, std::optional>(value_stats_cols), + external_path, first_row_id, write_cols, + /*column_max_sequence_numbers=*/std::nullopt); +} + +} // namespace paimon diff --git a/src/paimon/core/io/data_file_meta_write_cols_legacy_serializer.h b/src/paimon/core/io/data_file_meta_write_cols_legacy_serializer.h new file mode 100644 index 000000000..0a08d28cc --- /dev/null +++ b/src/paimon/core/io/data_file_meta_write_cols_legacy_serializer.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "paimon/common/data/binary_row.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/utils/object_serializer.h" +#include "paimon/result.h" + +namespace arrow { +class DataType; +} // namespace arrow + +namespace paimon { +class InternalRow; +class MemoryPool; + +/// Legacy serializer for `DataFileMeta` before column sequence numbers were introduced. +class DataFileMetaWriteColsLegacySerializer + : public ObjectSerializer> { + public: + static const std::shared_ptr& DataType(); + + explicit DataFileMetaWriteColsLegacySerializer(const std::shared_ptr& pool) + : ObjectSerializer>(DataType(), pool) {} + + Result ToRow(const std::shared_ptr& meta) const override; + + Result> FromRow(const InternalRow& row) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/data_file_path_factory_test.cpp b/src/paimon/core/io/data_file_path_factory_test.cpp index 12618a2ec..6283876df 100644 --- a/src/paimon/core/io/data_file_path_factory_test.cpp +++ b/src/paimon/core/io/data_file_path_factory_test.cpp @@ -97,7 +97,7 @@ TEST_F(DataFilePathFactoryTest, TestToPath) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/"file:/test/bucket-0/example.txt", /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); ASSERT_EQ(factory_.ToPath(file_meta), "file:/test/bucket-0/example.txt"); } @@ -118,7 +118,7 @@ TEST_F(DataFilePathFactoryTest, TestToAlignedPath) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/"file:/test/bucket-0/data-0.txt", /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); ASSERT_EQ(factory_.ToAlignedPath("index-0", file_meta), "file:/test/bucket-0/index-0"); @@ -135,7 +135,8 @@ TEST_F(DataFilePathFactoryTest, TestCollectFiles) { /*creation_time=*/Timestamp(1737111915429ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); ASSERT_EQ(factory_.CollectFiles(file_meta), std::vector({"/tmp/data-0.txt"})); file_meta->extra_files = {"data-0.index", "data-1.index"}; diff --git a/src/paimon/core/io/file_index_evaluator_test.cpp b/src/paimon/core/io/file_index_evaluator_test.cpp index d9730ade1..1824564e6 100644 --- a/src/paimon/core/io/file_index_evaluator_test.cpp +++ b/src/paimon/core/io/file_index_evaluator_test.cpp @@ -319,7 +319,8 @@ TEST_F(FileIndexEvaluatorTest, TestEvaluateEmbeddingIndex) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/embedded_index, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); CheckResult(/*data_file_path_factory=*/nullptr, data_file_meta); } @@ -339,7 +340,8 @@ TEST_F(FileIndexEvaluatorTest, TestEvaluateNoEmbeddingIndex) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto data_file_path_factory = std::make_shared(); ASSERT_OK(data_file_path_factory->Init(path + "/bucket-0/", /*format_identifier=*/"orc", /*data_file_prefix=*/"data-", nullptr)); @@ -374,7 +376,8 @@ TEST_F(FileIndexEvaluatorTest, TestTimestampType) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto data_file_path_factory = std::make_shared(); ASSERT_OK(data_file_path_factory->Init(path + "/bucket-0/", /*format_identifier=*/"orc", /*data_file_prefix=*/"data-", nullptr)); @@ -408,7 +411,8 @@ TEST_F(FileIndexEvaluatorTest, TestInvalidEvaluate) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto predicate = PredicateBuilder::IsNull(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT); ASSERT_NOK_WITH_MSG( diff --git a/src/paimon/core/io/key_value_data_file_writer.cpp b/src/paimon/core/io/key_value_data_file_writer.cpp index 9c32e0674..881712ebe 100644 --- a/src/paimon/core/io/key_value_data_file_writer.cpp +++ b/src/paimon/core/io/key_value_data_file_writer.cpp @@ -110,7 +110,8 @@ Result> KeyValueDataFileWriter::GetResult() { file_index.extra_files, Timestamp(/*millisecond=*/local_micro / 1000, /*nano_of_millisecond=*/0), delete_row_count_, file_index.embedded_index, file_source_, /*value_stats_cols=*/std::nullopt, final_path, - /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); } Status KeyValueDataFileWriter::GenerateMinMaxKey(BinaryRow* min_key, BinaryRow* max_key) const { diff --git a/src/paimon/core/io/rolling_blob_file_writer_test.cpp b/src/paimon/core/io/rolling_blob_file_writer_test.cpp index 654c91266..b88f4a329 100644 --- a/src/paimon/core/io/rolling_blob_file_writer_test.cpp +++ b/src/paimon/core/io/rolling_blob_file_writer_test.cpp @@ -57,7 +57,8 @@ TEST_F(RollingBlobFileWriterTest, ValidateFileConsistency) { /*creation_time=*/Timestamp(1724090888706ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0, - /*write_cols=*/std::vector({"f0", "f1"})); + /*write_cols=*/std::vector({"f0", "f1"}), + /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-xxx.blob", /*file_size=*/764, /*row_count=*/3, @@ -70,7 +71,8 @@ TEST_F(RollingBlobFileWriterTest, ValidateFileConsistency) { /*creation_time=*/Timestamp(1724090888706ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0, - /*write_cols=*/std::vector({"blob"})); + /*write_cols=*/std::vector({"blob"}), + /*column_max_sequence_numbers=*/std::nullopt); auto file_meta3 = std::make_shared( "data-xxx.blob", /*file_size=*/3023, /*row_count=*/1, @@ -83,7 +85,8 @@ TEST_F(RollingBlobFileWriterTest, ValidateFileConsistency) { /*creation_time=*/Timestamp(1724090888706ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/3, - /*write_cols=*/std::vector({"blob"})); + /*write_cols=*/std::vector({"blob"}), + /*column_max_sequence_numbers=*/std::nullopt); ASSERT_OK(RollingBlobFileWriter::ValidateFileConsistency(file_meta1, {file_meta2, file_meta3})); ASSERT_NOK_WITH_MSG(RollingBlobFileWriter::ValidateFileConsistency(file_meta1, {file_meta2}), "This is a bug: The row count of main file and blob file does not match."); diff --git a/src/paimon/core/manifest/manifest_entry_serializer_test.cpp b/src/paimon/core/manifest/manifest_entry_serializer_test.cpp index 2d8cffc37..a31f4bf73 100644 --- a/src/paimon/core/manifest/manifest_entry_serializer_test.cpp +++ b/src/paimon/core/manifest/manifest_entry_serializer_test.cpp @@ -43,7 +43,8 @@ class ManifestEntrySerializerTest : public testing::Test { /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/3, /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*external_path=*/std::optional(), - /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); } }; TEST_F(ManifestEntrySerializerTest, TestToFromRow) { diff --git a/src/paimon/core/manifest/manifest_entry_writer_test.cpp b/src/paimon/core/manifest/manifest_entry_writer_test.cpp index 3c3d00d7e..6c9291b71 100644 --- a/src/paimon/core/manifest/manifest_entry_writer_test.cpp +++ b/src/paimon/core/manifest/manifest_entry_writer_test.cpp @@ -64,7 +64,7 @@ class ManifestEntryWriterTest : public ::testing::Test { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, first_row_id, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return {FileKind::Add(), BinaryRowGenerator::GenerateRow({10}, pool_.get()), /*bucket=*/0, /*total_buckets=*/-1, meta}; } @@ -126,7 +126,8 @@ TEST_F(ManifestEntryWriterTest, TestSimple) { /*creation_time=*/Timestamp(1743525392885ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto meta2 = std::make_shared( "data-5858a84b-7081-4618-b828-ae3918c5e1f6-0.orc", /*file_size=*/943, /*row_count=*/4, @@ -144,7 +145,8 @@ TEST_F(ManifestEntryWriterTest, TestSimple) { /*creation_time=*/Timestamp(1743525392921ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto entry1 = ManifestEntry(FileKind::Add(), BinaryRowGenerator::GenerateRow({10}, pool_.get()), 0, 2, meta1); diff --git a/src/paimon/core/manifest/manifest_file_test.cpp b/src/paimon/core/manifest/manifest_file_test.cpp index a34f41524..df78fc00d 100644 --- a/src/paimon/core/manifest/manifest_file_test.cpp +++ b/src/paimon/core/manifest/manifest_file_test.cpp @@ -153,7 +153,7 @@ TEST_F(ManifestFileTest, TestSimple) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto manifest_entry1 = ManifestEntry(FileKind::Delete(), BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*total_buckets=*/2, file_meta1); @@ -171,7 +171,7 @@ TEST_F(ManifestFileTest, TestSimple) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto manifest_entry2 = ManifestEntry(FileKind::Delete(), BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*total_buckets=*/2, file_meta2); @@ -189,7 +189,7 @@ TEST_F(ManifestFileTest, TestSimple) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto manifest_entry3 = ManifestEntry(FileKind::Delete(), BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*total_buckets=*/2, file_meta3); @@ -207,7 +207,7 @@ TEST_F(ManifestFileTest, TestSimple) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto manifest_entry4 = ManifestEntry(FileKind::Delete(), BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*total_buckets=*/2, file_meta4); @@ -225,7 +225,7 @@ TEST_F(ManifestFileTest, TestSimple) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto manifest_entry5 = ManifestEntry(FileKind::Add(), BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*total_buckets=*/2, file_meta5); @@ -437,7 +437,7 @@ TEST_F(ManifestFileTest, TestWithNullCount) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto manifest_entry1 = ManifestEntry(FileKind::Add(), BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*total_buckets=*/2, file_meta1); @@ -464,7 +464,7 @@ TEST_F(ManifestFileTest, TestWithNullCount) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto manifest_entry2 = ManifestEntry(FileKind::Add(), BinaryRowGenerator::GenerateRow({20}, pool.get()), /*bucket=*/0, /*total_buckets=*/2, file_meta2); @@ -501,7 +501,7 @@ TEST_F(ManifestFileTest, TestManifestFileCompatibleWithJavaPaimon09) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto manifest_entry = ManifestEntry(FileKind::Add(), /*partition=*/BinaryRow::EmptyRow(), /*bucket=*/0, /*total_buckets=*/-1, file_meta); @@ -540,7 +540,7 @@ TEST_F(ManifestFileTest, TestManifestFileCompatibleWithJavaPaimon11) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto manifest_entry = ManifestEntry(FileKind::Add(), /*partition=*/BinaryRow::EmptyRow(), /*bucket=*/0, /*total_buckets=*/-1, file_meta); diff --git a/src/paimon/core/manifest/partition_entry_test.cpp b/src/paimon/core/manifest/partition_entry_test.cpp index 8b7dd6269..213e0dbaa 100644 --- a/src/paimon/core/manifest/partition_entry_test.cpp +++ b/src/paimon/core/manifest/partition_entry_test.cpp @@ -50,7 +50,7 @@ class PartitionEntryTest : public testing::Test { /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } }; diff --git a/src/paimon/core/mergetree/compact/compact_strategy_test.cpp b/src/paimon/core/mergetree/compact/compact_strategy_test.cpp index a0abebe33..e8508a794 100644 --- a/src/paimon/core/mergetree/compact/compact_strategy_test.cpp +++ b/src/paimon/core/mergetree/compact/compact_strategy_test.cpp @@ -38,7 +38,7 @@ class CompactStrategyTest : public testing::Test { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return {level, SortedRun::FromSingle(file_meta)}; } diff --git a/src/paimon/core/mergetree/compact/early_full_compaction_test.cpp b/src/paimon/core/mergetree/compact/early_full_compaction_test.cpp index a4cc9c663..37cc061f4 100644 --- a/src/paimon/core/mergetree/compact/early_full_compaction_test.cpp +++ b/src/paimon/core/mergetree/compact/early_full_compaction_test.cpp @@ -55,7 +55,7 @@ class EarlyFullCompactionTest : public testing::Test { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return {level, SortedRun::FromSingle(file_meta)}; } diff --git a/src/paimon/core/mergetree/compact/force_up_level0_compaction_test.cpp b/src/paimon/core/mergetree/compact/force_up_level0_compaction_test.cpp index 18a11229c..c1d26a569 100644 --- a/src/paimon/core/mergetree/compact/force_up_level0_compaction_test.cpp +++ b/src/paimon/core/mergetree/compact/force_up_level0_compaction_test.cpp @@ -39,7 +39,7 @@ class ForceUpLevel0CompactionTest : public testing::Test { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return {level, SortedRun::FromSingle(file_meta)}; } diff --git a/src/paimon/core/mergetree/compact/interval_partition_test.cpp b/src/paimon/core/mergetree/compact/interval_partition_test.cpp index 861a35c4c..9c68000ca 100644 --- a/src/paimon/core/mergetree/compact/interval_partition_test.cpp +++ b/src/paimon/core/mergetree/compact/interval_partition_test.cpp @@ -125,7 +125,7 @@ class IntervalPartitionTest : public testing::Test { /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } private: diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp index eeb7e0ce2..169170da1 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp @@ -460,7 +460,8 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowRewrite) { BinaryRowGenerator::GenerateStats({1, 5}, {5, 33}, {0, 0}, pool_.get()), /*min_sequence_number=*/0l, /*max_sequence_number=*/3l, /*schema_id=*/0, /*level=*/5, std::vector>(), Timestamp(0l, 0), /*delete_row_count=*/0, - nullptr, FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt); + nullptr, FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); ASSERT_TRUE(expected_file_meta->TEST_Equal(*compact_file_meta)); // check compact file exist @@ -564,7 +565,8 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowUpgrade) { BinaryRowGenerator::GenerateStats({1, 5}, {5, 33}, {0, 0}, pool_.get()), /*min_sequence_number=*/0l, /*max_sequence_number=*/2l, /*schema_id=*/0, /*level=*/5, std::vector>(), Timestamp(0l, 0), /*delete_row_count=*/0, - nullptr, FileSource::Append(), std::nullopt, std::nullopt, std::nullopt, std::nullopt); + nullptr, FileSource::Append(), std::nullopt, std::nullopt, std::nullopt, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); ASSERT_TRUE(expected_file_meta->TEST_Equal(*compact_file_meta)) << compact_file_meta->ToString(); } @@ -1108,7 +1110,8 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { BinaryRowGenerator::GenerateStats({1, 5}, {5, 33}, {0, 0}, pool_.get()), /*min_sequence_number=*/0l, /*max_sequence_number=*/3l, /*schema_id=*/0, level, std::vector>(), Timestamp(0l, 0), delete_row_count, nullptr, - FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt); + FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); }; { std::map options = {}; diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp index 7473b7c0b..91e3bb1cc 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp @@ -60,7 +60,8 @@ class MergeTreeCompactManagerFactoryStrategyTest : public ::testing::Test { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, - /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); return {level, SortedRun::FromSingle(file_meta)}; } diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp index 19b8bc363..9186eb1c4 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp @@ -142,7 +142,7 @@ class TestRewriter final : public CompactRewriter { /*max_sequence_number=*/max_sequence, /*schema_id=*/0, output_level, std::vector>(), Timestamp(1, 0), std::nullopt, nullptr, FileSource::Append(), std::nullopt, - std::nullopt, std::nullopt, std::nullopt); + std::nullopt, std::nullopt, std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return CompactResult(before, {after}); } @@ -208,7 +208,7 @@ class MergeTreeCompactManagerTest : public testing::Test { /*max_sequence_number=*/max_sequence, /*schema_id=*/0, minmax.level, std::vector>(), Timestamp(1, 0), std::nullopt, nullptr, FileSource::Append(), std::nullopt, - std::nullopt, std::nullopt, std::nullopt); + std::nullopt, std::nullopt, std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } StrategyFn TestStrategy() { diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp index cb26540aa..dc19c4726 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp @@ -148,7 +148,8 @@ TEST_F(MergeTreeCompactRewriterTest, TestSimple) { pool_.get()), /*min_sequence_number=*/0l, /*max_sequence_number=*/10l, /*schema_id=*/0, /*level=*/5, std::vector>(), Timestamp(0l, 0), /*delete_row_count=*/0, - nullptr, FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt); + nullptr, FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); ASSERT_TRUE(expected_file_meta->TEST_Equal(*compact_file_meta)); // check compact file exist std::string compact_file_name = @@ -245,7 +246,8 @@ TEST_F(MergeTreeCompactRewriterTest, TestNotDropDelete) { pool_.get()), /*min_sequence_number=*/0l, /*max_sequence_number=*/11l, /*schema_id=*/0, /*level=*/5, std::vector>(), Timestamp(0l, 0), /*delete_row_count=*/2, - nullptr, FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt); + nullptr, FileSource::Compact(), std::nullopt, std::nullopt, std::nullopt, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); ASSERT_TRUE(expected_file_meta->TEST_Equal(*compact_file_meta)); std::string compact_file_name = diff --git a/src/paimon/core/mergetree/compact/universal_compaction_test.cpp b/src/paimon/core/mergetree/compact/universal_compaction_test.cpp index 74abacaef..9b2e6cd1f 100644 --- a/src/paimon/core/mergetree/compact/universal_compaction_test.cpp +++ b/src/paimon/core/mergetree/compact/universal_compaction_test.cpp @@ -56,7 +56,7 @@ class UniversalCompactionTest : public testing::Test { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return {level, SortedRun::FromSingle(file_meta)}; } diff --git a/src/paimon/core/mergetree/levels_test.cpp b/src/paimon/core/mergetree/levels_test.cpp index 37a347f92..d31a31c92 100644 --- a/src/paimon/core/mergetree/levels_test.cpp +++ b/src/paimon/core/mergetree/levels_test.cpp @@ -45,7 +45,7 @@ class LevelsTest : public testing::Test { max_sequence_number, /*schema_id=*/0, level, std::vector>(), Timestamp(ts_second, 0l), std::nullopt, nullptr, FileSource::Append(), std::nullopt, - std::nullopt, std::nullopt, std::nullopt); + std::nullopt, std::nullopt, std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } std::shared_ptr CreateComparator() const { @@ -261,7 +261,8 @@ TEST_F(LevelsTest, TestUpdateDropFileCallbackExcludesUpgradeFiles) { file_level0->key_stats, file_level0->value_stats, file_level0->min_sequence_number, file_level0->max_sequence_number, file_level0->schema_id, /*level=*/1, file_level0->extra_files, file_level0->creation_time, std::nullopt, nullptr, - FileSource::Append(), std::nullopt, std::nullopt, std::nullopt, std::nullopt); + FileSource::Append(), std::nullopt, std::nullopt, std::nullopt, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); std::vector> before = {file_level0}; std::vector> after = {upgraded_file}; diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp index 2155647a1..c5a114f36 100644 --- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp +++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp @@ -192,7 +192,8 @@ class MergeTreeWriterTest : public ::testing::TestWithParam { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, - /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); } Result> CreateMergeWriter( @@ -289,7 +290,7 @@ TEST_P(MergeTreeWriterTest, TestSimple) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{}, /*changelog_files=*/{}); ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement()); @@ -370,7 +371,7 @@ TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{}, /*changelog_files=*/{}); ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement()); @@ -478,7 +479,8 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) { /*creation_time=*/actual_meta->creation_time, /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, - /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); ASSERT_TRUE(expected_data_file_meta->TEST_Equal(*actual_meta)); } @@ -658,7 +660,7 @@ TEST_P(MergeTreeWriterTest, TestWriteWithDeleteRow) { /*delete_row_count=*/1, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{}, /*changelog_files=*/{}); ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement()); @@ -769,7 +771,7 @@ TEST_P(MergeTreeWriterTest, TestMultiplePrepareCommit) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto expected_data_file_meta2 = std::make_shared( expected_data_file_name2, /*file_size=*/data_file_status2.GetLen(), /*row_count=*/3, @@ -788,7 +790,7 @@ TEST_P(MergeTreeWriterTest, TestMultiplePrepareCommit) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement expected_data_increment1({expected_data_file_meta1}, /*deleted_files=*/{}, /*changelog_files=*/{}); @@ -991,7 +993,7 @@ TEST_P(MergeTreeWriterTest, TestAutoFlush) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto expected_data_file_meta2 = std::make_shared( expected_data_file_name2, /*file_size=*/data_file_status2.GetLen(), /*row_count=*/3, @@ -1010,7 +1012,7 @@ TEST_P(MergeTreeWriterTest, TestAutoFlush) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement expected_data_increment({expected_data_file_meta1, expected_data_file_meta2}, /*deleted_files=*/{}, /*changelog_files=*/{}); @@ -1124,7 +1126,7 @@ TEST_P(MergeTreeWriterTest, TestBulkData) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); ASSERT_EQ(*commit_increment.GetNewFilesIncrement().NewFiles()[i], *expected_data_file_meta); } } diff --git a/src/paimon/core/mergetree/sorted_run_test.cpp b/src/paimon/core/mergetree/sorted_run_test.cpp index 7a19fac36..fc4dd08d6 100644 --- a/src/paimon/core/mergetree/sorted_run_test.cpp +++ b/src/paimon/core/mergetree/sorted_run_test.cpp @@ -53,7 +53,7 @@ class SortedRunTest : public testing::Test { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } }; diff --git a/src/paimon/core/migrate/file_meta_utils_test.cpp b/src/paimon/core/migrate/file_meta_utils_test.cpp index ee30680d8..caca9d2fc 100644 --- a/src/paimon/core/migrate/file_meta_utils_test.cpp +++ b/src/paimon/core/migrate/file_meta_utils_test.cpp @@ -132,7 +132,7 @@ TEST_F(FileMetaUtilsTest, TestSimple) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-db2b44c0-0d73-449d-82a0-4075bd2cb6e3-0.orc", /*file_size=*/541, @@ -148,7 +148,7 @@ TEST_F(FileMetaUtilsTest, TestSimple) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); CommitMessageImpl expected(BinaryRow::EmptyRow(), /*bucket=*/0, /*total_buckets=*/-1, DataIncrement({file_meta1, file_meta2}, {}, {}), CompactIncrement({}, {}, {})); @@ -211,7 +211,7 @@ TEST_F(FileMetaUtilsTest, TestFailover) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-db2b44c0-0d73-449d-82a0-4075bd2cb6e3-0.orc", /*file_size=*/541, @@ -227,7 +227,7 @@ TEST_F(FileMetaUtilsTest, TestFailover) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); CommitMessageImpl expected(BinaryRow::EmptyRow(), /*bucket=*/0, /*total_buckets=*/-1, DataIncrement({file_meta1, file_meta2}, {}, {}), CompactIncrement({}, {}, {})); @@ -291,7 +291,7 @@ TEST_F(FileMetaUtilsTest, TestWithPartition) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-1c547e5f-48b2-4917-a996-71d306377661-0.orc", /*file_size=*/589, @@ -307,7 +307,7 @@ TEST_F(FileMetaUtilsTest, TestWithPartition) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); CommitMessageImpl expected(BinaryRowGenerator::GenerateRow({10, 0}, pool_.get()), /*bucket=*/0, /*total_buckets=*/-1, DataIncrement({file_meta1, file_meta2}, {}, {}), @@ -370,7 +370,7 @@ TEST_F(FileMetaUtilsTest, TestWithNestedType) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); CommitMessageImpl expected(BinaryRow::EmptyRow(), /*bucket=*/0, /*total_buckets=*/-1, DataIncrement({file_meta1}, {}, {}), CompactIncrement({}, {}, {})); diff --git a/src/paimon/core/operation/commit/commit_changes_provider_test.cpp b/src/paimon/core/operation/commit/commit_changes_provider_test.cpp index edbb1a48b..f47e11262 100644 --- a/src/paimon/core/operation/commit/commit_changes_provider_test.cpp +++ b/src/paimon/core/operation/commit/commit_changes_provider_test.cpp @@ -63,7 +63,7 @@ ManifestEntry CreateManifestEntry(const std::string& file_name, const FileKind& /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return ManifestEntry(kind, CreateIntRow(partition_value), /*bucket=*/0, /*total_buckets=*/1, file_meta); diff --git a/src/paimon/core/operation/commit/conflict_detection_test.cpp b/src/paimon/core/operation/commit/conflict_detection_test.cpp index 66d57894b..3fb751acf 100644 --- a/src/paimon/core/operation/commit/conflict_detection_test.cpp +++ b/src/paimon/core/operation/commit/conflict_detection_test.cpp @@ -175,7 +175,7 @@ class ConflictDetectionTest : public testing::Test { /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, /*external_path=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return ManifestEntry(kind, partition, bucket, total_buckets, data_file_meta); } @@ -194,7 +194,7 @@ class ConflictDetectionTest : public testing::Test { /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, /*external_path=*/std::nullopt, /*value_stats_cols=*/std::nullopt, first_row_id, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return ManifestEntry(kind, partition, bucket, /*total_buckets=*/2, data_file_meta); } diff --git a/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp b/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp index 8680ff2c3..5eb1608c4 100644 --- a/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp +++ b/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp @@ -67,7 +67,7 @@ class ManifestEntryChangesTest : public testing::Test { /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, /*external_path=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } std::shared_ptr CreateIndexFileMeta( @@ -140,7 +140,7 @@ TEST_F(ManifestEntryChangesTest, TestDropStatsOnlyForDeleteEntries) { /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, /*value_stats_cols=*/std::vector({"f0"}), /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr after, before->Upgrade(/*new_level=*/1)); CompactIncrement compact_increment(/*compact_before=*/{before}, /*compact_after=*/{after}, diff --git a/src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp b/src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp index 8130fe65c..3552b6da1 100644 --- a/src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp +++ b/src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp @@ -64,7 +64,7 @@ ManifestEntry CreateManifestEntry(const std::string& file_name, const FileKind& /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return ManifestEntry(kind, CreateIntRow(partition_value), /*bucket=*/0, /*total_buckets=*/1, file_meta); diff --git a/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp b/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp index e9091b0fc..907aea44c 100644 --- a/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp +++ b/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp @@ -57,7 +57,8 @@ class RowIdColumnConflictCheckerTest : public testing::Test { /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, - /*first_row_id=*/first_row_id, write_cols); + /*first_row_id=*/first_row_id, write_cols, + /*column_max_sequence_numbers=*/std::nullopt); } Result> CreateChecker( diff --git a/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp b/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp index 02e79fc56..7d2b0fb02 100644 --- a/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp +++ b/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp @@ -61,7 +61,8 @@ class RowTrackingCommitUtilsTest : public testing::Test { /*embedded_index=*/nullptr, file_source, /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, - /*first_row_id=*/std::nullopt, write_cols); + /*first_row_id=*/std::nullopt, write_cols, + /*column_max_sequence_numbers=*/std::nullopt); return ManifestEntry(FileKind::Add(), CreateIntRow(1), /*bucket=*/0, /*total_buckets=*/1, file_meta); } @@ -81,7 +82,8 @@ class RowTrackingCommitUtilsTest : public testing::Test { /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, file_source, /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, first_row_id, write_cols); + /*external_path=*/std::nullopt, first_row_id, write_cols, + /*column_max_sequence_numbers=*/std::nullopt); return ManifestEntry(FileKind::Add(), CreateIntRow(1), /*bucket=*/0, /*total_buckets=*/1, file_meta); } diff --git a/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp b/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp index af572b720..5975eb1e8 100644 --- a/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp +++ b/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp @@ -74,7 +74,7 @@ class SequenceSnapshotPropertiesTest : public testing::Test { /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, /*external_path=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } ManifestEntry CreateEntry(const FileKind& kind, int64_t max_sequence_number) const { diff --git a/src/paimon/core/operation/commit_metrics_test.cpp b/src/paimon/core/operation/commit_metrics_test.cpp index 240d91345..9d1cf9cf2 100644 --- a/src/paimon/core/operation/commit_metrics_test.cpp +++ b/src/paimon/core/operation/commit_metrics_test.cpp @@ -62,7 +62,7 @@ ManifestEntry CreateEntry(const FileKind& kind, int32_t partition, int32_t bucke /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return ManifestEntry(kind, part, bucket, /*total_buckets=*/10, file_meta); } diff --git a/src/paimon/core/operation/data_evolution_file_store_scan_test.cpp b/src/paimon/core/operation/data_evolution_file_store_scan_test.cpp index b5581f85c..7d042814b 100644 --- a/src/paimon/core/operation/data_evolution_file_store_scan_test.cpp +++ b/src/paimon/core/operation/data_evolution_file_store_scan_test.cpp @@ -554,7 +554,8 @@ TEST_F(DataEvolutionFileStoreScanTest, TestFilterEntryByRowRanges) { /*creation_time=*/Timestamp(1737111915429ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); ManifestEntry entry(FileKind::Add(), BinaryRow::EmptyRow(), /*bucket=*/0, /*total_buckets=*/1, file); { @@ -574,7 +575,7 @@ TEST_F(DataEvolutionFileStoreScanTest, TestFilterEntryByRowRanges) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); ManifestEntry entry_without_first_row_id(FileKind::Add(), BinaryRow::EmptyRow(), /*bucket=*/0, /*total_buckets=*/1, file_without_first_row_id); diff --git a/src/paimon/core/operation/expire_snapshots_test.cpp b/src/paimon/core/operation/expire_snapshots_test.cpp index 6a9e46ba7..ffdd30feb 100644 --- a/src/paimon/core/operation/expire_snapshots_test.cpp +++ b/src/paimon/core/operation/expire_snapshots_test.cpp @@ -148,7 +148,7 @@ class ExpireSnapshotsTest : public testing::Test { /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, /*external_path=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return ManifestEntry(kind, row, bucket, /*total_buckets=*/3, data_file_meta); } diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index f32d86fac..b1b8677f8 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -198,7 +198,7 @@ class FileStoreCommitImplTest : public testing::Test { /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, /*external_path=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return ManifestEntry(kind, partition, bucket, total_buckets, data_file_meta); } @@ -223,7 +223,7 @@ class FileStoreCommitImplTest : public testing::Test { /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, /*external_path=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return ManifestEntry(kind, BinaryRow::EmptyRow(), 0, 2, data_file_meta); } @@ -278,7 +278,7 @@ class FileStoreCommitImplTest : public testing::Test { /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } std::shared_ptr CreateAppendDataFileMeta(const std::string& file_name, @@ -294,7 +294,7 @@ class FileStoreCommitImplTest : public testing::Test { /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } bool IsStringInSet(const std::set& strSet, const std::string& target) { diff --git a/src/paimon/core/operation/internal_read_context_test.cpp b/src/paimon/core/operation/internal_read_context_test.cpp index 286188570..20838122b 100644 --- a/src/paimon/core/operation/internal_read_context_test.cpp +++ b/src/paimon/core/operation/internal_read_context_test.cpp @@ -20,6 +20,7 @@ #include +#include "arrow/c/bridge.h" #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/common/table/special_fields.h" diff --git a/src/paimon/core/operation/key_value_file_store_scan_test.cpp b/src/paimon/core/operation/key_value_file_store_scan_test.cpp index b221715f5..703d7e493 100644 --- a/src/paimon/core/operation/key_value_file_store_scan_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_scan_test.cpp @@ -353,7 +353,8 @@ TEST_F(KeyValueFileStoreScanTest, TestNoOverlapping) { /*embedded_index=*/nullptr, /*file_source=*/FileSource::Append(), /*external_path=*/std::nullopt, /*value_stats_cols=*/std::nullopt, - /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt)); + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt)); } return entries; }; @@ -405,7 +406,7 @@ TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithValueStatsCols) { /*value_stats_cols=*/value_stats_cols, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt)); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt)); // max(v0)=50 > 30.1, should be kept. SimpleStats value_stats_keep = BinaryRowGenerator::GenerateStats( @@ -430,7 +431,7 @@ TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithValueStatsCols) { /*value_stats_cols=*/value_stats_cols, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt)); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt)); // max(v0)=20 <= 30.1, should be filtered out. ASSERT_OK_AND_ASSIGN(bool keep, scan->FilterByStats(entry)); @@ -482,7 +483,7 @@ TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithSchemaEvolution) { /*value_stats_cols=*/value_stats_cols, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt)); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt)); SimpleStats value_stats_keep = BinaryRowGenerator::GenerateStats( /*min=*/{40}, /*max=*/{50}, /*null=*/{0}, pool.get()); @@ -506,7 +507,7 @@ TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithSchemaEvolution) { /*value_stats_cols=*/value_stats_cols, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt)); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt)); ASSERT_OK_AND_ASSIGN(bool keep, scan->FilterByStats(entry)); ASSERT_FALSE(keep); @@ -555,7 +556,7 @@ TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithNewFieldUsesNullSta /*value_stats_cols=*/value_stats_cols, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt)); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt)); ASSERT_OK_AND_ASSIGN(bool keep, scan->FilterByStats(old_schema_entry)); ASSERT_FALSE(keep); diff --git a/src/paimon/core/operation/manifest_file_merger_test.cpp b/src/paimon/core/operation/manifest_file_merger_test.cpp index bff1659e3..d2617a654 100644 --- a/src/paimon/core/operation/manifest_file_merger_test.cpp +++ b/src/paimon/core/operation/manifest_file_merger_test.cpp @@ -101,7 +101,7 @@ class ManifestFileMergerTest : public testing::Test { nullptr, // not used FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt)); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt)); } ManifestFileMeta MakeManifest(const std::vector& entries) { diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp index 19cd96d67..f857d4876 100644 --- a/src/paimon/core/operation/merge_file_split_read_test.cpp +++ b/src/paimon/core/operation/merge_file_split_read_test.cpp @@ -137,7 +137,7 @@ class MergeFileSplitReadTest : public ::testing::Test, /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto meta1_2 = std::make_shared( "data-c80ccf0f-6387-4cbc-8889-ade8cef54c43-1.parquet", /*file_size=*/3370, /*row_count=*/4, @@ -154,7 +154,7 @@ class MergeFileSplitReadTest : public ::testing::Test, /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto meta1_3 = std::make_shared( "data-c80ccf0f-6387-4cbc-8889-ade8cef54c43-2.parquet", /*file_size=*/3252, /*row_count=*/1, @@ -173,7 +173,7 @@ class MergeFileSplitReadTest : public ::testing::Test, /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder1(BinaryRowGenerator::GenerateRow({0, 0}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ paimon::test::GetDataDir() + @@ -200,7 +200,7 @@ class MergeFileSplitReadTest : public ::testing::Test, /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto meta2_2 = std::make_shared( "data-24f8588c-d950-4e44-9d99-a023ea65a136-1.parquet", /*file_size=*/3229, /*row_count=*/1, @@ -218,7 +218,7 @@ class MergeFileSplitReadTest : public ::testing::Test, /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder2(BinaryRowGenerator::GenerateRow({0, 1}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ paimon::test::GetDataDir() + @@ -246,7 +246,7 @@ class MergeFileSplitReadTest : public ::testing::Test, /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto meta3_2 = std::make_shared( "data-184f2304-49fd-4916-ba07-037757e904eb-1.parquet", /*file_size=*/3259, /*row_count=*/1, @@ -264,7 +264,7 @@ class MergeFileSplitReadTest : public ::testing::Test, /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder3(BinaryRowGenerator::GenerateRow({1, 0}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ paimon::test::GetDataDir() + @@ -296,7 +296,7 @@ class MergeFileSplitReadTest : public ::testing::Test, /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto meta1_2 = std::make_shared( "data-d03e13e5-5e2e-463a-b53a-8d44e4dc9141-1.parquet", /*file_size=*/2623, /*row_count=*/ @@ -314,7 +314,7 @@ class MergeFileSplitReadTest : public ::testing::Test, /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder1( /*partition=*/BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/ @@ -1293,7 +1293,7 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { /*delete_row_count=*/1, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto meta2 = std::make_shared( "data-6871b960-edd9-40fc-9859-aaca9ea205cf-0.orc", /*file_size=*/887, /*row_count=*/5, /*min_key=*/BinaryRowGenerator::GenerateRow({std::string("Alex"), 0}, pool_.get()), @@ -1311,7 +1311,7 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( BinaryRowGenerator::GenerateRow({10}, pool_.get()), /*bucket=*/1, /*bucket_path=*/ diff --git a/src/paimon/core/operation/metrics/commit_stats_test.cpp b/src/paimon/core/operation/metrics/commit_stats_test.cpp index a18f88a61..b22f7a60f 100644 --- a/src/paimon/core/operation/metrics/commit_stats_test.cpp +++ b/src/paimon/core/operation/metrics/commit_stats_test.cpp @@ -59,7 +59,7 @@ ManifestEntry CreateEntry(const FileKind& kind, int32_t partition, int32_t bucke /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); return ManifestEntry(kind, part, bucket, /*total_buckets=*/10, file_meta); } diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index 6f7ae9781..51c478325 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -67,7 +67,7 @@ class RawFileSplitReadTest : public ::testing::Test { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder1(BinaryRowGenerator::GenerateRow({10, 0}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ paimon::test::GetDataDir() + @@ -91,7 +91,7 @@ class RawFileSplitReadTest : public ::testing::Test { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder2(BinaryRowGenerator::GenerateRow({20, 1}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ paimon::test::GetDataDir() + @@ -115,7 +115,7 @@ class RawFileSplitReadTest : public ::testing::Test { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder3(BinaryRowGenerator::GenerateRow({10, 1}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ paimon::test::GetDataDir() + @@ -462,7 +462,7 @@ TEST_F(RawFileSplitReadTest, TestMatch) { /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRowGenerator::GenerateRow({10, 0}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ paimon::test::GetDataDir() + diff --git a/src/paimon/core/operation/write_restore_test.cpp b/src/paimon/core/operation/write_restore_test.cpp index 43f070154..754dc6959 100644 --- a/src/paimon/core/operation/write_restore_test.cpp +++ b/src/paimon/core/operation/write_restore_test.cpp @@ -40,7 +40,8 @@ std::shared_ptr CreateDataFileMeta(const std::string& file_name) { /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, - /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); } ManifestEntry CreateManifestEntry(int32_t total_buckets, const std::string& file_name) { diff --git a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp index e932d40da..30357327a 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp +++ b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp @@ -226,7 +226,7 @@ TEST_P(PostponeBucketWriterTest, TestSimple) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{}, /*changelog_files=*/{}); ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement()); @@ -308,7 +308,7 @@ TEST_P(PostponeBucketWriterTest, TestNestedType) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{}, /*changelog_files=*/{}); ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement()); @@ -474,7 +474,7 @@ TEST_P(PostponeBucketWriterTest, TestWriteMultiBatch) { /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement expected_data_increment({expected_data_file_meta}, /*deleted_files=*/{}, /*changelog_files=*/{}); ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement()); @@ -624,7 +624,7 @@ TEST_P(PostponeBucketWriterTest, TestMultiplePrepareCommit) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement expected_data_increment1({expected_data_file_meta1}, /*deleted_files=*/{}, /*changelog_files=*/{}); ASSERT_EQ(expected_data_increment1, commit_increment1.GetNewFilesIncrement()); @@ -644,7 +644,7 @@ TEST_P(PostponeBucketWriterTest, TestMultiplePrepareCommit) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement expected_data_increment2({expected_data_file_meta2}, /*deleted_files=*/{}, /*changelog_files=*/{}); ASSERT_EQ(expected_data_increment2, commit_increment2.GetNewFilesIncrement()); diff --git a/src/paimon/core/table/sink/commit_message_serializer.cpp b/src/paimon/core/table/sink/commit_message_serializer.cpp index 1b0645f4e..e3424d4a2 100644 --- a/src/paimon/core/table/sink/commit_message_serializer.cpp +++ b/src/paimon/core/table/sink/commit_message_serializer.cpp @@ -38,6 +38,7 @@ #include "paimon/core/io/data_file_meta_12_serializer.h" #include "paimon/core/io/data_file_meta_first_row_id_legacy_serializer.h" #include "paimon/core/io/data_file_meta_serializer.h" +#include "paimon/core/io/data_file_meta_write_cols_legacy_serializer.h" #include "paimon/core/io/data_increment.h" #include "paimon/core/table/sink/commit_message_impl.h" #include "paimon/core/utils/object_serializer.h" @@ -46,7 +47,7 @@ namespace paimon { class MemoryPool; -const int32_t CommitMessageSerializer::CURRENT_VERSION = 12; +const int32_t CommitMessageSerializer::CURRENT_VERSION = 13; CommitMessageSerializer::CommitMessageSerializer(const std::shared_ptr& pool) : memory_pool_(pool), @@ -204,16 +205,25 @@ Result> CommitMessageSerializer::Deserialize(int3 DataInputStream* in) { if (version == CURRENT_VERSION) { return Deserialize(version, data_file_serializer_.get(), index_entry_serializer_.get(), in); + } else if (version == 12) { + auto data_file_meta_write_cols_legacy_serializer = + std::make_unique(memory_pool_); + return Deserialize(version, data_file_meta_write_cols_legacy_serializer.get(), + index_entry_serializer_.get(), in); } else if (version == 11) { + auto data_file_meta_write_cols_legacy_serializer = + std::make_unique(memory_pool_); auto index_entry_v4_deserializer = std::make_unique(memory_pool_); - return Deserialize(version, data_file_serializer_.get(), index_entry_v4_deserializer.get(), - in); + return Deserialize(version, data_file_meta_write_cols_legacy_serializer.get(), + index_entry_v4_deserializer.get(), in); } else if (version == 9 || version == 10) { + auto data_file_meta_write_cols_legacy_serializer = + std::make_unique(memory_pool_); auto index_entry_v3_deserializer = std::make_unique(memory_pool_); - return Deserialize(version, data_file_serializer_.get(), index_entry_v3_deserializer.get(), - in); + return Deserialize(version, data_file_meta_write_cols_legacy_serializer.get(), + index_entry_v3_deserializer.get(), in); } else if (version == 8) { auto data_file_meta_first_row_id_legacy_serializer = std::make_unique(memory_pool_); diff --git a/src/paimon/core/table/sink/commit_message_test.cpp b/src/paimon/core/table/sink/commit_message_test.cpp index d602da227..f372361cb 100644 --- a/src/paimon/core/table/sink/commit_message_test.cpp +++ b/src/paimon/core/table/sink/commit_message_test.cpp @@ -65,6 +65,44 @@ TEST(CommitMessageTest, TestCurrentVersion) { ASSERT_EQ(CommitMessageSerializer::CURRENT_VERSION, CommitMessage::CurrentVersion()); } +TEST(CommitMessageTest, TestDeserializeVersion13GeneratedByJava) { + // Generated by CommitMessageSerializer::serialize from Apache Paimon Java master. + std::string data_path = paimon::test::GetDataDir() + "/compatibility/commit_message-v13"; + auto file_system = std::make_shared(); + auto buffer_length = file_system->GetFileStatus(data_path).value().GetLen(); + ASSERT_GT(buffer_length, 0); + + std::vector buffer(buffer_length, 0); + ASSERT_OK_AND_ASSIGN(auto in_stream, file_system->Open(data_path)); + ASSERT_OK(in_stream->Read(buffer.data(), buffer.size())); + ASSERT_OK(in_stream->Close()); + + auto pool = GetDefaultPool(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + CommitMessage::Deserialize(CommitMessage::CurrentVersion(), buffer.data(), + buffer.size(), pool)); + auto result_message = std::dynamic_pointer_cast(result); + ASSERT_NE(result_message, nullptr); + ASSERT_EQ(result_message->Partition(), + BinaryRowGenerator::GenerateRow({std::string("aaaaa")}, pool.get())); + ASSERT_EQ(result_message->Bucket(), 20); + ASSERT_EQ(result_message->TotalBuckets(), std::optional(32)); + ASSERT_EQ(result_message->GetNewFilesIncrement().NewFiles().size(), 1); + ASSERT_TRUE(result_message->GetCompactIncrement().IsEmpty()); + + const std::shared_ptr& data_file = + result_message->GetNewFilesIncrement().NewFiles()[0]; + ASSERT_EQ(data_file->file_name, "my_file"); + ASSERT_TRUE(data_file->write_cols.has_value()); + ASSERT_EQ(data_file->write_cols.value(), (std::vector{"a", "b", "c", "f"})); + ASSERT_TRUE(data_file->column_max_sequence_numbers.has_value()); + ASSERT_EQ(data_file->column_max_sequence_numbers.value(), + (std::vector{15, 100, 150, 200})); + + ASSERT_OK_AND_ASSIGN(std::string serialized_bytes, CommitMessage::Serialize(result, pool)); + ASSERT_EQ(serialized_bytes, std::string(buffer.data(), buffer.size())); +} + TEST(CommitMessageTest, TestCompatibleWithVersion12) { // index file meta: add global index meta source meta int32_t version = 12; @@ -95,7 +133,13 @@ TEST(CommitMessageTest, TestCompatibleWithVersion12) { // check result ASSERT_OK_AND_ASSIGN(std::string serialized_bytes, CommitMessage::Serialize(ret, pool)); - ASSERT_EQ(serialized_bytes, std::string(reinterpret_cast(buffer.data()), buffer.size())); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr roundtrip, + CommitMessage::Deserialize(CommitMessage::CurrentVersion(), serialized_bytes.data(), + serialized_bytes.size(), pool)); + auto roundtrip_message = std::dynamic_pointer_cast(roundtrip); + ASSERT_NE(roundtrip_message, nullptr); + ASSERT_EQ(*roundtrip_message, *res_msg); } TEST(CommitMessageTest, TestCompatibleWithVersion11) { @@ -185,7 +229,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion10) { /*creation_time=*/Timestamp(1761242383412ll, 0), /*delete_row_count=*/1, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); LinkedHashMap dv_ranges; dv_ranges.insert_or_assign( @@ -249,7 +294,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion9) { /*creation_time=*/Timestamp(1757349273600ll, 0), /*delete_row_count=*/1, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); LinkedHashMap dv_ranges; dv_ranges.insert_or_assign( @@ -316,7 +362,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion9WithExternalPathForIndex) { /*value_stats_cols=*/std::nullopt, /*external_path=*/ "FILE:/tmp/external/f1=10/bucket-1/data-72b62a5f-d698-4db5-b51a-04c0dc027702-1.orc", - /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); LinkedHashMap dv_ranges; dv_ranges.insert_or_assign( @@ -379,7 +426,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion8) { /*creation_time=*/Timestamp(1754068646844ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); expected_msgs.emplace_back(/*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/0, /*total_bucket=*/2, DataIncrement({file_meta}, {}, {}), CompactIncrement({}, {}, {})); @@ -398,7 +446,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion8) { /*creation_time=*/Timestamp(1754068646864ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); expected_msgs.emplace_back(/*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*total_bucket=*/2, DataIncrement({file_meta2}, {}, {}), CompactIncrement({}, {}, {})); @@ -448,7 +497,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion7) { /*creation_time=*/Timestamp(1743525392885ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); expected_msgs.emplace_back(/*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/0, /*total_bucket=*/2, DataIncrement({file_meta}, {}, {}), CompactIncrement({}, {}, {})); @@ -469,7 +519,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion7) { /*creation_time=*/Timestamp(1743525392921ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); expected_msgs.emplace_back(/*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*total_bucket=*/2, DataIncrement({file_meta2}, {}, {}), CompactIncrement({}, {}, {})); @@ -515,7 +566,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion6) { /*creation_time=*/Timestamp(1737052260143ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); expected_msgs.emplace_back(BinaryRow::EmptyRow(), /*bucket=*/0, /*total_bucket=*/std::nullopt, DataIncrement({file_meta}, {}, {}), CompactIncrement({}, {}, {})); // check result @@ -562,7 +614,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion5) { /*creation_time=*/Timestamp(1734707236040ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::optional>({"f0", "f1", "f2"}), - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto file_meta1_after_compact = std::make_shared( "data-0d0f29cc-63c6-4fab-a594-71bd7d06fcde-1.orc", /*file_size=*/859, /*row_count=*/1, BinaryRowGenerator::GenerateRow({std::string("Alice"), 1}, pool.get()), @@ -576,7 +629,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion5) { /*creation_time=*/Timestamp(1734707236040ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::optional>({"f0", "f1", "f2"}), - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment1({file_meta1}, {}, {}, {}, {}); LinkedHashMap dv_metas1; @@ -605,7 +659,8 @@ TEST(CommitMessageTest, TestCompatibleWithVersion5) { /*creation_time=*/Timestamp(1734707236109ll, 0), /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::optional>({"f0", "f1", "f2"}), - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment2({file_meta2}, {}, {}, {}, {}); LinkedHashMap dv_metas2; dv_metas2.insert_or_assign( @@ -663,7 +718,7 @@ TEST(CommitMessageTest, TestCompatibleWithVersion4) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment1({file_meta1}, {}, {}); expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/0, /*total_bucket=*/std::nullopt, data_increment1, @@ -682,7 +737,7 @@ TEST(CommitMessageTest, TestCompatibleWithVersion4) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment2({file_meta2}, {}, {}); expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*total_bucket=*/std::nullopt, data_increment2, @@ -701,7 +756,7 @@ TEST(CommitMessageTest, TestCompatibleWithVersion4) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment3({file_meta3}, {}, {}); expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({20}, pool.get()), /*bucket=*/0, /*total_bucket=*/std::nullopt, data_increment3, @@ -751,7 +806,8 @@ TEST(CommitMessageTest, TestCompatibleWithJavaPaimon10WithStatsDenseStore) { /*creation_time=*/Timestamp(1731412938869ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::optional>({"f0", "f1", "f2"}), - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment1({file_meta1}, {}, {}); expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/0, /*total_bucket=*/std::nullopt, data_increment1, @@ -768,7 +824,8 @@ TEST(CommitMessageTest, TestCompatibleWithJavaPaimon10WithStatsDenseStore) { /*creation_time=*/Timestamp(1731412938891ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::optional>({"f0", "f1", "f2"}), - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment2({file_meta2}, {}, {}); expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*total_bucket=*/std::nullopt, data_increment2, @@ -785,7 +842,8 @@ TEST(CommitMessageTest, TestCompatibleWithJavaPaimon10WithStatsDenseStore) { /*creation_time=*/Timestamp(1731412938908ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::optional>({"f0", "f1", "f2"}), - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment3({file_meta3}, {}, {}); expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({20}, pool.get()), /*bucket=*/0, /*total_bucket=*/std::nullopt, data_increment3, @@ -835,7 +893,7 @@ TEST(CommitMessageTest, TestCompatibleWith09JavaPaimon1) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment1({file_meta1}, {}, {}); expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/0, /*total_bucket=*/std::nullopt, data_increment1, @@ -854,7 +912,7 @@ TEST(CommitMessageTest, TestCompatibleWith09JavaPaimon1) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment2({file_meta2}, {}, {}); expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*total_bucket=*/std::nullopt, data_increment2, @@ -873,7 +931,7 @@ TEST(CommitMessageTest, TestCompatibleWith09JavaPaimon1) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment3({file_meta3}, {}, {}); expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({20}, pool.get()), /*bucket=*/0, /*total_bucket=*/std::nullopt, data_increment3, @@ -923,7 +981,7 @@ TEST(CommitMessageTest, TestCompatibleWith09JavaPaimon2) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment1({file_meta1}, {}, {}); expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*total_bucket=*/std::nullopt, data_increment1, @@ -943,7 +1001,7 @@ TEST(CommitMessageTest, TestCompatibleWith09JavaPaimon2) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment2({file_meta2}, {}, {}); expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({20}, pool.get()), /*bucket=*/0, /*total_bucket=*/std::nullopt, data_increment2, @@ -993,7 +1051,7 @@ TEST(CommitMessageTest, TestCompatibleWith09JavaPaimon3) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment1({file_meta1}, {}, {}); expected_msgs.emplace_back(BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*total_bucket=*/std::nullopt, data_increment1, @@ -1046,7 +1104,7 @@ TEST(CommitMessageTest, TestPkTableCompatibleWithJavaPaimon09) { /*delete_row_count=*/1, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta_with_level = std::make_shared( "data-2eb2a766-97e4-4fe4-88ce-eb606675c101-0.orc", /*file_size=*/789, /*row_count=*/1, /*min_key=*/BinaryRowGenerator::GenerateRow({std::string("Bob"), 0}, pool.get()), @@ -1063,7 +1121,7 @@ TEST(CommitMessageTest, TestPkTableCompatibleWithJavaPaimon09) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment1({file_meta}, {}, {}, {}, {}); LinkedHashMap dv_ranges; @@ -1141,7 +1199,7 @@ TEST(CommitMessageTest, TestCompatibleWithComplexDataType) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment1({file_meta1}, {}, {}); expected_msgs.emplace_back(BinaryRow::EmptyRow(), /*bucket=*/0, /*total_bucket=*/std::nullopt, data_increment1, CompactIncrement({}, {}, {})); diff --git a/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp index 14fae3972..5fbd5f093 100644 --- a/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp +++ b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp @@ -49,7 +49,7 @@ std::shared_ptr NewAppendFile(const std::string& file_name, int64_ /*max_sequence_number=*/first_row_id + row_count - 1, /*schema_id=*/0, /*level=*/0, std::vector>(), Timestamp(0l, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), std::nullopt, std::nullopt, first_row_id, - std::nullopt); + std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } std::shared_ptr NewDataPlan(std::vector> files) { diff --git a/src/paimon/core/table/source/data_split_impl.cpp b/src/paimon/core/table/source/data_split_impl.cpp index 5b9128a73..be0ef4030 100644 --- a/src/paimon/core/table/source/data_split_impl.cpp +++ b/src/paimon/core/table/source/data_split_impl.cpp @@ -223,6 +223,8 @@ DataSplitImpl::GetFileMetaSerializer(int32_t version, const std::shared_ptr(pool); } else if (version == 7) { return std::make_unique(pool); + } else if (version == 8) { + return std::make_unique(pool); } else if (version == VERSION) { return std::make_unique(pool); } else { diff --git a/src/paimon/core/table/source/data_split_impl.h b/src/paimon/core/table/source/data_split_impl.h index 01c1231c8..2d357a163 100644 --- a/src/paimon/core/table/source/data_split_impl.h +++ b/src/paimon/core/table/source/data_split_impl.h @@ -35,6 +35,7 @@ #include "paimon/core/io/data_file_meta_12_serializer.h" #include "paimon/core/io/data_file_meta_first_row_id_legacy_serializer.h" #include "paimon/core/io/data_file_meta_serializer.h" +#include "paimon/core/io/data_file_meta_write_cols_legacy_serializer.h" #include "paimon/core/table/source/deletion_file.h" #include "paimon/table/source/data_split.h" @@ -44,7 +45,7 @@ namespace paimon { class DataSplitImpl : public DataSplit { public: static constexpr int64_t MAGIC = -2394839472490812314L; - static constexpr int32_t VERSION = 8; + static constexpr int32_t VERSION = 9; int64_t SnapshotId() const { return snapshot_id_; diff --git a/src/paimon/core/table/source/data_split_test.cpp b/src/paimon/core/table/source/data_split_test.cpp index b65988526..bb631be9a 100644 --- a/src/paimon/core/table/source/data_split_test.cpp +++ b/src/paimon/core/table/source/data_split_test.cpp @@ -46,6 +46,47 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { +TEST(DataSplitTest, TestDeserializeVersion9GeneratedByJava) { + // Generated by DataSplit::serialize from Apache Paimon Java master. + std::string file_name = paimon::test::GetDataDir() + "/compatibility/data_split-v9"; + auto file_system = std::make_unique(); + + ASSERT_OK_AND_ASSIGN(auto input_stream, file_system->Open(file_name)); + std::vector split_bytes(input_stream->Length().value_or(0), 0); + ASSERT_GT(split_bytes.size(), 0); + ASSERT_OK(input_stream->Read(split_bytes.data(), split_bytes.size())); + ASSERT_OK(input_stream->Close()); + + auto pool = GetDefaultPool(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + Split::Deserialize(split_bytes.data(), split_bytes.size(), pool)); + auto result_data_split = std::dynamic_pointer_cast(result); + ASSERT_NE(result_data_split, nullptr); + ASSERT_EQ(result_data_split->SnapshotId(), 18); + ASSERT_EQ(result_data_split->Partition(), + BinaryRowGenerator::GenerateRow({std::string("aaaaa")}, pool.get())); + ASSERT_EQ(result_data_split->Bucket(), 20); + ASSERT_EQ(result_data_split->BucketPath(), "my path"); + ASSERT_EQ(result_data_split->TotalBuckets(), std::optional(32)); + ASSERT_EQ(result_data_split->DataFiles().size(), 1); + ASSERT_EQ( + result_data_split->DeletionFiles(), + (std::vector>{DeletionFile("deletion_file", 100, 22, 33)})); + ASSERT_FALSE(result_data_split->IsStreaming()); + ASSERT_FALSE(result_data_split->RawConvertible()); + + const std::shared_ptr& data_file = result_data_split->DataFiles()[0]; + ASSERT_EQ(data_file->file_name, "my_file"); + ASSERT_TRUE(data_file->write_cols.has_value()); + ASSERT_EQ(data_file->write_cols.value(), (std::vector{"a", "b", "c", "f"})); + ASSERT_TRUE(data_file->column_max_sequence_numbers.has_value()); + ASSERT_EQ(data_file->column_max_sequence_numbers.value(), + (std::vector{15, 100, 150, 200})); + + ASSERT_OK_AND_ASSIGN(std::string serialized_bytes, Split::Serialize(result, pool)); + ASSERT_EQ(serialized_bytes, std::string(split_bytes.data(), split_bytes.size())); +} + TEST(DataSplitTest, TestDeserializeVersion8WithWriteColsAndExternalPath) { std::string file_name = paimon::test::GetDataDir() + "/orc/pk_dv_index_in_data_with_external.db/" @@ -86,7 +127,8 @@ TEST(DataSplitTest, TestDeserializeVersion8WithWriteColsAndExternalPath) { /*value_stats_cols=*/std::nullopt, /*external_path=*/ "FILE:/tmp/external/f1=10/bucket-1/data-72b62a5f-d698-4db5-b51a-04c0dc027702-0.orc", - /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), @@ -108,7 +150,10 @@ TEST(DataSplitTest, TestDeserializeVersion8WithWriteColsAndExternalPath) { .value()); ASSERT_EQ(*result_data_split, *expected_data_split) << result_data_split->ToString(); ASSERT_OK_AND_ASSIGN(std::string serialize_bytes, Split::Serialize(result_data_split, pool)); - ASSERT_EQ(serialize_bytes, std::string(split_bytes.data(), split_bytes.size())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr roundtrip, + Split::Deserialize(serialize_bytes.data(), serialize_bytes.size(), pool)); + auto roundtrip_data_split = std::dynamic_pointer_cast(roundtrip); + ASSERT_EQ(*roundtrip_data_split, *expected_data_split) << roundtrip_data_split->ToString(); } TEST(DataSplitTest, TestDeserializeVersion8WithWriteCols) { @@ -149,7 +194,8 @@ TEST(DataSplitTest, TestDeserializeVersion8WithWriteCols) { /*creation_time=*/Timestamp(1757349273246ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), @@ -172,7 +218,10 @@ TEST(DataSplitTest, TestDeserializeVersion8WithWriteCols) { .value()); ASSERT_EQ(*result_data_split, *expected_data_split) << result_data_split->ToString(); ASSERT_OK_AND_ASSIGN(std::string serialize_bytes, Split::Serialize(result_data_split, pool)); - ASSERT_EQ(serialize_bytes, std::string(split_bytes.data(), split_bytes.size())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr roundtrip, + Split::Deserialize(serialize_bytes.data(), serialize_bytes.size(), pool)); + auto roundtrip_data_split = std::dynamic_pointer_cast(roundtrip); + ASSERT_EQ(*roundtrip_data_split, *expected_data_split) << roundtrip_data_split->ToString(); } TEST(DataSplitTest, TestDeserializeVersion7WithFirstRowId) { @@ -209,7 +258,8 @@ TEST(DataSplitTest, TestDeserializeVersion7WithFirstRowId) { /*creation_time=*/Timestamp(1754073518741ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/5, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/5, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRow::EmptyRow(), @@ -261,7 +311,8 @@ TEST(DataSplitTest, TestDeserializeVersion7WithNullFirstRowId) { /*creation_time=*/Timestamp(1754068646844ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), @@ -312,7 +363,8 @@ TEST(DataSplitTest, TestDeserializeVersion6PkWithTotalBuckets) { /*creation_time=*/Timestamp(1743525392885ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), @@ -358,7 +410,8 @@ TEST(DataSplitTest, TestDeserializeVersion5PkWithExternalPath) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/"file:/tmp/bucket-0/data-80110e15-97b5-4bcf-ac09-6ca2659a4950-0.orc", - /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRow::EmptyRow(), @@ -407,7 +460,8 @@ TEST(DataSplitTest, TestDeserializeVersion5PkWithEmptyExternalPath) { /*creation_time=*/Timestamp(1737052260143ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRow::EmptyRow(), @@ -463,7 +517,8 @@ TEST(DataSplitTest, TestDeserializeVersion4PkWithSnapshot4WithDvCardinality) { /*creation_time=*/Timestamp(1734707235578ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::optional>({"f0", "f1", "f2"}), - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*bucket_path=*/ @@ -516,7 +571,7 @@ TEST(DataSplitTest, TestDeserializeVersion3AppendWithSnapshot1) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(/*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*bucket_path=*/ "data/append_10.db/append_10/f1=10/bucket-1", {file_meta}); @@ -555,7 +610,8 @@ TEST(DataSplitTest, TestDeserializeVersion3AppendWithSnapshot1WithStatsDenseStor /*creation_time=*/Timestamp(1731412938891ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::optional>({"f0", "f1", "f2"}), - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*bucket_path=*/ @@ -596,7 +652,7 @@ TEST(DataSplitTest, TestDeserializeAppendWithSnapshot1) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*bucket_path=*/ @@ -636,7 +692,7 @@ TEST(DataSplitTest, TestDeserializeAppendWithSnapshot3) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-b913a160-a4d1-4084-af2a-18333c35668e-0.orc", /*file_size=*/506, /*row_count=*/1, @@ -651,7 +707,7 @@ TEST(DataSplitTest, TestDeserializeAppendWithSnapshot3) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({20}, pool.get()), @@ -709,7 +765,7 @@ TEST(DataSplitTest, TestDeserializeAppendWithSnapshot5) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), @@ -759,7 +815,7 @@ TEST(DataSplitTest, TestDeserializePkWithSnapshot2) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/0, /*bucket_path=*/ @@ -808,7 +864,7 @@ TEST(DataSplitTest, TestDeserializePkWithSnapshot6OfSingleFile) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(/*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/1, /*bucket_path=*/ "data/pk_09.db/pk_09/f1=10/bucket-1", {file_meta}); @@ -866,7 +922,7 @@ TEST(DataSplitTest, TestDeserializePkWithSnapshot6OfMultiFiles) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-1c7a85f1-55bd-424f-b503-34a33be0fb96-0.orc", /*file_size=*/1148, /*row_count=*/2, @@ -888,7 +944,7 @@ TEST(DataSplitTest, TestDeserializePkWithSnapshot6OfMultiFiles) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta3 = std::make_shared( "data-8cdb8b8d-5830-4b3b-aa94-8a30c449277a-0.orc", /*file_size=*/810, /*row_count=*/1, @@ -907,7 +963,7 @@ TEST(DataSplitTest, TestDeserializePkWithSnapshot6OfMultiFiles) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), @@ -967,7 +1023,7 @@ TEST(DataSplitTest, TestDeserializePkWithSnapshot8) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(/*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/0, /*bucket_path=*/ "data/pk_09.db/pk_09/f1=10/bucket-0", {file_meta}); @@ -1018,7 +1074,7 @@ TEST(DataSplitTest, TestDeserializePk10WithSnapshot6) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-d6d370f3-242b-45c9-8739-44bf31b2b449-0.orc", /*file_size=*/924, /*row_count=*/1, /*min_key=*/BinaryRowGenerator::GenerateRow({52}, pool.get()), /*max_key=*/ @@ -1035,7 +1091,7 @@ TEST(DataSplitTest, TestDeserializePk10WithSnapshot6) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({1, 1}, pool.get()), /*bucket=*/0, @@ -1068,7 +1124,7 @@ TEST(DataSplitTest, TestPartialMergedRowCount) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), @@ -1078,7 +1134,7 @@ TEST(DataSplitTest, TestPartialMergedRowCount) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), /*bucket=*/0, /*bucket_path=*/ @@ -1112,7 +1168,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithoutDeletionFiles) /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-1.orc", /*file_size=*/100, /*row_count=*/4, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), @@ -1122,7 +1178,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithoutDeletionFiles) /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), @@ -1147,7 +1203,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithCardinality) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), @@ -1157,7 +1213,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithCardinality) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta3 = std::make_shared( "data-2.orc", /*file_size=*/100, /*row_count=*/3, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), @@ -1167,7 +1223,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithCardinality) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), @@ -1202,7 +1258,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountMixedCardinalityReturnsNullopt) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), @@ -1212,7 +1268,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountMixedCardinalityReturnsNullopt) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), @@ -1245,7 +1301,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountUnknownDeleteRowCountDoesNotBlockRa /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), @@ -1255,7 +1311,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountUnknownDeleteRowCountDoesNotBlockRa /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), @@ -1284,7 +1340,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountFallsBackToDataEvolution) { /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/100, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-1.orc", /*file_size=*/100, /*row_count=*/5, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), @@ -1294,7 +1350,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountFallsBackToDataEvolution) { /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/100, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta3 = std::make_shared( "data-2.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), @@ -1304,7 +1360,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountFallsBackToDataEvolution) { /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/200, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), @@ -1335,7 +1391,7 @@ TEST(DataSplitTest, TestDataEvolutionMergedRowCountSubtractsDeletionFileCardinal /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/100, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-1.orc", /*file_size=*/100, /*row_count=*/5, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), @@ -1345,7 +1401,7 @@ TEST(DataSplitTest, TestDataEvolutionMergedRowCountSubtractsDeletionFileCardinal /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/100, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), @@ -1380,7 +1436,7 @@ TEST(DataSplitTest, TestSerializeDataEvolutionSplitWithDeletionFiles) { /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-1.orc", /*file_size=*/100, /*row_count=*/10, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), @@ -1390,7 +1446,7 @@ TEST(DataSplitTest, TestSerializeDataEvolutionSplitWithDeletionFiles) { /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), @@ -1438,7 +1494,7 @@ TEST(DataSplitTest, TestDataEvolutionMergedRowCountUnavailableWithoutCardinality /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/100, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), @@ -1471,7 +1527,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountResolvesMissingCardinalityViaFactor /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-1.orc", /*file_size=*/100, /*row_count=*/5, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), @@ -1481,7 +1537,7 @@ TEST(DataSplitTest, TestPartialMergedRowCountResolvesMissingCardinalityViaFactor /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), diff --git a/src/paimon/core/table/source/fallback_data_split_test.cpp b/src/paimon/core/table/source/fallback_data_split_test.cpp index 80d290560..5ae97e19e 100644 --- a/src/paimon/core/table/source/fallback_data_split_test.cpp +++ b/src/paimon/core/table/source/fallback_data_split_test.cpp @@ -115,7 +115,8 @@ TEST(FallbackDataSplitTest, TestDeserialize) { /*creation_time=*/Timestamp(1755880762233ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-43880011-d066-4255-ad65-891d79cde23b-0.parquet", /*file_size=*/891, /*row_count=*/1, /*min_key=*/BinaryRow::EmptyRow(), @@ -129,7 +130,8 @@ TEST(FallbackDataSplitTest, TestDeserialize) { /*creation_time=*/Timestamp(1755884315482ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( /*partition=*/BinaryRowGenerator::GenerateRow({1}, pool.get()), @@ -185,7 +187,8 @@ TEST(FallbackDataSplitTest, TestDeserialize2) { /*creation_time=*/Timestamp(1755880762585ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-625b3277-84d3-4320-80b9-89a5075bf5fd-0.parquet", /*file_size=*/891, /*row_count=*/1, /*min_key=*/BinaryRow::EmptyRow(), @@ -199,7 +202,8 @@ TEST(FallbackDataSplitTest, TestDeserialize2) { /*creation_time=*/Timestamp(1755884315889ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); std::vector file_list; file_list.emplace_back( diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp index a56469656..4126232a0 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp @@ -220,7 +220,8 @@ class PrimaryKeySortedIndexScanTest : public ::testing::Test { /*creation_time=*/Timestamp(1721643142456LL, 0), delete_row_count, /*embedded_index=*/nullptr, file_source, /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, - /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); } Result> BuildPayload(std::vector ordinals, diff --git a/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp index cebc644cf..f3174fc6e 100644 --- a/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp +++ b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp @@ -52,7 +52,7 @@ class SnapshotReaderTest : public testing::Test { /*min_sequence_number=*/0, /*max_sequence_number=*/0, /*schema_id=*/0, DataFileMeta::DUMMY_LEVEL, std::vector>{}, Timestamp(0, 0), std::nullopt, nullptr, FileSource::Append(), std::nullopt, std::nullopt, std::nullopt, - std::nullopt); + std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } std::shared_ptr CreateIndexFileMeta(const std::string& index_file_name, diff --git a/src/paimon/core/table/source/split_generator_test.cpp b/src/paimon/core/table/source/split_generator_test.cpp index ac80eb3f1..9f1368700 100644 --- a/src/paimon/core/table/source/split_generator_test.cpp +++ b/src/paimon/core/table/source/split_generator_test.cpp @@ -74,7 +74,7 @@ class SplitGeneratorTest : public testing::Test { /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::optional(), /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } std::shared_ptr CreateDataFileMeta(const std::string& file_name, int32_t level, @@ -99,7 +99,7 @@ class SplitGeneratorTest : public testing::Test { /*embedded_index=*/nullptr, FileSource::Append(), /*external_path=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } std::shared_ptr CreateDataFileMeta(const std::string& file_name, int32_t min_key, @@ -117,7 +117,7 @@ class SplitGeneratorTest : public testing::Test { /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*external_path=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } std::shared_ptr CreateDataFileMetaWithRowId(const std::string& file_name, @@ -132,7 +132,7 @@ class SplitGeneratorTest : public testing::Test { /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, first_row_id, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); } static void CheckResult(const std::vector& result_groups, diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index a2a52d343..343d99605 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -183,7 +183,7 @@ class ReadInteTest : public testing::Test, public ::testing::WithParamInterface< /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt)); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt)); } auto bucket_str = bucket_path.substr(bucket_path.find("bucket-") + 7); int32_t bucket = std::stoi(bucket_str); diff --git a/test/inte/read_inte_with_index_test.cpp b/test/inte/read_inte_with_index_test.cpp index 7a4439734..1442f6263 100644 --- a/test/inte/read_inte_with_index_test.cpp +++ b/test/inte/read_inte_with_index_test.cpp @@ -945,7 +945,8 @@ TEST_P(ReadInteWithIndexTest, TestSimple) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); ASSERT_OK_AND_ASSIGN(auto split, @@ -1019,7 +1020,8 @@ TEST_P(ReadInteWithIndexTest, TestReadWithLimits) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); ASSERT_OK_AND_ASSIGN(auto split, @@ -1123,7 +1125,8 @@ TEST_P(ReadInteWithIndexTest, TestEmbeddingBitmapIndex) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/embedded_index, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); ASSERT_OK_AND_ASSIGN(auto split, @@ -1186,7 +1189,8 @@ TEST_P(ReadInteWithIndexTest, TestBitmapWithV1) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/embedded_index, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); ASSERT_OK_AND_ASSIGN(auto split, @@ -1225,7 +1229,8 @@ TEST_P(ReadInteWithIndexTest, TestNoEmbeddingBitmapIndex) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); ASSERT_OK_AND_ASSIGN(auto split, @@ -1264,7 +1269,8 @@ TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); ASSERT_OK_AND_ASSIGN(auto split, @@ -1339,7 +1345,7 @@ TEST_P(ReadInteWithIndexTest, TestNoEmbeddingBitmapIndexWithExternalPath) { /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/external_file_path, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); ASSERT_OK_AND_ASSIGN(auto split, @@ -1381,7 +1387,8 @@ TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithDv) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DeletionFile deletion_file(deletion_file_path, /*offset=*/1, /*length=*/24, /*cardinality=*/2); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, @@ -1487,7 +1494,7 @@ TEST_P(ReadInteWithIndexTest, TestWithAlterTable) { /*embedded_index=*/embedded_index, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); }; std::vector embedded_bytes1 = { @@ -1870,7 +1877,8 @@ TEST_P(ReadInteWithIndexTest, TestWithBsiIndex) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); ASSERT_OK_AND_ASSIGN(auto split, @@ -1930,7 +1938,8 @@ TEST_P(ReadInteWithIndexTest, TestWithBloomFilterIndex) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); ASSERT_OK_AND_ASSIGN(auto split, @@ -2136,7 +2145,8 @@ TEST_P(ReadInteWithIndexTest, TestBitmapPushDownWithMultiStripes) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); ASSERT_OK_AND_ASSIGN(auto split, @@ -2246,7 +2256,8 @@ TEST_P(ReadInteWithIndexTest, TestWithBitmapAndBsiAndBloomFilterIndex) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); ASSERT_OK_AND_ASSIGN(auto split, @@ -2328,7 +2339,8 @@ TEST_P(ReadInteWithIndexTest, TestWithIndexWithoutRegistered) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); ASSERT_OK_AND_ASSIGN(auto split, @@ -2465,7 +2477,8 @@ TEST_P(ReadInteWithIndexTest, TestRangeBitmapIndex) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); @@ -2510,7 +2523,8 @@ TEST_P(ReadInteWithIndexTest, TestRangeBitmapIndexMultiChunk) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); @@ -2552,7 +2566,8 @@ TEST_P(ReadInteWithIndexTest, TestWithIOException) { /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/path + "bucket-0/", {data_file_meta}); ASSERT_OK_AND_ASSIGN(auto split, diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index fdc38ea9f..b33fcf36b 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -148,7 +148,7 @@ class ScanInteTest : public testing::TestWithParam { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); std::shared_ptr meta_snapshot1_partition10_bucket1_ = std::make_shared( @@ -164,7 +164,7 @@ class ScanInteTest : public testing::TestWithParam { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); std::shared_ptr meta_snapshot1_partition20_bucket0_ = std::make_shared( @@ -180,7 +180,7 @@ class ScanInteTest : public testing::TestWithParam { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); std::shared_ptr meta_snapshot2_partition10_bucket1_ = std::make_shared( @@ -196,7 +196,7 @@ class ScanInteTest : public testing::TestWithParam { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); std::shared_ptr meta_snapshot2_partition20_bucket0_ = std::make_shared( @@ -212,7 +212,7 @@ class ScanInteTest : public testing::TestWithParam { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); std::shared_ptr meta_snapshot3_partition10_bucket1_ = std::make_shared( @@ -228,7 +228,7 @@ class ScanInteTest : public testing::TestWithParam { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); std::shared_ptr meta_snapshot4_partition10_bucket1_ = std::make_shared( @@ -244,7 +244,7 @@ class ScanInteTest : public testing::TestWithParam { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); std::shared_ptr meta_snapshot5_partition10_bucket1_ = std::make_shared( @@ -260,7 +260,7 @@ class ScanInteTest : public testing::TestWithParam { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); }; TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { @@ -1422,7 +1422,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder1( BinaryRowGenerator::GenerateRow({10, 0}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ @@ -1483,7 +1483,7 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ @@ -1549,7 +1549,7 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Compact(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ @@ -1591,7 +1591,8 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { /*creation_time=*/Timestamp(1731412938869ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::optional>({"f0", "f1", "f2"}), - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto file_meta2 = std::make_shared( "data-c2613568-0412-4cd9-a0c4-1eae8e4ca89b-0.orc", /*file_size=*/575, /*row_count=*/3, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), @@ -1603,7 +1604,8 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { /*creation_time=*/Timestamp(1731412938891ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::optional>({"f0", "f1", "f2"}), - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); auto file_meta3 = std::make_shared( "data-a6d1261a-f798-4fbd-a251-6d6c7d8060dd-0.orc", /*file_size=*/541, /*row_count=*/1, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), @@ -1615,7 +1617,8 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { /*creation_time=*/Timestamp(1731412938908ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::optional>({"f0", "f1", "f2"}), - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -1689,7 +1692,8 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { /*creation_time=*/Timestamp(1731412938891ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::optional>({"f0", "f1", "f2"}), - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -1787,7 +1791,8 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { /*creation_time=*/Timestamp(1732635461460ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRowGenerator::GenerateRow({1, 1}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ @@ -1833,7 +1838,8 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { /*creation_time=*/Timestamp(1730458825047ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder1(BinaryRowGenerator::GenerateRow({1, 1}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ @@ -1860,7 +1866,8 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { /*creation_time=*/Timestamp(1730459969493ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder2(BinaryRowGenerator::GenerateRow({0, 1}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ @@ -1910,7 +1917,8 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { /*creation_time=*/Timestamp(1751647880163ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::optional>({"key0", "f0", "f1", "f2"}), - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder( BinaryRowGenerator::GenerateRow({1}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ @@ -1992,7 +2000,8 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { /*creation_time=*/Timestamp(1745000702835ll, 0), /*delete_row_count=*/0, /*embedded_index=*/embedded_index, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/table_path + "bucket-0", {file_meta}); ASSERT_OK_AND_ASSIGN(auto expected_data_split, builder.WithTotalBuckets(-1) @@ -2057,7 +2066,8 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { /*creation_time=*/Timestamp(1745235371029ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/table_path + "bucket-0", {file_meta}); ASSERT_OK_AND_ASSIGN(auto expected_data_split, builder.WithTotalBuckets(-1) @@ -2124,7 +2134,8 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { /*creation_time=*/Timestamp(1745253323731ll, 0), /*delete_row_count=*/0, /*embedded_index=*/embedded_index, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/table_path + "bucket-0", {file_meta}); @@ -2198,7 +2209,8 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { /*creation_time=*/Timestamp(1745251357742ll, 0), /*delete_row_count=*/0, /*embedded_index=*/embedded_index, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/table_path + "bucket-0", {file_meta}); @@ -2274,7 +2286,8 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { /*creation_time=*/Timestamp(1745251357742ll, 0), /*delete_row_count=*/0, /*embedded_index=*/embedded_index, FileSource::Append(), /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/table_path + "bucket-0", {file_meta}); diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index 451c3ce29..9035ae6e5 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -208,7 +208,7 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface file_meta->level, file_meta->extra_files, file_meta->creation_time, file_meta->delete_row_count, file_meta->embedded_index, file_meta->file_source, file_meta->value_stats_cols, file_meta->external_path, file_meta->first_row_id, - file_meta->write_cols); + file_meta->write_cols, /*column_max_sequence_numbers=*/std::nullopt); auto generate_null_stats = [this](const SimpleStats& stats) -> SimpleStats { if (stats == SimpleStats::EmptyStats()) { return stats; @@ -445,7 +445,7 @@ TEST_P(WriteInteTest, TestAppendTableBatchWrite) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta = ReconstructDataFileMeta(file_meta); DataIncrement data_increment({file_meta}, {}, {}); std::shared_ptr expected_commit_message_1 = std::make_shared( @@ -549,7 +549,7 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithOneBucket) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta = ReconstructDataFileMeta(file_meta); DataIncrement data_increment({file_meta}, {}, {}); std::shared_ptr expected_commit_message_1 = std::make_shared( @@ -619,7 +619,7 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithOneBucket) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_2 = ReconstructDataFileMeta(file_meta_2); DataIncrement data_increment_2({file_meta_2}, {}, {}); std::shared_ptr expected_commit_message_2 = std::make_shared( @@ -862,7 +862,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithComplexType) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta = ReconstructDataFileMeta(file_meta); DataIncrement data_increment({file_meta}, {}, {}); std::shared_ptr expected_commit_message_1 = std::make_shared( @@ -926,7 +926,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithComplexType) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_2 = ReconstructDataFileMeta(file_meta_2); DataIncrement data_increment_2({file_meta_2}, {}, {}); std::shared_ptr expected_commit_message_2 = std::make_shared( @@ -1011,7 +1011,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_1 = ReconstructDataFileMeta(file_meta_1); DataIncrement data_increment_1({file_meta_1}, {}, {}); std::shared_ptr expected_commit_message_1 = std::make_shared( @@ -1037,7 +1037,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_2 = ReconstructDataFileMeta(file_meta_2); DataIncrement data_increment_2({file_meta_2}, {}, {}); std::shared_ptr expected_commit_message_2 = std::make_shared( @@ -1063,7 +1063,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_3 = ReconstructDataFileMeta(file_meta_3); DataIncrement data_increment_3({file_meta_3}, {}, {}); std::shared_ptr expected_commit_message_3 = std::make_shared( @@ -1135,7 +1135,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_4 = ReconstructDataFileMeta(file_meta_4); DataIncrement data_increment_4({file_meta_4}, {}, {}); std::shared_ptr expected_commit_message_4 = std::make_shared( @@ -1161,7 +1161,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_5 = ReconstructDataFileMeta(file_meta_5); DataIncrement data_increment_5({file_meta_5}, {}, {}); std::shared_ptr expected_commit_message_5 = std::make_shared( @@ -1187,7 +1187,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_6 = ReconstructDataFileMeta(file_meta_6); DataIncrement data_increment_6({file_meta_6}, {}, {}); std::shared_ptr expected_commit_message_6 = std::make_shared( @@ -1285,7 +1285,7 @@ TEST_P(WriteInteTest, TestPkTableBatchWrite) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_1 = ReconstructDataFileMeta(file_meta_1); DataIncrement data_increment_1({file_meta_1}, {}, {}); std::shared_ptr expected_commit_message_1 = std::make_shared( @@ -1311,7 +1311,7 @@ TEST_P(WriteInteTest, TestPkTableBatchWrite) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_2 = ReconstructDataFileMeta(file_meta_2); DataIncrement data_increment_2({file_meta_2}, {}, {}); std::shared_ptr expected_commit_message_2 = std::make_shared( @@ -1337,7 +1337,7 @@ TEST_P(WriteInteTest, TestPkTableBatchWrite) { /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_3 = ReconstructDataFileMeta(file_meta_3); DataIncrement data_increment_3({file_meta_3}, {}, {}); std::shared_ptr expected_commit_message_3 = std::make_shared( @@ -1452,7 +1452,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_1 = ReconstructDataFileMeta(file_meta_1); DataIncrement data_increment_1({file_meta_1}, {}, {}); std::shared_ptr expected_commit_message_1 = std::make_shared( @@ -1482,7 +1482,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) { /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_2 = ReconstructDataFileMeta(file_meta_2); DataIncrement data_increment_2({file_meta_2}, {}, {}); std::shared_ptr expected_commit_message_2 = std::make_shared( @@ -1558,7 +1558,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_3 = ReconstructDataFileMeta(file_meta_3); DataIncrement data_increment_3({file_meta_3}, {}, {}); std::shared_ptr expected_commit_message_3 = std::make_shared( @@ -1586,7 +1586,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_4 = ReconstructDataFileMeta(file_meta_4); DataIncrement data_increment_4({file_meta_4}, {}, {}); std::shared_ptr expected_commit_message_4 = std::make_shared( @@ -1696,7 +1696,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithComplexType) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta = ReconstructDataFileMeta(file_meta); DataIncrement data_increment({file_meta}, {}, {}); std::shared_ptr expected_commit_message_1 = std::make_shared( @@ -1769,7 +1769,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithComplexType) { /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_2 = ReconstructDataFileMeta(file_meta_2); DataIncrement data_increment_2({file_meta_2}, {}, {}); std::shared_ptr expected_commit_message_2 = std::make_shared( @@ -2036,7 +2036,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_1 = ReconstructDataFileMeta(file_meta_1); DataIncrement data_increment_1({file_meta_1}, {}, {}); std::shared_ptr expected_commit_message_1 = @@ -2065,7 +2065,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_2 = ReconstructDataFileMeta(file_meta_2); DataIncrement data_increment_2({file_meta_2}, {}, {}); std::shared_ptr expected_commit_message_2 = @@ -2094,7 +2094,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { /*delete_row_count=*/2, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_3 = ReconstructDataFileMeta(file_meta_3); DataIncrement data_increment_3({file_meta_3}, {}, {}); std::shared_ptr expected_commit_message_3 = @@ -2126,7 +2126,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_4 = ReconstructDataFileMeta(file_meta_4); DataIncrement data_increment_4({file_meta_4}, {}, {}); std::shared_ptr expected_commit_message_4 = @@ -2154,7 +2154,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_5 = ReconstructDataFileMeta(file_meta_5); DataIncrement data_increment_5({file_meta_5}, {}, {}); std::shared_ptr expected_commit_message_5 = @@ -2183,7 +2183,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta_6 = ReconstructDataFileMeta(file_meta_6); DataIncrement data_increment_6({file_meta_6}, {}, {}); std::shared_ptr expected_commit_message_6 = @@ -2257,7 +2257,7 @@ TEST_F(WriteInteTest, TestAppendTableWriteWithAlterTable) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment({file_meta}, {}, {}); std::shared_ptr expected_commit_message = std::make_shared( BinaryRowGenerator::GenerateRow({1, 1}, pool_.get()), /*bucket=*/0, @@ -2337,7 +2337,7 @@ TEST_F(WriteInteTest, TestPKTableWriteWithAlterTable) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment({file_meta}, {}, {}); std::shared_ptr expected_commit_message = std::make_shared( BinaryRowGenerator::GenerateRow({0, 0}, pool_.get()), /*bucket=*/0, @@ -2775,7 +2775,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithExternalPath) { /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(1724090888706ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), - /*value_stats_cols=*/std::nullopt, "FILE:/tmp/xxx", std::nullopt, std::nullopt); + /*value_stats_cols=*/std::nullopt, "FILE:/tmp/xxx", std::nullopt, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); file_meta = ReconstructDataFileMeta(file_meta); DataIncrement data_increment({file_meta}, {}, {}); std::shared_ptr expected_commit_message_1 = std::make_shared( @@ -2844,7 +2845,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithExternalPath) { /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(1724090888706ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), - /*value_stats_cols=*/std::nullopt, "FILE:/tmp/xxx", std::nullopt, std::nullopt); + /*value_stats_cols=*/std::nullopt, "FILE:/tmp/xxx", std::nullopt, std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); file_meta_2 = ReconstructDataFileMeta(file_meta_2); DataIncrement data_increment_2({file_meta_2}, {}, {}); std::shared_ptr expected_commit_message_2 = std::make_shared( @@ -3443,7 +3445,7 @@ TEST_P(WriteInteTest, TestPkTablePostponeBucket) { /*delete_row_count=*/1, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment({file_meta}, {}, {}); std::shared_ptr expected_commit_message = std::make_shared( /*partition_map=*/BinaryRow::EmptyRow(), /*bucket=*/-2, @@ -3532,7 +3534,7 @@ TEST_F(WriteInteTest, TestBranchWrite) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment_0({file_meta_0}, {}, {}); auto expected_commit_message_0 = std::make_shared( BinaryRowGenerator::GenerateRow({std::string("20240726")}, pool_.get()), /*bucket=*/0, @@ -3574,7 +3576,7 @@ TEST_F(WriteInteTest, TestBranchWrite) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); DataIncrement data_increment_1({file_meta_1}, {}, {}); auto expected_commit_message_1 = std::make_shared( BinaryRowGenerator::GenerateRow({std::string("20240725")}, pool_.get()), /*bucket=*/0, @@ -3704,7 +3706,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); check_meta(commit_msgs1[0], {file_meta1}); commit(commit_msgs1, /*latest_snapshot_id=*/1, /*next_row_id=*/2); check_committed_meta({{0, std::nullopt, 1, 1}}); @@ -3736,7 +3738,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/write_cols2); + /*write_cols=*/write_cols2, /*column_max_sequence_numbers=*/std::nullopt); check_meta(commit_msgs2[0], {file_meta2}); commit(commit_msgs2, /*latest_snapshot_id=*/2, /*next_row_id=*/7); check_committed_meta({{0, std::nullopt, 1, 1}, {2, write_cols2, 2, 2}}); @@ -3762,7 +3764,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/write_cols3); + /*write_cols=*/write_cols3, /*column_max_sequence_numbers=*/std::nullopt); check_meta(commit_msgs3[0], {file_meta3}); auto commit_msg_impl = std::dynamic_pointer_cast(commit_msgs3[0]); ASSERT_TRUE(commit_msg_impl); @@ -3834,7 +3836,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { /*creation_time=*/Timestamp(1724090888706ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0, - /*write_cols=*/std::vector({"f0", "f1"})); + /*write_cols=*/std::vector({"f0", "f1"}), + /*column_max_sequence_numbers=*/std::nullopt); file_meta1 = ReconstructDataFileMeta(file_meta1); auto file_meta2 = std::make_shared( "data-xxx.blob", /*file_size=*/764, /*row_count=*/3, @@ -3845,7 +3848,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { /*creation_time=*/Timestamp(1724090888706ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0, - /*write_cols=*/std::vector({"blob"})); + /*write_cols=*/std::vector({"blob"}), + /*column_max_sequence_numbers=*/std::nullopt); auto file_meta3 = std::make_shared( "data-xxx.blob", /*file_size=*/3023, /*row_count=*/1, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), @@ -3855,7 +3859,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { /*creation_time=*/Timestamp(1724090888706ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/3, - /*write_cols=*/std::vector({"blob"})); + /*write_cols=*/std::vector({"blob"}), + /*column_max_sequence_numbers=*/std::nullopt); std::vector> expected_meta = {file_meta1, file_meta2, file_meta3}; // NOTE: Due to the write logic of C++ Paimon and Java Paimon is different, the first_row_id in @@ -3924,7 +3929,7 @@ TEST_P(WriteInteTest, TestAppendTableWithDateFieldAsPartitionField) { /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt); + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); file_meta = ReconstructDataFileMeta(file_meta); DataIncrement data_increment({file_meta}, {}, {}); std::shared_ptr expected_commit_message_1 = std::make_shared( @@ -4728,7 +4733,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0, - /*write_cols=*/std::vector({"f0", "f1"})); + /*write_cols=*/std::vector({"f0", "f1"}), + /*column_max_sequence_numbers=*/std::nullopt); expected_main = ReconstructDataFileMeta(expected_main); // blob1 file: 3 rows, write_cols={"blob1"}, first_row_id=0 @@ -4741,7 +4747,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0, - /*write_cols=*/std::vector({"blob1"})); + /*write_cols=*/std::vector({"blob1"}), + /*column_max_sequence_numbers=*/std::nullopt); // blob2 file: 3 rows, write_cols={"blob2"}, first_row_id=0 auto expected_blob2 = std::make_shared( @@ -4753,7 +4760,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/0, - /*write_cols=*/std::vector({"blob2"})); + /*write_cols=*/std::vector({"blob2"}), + /*column_max_sequence_numbers=*/std::nullopt); ASSERT_OK_AND_ASSIGN(auto commit_msgs, helper->WriteAndCommit(std::move(batch), commit_identifier++, diff --git a/test/test_data/compatibility/commit_message-v13 b/test/test_data/compatibility/commit_message-v13 new file mode 100644 index 000000000..568705a59 Binary files /dev/null and b/test/test_data/compatibility/commit_message-v13 differ diff --git a/test/test_data/compatibility/data_split-v9 b/test/test_data/compatibility/data_split-v9 new file mode 100644 index 000000000..2bfd4ac19 Binary files /dev/null and b/test/test_data/compatibility/data_split-v9 differ