From 335fa2e075024b3cf52755379d869b500d0c20b8 Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Wed, 26 Aug 2026 17:23:07 +0800 Subject: [PATCH 1/9] feat(core): support input and lookup changelog producers --- include/paimon/defs.h | 24 +- src/paimon/CMakeLists.txt | 1 + src/paimon/common/defs.cpp | 6 + src/paimon/common/utils/fields_comparator.cpp | 7 +- .../common/utils/fields_comparator_test.cpp | 45 ++ src/paimon/core/core_options.cpp | 42 ++ src/paimon/core/core_options.h | 5 + src/paimon/core/core_options_test.cpp | 16 + .../async_key_value_producer_and_consumer.cpp | 116 ++- .../async_key_value_producer_and_consumer.h | 66 +- src/paimon/core/io/data_file_path_factory.h | 5 +- .../core/io/data_file_path_factory_test.cpp | 7 + .../io/key_value_data_file_writer_factory.cpp | 54 +- .../io/key_value_data_file_writer_factory.h | 8 +- ...ing_key_value_data_file_writer_factory.cpp | 31 +- ...dding_key_value_data_file_writer_factory.h | 2 +- src/paimon/core/manifest/manifest_list.h | 3 +- .../core/manifest/manifest_list_test.cpp | 28 + .../compact/changelog_merge_tree_rewriter.cpp | 224 +++++- .../compact/changelog_merge_tree_rewriter.h | 9 + .../core/mergetree/compact/changelog_result.h | 32 + .../first_row_merge_function_wrapper.h | 34 +- .../first_row_merge_function_wrapper_test.cpp | 31 +- .../compact/internal_row_equalizer.h | 245 ++++++ .../compact/internal_row_equalizer_test.cpp | 201 +++++ .../lookup_changelog_merge_function_wrapper.h | 99 ++- ..._changelog_merge_function_wrapper_test.cpp | 196 ++++- .../mergetree/compact/lookup_merge_function.h | 5 + .../lookup_merge_tree_compact_rewriter.cpp | 48 +- .../lookup_merge_tree_compact_rewriter.h | 26 +- ...ookup_merge_tree_compact_rewriter_test.cpp | 320 +++++++- .../merge_tree_compact_manager_factory.cpp | 155 ++-- .../merge_tree_compact_manager_factory.h | 11 +- ...erge_tree_compact_manager_factory_test.cpp | 17 +- .../compact/merge_tree_compact_rewriter.cpp | 46 +- .../compact/merge_tree_compact_rewriter.h | 10 +- src/paimon/core/mergetree/lookup_levels.cpp | 14 +- .../core/mergetree/merge_tree_writer.cpp | 114 ++- src/paimon/core/mergetree/merge_tree_writer.h | 5 + .../core/mergetree/merge_tree_writer_test.cpp | 214 +++++- src/paimon/core/mergetree/write_buffer.cpp | 4 + src/paimon/core/mergetree/write_buffer.h | 4 + src/paimon/core/operation/file_store_scan.cpp | 2 + .../operation/key_value_file_store_scan.cpp | 3 +- .../operation/key_value_file_store_write.cpp | 2 +- .../core/operation/merge_file_split_read.cpp | 17 + .../core/operation/merge_file_split_read.h | 6 + .../core/postpone/postpone_bucket_writer.cpp | 5 +- src/paimon/core/schema/schema_validation.cpp | 58 +- src/paimon/core/schema/schema_validation.h | 2 +- .../core/schema/schema_validation_test.cpp | 79 +- .../table/source/data_table_stream_scan.cpp | 30 +- src/paimon/core/table/source/scan_mode.h | 4 +- .../snapshot/changelog_follow_up_scanner.h | 50 ++ .../source/snapshot/snapshot_reader_test.cpp | 21 + test/inte/scan_inte_test.cpp | 4 +- test/inte/write_and_read_inte_test.cpp | 713 ++++++++++++++++++ 57 files changed, 3216 insertions(+), 310 deletions(-) create mode 100644 src/paimon/core/mergetree/compact/changelog_result.h create mode 100644 src/paimon/core/mergetree/compact/internal_row_equalizer.h create mode 100644 src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp create mode 100644 src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 8062d3d2d..450ce036c 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -378,14 +378,34 @@ struct PAIMON_EXPORT Options { /// @note: bitmap64 dv is not supported. static const char DELETION_VECTOR_BITMAP64[]; - /// @note `CHANGELOG_PRODUCER` currently only support `none` - /// /// "changelog-producer" - Whether to double write to a changelog file. This changelog file /// keeps the details of data changes, it can be read directly during stream reads. This can be /// applied to tables with primary keys. Values can be "none", "input", "lookup", /// "full-compaction". Default value is "none". + /// @note C++ Paimon currently supports "none" and "input". static const char CHANGELOG_PRODUCER[]; + /// "changelog-producer.row-deduplicate" - Whether to generate update-before and update-after + /// changelog records when the row has not changed. This option is only valid for "lookup" or + /// "full-compaction" changelog producers. Default value is "false". + static const char CHANGELOG_PRODUCER_ROW_DEDUPLICATE[]; + + /// "changelog-producer.row-deduplicate-ignore-fields" - Comma-separated fields to ignore when + /// comparing rows for changelog deduplication. This option is only valid when + /// "changelog-producer.row-deduplicate" is "true". + static const char CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS[]; + + /// "changelog-file.prefix" - Specify the file name prefix of changelog files. Default value is + /// "changelog-". + static const char CHANGELOG_FILE_PREFIX[]; + + /// "changelog-file.format" - Specify the file format of changelog files. No default value. + static const char CHANGELOG_FILE_FORMAT[]; + + /// "changelog-file.compression" - Specify the compression of changelog files. No default + /// value. + static const char CHANGELOG_FILE_COMPRESSION[]; + /// "force-lookup" - Whether to force the use of lookup for compaction. Default value is /// "false". static const char FORCE_LOOKUP[]; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 61fd7c96d..7f001216f 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -811,6 +811,7 @@ if(PAIMON_BUILD_TESTS) core/mergetree/compact/deduplicate_merge_function_test.cpp core/mergetree/compact/first_row_merge_function_test.cpp core/mergetree/compact/first_row_merge_function_wrapper_test.cpp + core/mergetree/compact/internal_row_equalizer_test.cpp core/mergetree/compact/interval_partition_test.cpp core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 36b9c1c99..a5a876637 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -94,6 +94,12 @@ const char Options::DELETION_VECTOR_INDEX_FILE_TARGET_SIZE[] = "deletion-vector.index-file.target-size"; const char Options::DELETION_VECTOR_BITMAP64[] = "deletion-vectors.bitmap64"; const char Options::CHANGELOG_PRODUCER[] = "changelog-producer"; +const char Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE[] = "changelog-producer.row-deduplicate"; +const char Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS[] = + "changelog-producer.row-deduplicate-ignore-fields"; +const char Options::CHANGELOG_FILE_PREFIX[] = "changelog-file.prefix"; +const char Options::CHANGELOG_FILE_FORMAT[] = "changelog-file.format"; +const char Options::CHANGELOG_FILE_COMPRESSION[] = "changelog-file.compression"; const char Options::FORCE_LOOKUP[] = "force-lookup"; const char Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE[] = "partial-update.remove-record-on-delete"; diff --git a/src/paimon/common/utils/fields_comparator.cpp b/src/paimon/common/utils/fields_comparator.cpp index 224ec131f..523491184 100644 --- a/src/paimon/common/utils/fields_comparator.cpp +++ b/src/paimon/common/utils/fields_comparator.cpp @@ -128,21 +128,18 @@ Result FieldsComparator::CompareField( return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); }); case arrow::Type::type::FLOAT: - // TODO(xinyu.lxy): - // currently in java KeyComparatorSupplier: -inf < -0.0 == +0.0 < +inf = nan - // paimon-cpp: -inf < -0.0 == +0.0 < +inf and nan cannot be compared return FieldsComparator::FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { float lvalue = lhs.GetFloat(field_idx); float rvalue = rhs.GetFloat(field_idx); - return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); + return CompareFloatingPoint(lvalue, rvalue); }); case arrow::Type::type::DOUBLE: return FieldsComparator::FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { double lvalue = lhs.GetDouble(field_idx); double rvalue = rhs.GetDouble(field_idx); - return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); + return CompareFloatingPoint(lvalue, rvalue); }); case arrow::Type::type::STRING: case arrow::Type::type::BINARY: { diff --git a/src/paimon/common/utils/fields_comparator_test.cpp b/src/paimon/common/utils/fields_comparator_test.cpp index 2f64f813c..111de404c 100644 --- a/src/paimon/common/utils/fields_comparator_test.cpp +++ b/src/paimon/common/utils/fields_comparator_test.cpp @@ -20,8 +20,10 @@ #include "paimon/common/utils/fields_comparator.h" #include +#include #include #include +#include #include "arrow/api.h" #include "gtest/gtest.h" @@ -80,6 +82,44 @@ class FieldsComparatorTest : public ::testing::Test { } CheckResult(row1, row2, input_types, sort_fields, has_null); } + + template + void CheckFloatingPointOrder(const std::shared_ptr& type) { + std::shared_ptr pool = GetDefaultPool(); + std::vector data_fields = {DataField(0, arrow::field("f0", type))}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr ascending_comparator, + FieldsComparator::Create(data_fields, + /*is_ascending_order=*/true)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr descending_comparator, + FieldsComparator::Create(data_fields, + /*is_ascending_order=*/false)); + + const T nan = std::numeric_limits::quiet_NaN(); + const std::vector values = {-std::numeric_limits::infinity(), + static_cast(-1), + static_cast(-0.0), + static_cast(0.0), + static_cast(1), + std::numeric_limits::infinity(), + nan}; + std::vector rows; + rows.reserve(values.size()); + for (T value : values) { + rows.emplace_back(BinaryRowGenerator::GenerateRow({value}, pool.get())); + } + + for (size_t i = 0; i < rows.size(); ++i) { + for (size_t j = 0; j < rows.size(); ++j) { + int32_t expected = i == j ? 0 : (i < j ? -1 : 1); + ASSERT_EQ(expected, ascending_comparator->CompareTo(rows[i], rows[j])); + ASSERT_EQ(-expected, descending_comparator->CompareTo(rows[i], rows[j])); + } + } + + BinaryRow negative_nan_row = BinaryRowGenerator::GenerateRow({-nan}, pool.get()); + ASSERT_EQ(0, ascending_comparator->CompareTo(rows.back(), negative_nan_row)); + ASSERT_EQ(0, ascending_comparator->CompareTo(negative_nan_row, rows.back())); + } }; TEST_F(FieldsComparatorTest, TestSimple) { @@ -202,6 +242,11 @@ TEST_F(FieldsComparatorTest, TestSimple) { } } +TEST_F(FieldsComparatorTest, TestFloatingPointOrder) { + CheckFloatingPointOrder(arrow::float32()); + CheckFloatingPointOrder(arrow::float64()); +} + TEST_F(FieldsComparatorTest, TestTimestampType) { auto pool = GetDefaultPool(); // test ts with different precision diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 6320ec577..7e4d1e1a3 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -406,6 +406,7 @@ struct CoreOptions::Impl { ExpireConfig expire_config; std::vector sequence_field; std::vector remove_record_on_sequence_group; + std::vector changelog_row_deduplicate_ignore_fields; std::vector blob_fields; std::vector blob_descriptor_fields; std::vector blob_view_fields; @@ -416,12 +417,14 @@ struct CoreOptions::Impl { std::string manifest_compression = "zstd"; std::string branch = BranchManager::DEFAULT_MAIN_BRANCH; std::string data_file_prefix = "data-"; + std::string changelog_file_prefix = "changelog-"; std::string file_system_scheme_to_identifier_map_str; std::optional field_default_func; std::optional scan_fallback_branch; std::optional data_file_external_paths; std::optional blob_view_upstream_warehouse; + std::optional changelog_file_compression; std::map raw_options; @@ -464,6 +467,7 @@ struct CoreOptions::Impl { bool deletion_vectors_bitmap64 = false; bool force_lookup = false; bool lookup_wait = true; + bool changelog_row_deduplicate = false; bool partial_update_remove_record_on_delete = false; bool aggregation_remove_record_on_delete = false; bool table_read_sequence_number_enabled = false; @@ -513,6 +517,7 @@ struct CoreOptions::Impl { int64_t cache_page_size = 64 * 1024; // 64KB std::map> file_format_per_level; std::map file_compression_per_level; + std::shared_ptr changelog_file_format; int64_t lookup_cache_max_memory = 256 * 1024 * 1024; double lookup_cache_high_prio_pool_ratio = 0.25; int64_t lookup_cache_file_retention_ms = 1 * 3600 * 1000; // 1 hour @@ -590,6 +595,8 @@ struct CoreOptions::Impl { PAIMON_RETURN_NOT_OK(parser.ParseExternalPathStrategy(&external_path_strategy)); // Parse data-file.prefix - file name prefix of data files, default "data-" PAIMON_RETURN_NOT_OK(parser.Parse(Options::DATA_FILE_PREFIX, &data_file_prefix)); + // Parse changelog-file.prefix - file name prefix of changelog files, default "changelog-" + PAIMON_RETURN_NOT_OK(parser.Parse(Options::CHANGELOG_FILE_PREFIX, &changelog_file_prefix)); // Parse row-tracking.enabled - whether to enable unique row id for append table PAIMON_RETURN_NOT_OK( parser.Parse(Options::ROW_TRACKING_ENABLED, &row_tracking_enabled)); @@ -638,6 +645,14 @@ struct CoreOptions::Impl { Options::FILE_FORMAT, /*default_identifier=*/"parquet", &file_format)); // Parse file.compression - default file compression, default "zstd" PAIMON_RETURN_NOT_OK(parser.Parse(Options::FILE_COMPRESSION, &file_compression)); + // Parse changelog-file.format - no default value + if (parser.ContainsKey(Options::CHANGELOG_FILE_FORMAT)) { + PAIMON_RETURN_NOT_OK(parser.ParseObject( + Options::CHANGELOG_FILE_FORMAT, file_format->Identifier(), &changelog_file_format)); + } + // Parse changelog-file.compression - no default value + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::CHANGELOG_FILE_COMPRESSION, &changelog_file_compression)); // Parse file.compression.zstd-level - zstd compression level, default 1 PAIMON_RETURN_NOT_OK( parser.Parse(Options::FILE_COMPRESSION_ZSTD_LEVEL, &file_compression_zstd_level)); @@ -768,6 +783,13 @@ struct CoreOptions::Impl { PAIMON_RETURN_NOT_OK(parser.Parse(Options::FIELDS_DEFAULT_AGG_FUNC, &field_default_func)); // Parse changelog-producer - whether to double write to a changelog file, default "none" PAIMON_RETURN_NOT_OK(parser.ParseChangelogProducer(&changelog_producer)); + // Parse changelog-producer.row-deduplicate - skip unchanged row changelogs + PAIMON_RETURN_NOT_OK(parser.Parse(Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, + &changelog_row_deduplicate)); + // Parse changelog-producer.row-deduplicate-ignore-fields - ignored comparison fields + PAIMON_RETURN_NOT_OK(parser.ParseList( + Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, Options::FIELDS_SEPARATOR, + &changelog_row_deduplicate_ignore_fields, /*need_trim=*/true)); // Parse partial-update.remove-record-on-delete - remove whole row on delete PAIMON_RETURN_NOT_OK(parser.Parse(Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, &partial_update_remove_record_on_delete)); @@ -1620,6 +1642,26 @@ ChangelogProducer CoreOptions::GetChangelogProducer() const { return impl_->changelog_producer; } +bool CoreOptions::ChangelogRowDeduplicate() const { + return impl_->changelog_row_deduplicate; +} + +const std::vector& CoreOptions::GetChangelogRowDeduplicateIgnoreFields() const { + return impl_->changelog_row_deduplicate_ignore_fields; +} + +std::string CoreOptions::ChangelogFilePrefix() const { + return impl_->changelog_file_prefix; +} + +std::shared_ptr CoreOptions::GetChangelogFileFormat() const { + return impl_->changelog_file_format; +} + +std::optional CoreOptions::GetChangelogFileCompression() const { + return impl_->changelog_file_compression; +} + LookupStrategy CoreOptions::GetLookupStrategy() const { return LookupStrategy::From( /*is_first_row=*/GetMergeEngine() == MergeEngine::FIRST_ROW, diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index 85a4a7fdf..a2ea7e37c 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -207,6 +207,11 @@ class PAIMON_EXPORT CoreOptions { bool DeletionVectorsBitmap64() const; int64_t DeletionVectorTargetFileSize() const; ChangelogProducer GetChangelogProducer() const; + bool ChangelogRowDeduplicate() const; + const std::vector& GetChangelogRowDeduplicateIgnoreFields() const; + std::string ChangelogFilePrefix() const; + std::shared_ptr GetChangelogFileFormat() const; + std::optional GetChangelogFileCompression() const; LookupStrategy GetLookupStrategy() const; bool NeedLookup() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index c110623d7..4d98c05f3 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -39,6 +39,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); ASSERT_EQ(core_options.GetManifestFormat()->Identifier(), "avro"); ASSERT_EQ(core_options.GetFileFormat()->Identifier(), "parquet"); + ASSERT_EQ(nullptr, core_options.GetChangelogFileFormat()); ASSERT_EQ(core_options.GetWriteFileFormat(0)->Identifier(), "parquet"); ASSERT_EQ(core_options.GetWriteFileFormat(3)->Identifier(), "parquet"); ASSERT_TRUE(core_options.GetFileSystem()); @@ -58,6 +59,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_FALSE(core_options.RealtimeEnabled()); ASSERT_EQ(StatisticsMode::NONE, core_options.GetRealtimeStoreStatisticsMode()); ASSERT_EQ("zstd", core_options.GetFileCompression()); + ASSERT_EQ(std::nullopt, core_options.GetChangelogFileCompression()); ASSERT_EQ("zstd", core_options.GetWriteFileCompression(0)); ASSERT_EQ("zstd", core_options.GetWriteFileCompression(3)); ASSERT_EQ("zstd", core_options.GetManifestCompression()); @@ -125,6 +127,9 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_FALSE(core_options.DeletionVectorsBitmap64()); ASSERT_EQ(2 * 1024 * 1024, core_options.DeletionVectorTargetFileSize()); ASSERT_EQ(ChangelogProducer::NONE, core_options.GetChangelogProducer()); + ASSERT_FALSE(core_options.ChangelogRowDeduplicate()); + ASSERT_TRUE(core_options.GetChangelogRowDeduplicateIgnoreFields().empty()); + ASSERT_EQ("changelog-", core_options.ChangelogFilePrefix()); ASSERT_FALSE(core_options.NeedLookup()); ASSERT_FALSE(core_options.PrepareCommitWaitCompaction()); LookupStrategy expected_lookup_strategy = {/*is_first_row=*/false, @@ -191,6 +196,7 @@ TEST(CoreOptionsTest, TestFromMap) { std::map options = { {Options::FILE_SYSTEM, "Local"}, {Options::FILE_FORMAT, "ORC"}, + {Options::CHANGELOG_FILE_FORMAT, "avro"}, {Options::MANIFEST_FORMAT, "avRo"}, {Options::BUCKET, "3"}, {Options::PAGE_SIZE, "128 kb"}, @@ -246,6 +252,10 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::DELETION_VECTOR_BITMAP64, "true"}, {Options::DELETION_VECTOR_INDEX_FILE_TARGET_SIZE, "4MB"}, {Options::CHANGELOG_PRODUCER, "full-compaction"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, "f0, f2"}, + {Options::CHANGELOG_FILE_PREFIX, "test-changelog-"}, + {Options::CHANGELOG_FILE_COMPRESSION, "lz4"}, {Options::FORCE_LOOKUP, "true"}, {"fields.g_1,g_3.sequence-group", "c,d"}, {Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, "true"}, @@ -322,6 +332,7 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_TRUE(fs); ASSERT_EQ(core_options.GetFileFormat()->Identifier(), "orc"); + ASSERT_EQ(core_options.GetChangelogFileFormat()->Identifier(), "avro"); ASSERT_EQ(core_options.GetWriteFileFormat(0)->Identifier(), "avro"); ASSERT_EQ(core_options.GetWriteFileFormat(1)->Identifier(), "orc"); ASSERT_EQ(core_options.GetWriteFileFormat(3)->Identifier(), "parquet"); @@ -390,6 +401,11 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_TRUE(core_options.DeletionVectorsBitmap64()); ASSERT_EQ(4 * 1024 * 1024, core_options.DeletionVectorTargetFileSize()); ASSERT_EQ(ChangelogProducer::FULL_COMPACTION, core_options.GetChangelogProducer()); + ASSERT_TRUE(core_options.ChangelogRowDeduplicate()); + ASSERT_EQ(std::vector({"f0", "f2"}), + core_options.GetChangelogRowDeduplicateIgnoreFields()); + ASSERT_EQ("test-changelog-", core_options.ChangelogFilePrefix()); + ASSERT_EQ(std::optional("lz4"), core_options.GetChangelogFileCompression()); ASSERT_TRUE(core_options.NeedLookup()); ASSERT_TRUE(core_options.PrepareCommitWaitCompaction()); LookupStrategy expected_lookup_strategy = {/*is_first_row=*/false, diff --git a/src/paimon/core/io/async_key_value_producer_and_consumer.cpp b/src/paimon/core/io/async_key_value_producer_and_consumer.cpp index 1792b43cd..96a0a9d45 100644 --- a/src/paimon/core/io/async_key_value_producer_and_consumer.cpp +++ b/src/paimon/core/io/async_key_value_producer_and_consumer.cpp @@ -31,11 +31,34 @@ namespace paimon { class MemoryPool; +namespace { + +class AsyncKeyValueQueueBatchSink : public AsyncKeyValueBatchSink { + public: + AsyncKeyValueQueueBatchSink(std::atomic* consume_finished, + tbb::concurrent_bounded_queue* kv_queue) + : consume_finished_(consume_finished), kv_queue_(kv_queue) {} + + Status Write(AsyncKeyValueBatchType type, std::vector&& rows) override { + if (*consume_finished_) { + return Status::Cancelled("Key value conversion is cancelled"); + } + kv_queue_->push(AsyncKeyValueRowsBatch{type, std::move(rows)}); + return Status::OK(); + } + + private: + std::atomic* consume_finished_; + tbb::concurrent_bounded_queue* kv_queue_; +}; + +} // namespace + template AsyncKeyValueProducerAndConsumer::AsyncKeyValueProducerAndConsumer( std::unique_ptr&& sort_merge_reader, ConsumerCreator create_consumer, int32_t batch_size, int32_t consumer_thread_num, const std::shared_ptr& pool) - : batch_size_(std::min(batch_size, MAX_PROJECTION_BATCH_SIZE)), + : batch_size_(NormalizeProjectionBatchSize(batch_size)), consumer_thread_num_(consumer_thread_num), pool_(pool), sort_merge_reader_(std::move(sort_merge_reader)), @@ -44,6 +67,19 @@ AsyncKeyValueProducerAndConsumer::AsyncKeyValueProducerAndConsumer( result_queue_.set_capacity(RESULT_BATCH_COUNT); } +template +AsyncKeyValueProducerAndConsumer::AsyncKeyValueProducerAndConsumer( + std::unique_ptr&& producer, ConsumerCreator data_consumer_creator, + ConsumerCreator changelog_consumer_creator) + : batch_size_(0), + consumer_thread_num_(1), + create_consumer_(std::move(data_consumer_creator)), + create_changelog_consumer_(std::move(changelog_consumer_creator)), + producer_(std::move(producer)) { + kv_queue_.set_capacity(consumer_thread_num_ * 2); + result_queue_.set_capacity(RESULT_BATCH_COUNT); +} + template Status AsyncKeyValueProducerAndConsumer::CheckStatus() const { // check producer status @@ -70,6 +106,12 @@ Status AsyncKeyValueProducerAndConsumer::CheckStatusAndCleanUp() { template Result AsyncKeyValueProducerAndConsumer::NextBatch() { + PAIMON_ASSIGN_OR_RAISE(AsyncKeyValueResultBatch result, NextBatchWithType()); + return std::move(result.result); +} + +template +Result> AsyncKeyValueProducerAndConsumer::NextBatchWithType() { if (!producer_future_.valid()) { producer_future_ = std::async(std::launch::async, &AsyncKeyValueProducerAndConsumer::ProduceLoop, this) @@ -78,11 +120,17 @@ Result AsyncKeyValueProducerAndConsumer::NextBatch() { if (consumers_.empty()) { consumers_.reserve(consumer_thread_num_); for (int32_t i = 0; i < consumer_thread_num_; i++) { - Result>> consumer = create_consumer_(); - PAIMON_RETURN_NOT_OK(consumer.status()); + std::unique_ptr> consumer; + if (create_consumer_) { + PAIMON_ASSIGN_OR_RAISE(consumer, create_consumer_()); + } + std::unique_ptr> changelog_consumer; + if (create_changelog_consumer_) { + PAIMON_ASSIGN_OR_RAISE(changelog_consumer, create_changelog_consumer_()); + } auto async_consumer = std::make_unique>( - std::move(consumer).value(), consume_finished_, consumer_finished_count_, kv_queue_, - result_queue_); + std::move(consumer), std::move(changelog_consumer), consume_finished_, + consumer_finished_count_, kv_queue_, result_queue_); consumers_.push_back(std::move(async_consumer)); } } @@ -90,16 +138,16 @@ Result AsyncKeyValueProducerAndConsumer::NextBatch() { if (next_batch_finished_) { // projection reader is eof - return R(); + return AsyncKeyValueResultBatch(); } - R result; + AsyncKeyValueResultBatch result; while (!result_queue_.try_pop(result)) { PAIMON_RETURN_NOT_OK(CheckStatusAndCleanUp()); if (consumer_finished_count_ == consumer_thread_num_ && result_queue_.empty()) { // all consume thread finished next_batch_finished_ = true; - return R(); + return AsyncKeyValueResultBatch(); } usleep(1000); } @@ -109,6 +157,13 @@ Result AsyncKeyValueProducerAndConsumer::NextBatch() { template Status AsyncKeyValueProducerAndConsumer::ProduceLoop() { + if (producer_) { + AsyncKeyValueQueueBatchSink sink(&consume_finished_, &kv_queue_); + PAIMON_RETURN_NOT_OK(producer_->Produce(&sink)); + kv_queue_.push(AsyncKeyValueRowsBatch()); + return Status::OK(); + } + std::vector batch; batch.reserve(batch_size_); while (!consume_finished_) { @@ -124,7 +179,8 @@ Status AsyncKeyValueProducerAndConsumer::ProduceLoop() { } batch.push_back(std::move(iterator->Next())); if (static_cast(batch.size()) >= batch_size_) { - kv_queue_.push(std::move(batch)); + kv_queue_.push( + AsyncKeyValueRowsBatch{AsyncKeyValueBatchType::DATA, std::move(batch)}); batch = std::vector(); batch.reserve(batch_size_); } @@ -132,10 +188,10 @@ Status AsyncKeyValueProducerAndConsumer::ProduceLoop() { } // Push remaining rows if (!batch.empty()) { - kv_queue_.push(std::move(batch)); + kv_queue_.push(AsyncKeyValueRowsBatch{AsyncKeyValueBatchType::DATA, std::move(batch)}); } // Push empty batch as EOF signal - kv_queue_.push(std::vector()); + kv_queue_.push(AsyncKeyValueRowsBatch()); return Status::OK(); } @@ -155,8 +211,9 @@ void AsyncKeyValueProducerAndConsumer::CleanUp() { template void AsyncKeyValueProducerAndConsumer::CleanUpQueue() { - R read_batch; - while (result_queue_.try_pop(read_batch)) { + AsyncKeyValueResultBatch tagged_batch; + while (result_queue_.try_pop(tagged_batch)) { + R& read_batch = tagged_batch.result; if constexpr (std::is_same_v) { if (!BatchReader::IsEofBatch(read_batch)) { ReaderUtils::ReleaseReadBatch(std::move(read_batch)); @@ -168,7 +225,7 @@ void AsyncKeyValueProducerAndConsumer::CleanUpQueue() { } } - std::vector kv_batch; + AsyncKeyValueRowsBatch kv_batch; while (kv_queue_.try_pop(kv_batch)) { } } @@ -179,10 +236,12 @@ template class AsyncKeyValueProducerAndConsumer; template AsyncKeyValueConsumer::AsyncKeyValueConsumer( std::unique_ptr>&& key_value_consumer, + std::unique_ptr>&& changelog_consumer, std::atomic& consume_finished, std::atomic& consumer_finished_count, - tbb::concurrent_bounded_queue>& kv_queue, - tbb::concurrent_bounded_queue& result_queue) + tbb::concurrent_bounded_queue& kv_queue, + tbb::concurrent_bounded_queue>& result_queue) : key_value_consumer_(std::move(key_value_consumer)), + changelog_consumer_(std::move(changelog_consumer)), consume_finished_(consume_finished), consumer_finished_count_(consumer_finished_count), kv_queue_(kv_queue), @@ -205,18 +264,24 @@ Status AsyncKeyValueConsumer::GetStatus() const { template Status AsyncKeyValueConsumer::ConsumeLoop() { while (!consume_finished_) { - std::vector key_value_vec; - if (!kv_queue_.try_pop(key_value_vec)) { + AsyncKeyValueRowsBatch rows_batch; + if (!kv_queue_.try_pop(rows_batch)) { usleep(100); continue; } - if (key_value_vec.empty()) { + if (rows_batch.rows.empty()) { // Empty batch is EOF signal; re-push for other consumers - kv_queue_.push(std::move(key_value_vec)); + kv_queue_.push(std::move(rows_batch)); break; } - PAIMON_ASSIGN_OR_RAISE(R result, key_value_consumer_->NextBatch(key_value_vec)); - result_queue_.push(std::move(result)); + RowToArrowArrayConverter* consumer = + rows_batch.type == AsyncKeyValueBatchType::CHANGELOG ? changelog_consumer_.get() + : key_value_consumer_.get(); + if (consumer == nullptr) { + return Status::Invalid("Missing consumer for asynchronous key value batch"); + } + PAIMON_ASSIGN_OR_RAISE(R result, consumer->NextBatch(rows_batch.rows)); + result_queue_.push(AsyncKeyValueResultBatch{rows_batch.type, std::move(result)}); } consumer_finished_count_++; return Status::OK(); @@ -227,7 +292,12 @@ void AsyncKeyValueConsumer::CleanUp() { if (consumer_future_.valid()) { [[maybe_unused]] Status status = consumer_future_.get(); } - key_value_consumer_->CleanUp(); + if (key_value_consumer_) { + key_value_consumer_->CleanUp(); + } + if (changelog_consumer_) { + changelog_consumer_->CleanUp(); + } } template class AsyncKeyValueConsumer; diff --git a/src/paimon/core/io/async_key_value_producer_and_consumer.h b/src/paimon/core/io/async_key_value_producer_and_consumer.h index af8bbed68..bf6249c99 100644 --- a/src/paimon/core/io/async_key_value_producer_and_consumer.h +++ b/src/paimon/core/io/async_key_value_producer_and_consumer.h @@ -18,6 +18,7 @@ #pragma once +#include #include #include #include @@ -40,6 +41,36 @@ class AsyncKeyValueConsumer; class MemoryPool; class Metrics; +enum class AsyncKeyValueBatchType { + DATA, + CHANGELOG, +}; + +struct AsyncKeyValueRowsBatch { + AsyncKeyValueBatchType type = AsyncKeyValueBatchType::DATA; + std::vector rows; +}; + +class AsyncKeyValueBatchSink { + public: + virtual ~AsyncKeyValueBatchSink() = default; + + virtual Status Write(AsyncKeyValueBatchType type, std::vector&& rows) = 0; +}; + +class AsyncKeyValueBatchProducer { + public: + virtual ~AsyncKeyValueBatchProducer() = default; + + virtual Status Produce(AsyncKeyValueBatchSink* sink) = 0; +}; + +template +struct AsyncKeyValueResultBatch { + AsyncKeyValueBatchType type = AsyncKeyValueBatchType::DATA; + R result; +}; + // Asynchronous iterate SortMergeReader (producer) and row-to-array conversion (consumer), support // multi-threaded conversion, R can be BatchReader::ReadBatch, KeyValueBatch template @@ -48,24 +79,39 @@ class AsyncKeyValueProducerAndConsumer { using ConsumerCreator = std::function>>()>; + // Limits the number of rows sent to one Arrow projection call. + static int32_t NormalizeProjectionBatchSize(int32_t batch_size) { + return std::min(batch_size, MAX_PROJECTION_BATCH_SIZE); + } + AsyncKeyValueProducerAndConsumer(std::unique_ptr&& sort_merge_reader, ConsumerCreator create_consumer, int32_t batch_size, int32_t consumer_thread_num, const std::shared_ptr& pool); + /// Creates a single-conversion-thread pipeline whose producer emits tagged data and changelog + /// row batches. Converted batches retain the producer order and are returned with their tag. + AsyncKeyValueProducerAndConsumer(std::unique_ptr&& producer, + ConsumerCreator data_consumer_creator, + ConsumerCreator changelog_consumer_creator); + ~AsyncKeyValueProducerAndConsumer() { CleanUp(); } Result NextBatch(); + Result> NextBatchWithType(); + std::shared_ptr GetReaderMetrics() const { - return sort_merge_reader_->GetReaderMetrics(); + return sort_merge_reader_ ? sort_merge_reader_->GetReaderMetrics() : nullptr; } void Close() { CleanUp(); - sort_merge_reader_->Close(); + if (sort_merge_reader_) { + sort_merge_reader_->Close(); + } } private: @@ -86,6 +132,8 @@ class AsyncKeyValueProducerAndConsumer { std::shared_ptr pool_; std::unique_ptr sort_merge_reader_; ConsumerCreator create_consumer_; + ConsumerCreator create_changelog_consumer_; + std::unique_ptr producer_; // produce: merge sort KeyValue and push result KeyValue to kv_queue_, consume: project KeyValue // to arrow array and push result array to result_queue_ @@ -94,18 +142,19 @@ class AsyncKeyValueProducerAndConsumer { std::shared_future producer_future_; std::vector>> consumers_; std::atomic consumer_finished_count_ = 0; - tbb::concurrent_bounded_queue> kv_queue_; - tbb::concurrent_bounded_queue result_queue_; + tbb::concurrent_bounded_queue kv_queue_; + tbb::concurrent_bounded_queue> result_queue_; }; template class AsyncKeyValueConsumer { public: AsyncKeyValueConsumer(std::unique_ptr>&& key_value_consumer, + std::unique_ptr>&& changelog_consumer, std::atomic& consume_finished, std::atomic& consumer_finished_count, - tbb::concurrent_bounded_queue>& kv_queue, - tbb::concurrent_bounded_queue& result_queue); + tbb::concurrent_bounded_queue& kv_queue, + tbb::concurrent_bounded_queue>& result_queue); ~AsyncKeyValueConsumer() { CleanUp(); @@ -119,11 +168,12 @@ class AsyncKeyValueConsumer { private: std::unique_ptr> key_value_consumer_; + std::unique_ptr> changelog_consumer_; std::shared_future consumer_future_; std::atomic& consume_finished_; std::atomic& consumer_finished_count_; - tbb::concurrent_bounded_queue>& kv_queue_; - tbb::concurrent_bounded_queue& result_queue_; + tbb::concurrent_bounded_queue& kv_queue_; + tbb::concurrent_bounded_queue>& result_queue_; }; } // namespace paimon diff --git a/src/paimon/core/io/data_file_path_factory.h b/src/paimon/core/io/data_file_path_factory.h index b49154f1b..a9050eeee 100644 --- a/src/paimon/core/io/data_file_path_factory.h +++ b/src/paimon/core/io/data_file_path_factory.h @@ -56,8 +56,9 @@ class DataFilePathFactory : public PathFactory { return NewPath(data_file_prefix_); } - std::string NewChangelogPath() const { - return NewPath(std::string(CHANGELOG_FILE_PREFIX)); + std::string NewChangelogPath(const std::string& changelog_file_prefix, + const std::string& format_identifier) const { + return NewPathFromName(NewFileName(changelog_file_prefix, "." + format_identifier)); } std::string NewBlobPath() const { diff --git a/src/paimon/core/io/data_file_path_factory_test.cpp b/src/paimon/core/io/data_file_path_factory_test.cpp index 12618a2ec..f3e311dbc 100644 --- a/src/paimon/core/io/data_file_path_factory_test.cpp +++ b/src/paimon/core/io/data_file_path_factory_test.cpp @@ -59,6 +59,13 @@ TEST_F(DataFilePathFactoryTest, TestNewPath) { ASSERT_EQ(factory_.NewPathFromName("index-file"), "/tmp/index-file"); } +TEST_F(DataFilePathFactoryTest, TestNewChangelogPath) { + std::string path = factory_.NewChangelogPath("changes-", "parquet"); + + ASSERT_TRUE(path.find("/tmp/changes-") != std::string::npos); + ASSERT_TRUE(StringUtils::EndsWith(path, ".parquet")); +} + TEST_F(DataFilePathFactoryTest, TestNewPathWithDataFilePrefixAndExternalPath) { DataFilePathFactory factory; ASSERT_OK_AND_ASSIGN( diff --git a/src/paimon/core/io/key_value_data_file_writer_factory.cpp b/src/paimon/core/io/key_value_data_file_writer_factory.cpp index 8f3885591..e0640ab61 100644 --- a/src/paimon/core/io/key_value_data_file_writer_factory.cpp +++ b/src/paimon/core/io/key_value_data_file_writer_factory.cpp @@ -20,6 +20,7 @@ #include "paimon/core/io/key_value_data_file_writer_factory.h" #include +#include #include #include "arrow/c/helpers.h" @@ -37,14 +38,15 @@ KeyValueDataFileWriterFactory::KeyValueDataFileWriterFactory( const std::shared_ptr& write_schema, int32_t level, FileSource file_source, const std::vector& primary_keys, const std::shared_ptr& path_factory, bool create_stats_extractor, - const std::shared_ptr& pool) + bool is_changelog, const std::shared_ptr& pool) : DataFileWriterFactory(options, schema_id, pool), write_schema_(write_schema), level_(level), file_source_(file_source), primary_keys_(primary_keys), path_factory_(path_factory), - create_stats_extractor_(create_stats_extractor) {} + create_stats_extractor_(create_stats_extractor), + is_changelog_(is_changelog) {} Result>>> KeyValueDataFileWriterFactory::CreateWriter() const { @@ -54,22 +56,50 @@ KeyValueDataFileWriterFactory::CreateWriter() const { return Status::OK(); }; - auto format = options_.GetWriteFileFormat(level_); + std::shared_ptr format = GetFileFormat(); PAIMON_ASSIGN_OR_RAISE(WriterResources resources, CreateWriterResources(*format, write_schema_, create_stats_extractor_)); auto writer = std::make_unique( - options_.GetWriteFileCompression(level_), std::move(converter), schema_id_, level_, - file_source_, primary_keys_, resources.stats_extractor, write_schema_, - path_factory_->IsExternalPath(), pool_); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_index_writer, - CreateFileIndexWriter(write_schema_, path_factory_)); - if (file_index_writer) { - writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_); + GetFileCompression(), std::move(converter), schema_id_, level_, file_source_, primary_keys_, + resources.stats_extractor, write_schema_, path_factory_->IsExternalPath(), pool_); + if (!is_changelog_) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_index_writer, + CreateFileIndexWriter(write_schema_, path_factory_)); + if (file_index_writer) { + writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_); + } } - PAIMON_RETURN_NOT_OK( - writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); + PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), NewFilePath(format->Identifier()), + resources.writer_builder)); return std::unique_ptr>>( std::move(writer)); } +std::shared_ptr KeyValueDataFileWriterFactory::GetFileFormat() const { + if (is_changelog_) { + std::shared_ptr changelog_format = options_.GetChangelogFileFormat(); + if (changelog_format) { + return changelog_format; + } + } + return options_.GetWriteFileFormat(level_); +} + +std::string KeyValueDataFileWriterFactory::GetFileCompression() const { + if (is_changelog_) { + std::optional changelog_compression = options_.GetChangelogFileCompression(); + if (changelog_compression) { + return changelog_compression.value(); + } + } + return options_.GetWriteFileCompression(level_); +} + +std::string KeyValueDataFileWriterFactory::NewFilePath(const std::string& format_identifier) const { + if (is_changelog_) { + return path_factory_->NewChangelogPath(options_.ChangelogFilePrefix(), format_identifier); + } + return path_factory_->NewPath(); +} + } // namespace paimon diff --git a/src/paimon/core/io/key_value_data_file_writer_factory.h b/src/paimon/core/io/key_value_data_file_writer_factory.h index 6ac50aba3..533b84c6e 100644 --- a/src/paimon/core/io/key_value_data_file_writer_factory.h +++ b/src/paimon/core/io/key_value_data_file_writer_factory.h @@ -38,6 +38,7 @@ namespace paimon { class CoreOptions; class DataFilePathFactory; +class FileFormat; class MemoryPool; class KeyValueDataFileWriterFactory @@ -49,19 +50,24 @@ class KeyValueDataFileWriterFactory FileSource file_source, const std::vector& primary_keys, const std::shared_ptr& path_factory, - bool create_stats_extractor, + bool create_stats_extractor, bool is_changelog, const std::shared_ptr& pool); Result>>> CreateWriter() const override; protected: + std::shared_ptr GetFileFormat() const; + std::string GetFileCompression() const; + std::string NewFilePath(const std::string& format_identifier) const; + std::shared_ptr write_schema_; int32_t level_; FileSource file_source_; std::vector primary_keys_; std::shared_ptr path_factory_; bool create_stats_extractor_; + bool is_changelog_; }; } // namespace paimon diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp index 30d4c9fce..f56ba8d14 100644 --- a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp +++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp @@ -37,10 +37,11 @@ ShreddingKeyValueDataFileWriterFactory::ShreddingKeyValueDataFileWriterFactory( const std::shared_ptr& write_schema, int32_t level, FileSource file_source, const std::vector& primary_keys, const std::shared_ptr& path_factory, bool create_stats_extractor, - const std::shared_ptr& plan_factory, + const std::shared_ptr& plan_factory, bool is_changelog, const std::shared_ptr& pool) : KeyValueDataFileWriterFactory(options, schema_id, write_schema, level, file_source, - primary_keys, path_factory, create_stats_extractor, pool), + primary_keys, path_factory, create_stats_extractor, + is_changelog, pool), plan_factory_(plan_factory) {} Result>>> @@ -48,7 +49,7 @@ ShreddingKeyValueDataFileWriterFactory::CreateWriter() const { if (!plan_factory_) { return Status::Invalid("Shredding key-value writer requires a write-plan factory."); } - const std::string format_identifier = options_.GetWriteFileFormat(level_)->Identifier(); + const std::string format_identifier = GetFileFormat()->Identifier(); if (plan_factory_->ShouldInferWritePlan()) { auto create_inner = [this](const std::shared_ptr& converter) { return CreateShreddedWriter(converter); @@ -74,7 +75,8 @@ ShreddingKeyValueDataFileWriterFactory::CreateShreddedWriter( [factory = plan_factory_, converter]() { return factory->OnFileCompleted(converter); }); return writer; } - auto format = options_.GetWriteFileFormat(level_); + std::shared_ptr format = GetFileFormat(); + std::string compression = GetFileCompression(); std::shared_ptr file_schema = converter->GetPhysicalSchema(); std::function batch_converter = [converter](KeyValueBatch key_value_batch, ::ArrowArray* array) -> Status { @@ -86,18 +88,19 @@ ShreddingKeyValueDataFileWriterFactory::CreateShreddedWriter( PAIMON_ASSIGN_OR_RAISE(WriterResources resources, CreateWriterResources(*format, file_schema, create_stats_extractor_)); auto writer = std::make_unique( - options_.GetWriteFileCompression(level_), std::move(batch_converter), schema_id_, level_, - file_source_, primary_keys_, resources.stats_extractor, file_schema, - path_factory_->IsExternalPath(), pool_); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_index_writer, - CreateFileIndexWriter(write_schema_, path_factory_)); - if (file_index_writer) { - writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_); + compression, std::move(batch_converter), schema_id_, level_, file_source_, primary_keys_, + resources.stats_extractor, file_schema, path_factory_->IsExternalPath(), pool_); + if (!is_changelog_) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_index_writer, + CreateFileIndexWriter(write_schema_, path_factory_)); + if (file_index_writer) { + writer->SetFileIndexWriter(std::move(file_index_writer), write_schema_); + } } - PAIMON_RETURN_NOT_OK( - writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); + PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), NewFilePath(format->Identifier()), + resources.writer_builder)); ShreddingWritePlanFactory::MetadataFinalizer finalizer = - plan_factory_->CreateMetadataFinalizer(converter, options_.GetWriteFileCompression(level_)); + plan_factory_->CreateMetadataFinalizer(converter, compression); if (finalizer) { writer->SetMetadataFinalizer(std::move(finalizer)); } diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h index fb967588e..e32dc8a04 100644 --- a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h +++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h @@ -41,7 +41,7 @@ class ShreddingKeyValueDataFileWriterFactory : public KeyValueDataFileWriterFact const std::shared_ptr& write_schema, int32_t level, FileSource file_source, const std::vector& primary_keys, const std::shared_ptr& path_factory, bool create_stats_extractor, - const std::shared_ptr& plan_factory, + const std::shared_ptr& plan_factory, bool is_changelog, const std::shared_ptr& pool); Result>>> diff --git a/src/paimon/core/manifest/manifest_list.h b/src/paimon/core/manifest/manifest_list.h index 31959a5c9..ff67e8080 100644 --- a/src/paimon/core/manifest/manifest_list.h +++ b/src/paimon/core/manifest/manifest_list.h @@ -113,8 +113,7 @@ class ManifestList : public ObjectsFile { const std::optional& changelog_manifest_list = snapshot.ChangelogManifestList(); if (changelog_manifest_list) { - return Status::NotImplemented("do not support read changelog manifest list"); - // return Read(changelog_manifest_list.value(), /*filter=*/nullptr, manifests); + return Read(changelog_manifest_list.value(), /*filter=*/nullptr, manifests); } else { return Status::OK(); } diff --git a/src/paimon/core/manifest/manifest_list_test.cpp b/src/paimon/core/manifest/manifest_list_test.cpp index 889f18f2b..23949bfdd 100644 --- a/src/paimon/core/manifest/manifest_list_test.cpp +++ b/src/paimon/core/manifest/manifest_list_test.cpp @@ -25,6 +25,7 @@ #include "gtest/gtest.h" #include "paimon/core/core_options.h" #include "paimon/core/manifest/manifest_file_meta.h" +#include "paimon/core/snapshot.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/format/file_format.h" @@ -141,6 +142,33 @@ TEST_F(ManifestListTest, TestEmptyManifestList) { ASSERT_EQ(manifest_file_metas.size(), 0); } +TEST_F(ManifestListTest, TestReadChangelogManifests) { + auto pool = GetDefaultPool(); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto manifest_list = CreateManifestList("orc", dir->Str(), pool); + ManifestFileMeta expected_meta( + "changelog-manifest", /*file_size=*/100, /*num_added_files=*/1, + /*num_deleted_files=*/0, SimpleStats::EmptyStats(), /*schema_id=*/0, + /*min_bucket=*/0, /*max_bucket=*/0, /*min_level=*/0, /*max_level=*/0, + /*min_row_id=*/std::nullopt, /*max_row_id=*/std::nullopt); + ASSERT_OK_AND_ASSIGN(auto changelog_manifest_list, manifest_list->Write({expected_meta})); + Snapshot snapshot( + /*id=*/1, /*schema_id=*/0, /*base_manifest_list=*/"", + /*base_manifest_list_size=*/std::nullopt, /*delta_manifest_list=*/"", + /*delta_manifest_list_size=*/std::nullopt, + /*changelog_manifest_list=*/changelog_manifest_list.first, + /*changelog_manifest_list_size=*/changelog_manifest_list.second, + /*index_manifest=*/std::nullopt, /*commit_user=*/"user", /*commit_identifier=*/1, + Snapshot::CommitKind::Append(), /*time_millis=*/0, /*total_record_count=*/1, + /*delta_record_count=*/1, /*changelog_record_count=*/1, /*watermark=*/std::nullopt, + /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/std::nullopt); + + std::vector actual_metas; + ASSERT_OK(manifest_list->ReadChangelogManifests(snapshot, &actual_metas)); + ASSERT_EQ(std::vector({expected_meta}), actual_metas); +} + TEST_F(ManifestListTest, TestManifestListCompatibleWithJavaPaimon09) { auto pool = GetDefaultPool(); auto manifest_file_metas = ReadManifestFileMeta("avro", paimon::test::GetDataDir() + "/avro", diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp index 3efd40f64..abf797dde 100644 --- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp @@ -17,7 +17,152 @@ */ #include "paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h" + +#include + +#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/io/key_value_meta_projection_consumer.h" +#include "paimon/core/io/row_to_arrow_array_converter.h" +#include "paimon/format/file_format.h" namespace paimon { + +namespace { + +using SortMergeReaderFactory = + std::function>(const std::vector& section)>; +using BoundChangelogMergeFunctionWrapperFactory = + std::function>>()>; +using KeyComparator = std::function; +using CancellationChecker = std::function; + +class ChangelogCompactionBatchProducer : public AsyncKeyValueBatchProducer { + public: + ChangelogCompactionBatchProducer( + const std::vector>& sections, + std::vector>& reader_holders, int32_t write_batch_size, + SortMergeReaderFactory reader_factory, + BoundChangelogMergeFunctionWrapperFactory merge_function_wrapper_factory, + KeyComparator key_comparator, CancellationChecker cancellation_checker, bool drop_delete, + bool produce_data, bool produce_changelog) + : sections_(sections), + reader_holders_(reader_holders), + write_batch_size_(write_batch_size), + reader_factory_(std::move(reader_factory)), + merge_function_wrapper_factory_(std::move(merge_function_wrapper_factory)), + key_comparator_(std::move(key_comparator)), + cancellation_checker_(std::move(cancellation_checker)), + drop_delete_(drop_delete), + produce_data_(produce_data), + produce_changelog_(produce_changelog) {} + + Status Produce(AsyncKeyValueBatchSink* sink) override { + std::vector compact_buffer; + std::vector changelog_buffer; + compact_buffer.reserve(write_batch_size_); + changelog_buffer.reserve(write_batch_size_); + + auto flush = [&](AsyncKeyValueBatchType type, std::vector* buffer) -> Status { + if (buffer->empty()) { + return Status::OK(); + } + std::vector rows = std::move(*buffer); + buffer->clear(); + buffer->reserve(write_batch_size_); + return sink->Write(type, std::move(rows)); + }; + + auto emit_result = [&](ChangelogResult&& result) -> Status { + if (produce_data_ && result.result && + (!drop_delete_ || result.result->value_kind->IsAdd())) { + compact_buffer.emplace_back(std::move(result.result).value()); + if (static_cast(compact_buffer.size()) >= write_batch_size_) { + PAIMON_RETURN_NOT_OK(flush(AsyncKeyValueBatchType::DATA, &compact_buffer)); + } + } + if (produce_changelog_) { + for (auto& changelog : result.changelogs) { + changelog_buffer.emplace_back(std::move(changelog)); + if (static_cast(changelog_buffer.size()) >= write_batch_size_) { + PAIMON_RETURN_NOT_OK( + flush(AsyncKeyValueBatchType::CHANGELOG, &changelog_buffer)); + } + } + } + return Status::OK(); + }; + + for (const auto& section : sections_) { + if (cancellation_checker_()) { + return Status::Cancelled("Compaction is cancelled"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr sort_merge_reader, + reader_factory_(section)); + reader_holders_.emplace_back(std::move(sort_merge_reader)); + SortMergeReader* reader = reader_holders_.back().get(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr> wrapper, + merge_function_wrapper_factory_()); + + std::shared_ptr current_key; + auto finish_group = [&]() -> Status { + if (!current_key) { + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE(std::optional result, wrapper->GetResult()); + current_key.reset(); + if (result) { + PAIMON_RETURN_NOT_OK(emit_result(std::move(result).value())); + } + return Status::OK(); + }; + + while (true) { + if (cancellation_checker_()) { + return Status::Cancelled("Compaction is cancelled"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, + reader->NextBatch()); + if (!iterator) { + break; + } + while (true) { + PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext()); + if (!has_next) { + break; + } + KeyValue key_value = iterator->Next(); + if (current_key && key_comparator_(*current_key, *key_value.key) != 0) { + PAIMON_RETURN_NOT_OK(finish_group()); + } + if (!current_key) { + wrapper->Reset(); + current_key = key_value.key; + } + PAIMON_RETURN_NOT_OK(wrapper->Add(std::move(key_value))); + } + } + PAIMON_RETURN_NOT_OK(finish_group()); + reader->Close(); + } + PAIMON_RETURN_NOT_OK(flush(AsyncKeyValueBatchType::DATA, &compact_buffer)); + PAIMON_RETURN_NOT_OK(flush(AsyncKeyValueBatchType::CHANGELOG, &changelog_buffer)); + return Status::OK(); + } + + private: + const std::vector>& sections_; + std::vector>& reader_holders_; + int32_t write_batch_size_; + SortMergeReaderFactory reader_factory_; + BoundChangelogMergeFunctionWrapperFactory merge_function_wrapper_factory_; + KeyComparator key_comparator_; + CancellationChecker cancellation_checker_; + bool drop_delete_; + bool produce_data_; + bool produce_changelog_; +}; + +} // namespace + ChangelogMergeTreeRewriter::ChangelogMergeTreeRewriter( int32_t max_level, bool force_drop_delete, const BinaryRow& partition, int32_t bucket, int64_t schema_id, const std::vector& trimmed_primary_keys, @@ -26,14 +171,18 @@ ChangelogMergeTreeRewriter::ChangelogMergeTreeRewriter( const std::shared_ptr& path_factory_cache, std::unique_ptr&& merge_file_split_read, MergeFunctionWrapperFactory merge_function_wrapper_factory, - const std::shared_ptr& cancellation_controller, + ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory, + bool produce_changelog, const std::shared_ptr& cancellation_controller, const std::shared_ptr& pool) : MergeTreeCompactRewriter( partition, bucket, schema_id, trimmed_primary_keys, options, data_schema, write_schema, std::move(dv_factory), path_factory_cache, std::move(merge_file_split_read), std::move(merge_function_wrapper_factory), cancellation_controller, pool), max_level_(max_level), - force_drop_delete_(force_drop_delete) {} + force_drop_delete_(force_drop_delete), + changelog_merge_function_wrapper_factory_( + std::move(changelog_merge_function_wrapper_factory)), + produce_changelog_(produce_changelog) {} Result ChangelogMergeTreeRewriter::Rewrite( int32_t output_level, bool drop_delete, const std::vector>& sections) { @@ -78,31 +227,82 @@ Result ChangelogMergeTreeRewriter::RewriteOrProduceChangelog( bool rewrite_compact_file) { PAIMON_ASSIGN_OR_RAISE(MergeTreeCompactRewriter::KeyValueConsumerCreator create_consumer, GenerateKeyValueConsumer()); - std::vector> reader_holders; - auto before = ExtractFilesFromSections(sections); std::unique_ptr compact_file_writer; if (rewrite_compact_file) { PAIMON_ASSIGN_OR_RAISE(compact_file_writer, CreateRollingRowWriter(output_level)); } - // TODO(xinyu.lxy): produce changelog + std::unique_ptr changelog_file_writer; + if (produce_changelog_) { + PAIMON_ASSIGN_OR_RAISE(changelog_file_writer, CreateRollingChangelogWriter(output_level)); + } + + std::vector> reader_holders; ScopeGuard write_guard([&]() -> void { if (compact_file_writer) { compact_file_writer->Abort(); + compact_file_writer.reset(); + } + if (changelog_file_writer) { + changelog_file_writer->Abort(); + changelog_file_writer.reset(); } - merge_file_split_read_.reset(); for (const auto& reader : reader_holders) { reader->Close(); } + merge_file_split_read_.reset(); }); - for (const auto& section : sections) { - PAIMON_RETURN_NOT_OK(MergeReadAndWrite(output_level, drop_delete, section, create_consumer, - compact_file_writer.get(), &reader_holders)); + bool produce_data = compact_file_writer != nullptr; + bool produce_changelog = changelog_file_writer != nullptr; + using AsyncPipeline = AsyncKeyValueProducerAndConsumer; + SortMergeReaderFactory reader_factory = [this](const std::vector& section) { + return CreateRawSortMergeReaderForSection(section); + }; + BoundChangelogMergeFunctionWrapperFactory merge_function_wrapper_factory = + [factory = changelog_merge_function_wrapper_factory_, output_level]() { + return factory(output_level); + }; + KeyComparator key_comparator = [this](const InternalRow& lhs, const InternalRow& rhs) { + return merge_file_split_read_->GetKeyComparator()->CompareTo(lhs, rhs); + }; + CancellationChecker cancellation_checker = [this]() { return IsCancelled(); }; + std::unique_ptr producer = + std::make_unique( + sections, reader_holders, + AsyncPipeline::NormalizeProjectionBatchSize(options_.GetWriteBatchSize()), + std::move(reader_factory), std::move(merge_function_wrapper_factory), + std::move(key_comparator), std::move(cancellation_checker), drop_delete, produce_data, + produce_changelog); + + AsyncPipeline::ConsumerCreator data_consumer = + produce_data ? create_consumer : AsyncPipeline::ConsumerCreator(); + AsyncPipeline::ConsumerCreator changelog_consumer = + produce_changelog ? create_consumer : AsyncPipeline::ConsumerCreator(); + auto producer_and_consumer = std::make_unique( + std::move(producer), std::move(data_consumer), std::move(changelog_consumer)); + while (true) { + if (IsCancelled()) { + return Status::Cancelled("Compaction is cancelled"); + } + PAIMON_ASSIGN_OR_RAISE(AsyncKeyValueResultBatch output, + producer_and_consumer->NextBatchWithType()); + if (output.result.batch == nullptr) { + break; + } + if (output.type == AsyncKeyValueBatchType::DATA) { + PAIMON_RETURN_NOT_OK(compact_file_writer->Write(std::move(output.result))); + } else { + PAIMON_RETURN_NOT_OK(changelog_file_writer->Write(std::move(output.result))); + } } + producer_and_consumer->Close(); if (compact_file_writer) { PAIMON_RETURN_NOT_OK(compact_file_writer->Close()); } + if (changelog_file_writer) { + PAIMON_RETURN_NOT_OK(changelog_file_writer->Close()); + } std::vector> after; if (compact_file_writer) { PAIMON_ASSIGN_OR_RAISE(after, compact_file_writer->GetResult()); @@ -118,8 +318,12 @@ Result ChangelogMergeTreeRewriter::RewriteOrProduceChangelog( NotifyRewriteCompactBefore(before); } PAIMON_ASSIGN_OR_RAISE(after, NotifyRewriteCompactAfter(after)); + std::vector> changelog_files; + if (changelog_file_writer) { + PAIMON_ASSIGN_OR_RAISE(changelog_files, changelog_file_writer->GetResult()); + } write_guard.Release(); - return CompactResult(before, after); + return CompactResult(before, after, changelog_files); } } // namespace paimon diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h index c5d8e8914..c080b30a7 100644 --- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h +++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h @@ -22,11 +22,15 @@ #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/key_value.h" +#include "paimon/core/mergetree/compact/changelog_result.h" #include "paimon/core/mergetree/compact/merge_tree_compact_rewriter.h" namespace paimon { /// A `MergeTreeCompactRewriter` which produces changelog files while performing compaction. class ChangelogMergeTreeRewriter : public MergeTreeCompactRewriter { public: + using ChangelogMergeFunctionWrapperFactory = + std::function>>(int32_t)>; + Result Rewrite(int32_t output_level, bool drop_delete, const std::vector>& sections) override; @@ -42,6 +46,8 @@ class ChangelogMergeTreeRewriter : public MergeTreeCompactRewriter { const std::shared_ptr& path_factory_cache, std::unique_ptr&& merge_file_split_read, MergeFunctionWrapperFactory merge_function_wrapper_factory, + ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory, + bool produce_changelog, const std::shared_ptr& cancellation_controller, const std::shared_ptr& pool); @@ -85,5 +91,8 @@ class ChangelogMergeTreeRewriter : public MergeTreeCompactRewriter { Result RewriteOrProduceChangelog( int32_t output_level, const std::vector>& sections, bool drop_delete, bool rewrite_compact_file); + + ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory_; + bool produce_changelog_; }; } // namespace paimon diff --git a/src/paimon/core/mergetree/compact/changelog_result.h b/src/paimon/core/mergetree/compact/changelog_result.h new file mode 100644 index 000000000..5b9b78008 --- /dev/null +++ b/src/paimon/core/mergetree/compact/changelog_result.h @@ -0,0 +1,32 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/core/key_value.h" + +namespace paimon { + +/// The result of merging all records with the same primary key while producing changelog. +struct ChangelogResult { + std::optional result; + std::vector changelogs; +}; + +} // namespace paimon diff --git a/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h b/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h index 319ed5ab6..0baed203e 100644 --- a/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h +++ b/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h @@ -24,7 +24,9 @@ #include #include +#include "paimon/common/data/serializer/row_compacted_serializer.h" #include "paimon/core/key_value.h" +#include "paimon/core/mergetree/compact/changelog_result.h" #include "paimon/core/mergetree/compact/first_row_merge_function.h" #include "paimon/core/mergetree/compact/merge_function_wrapper.h" #include "paimon/result.h" @@ -32,12 +34,15 @@ namespace paimon { /// Wrapper for `MergeFunction`s to produce changelog by lookup for first row. -class FirstRowMergeFunctionWrapper : public MergeFunctionWrapper { +class FirstRowMergeFunctionWrapper : public MergeFunctionWrapper { public: FirstRowMergeFunctionWrapper( std::unique_ptr&& merge_function, - std::function(const std::shared_ptr&)> contains) - : merge_function_(std::move(merge_function)), contains_(std::move(contains)) {} + std::function(const std::shared_ptr&)> contains, + std::unique_ptr&& value_serializer) + : merge_function_(std::move(merge_function)), + contains_(std::move(contains)), + value_serializer_(std::move(value_serializer)) {} void Reset() override { merge_function_->Reset(); @@ -47,11 +52,13 @@ class FirstRowMergeFunctionWrapper : public MergeFunctionWrapper { return merge_function_->Add(std::move(kv)); } - Result> GetResult() override { + Result> GetResult() override { PAIMON_ASSIGN_OR_RAISE(std::optional result, merge_function_->GetResult()); + ChangelogResult changelog_result; if (merge_function_->ContainsHighLevel()) { + changelog_result.result = std::move(result); Reset(); - return result; + return std::optional(std::move(changelog_result)); } if (!result) { Reset(); @@ -62,17 +69,26 @@ class FirstRowMergeFunctionWrapper : public MergeFunctionWrapper { if (contains) { // empty Reset(); - return std::optional(); + return std::optional(std::move(changelog_result)); } - // new record, output changelog - // TODO(xinyu.lxy) support changelog + if (value_serializer_) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, + value_serializer_->SerializeToBytes(*result->value)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr changelog_value, + value_serializer_->Deserialize(bytes)); + changelog_result.changelogs.emplace_back(result->value_kind, result->sequence_number, + result->level, result->key, + std::move(changelog_value)); + } + changelog_result.result = std::move(result); Reset(); - return result; + return std::optional(std::move(changelog_result)); } private: std::unique_ptr merge_function_; std::function(const std::shared_ptr&)> contains_; + std::unique_ptr value_serializer_; }; } // namespace paimon diff --git a/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper_test.cpp b/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper_test.cpp index 36a2ef01e..47a2d38ed 100644 --- a/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper_test.cpp +++ b/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper_test.cpp @@ -29,6 +29,15 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { +namespace { +std::unique_ptr CreateValueSerializer( + const std::shared_ptr& pool) { + return RowCompactedSerializer::Create(arrow::schema({arrow::field("value", arrow::int32())}), + pool) + .value(); +} +} // namespace + TEST(FirstRowMergeFunctionWrapperTest, TestSimple) { auto pool = GetDefaultPool(); KeyValue kv1(RowKind::Insert(), /*sequence_number=*/0, /*level=*/0, /*key=*/ @@ -45,14 +54,17 @@ TEST(FirstRowMergeFunctionWrapperTest, TestSimple) { auto contains = [](const std::shared_ptr& row) { return true; }; - FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains)); + FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains), + CreateValueSerializer(pool)); wrapper.Reset(); ASSERT_OK(wrapper.Add(std::move(kv1))); ASSERT_OK(wrapper.Add(std::move(kv2))); ASSERT_OK(wrapper.Add(std::move(kv3))); ASSERT_OK_AND_ASSIGN(auto result, wrapper.GetResult()); ASSERT_TRUE(result); - ASSERT_EQ(result.value().sequence_number, 0); + ASSERT_TRUE(result->result); + ASSERT_EQ(result->result->sequence_number, 0); + ASSERT_TRUE(result->changelogs.empty()); } TEST(FirstRowMergeFunctionWrapperTest, TestAllLevel0WithContain) { @@ -71,13 +83,16 @@ TEST(FirstRowMergeFunctionWrapperTest, TestAllLevel0WithContain) { auto contains = [](const std::shared_ptr& row) { return true; }; - FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains)); + FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains), + CreateValueSerializer(pool)); wrapper.Reset(); ASSERT_OK(wrapper.Add(std::move(kv1))); ASSERT_OK(wrapper.Add(std::move(kv2))); ASSERT_OK(wrapper.Add(std::move(kv3))); ASSERT_OK_AND_ASSIGN(auto result, wrapper.GetResult()); - ASSERT_FALSE(result); + ASSERT_TRUE(result); + ASSERT_FALSE(result->result); + ASSERT_TRUE(result->changelogs.empty()); } TEST(FirstRowMergeFunctionWrapperTest, TestAllLevel0WithoutContain) { @@ -96,14 +111,18 @@ TEST(FirstRowMergeFunctionWrapperTest, TestAllLevel0WithoutContain) { auto contains = [](const std::shared_ptr& row) { return false; }; - FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains)); + FirstRowMergeFunctionWrapper wrapper(std::move(mfunc), std::move(contains), + CreateValueSerializer(pool)); wrapper.Reset(); ASSERT_OK(wrapper.Add(std::move(kv1))); ASSERT_OK(wrapper.Add(std::move(kv2))); ASSERT_OK(wrapper.Add(std::move(kv3))); ASSERT_OK_AND_ASSIGN(auto result, wrapper.GetResult()); ASSERT_TRUE(result); - ASSERT_EQ(result.value().sequence_number, 0); + ASSERT_TRUE(result->result); + ASSERT_EQ(result->result->sequence_number, 0); + ASSERT_EQ(result->changelogs.size(), 1); + ASSERT_EQ(result->changelogs[0].sequence_number, 0); } } // namespace paimon::test diff --git a/src/paimon/core/mergetree/compact/internal_row_equalizer.h b/src/paimon/core/mergetree/compact/internal_row_equalizer.h new file mode 100644 index 000000000..d5336999f --- /dev/null +++ b/src/paimon/core/mergetree/compact/internal_row_equalizer.h @@ -0,0 +1,245 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "fmt/format.h" +#include "paimon/common/data/data_getters.h" +#include "paimon/common/data/internal_array.h" +#include "paimon/common/data/internal_map.h" +#include "paimon/common/data/internal_row.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/date_time_utils.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// Creates equality functions for internal rows, including nested values. +class InternalRowEqualizer { + public: + using Equalizer = std::function; + + static Result Create(const std::shared_ptr& schema, + const std::vector& ignore_fields) { + std::set ignored(ignore_fields.begin(), ignore_fields.end()); + std::vector> equalizers; + for (int32_t i = 0; i < schema->num_fields(); ++i) { + if (ignored.find(schema->field(i)->name()) != ignored.end()) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(ValueEqualizer equalizer, + CreateValueEqualizer(schema->field(i)->type())); + equalizers.emplace_back(i, std::move(equalizer)); + } + return Equalizer( + [equalizers = std::move(equalizers)](const InternalRow& lhs, const InternalRow& rhs) { + for (const auto& [field_idx, equalizer] : equalizers) { + if (!EqualAt(lhs, field_idx, rhs, field_idx, equalizer)) { + return false; + } + } + return true; + }); + } + + private: + using ValueEqualizer = + std::function; + + static bool EqualAt(const DataGetters& lhs, int32_t lhs_pos, const DataGetters& rhs, + int32_t rhs_pos, const ValueEqualizer& equalizer) { + bool lhs_null = lhs.IsNullAt(lhs_pos); + bool rhs_null = rhs.IsNullAt(rhs_pos); + if (lhs_null || rhs_null) { + return lhs_null == rhs_null; + } + return equalizer(lhs, lhs_pos, rhs, rhs_pos); + } + + static Result CreateValueEqualizer( + const std::shared_ptr& type) { + switch (type->id()) { + case arrow::Type::BOOL: + return PrimitiveEqualizer( + [](const DataGetters& row, int32_t pos) { return row.GetBoolean(pos); }); + case arrow::Type::INT8: + return PrimitiveEqualizer( + [](const DataGetters& row, int32_t pos) { return row.GetByte(pos); }); + case arrow::Type::INT16: + return PrimitiveEqualizer( + [](const DataGetters& row, int32_t pos) { return row.GetShort(pos); }); + case arrow::Type::INT32: + return PrimitiveEqualizer( + [](const DataGetters& row, int32_t pos) { return row.GetInt(pos); }); + case arrow::Type::DATE32: + return PrimitiveEqualizer( + [](const DataGetters& row, int32_t pos) { return row.GetDate(pos); }); + case arrow::Type::INT64: + return PrimitiveEqualizer( + [](const DataGetters& row, int32_t pos) { return row.GetLong(pos); }); + case arrow::Type::FLOAT: + return ValueEqualizer([](const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + return FieldsComparator::CompareFloatingPoint(lhs.GetFloat(lhs_pos), + rhs.GetFloat(rhs_pos)) == 0; + }); + case arrow::Type::DOUBLE: + return ValueEqualizer([](const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + return FieldsComparator::CompareFloatingPoint(lhs.GetDouble(lhs_pos), + rhs.GetDouble(rhs_pos)) == 0; + }); + case arrow::Type::STRING: + case arrow::Type::BINARY: + return ValueEqualizer([](const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + return lhs.GetStringView(lhs_pos) == rhs.GetStringView(rhs_pos); + }); + case arrow::Type::TIMESTAMP: { + std::shared_ptr timestamp_type = + checked_pointer_cast(type); + int32_t precision = DateTimeUtils::GetPrecisionFromType(timestamp_type); + return ValueEqualizer([precision](const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + return lhs.GetTimestamp(lhs_pos, precision) == + rhs.GetTimestamp(rhs_pos, precision); + }); + } + case arrow::Type::DECIMAL128: { + std::shared_ptr decimal_type = + checked_pointer_cast(type); + int32_t precision = decimal_type->precision(); + int32_t scale = decimal_type->scale(); + return ValueEqualizer([precision, scale](const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + return lhs.GetDecimal(lhs_pos, precision, scale) + .CompareTo(rhs.GetDecimal(rhs_pos, precision, scale)) == 0; + }); + } + case arrow::Type::LIST: { + std::shared_ptr list_type = + checked_pointer_cast(type); + PAIMON_ASSIGN_OR_RAISE(ValueEqualizer element_equalizer, + CreateValueEqualizer(list_type->value_type())); + return ValueEqualizer([element_equalizer = std::move(element_equalizer)]( + const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + std::shared_ptr lhs_array = lhs.GetArray(lhs_pos); + std::shared_ptr rhs_array = rhs.GetArray(rhs_pos); + if (lhs_array->Size() != rhs_array->Size()) { + return false; + } + for (int32_t i = 0; i < lhs_array->Size(); ++i) { + if (!EqualAt(*lhs_array, i, *rhs_array, i, element_equalizer)) { + return false; + } + } + return true; + }); + } + case arrow::Type::MAP: { + std::shared_ptr map_type = + checked_pointer_cast(type); + PAIMON_ASSIGN_OR_RAISE(ValueEqualizer key_equalizer, + CreateValueEqualizer(map_type->key_type())); + PAIMON_ASSIGN_OR_RAISE(ValueEqualizer item_equalizer, + CreateValueEqualizer(map_type->item_type())); + return ValueEqualizer([key_equalizer = std::move(key_equalizer), + item_equalizer = std::move(item_equalizer)]( + const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + std::shared_ptr lhs_map = lhs.GetMap(lhs_pos); + std::shared_ptr rhs_map = rhs.GetMap(rhs_pos); + if (lhs_map->Size() != rhs_map->Size()) { + return false; + } + std::shared_ptr lhs_keys = lhs_map->KeyArray(); + std::shared_ptr rhs_keys = rhs_map->KeyArray(); + std::shared_ptr lhs_values = lhs_map->ValueArray(); + std::shared_ptr rhs_values = rhs_map->ValueArray(); + std::vector matched(rhs_map->Size(), false); + for (int32_t lhs_index = 0; lhs_index < lhs_map->Size(); ++lhs_index) { + bool found = false; + for (int32_t rhs_index = 0; rhs_index < rhs_map->Size(); ++rhs_index) { + if (matched[rhs_index]) { + continue; + } + if (EqualAt(*lhs_keys, lhs_index, *rhs_keys, rhs_index, + key_equalizer) && + EqualAt(*lhs_values, lhs_index, *rhs_values, rhs_index, + item_equalizer)) { + matched[rhs_index] = true; + found = true; + break; + } + } + if (!found) { + return false; + } + } + return true; + }); + } + case arrow::Type::STRUCT: { + std::shared_ptr struct_type = + checked_pointer_cast(type); + std::vector field_equalizers; + field_equalizers.reserve(struct_type->num_fields()); + for (const auto& field : struct_type->fields()) { + PAIMON_ASSIGN_OR_RAISE(ValueEqualizer field_equalizer, + CreateValueEqualizer(field->type())); + field_equalizers.emplace_back(std::move(field_equalizer)); + } + int32_t field_count = struct_type->num_fields(); + return ValueEqualizer([field_equalizers = std::move(field_equalizers), field_count]( + const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + std::shared_ptr lhs_row = lhs.GetRow(lhs_pos, field_count); + std::shared_ptr rhs_row = rhs.GetRow(rhs_pos, field_count); + for (int32_t i = 0; i < field_count; ++i) { + if (!EqualAt(*lhs_row, i, *rhs_row, i, field_equalizers[i])) { + return false; + } + } + return true; + }); + } + default: + return Status::NotImplemented( + fmt::format("Do not support equality for type {}", type->ToString())); + } + } + + template + static ValueEqualizer PrimitiveEqualizer(Getter getter) { + return [getter = std::move(getter)](const DataGetters& lhs, int32_t lhs_pos, + const DataGetters& rhs, int32_t rhs_pos) { + return static_cast(getter(lhs, lhs_pos)) == static_cast(getter(rhs, rhs_pos)); + }; + } +}; + +} // namespace paimon diff --git a/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp b/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp new file mode 100644 index 000000000..d655513ec --- /dev/null +++ b/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp @@ -0,0 +1,201 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/mergetree/compact/internal_row_equalizer.h" + +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/data/binary_array.h" +#include "paimon/common/data/binary_array_writer.h" +#include "paimon/common/data/binary_map.h" +#include "paimon/common/data/generic_row.h" +#include "paimon/common/data/internal_array.h" +#include "paimon/common/data/internal_map.h" +#include "paimon/common/data/internal_row.h" +#include "paimon/common/utils/decimal_utils.h" +#include "paimon/data/decimal.h" +#include "paimon/data/timestamp.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { + +std::shared_ptr CreateIntArray(const std::vector& values, + MemoryPool* pool) { + return std::make_shared(BinaryArray::FromIntArray(values, pool)); +} + +std::shared_ptr CreateNullableIntArray(const std::vector& values, + int32_t null_pos, MemoryPool* pool) { + auto array = std::make_shared(); + BinaryArrayWriter writer(array.get(), static_cast(values.size()), sizeof(int32_t), + pool); + for (int32_t i = 0; i < static_cast(values.size()); ++i) { + if (i == null_pos) { + writer.SetNullValue(i); + } else { + writer.WriteInt(i, values[i]); + } + } + writer.Complete(); + return array; +} + +std::shared_ptr CreateIntMap(const std::vector& keys, + const std::vector& values, MemoryPool* pool) { + BinaryArray key_array = BinaryArray::FromIntArray(keys, pool); + BinaryArray value_array = BinaryArray::FromIntArray(values, pool); + return BinaryMap::ValueOf(key_array, value_array, pool); +} + +std::shared_ptr CreateNestedRow(int32_t value, double floating_point) { + return GenericRow::Of({value, floating_point}); +} + +} // namespace + +TEST(InternalRowEqualizerTest, PrimitiveTypesAndIgnoreFields) { + std::shared_ptr pool = GetDefaultPool(); + std::shared_ptr schema = arrow::schema( + {arrow::field("boolean", arrow::boolean()), arrow::field("tinyint", arrow::int8()), + arrow::field("smallint", arrow::int16()), arrow::field("int", arrow::int32()), + arrow::field("date", arrow::date32()), arrow::field("bigint", arrow::int64()), + arrow::field("float", arrow::float32()), arrow::field("double", arrow::float64()), + arrow::field("string", arrow::utf8()), arrow::field("binary", arrow::binary()), + arrow::field("timestamp", arrow::timestamp(arrow::TimeUnit::MICRO)), + arrow::field("decimal", arrow::decimal128(10, 2)), + arrow::field("ignored", arrow::int32())}); + + const float float_nan = std::numeric_limits::quiet_NaN(); + const double double_nan = std::numeric_limits::quiet_NaN(); + auto binary = std::make_shared("binary", pool.get()); + Decimal decimal(10, 2, DecimalUtils::StrToInt128("12345").value()); + std::vector left_values = {true, + static_cast(1), + static_cast(2), + static_cast(3), + int32_t{4}, + int64_t{5}, + float_nan, + double_nan, + std::string_view("string"), + binary, + Timestamp(1234, 567000), + decimal, + int32_t{10}}; + std::vector right_values = left_values; + right_values.back() = int32_t{20}; + + std::unique_ptr left = GenericRow::Of(left_values); + std::unique_ptr right = GenericRow::Of(right_values); + ASSERT_OK_AND_ASSIGN(InternalRowEqualizer::Equalizer equalizer, + InternalRowEqualizer::Create(schema, {"ignored"})); + ASSERT_TRUE(equalizer(*left, *right)); + + right->SetField(/*pos=*/3, int32_t{30}); + ASSERT_FALSE(equalizer(*left, *right)); +} + +TEST(InternalRowEqualizerTest, NullAndFloatingPointSemantics) { + std::shared_ptr schema = arrow::schema( + {arrow::field("value", arrow::float64()), arrow::field("nullable", arrow::int32())}); + ASSERT_OK_AND_ASSIGN(InternalRowEqualizer::Equalizer equalizer, + InternalRowEqualizer::Create(schema, {})); + + std::unique_ptr negative_zero = + GenericRow::Of({static_cast(-0.0), NullType()}); + std::unique_ptr positive_zero = + GenericRow::Of({static_cast(0.0), NullType()}); + ASSERT_FALSE(equalizer(*negative_zero, *positive_zero)); + + double nan1 = std::numeric_limits::quiet_NaN(); + double nan2 = -std::numeric_limits::quiet_NaN(); + std::unique_ptr left_nan = GenericRow::Of({nan1, NullType()}); + std::unique_ptr right_nan = GenericRow::Of({nan2, NullType()}); + ASSERT_TRUE(equalizer(*left_nan, *right_nan)); + + right_nan->SetField(/*pos=*/1, int32_t{1}); + ASSERT_FALSE(equalizer(*left_nan, *right_nan)); +} + +TEST(InternalRowEqualizerTest, NestedTypes) { + std::shared_ptr pool = GetDefaultPool(); + std::shared_ptr schema = arrow::schema( + {arrow::field("array", arrow::list(arrow::int32())), + arrow::field("map", arrow::map(arrow::int32(), arrow::int32())), + arrow::field("row", arrow::struct_({arrow::field("value", arrow::int32()), + arrow::field("floating", arrow::float64())}))}); + ASSERT_OK_AND_ASSIGN(InternalRowEqualizer::Equalizer equalizer, + InternalRowEqualizer::Create(schema, {})); + + std::unique_ptr left = + GenericRow::Of({CreateNullableIntArray({1, 0, 3}, /*null_pos=*/1, pool.get()), + CreateIntMap({1, 2}, {10, 20}, pool.get()), + CreateNestedRow(100, std::numeric_limits::quiet_NaN())}); + std::unique_ptr right = + GenericRow::Of({CreateNullableIntArray({1, 9, 3}, /*null_pos=*/1, pool.get()), + CreateIntMap({1, 2}, {10, 20}, pool.get()), + CreateNestedRow(100, -std::numeric_limits::quiet_NaN())}); + ASSERT_TRUE(equalizer(*left, *right)); + + right->SetField(/*pos=*/0, CreateIntArray({1, 2}, pool.get())); + ASSERT_FALSE(equalizer(*left, *right)); + + right->SetField(/*pos=*/0, CreateNullableIntArray({1, 0, 3}, /*null_pos=*/1, pool.get())); + right->SetField(/*pos=*/0, CreateNullableIntArray({1, 0, 4}, /*null_pos=*/1, pool.get())); + ASSERT_FALSE(equalizer(*left, *right)); + + right->SetField(/*pos=*/0, CreateNullableIntArray({1, 0, 3}, /*null_pos=*/1, pool.get())); + right->SetField(/*pos=*/1, CreateIntMap({1, 3}, {10, 20}, pool.get())); + ASSERT_FALSE(equalizer(*left, *right)); + + right->SetField(/*pos=*/1, CreateIntMap({1, 2}, {10, 20}, pool.get())); + right->SetField(/*pos=*/2, CreateNestedRow(101, 1.0)); + ASSERT_FALSE(equalizer(*left, *right)); +} + +TEST(InternalRowEqualizerTest, MapEqualityDoesNotDependOnEntryOrder) { + std::shared_ptr pool = GetDefaultPool(); + std::shared_ptr schema = + arrow::schema({arrow::field("map", arrow::map(arrow::int32(), arrow::int32()))}); + ASSERT_OK_AND_ASSIGN(InternalRowEqualizer::Equalizer equalizer, + InternalRowEqualizer::Create(schema, {})); + + std::unique_ptr left = + GenericRow::Of({CreateIntMap({1, 2, 3}, {10, 20, 30}, pool.get())}); + std::unique_ptr reordered = + GenericRow::Of({CreateIntMap({3, 1, 2}, {30, 10, 20}, pool.get())}); + ASSERT_TRUE(equalizer(*left, *reordered)); + + reordered->SetField(/*pos=*/0, CreateIntMap({3, 1, 2}, {30, 10, 21}, pool.get())); + ASSERT_FALSE(equalizer(*left, *reordered)); +} + +TEST(InternalRowEqualizerTest, UnsupportedType) { + ASSERT_NOK_WITH_MSG(InternalRowEqualizer::Create( + arrow::schema({arrow::field("unsupported", arrow::null())}), {}), + "Do not support equality for type null"); +} + +} // namespace paimon::test diff --git a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h index d2b21b2c0..51fa4aee9 100644 --- a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h +++ b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h @@ -25,9 +25,11 @@ #include #include +#include "paimon/common/data/serializer/row_compacted_serializer.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" #include "paimon/core/key_value.h" +#include "paimon/core/mergetree/compact/changelog_result.h" #include "paimon/core/mergetree/compact/lookup_merge_function.h" #include "paimon/core/mergetree/compact/merge_function_wrapper.h" #include "paimon/core/mergetree/lookup/file_position.h" @@ -47,23 +49,30 @@ namespace paimon { /// should be AFTER. /// With level-0 record, without level-x record, need to lookup the history value of the upper /// level as BEFORE. -/// TODO(xinyu.lxy) : add changelog template -class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper { +class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper { public: + using ValueEqualizer = std::function; + static Result> Create( std::unique_ptr&& merge_function, std::function>(const std::shared_ptr&)> lookup, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& deletion_vectors_maintainer, - const std::shared_ptr& comparator) { + const std::shared_ptr& comparator, + std::unique_ptr&& value_serializer, + ValueEqualizer value_equalizer) { if (lookup_strategy.deletion_vector && !deletion_vectors_maintainer) { return Status::Invalid("deletionVectorsMaintainer should not be null, there is a bug."); } + if (should_produce_changelog && !value_serializer) { + return Status::Invalid("valueSerializer is required when producing changelog."); + } return std::unique_ptr( - new LookupChangelogMergeFunctionWrapper(std::move(merge_function), std::move(lookup), - lookup_strategy, deletion_vectors_maintainer, - comparator)); + new LookupChangelogMergeFunctionWrapper( + std::move(merge_function), std::move(lookup), lookup_strategy, + should_produce_changelog, deletion_vectors_maintainer, comparator, + std::move(value_serializer), std::move(value_equalizer))); } void Reset() override { merge_function_->Reset(); @@ -73,12 +82,17 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapperAdd(std::move(kv)); } - Result> GetResult() override { + Result> GetResult() override { // 1. Find the latest high level record and compute containLevel0 - std::optional high_level_idx = merge_function_->PickHighLevelIdx(); + const KeyValue* high_level = merge_function_->PickHighLevel(); + bool contain_level0 = merge_function_->ContainLevel0(); + std::optional before; + if (contain_level0 && should_produce_changelog_ && high_level != nullptr) { + PAIMON_ASSIGN_OR_RAISE(before, CloneKeyValue(*high_level, high_level->value_kind)); + } // 2. Lookup if latest high level record is absent - if (high_level_idx == std::nullopt) { + if (high_level == nullptr) { std::optional lookup_high_level; PAIMON_ASSIGN_OR_RAISE(std::optional lookup_result, lookup_(merge_function_->GetKey())); @@ -105,30 +119,80 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapperInsertInto(std::move(lookup_high_level), comparator_); } } // 3. Calculate result PAIMON_ASSIGN_OR_RAISE(std::optional result, merge_function_->GetResult()); - Reset(); + // 4. Set changelog when there's level-0 records - // TODO(liancheng.lsz): setChangelog - return result; + ChangelogResult changelog_result; + if (contain_level0 && should_produce_changelog_) { + PAIMON_RETURN_NOT_OK( + SetChangelog(std::move(before), result, &changelog_result.changelogs)); + } + changelog_result.result = std::move(result); + Reset(); + return std::optional(std::move(changelog_result)); } private: LookupChangelogMergeFunctionWrapper( std::unique_ptr&& merge_function, std::function>(const std::shared_ptr&)> lookup, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& deletion_vectors_maintainer, - const std::shared_ptr& user_defined_seq_comparator) + const std::shared_ptr& user_defined_seq_comparator, + std::unique_ptr&& value_serializer, ValueEqualizer value_equalizer) : merge_function_(std::move(merge_function)), lookup_(std::move(lookup)), lookup_strategy_(lookup_strategy), + should_produce_changelog_(should_produce_changelog), deletion_vectors_maintainer_(deletion_vectors_maintainer), - comparator_(CreateSequenceComparator(user_defined_seq_comparator)) {} + comparator_(CreateSequenceComparator(user_defined_seq_comparator)), + value_serializer_(std::move(value_serializer)), + value_equalizer_(std::move(value_equalizer)) {} + + Result CloneKeyValue(const KeyValue& from, const RowKind* value_kind) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, + value_serializer_->SerializeToBytes(*from.value)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr value, + value_serializer_->Deserialize(bytes)); + return KeyValue(value_kind, from.sequence_number, from.level, from.key, std::move(value)); + } + + Status SetChangelog(std::optional&& before, const std::optional& after, + std::vector* changelogs) { + if (!before || !before->value_kind->IsAdd()) { + if (after && after->value_kind->IsAdd()) { + PAIMON_ASSIGN_OR_RAISE(KeyValue insert, + CloneKeyValue(after.value(), RowKind::Insert())); + changelogs->emplace_back(std::move(insert)); + } + return Status::OK(); + } + + if (!after || !after->value_kind->IsAdd()) { + before->value_kind = RowKind::Delete(); + changelogs->emplace_back(std::move(before.value())); + return Status::OK(); + } + + if (!value_equalizer_ || !value_equalizer_(*before->value, *after->value)) { + before->value_kind = RowKind::UpdateBefore(); + PAIMON_ASSIGN_OR_RAISE(KeyValue update_after, + CloneKeyValue(after.value(), RowKind::UpdateAfter())); + changelogs->emplace_back(std::move(before.value())); + changelogs->emplace_back(std::move(update_after)); + } + return Status::OK(); + } static std::function CreateSequenceComparator( const std::shared_ptr& user_defined_seq_comparator) { @@ -150,8 +214,11 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper merge_function_; std::function>(const std::shared_ptr&)> lookup_; LookupStrategy lookup_strategy_; + bool should_produce_changelog_; std::shared_ptr deletion_vectors_maintainer_; std::function comparator_; + std::unique_ptr value_serializer_; + ValueEqualizer value_equalizer_; }; } // namespace paimon diff --git a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp index 446beb091..15e2223b0 100644 --- a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp +++ b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp @@ -26,12 +26,22 @@ #include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" #include "paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h" #include "paimon/core/mergetree/compact/deduplicate_merge_function.h" +#include "paimon/core/mergetree/compact/internal_row_equalizer.h" #include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/key_value_checker.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { +namespace { +std::unique_ptr CreateValueSerializer( + const std::shared_ptr& pool) { + return RowCompactedSerializer::Create(arrow::schema({arrow::field("value", arrow::int32())}), + pool) + .value(); +} +} // namespace + TEST(LookupChangelogMergeFunctionWrapperTest, TestCreateInvalid) { auto pool = GetDefaultPool(); auto mfunc = std::make_unique(/*ignore_delete=*/true); @@ -45,8 +55,10 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestCreateInvalid) { /*deletion_vector=*/true, /*force_lookup=*/true); ASSERT_NOK_WITH_MSG(LookupChangelogMergeFunctionWrapper::Create( std::move(lookup_mfunc), lookup, lookup_strategy, + /*should_produce_changelog=*/false, /*deletion_vectors_maintainer=*/nullptr, - /*comparator=*/nullptr), + /*comparator=*/nullptr, CreateValueSerializer(pool), + /*value_equalizer=*/{}), "deletionVectorsMaintainer should not be null, there is a bug."); } @@ -69,12 +81,15 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestSimple) { KeyValue(RowKind::Insert(), /*sequence_number=*/1000, /*level=*/3, key, BinaryRowGenerator::GenerateRowPtr({1001}, pool.get()))); }; - LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/false, + LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true, /*deletion_vector=*/false, /*force_lookup=*/true); ASSERT_OK_AND_ASSIGN(auto wrapper, LookupChangelogMergeFunctionWrapper::Create( std::move(lookup_mfunc), lookup, lookup_strategy, + /*should_produce_changelog=*/false, /*deletion_vectors_maintainer=*/nullptr, - /*comparator=*/nullptr)); + /*comparator=*/nullptr, + /*value_serializer=*/nullptr, + /*value_equalizer=*/{})); wrapper->Reset(); ASSERT_OK(wrapper->Add(std::move(kv1))); @@ -82,7 +97,9 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestSimple) { ASSERT_OK(wrapper->Add(std::move(kv3))); ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult()); ASSERT_TRUE(result); - ASSERT_EQ(result.value().sequence_number, 2); + ASSERT_TRUE(result->result); + ASSERT_EQ(result->result->sequence_number, 2); + ASSERT_TRUE(result->changelogs.empty()); } TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookup) { @@ -104,12 +121,14 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookup) { KeyValue(RowKind::Insert(), /*sequence_number=*/0, /*level=*/3, key, BinaryRowGenerator::GenerateRowPtr({1001}, pool.get()))); }; - LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/false, + LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true, /*deletion_vector=*/false, /*force_lookup=*/true); ASSERT_OK_AND_ASSIGN(auto wrapper, LookupChangelogMergeFunctionWrapper::Create( std::move(lookup_mfunc), lookup, lookup_strategy, + /*should_produce_changelog=*/true, /*deletion_vectors_maintainer=*/nullptr, - /*comparator=*/nullptr)); + /*comparator=*/nullptr, CreateValueSerializer(pool), + /*value_equalizer=*/{})); wrapper->Reset(); ASSERT_OK(wrapper->Add(std::move(kv1))); @@ -117,7 +136,96 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookup) { ASSERT_OK(wrapper->Add(std::move(kv3))); ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult()); ASSERT_TRUE(result); - ASSERT_EQ(result.value().sequence_number, 3); + ASSERT_TRUE(result->result); + ASSERT_EQ(result->result->sequence_number, 3); + ASSERT_EQ(result->changelogs.size(), 2); + ASSERT_EQ(result->changelogs[0].value_kind, RowKind::UpdateBefore()); + ASSERT_EQ(result->changelogs[0].value->GetInt(0), 1001); + ASSERT_EQ(result->changelogs[1].value_kind, RowKind::UpdateAfter()); + ASSERT_EQ(result->changelogs[1].value->GetInt(0), 300); +} + +TEST(LookupChangelogMergeFunctionWrapperTest, TestRowDeduplicate) { + auto pool = GetDefaultPool(); + KeyValue kv(RowKind::Insert(), /*sequence_number=*/1, /*level=*/0, + BinaryRowGenerator::GenerateRowPtr({10}, pool.get()), + BinaryRowGenerator::GenerateRowPtr({300}, pool.get())); + + auto merge_function = std::make_unique(/*ignore_delete=*/true); + auto lookup_merge_function = std::make_unique(std::move(merge_function)); + auto lookup = [&](const std::shared_ptr& key) -> Result> { + return std::optional( + KeyValue(RowKind::Insert(), /*sequence_number=*/0, /*level=*/3, key, + BinaryRowGenerator::GenerateRowPtr({300}, pool.get()))); + }; + LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true, + /*deletion_vector=*/false, /*force_lookup=*/true); + auto value_equalizer = [](const InternalRow& lhs, const InternalRow& rhs) { + return lhs.GetInt(0) == rhs.GetInt(0); + }; + ASSERT_OK_AND_ASSIGN(auto wrapper, + LookupChangelogMergeFunctionWrapper::Create( + std::move(lookup_merge_function), lookup, lookup_strategy, + /*should_produce_changelog=*/true, + /*deletion_vectors_maintainer=*/nullptr, + /*comparator=*/nullptr, CreateValueSerializer(pool), value_equalizer)); + + wrapper->Reset(); + ASSERT_OK(wrapper->Add(std::move(kv))); + ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult()); + ASSERT_TRUE(result); + ASSERT_TRUE(result->result); + ASSERT_TRUE(result->changelogs.empty()); +} + +TEST(LookupChangelogMergeFunctionWrapperTest, TestRowDeduplicateWithIgnoreFields) { + auto pool = GetDefaultPool(); + auto value_schema = arrow::schema( + {arrow::field("value", arrow::int32()), arrow::field("ignored", arrow::int32())}); + auto merge_function = std::make_unique(/*ignore_delete=*/true); + auto lookup_merge_function = std::make_unique(std::move(merge_function)); + auto lookup = [&](const std::shared_ptr& key) -> Result> { + return std::optional( + KeyValue(RowKind::Insert(), /*sequence_number=*/0, /*level=*/3, key, + BinaryRowGenerator::GenerateRowPtr({300, 1}, pool.get()))); + }; + LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true, + /*deletion_vector=*/false, /*force_lookup=*/true); + ASSERT_OK_AND_ASSIGN(auto value_equalizer, + InternalRowEqualizer::Create(value_schema, {"ignored"})); + ASSERT_OK_AND_ASSIGN(auto value_serializer, RowCompactedSerializer::Create(value_schema, pool)); + ASSERT_OK_AND_ASSIGN(auto wrapper, + LookupChangelogMergeFunctionWrapper::Create( + std::move(lookup_merge_function), lookup, lookup_strategy, + /*should_produce_changelog=*/true, + /*deletion_vectors_maintainer=*/nullptr, + /*comparator=*/nullptr, std::move(value_serializer), value_equalizer)); + + // A change to an ignored field does not produce an update changelog. + wrapper->Reset(); + ASSERT_OK(wrapper->Add(KeyValue(RowKind::Insert(), /*sequence_number=*/1, /*level=*/0, + BinaryRowGenerator::GenerateRowPtr({10}, pool.get()), + BinaryRowGenerator::GenerateRowPtr({300, 2}, pool.get())))); + ASSERT_OK_AND_ASSIGN(auto ignored_field_result, wrapper->GetResult()); + ASSERT_TRUE(ignored_field_result); + ASSERT_TRUE(ignored_field_result->result); + ASSERT_TRUE(ignored_field_result->changelogs.empty()); + + // A change to a non-ignored field still produces update-before and update-after. + wrapper->Reset(); + ASSERT_OK(wrapper->Add(KeyValue(RowKind::Insert(), /*sequence_number=*/2, /*level=*/0, + BinaryRowGenerator::GenerateRowPtr({20}, pool.get()), + BinaryRowGenerator::GenerateRowPtr({301, 2}, pool.get())))); + ASSERT_OK_AND_ASSIGN(auto value_field_result, wrapper->GetResult()); + ASSERT_TRUE(value_field_result); + ASSERT_TRUE(value_field_result->result); + ASSERT_EQ(value_field_result->changelogs.size(), 2); + ASSERT_EQ(value_field_result->changelogs[0].value_kind, RowKind::UpdateBefore()); + ASSERT_EQ(value_field_result->changelogs[0].value->GetInt(0), 300); + ASSERT_EQ(value_field_result->changelogs[0].value->GetInt(1), 1); + ASSERT_EQ(value_field_result->changelogs[1].value_kind, RowKind::UpdateAfter()); + ASSERT_EQ(value_field_result->changelogs[1].value->GetInt(0), 301); + ASSERT_EQ(value_field_result->changelogs[1].value->GetInt(1), 2); } TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookupWithDv) { @@ -154,8 +262,66 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookupWithDv) { /*deletion_vector=*/true, /*force_lookup=*/false); ASSERT_OK_AND_ASSIGN(auto wrapper, LookupChangelogMergeFunctionWrapper::Create( - std::move(lookup_mfunc), lookup, lookup_strategy, dv_maintainer, - /*comparator=*/nullptr)); + std::move(lookup_mfunc), lookup, lookup_strategy, + /*should_produce_changelog=*/false, dv_maintainer, + /*comparator=*/nullptr, + /*value_serializer=*/nullptr, + /*value_equalizer=*/{})); + + wrapper->Reset(); + ASSERT_OK(wrapper->Add(std::move(kv1))); + ASSERT_OK(wrapper->Add(std::move(kv2))); + ASSERT_OK(wrapper->Add(std::move(kv3))); + ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult()); + ASSERT_TRUE(result); + ASSERT_TRUE(result->result); + ASSERT_EQ(result->result->sequence_number, 3); + ASSERT_EQ(result->result->value->GetInt(0), 100 + 200 + 300 + 1001); + + auto dv = dv_maintainer->DeletionVectorOf("data.file"); + ASSERT_TRUE(dv); + ASSERT_FALSE(dv.value()->IsDeleted(0).value()); + ASSERT_TRUE(dv.value()->IsDeleted(10).value()); +} + +TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookupWithDvAndChangelog) { + auto pool = GetDefaultPool(); + KeyValue kv1(RowKind::Insert(), /*sequence_number=*/1, /*level=*/0, /*key=*/ + BinaryRowGenerator::GenerateRowPtr({10}, pool.get()), + /*value=*/BinaryRowGenerator::GenerateRowPtr({100}, pool.get())); + KeyValue kv2(RowKind::Insert(), /*sequence_number=*/2, /*level=*/0, + /*key=*/BinaryRowGenerator::GenerateRowPtr({10}, pool.get()), + /*value=*/BinaryRowGenerator::GenerateRowPtr({200}, pool.get())); + KeyValue kv3(RowKind::Insert(), /*sequence_number=*/3, /*level=*/0, + /*key=*/BinaryRowGenerator::GenerateRowPtr({10}, pool.get()), + /*value=*/BinaryRowGenerator::GenerateRowPtr({300}, pool.get())); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::FIELDS_DEFAULT_AGG_FUNC, "sum"}})); + ASSERT_OK_AND_ASSIGN(auto mfunc, AggregateMergeFunction::Create( + arrow::schema({arrow::field("value", arrow::int32())}), + {"key"}, core_options, pool)); + auto lookup_mfunc = std::make_unique(std::move(mfunc)); + auto lookup = + [&](const std::shared_ptr& key) -> Result> { + return std::optional( + {KeyValue(RowKind::Insert(), /*sequence_number=*/0, /*level=*/3, key, + BinaryRowGenerator::GenerateRowPtr({1001}, pool.get())), + "data.file", /*row_position=*/10}); + }; + auto dv_index_file = + std::make_shared(/*fs=*/nullptr, /*path_factory=*/nullptr, + /*bitmap64=*/false, pool); + std::map> deletion_vectors; + auto dv_maintainer = std::make_shared(dv_index_file, deletion_vectors); + + LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true, + /*deletion_vector=*/true, /*force_lookup=*/false); + ASSERT_OK_AND_ASSIGN(auto wrapper, + LookupChangelogMergeFunctionWrapper::Create( + std::move(lookup_mfunc), lookup, lookup_strategy, + /*should_produce_changelog=*/true, dv_maintainer, + /*comparator=*/nullptr, CreateValueSerializer(pool), + /*value_equalizer=*/{})); wrapper->Reset(); ASSERT_OK(wrapper->Add(std::move(kv1))); @@ -163,8 +329,16 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookupWithDv) { ASSERT_OK(wrapper->Add(std::move(kv3))); ASSERT_OK_AND_ASSIGN(auto result, wrapper->GetResult()); ASSERT_TRUE(result); - ASSERT_EQ(result.value().sequence_number, 3); - ASSERT_EQ(result.value().value->GetInt(0), 100 + 200 + 300 + 1001); + ASSERT_TRUE(result->result); + ASSERT_EQ(result->result->sequence_number, 3); + ASSERT_EQ(result->result->value->GetInt(0), 100 + 200 + 300 + 1001); + ASSERT_EQ(result->changelogs.size(), 2); + ASSERT_EQ(result->changelogs[0].value_kind, RowKind::UpdateBefore()); + ASSERT_EQ(result->changelogs[0].sequence_number, 0); + ASSERT_EQ(result->changelogs[0].value->GetInt(0), 1001); + ASSERT_EQ(result->changelogs[1].value_kind, RowKind::UpdateAfter()); + ASSERT_EQ(result->changelogs[1].sequence_number, 3); + ASSERT_EQ(result->changelogs[1].value->GetInt(0), 100 + 200 + 300 + 1001); auto dv = dv_maintainer->DeletionVectorOf("data.file"); ASSERT_TRUE(dv); diff --git a/src/paimon/core/mergetree/compact/lookup_merge_function.h b/src/paimon/core/mergetree/compact/lookup_merge_function.h index e50d85eb2..7d8ee1a02 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_function.h +++ b/src/paimon/core/mergetree/compact/lookup_merge_function.h @@ -100,6 +100,11 @@ class LookupMergeFunction : public MergeFunction { return high_level_idx; } + const KeyValue* PickHighLevel() const { + std::optional high_level_idx = PickHighLevelIdx(); + return high_level_idx ? &candidates_[high_level_idx.value()] : nullptr; + } + private: std::unique_ptr merge_function_; std::vector candidates_; diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp index 994b0e3fc..30b7b9ac4 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp @@ -23,8 +23,10 @@ #include "paimon/common/table/special_fields.h" #include "paimon/core/mergetree/compact/first_row_merge_function_wrapper.h" #include "paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h" +#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" #include "paimon/core/mergetree/lookup/file_position.h" #include "paimon/core/mergetree/lookup/positioned_key_value.h" +#include "paimon/core/utils/primary_key_table_utils.h" namespace paimon { template @@ -38,7 +40,8 @@ LookupMergeTreeCompactRewriter::LookupMergeTreeCompactRewriter( const std::shared_ptr& path_factory_cache, std::unique_ptr&& merge_file_split_read, MergeFunctionWrapperFactory merge_function_wrapper_factory, - const std::shared_ptr& cancellation_controller, + ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory, + bool produce_changelog, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager, const std::shared_ptr& pool) : ChangelogMergeTreeRewriter( @@ -46,6 +49,7 @@ LookupMergeTreeCompactRewriter::LookupMergeTreeCompactRewriter( trimmed_primary_keys, options, data_schema, write_schema, DeletionVector::CreateFactory(dv_maintainer), path_factory_cache, std::move(merge_file_split_read), std::move(merge_function_wrapper_factory), + std::move(changelog_merge_function_wrapper_factory), produce_changelog, cancellation_controller, pool), lookup_levels_(std::move(lookup_levels)), dv_maintainer_(dv_maintainer), @@ -56,10 +60,10 @@ Result>> LookupMergeTreeCompactRewriter::Create( int32_t max_level, std::unique_ptr>&& lookup_levels, const std::shared_ptr& dv_maintainer, - MergeFunctionWrapperFactory merge_function_wrapper_factory, int32_t bucket, + ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory, int32_t bucket, const BinaryRow& partition, const std::shared_ptr& table_schema, const std::shared_ptr& path_factory_cache, - const CoreOptions& options, + const CoreOptions& options, bool produce_changelog, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager, const std::shared_ptr& pool) { @@ -93,44 +97,60 @@ LookupMergeTreeCompactRewriter::Create( std::unique_ptr merge_file_split_read, MergeFileSplitRead::Create(path_factory, internal_context, pool, CreateDefaultExecutor())); + MergeFunctionWrapperFactory merge_function_wrapper_factory = + [data_schema, options, trimmed_primary_keys, pool]( + int32_t /*output_level*/) -> Result>> { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, + PrimaryKeyTableUtils::CreateMergeFunction( + data_schema, trimmed_primary_keys, options, pool)); + if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { + merge_function = std::make_unique(std::move(merge_function)); + } + return std::make_shared(std::move(merge_function)); + }; + return std::unique_ptr(new LookupMergeTreeCompactRewriter( std::move(lookup_levels), dv_maintainer, max_level, partition, bucket, table_schema->Id(), trimmed_primary_keys, options, data_schema, write_schema, path_factory_cache, std::move(merge_file_split_read), std::move(merge_function_wrapper_factory), + std::move(changelog_merge_function_wrapper_factory), produce_changelog, cancellation_controller, remote_lookup_file_manager, pool)); } template -std::shared_ptr> +std::shared_ptr> LookupMergeTreeCompactRewriter::CreateFirstRowMergeFunctionWrapper( std::unique_ptr&& merge_func, int32_t output_level, - LookupLevels* lookup_levels) { + LookupLevels* lookup_levels, std::unique_ptr&& value_serializer) { auto contains = [output_level, lookup_levels](const std::shared_ptr& key) -> Result { PAIMON_ASSIGN_OR_RAISE(std::optional contain, lookup_levels->Lookup(key, output_level + 1)); return contain != std::nullopt; }; - return std::make_shared(std::move(merge_func), - std::move(contains)); + return std::make_shared( + std::move(merge_func), std::move(contains), std::move(value_serializer)); } template -Result>> +Result>> LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( std::unique_ptr&& merge_func, int32_t output_level, const std::shared_ptr& deletion_vectors_maintainer, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& user_defined_seq_comparator, - LookupLevels* lookup_levels) { + LookupLevels* lookup_levels, std::unique_ptr&& value_serializer, + typename LookupChangelogMergeFunctionWrapper::ValueEqualizer value_equalizer) { auto lookup = [output_level, lookup_levels]( const std::shared_ptr& key) -> Result> { return lookup_levels->Lookup(key, output_level + 1); }; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr> wrapper, - LookupChangelogMergeFunctionWrapper::Create( - std::move(merge_func), std::move(lookup), lookup_strategy, - deletion_vectors_maintainer, user_defined_seq_comparator)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr> wrapper, + LookupChangelogMergeFunctionWrapper::Create( + std::move(merge_func), std::move(lookup), lookup_strategy, should_produce_changelog, + deletion_vectors_maintainer, user_defined_seq_comparator, std::move(value_serializer), + std::move(value_equalizer))); return wrapper; } diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h index 06c5f47f4..5870ee805 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h @@ -19,10 +19,12 @@ #pragma once #include "arrow/api.h" +#include "paimon/common/data/serializer/row_compacted_serializer.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h" #include "paimon/core/mergetree/compact/first_row_merge_function.h" +#include "paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h" #include "paimon/core/mergetree/compact/lookup_merge_function.h" #include "paimon/core/mergetree/lookup/remote_lookup_file_manager.h" #include "paimon/core/mergetree/lookup_levels.h" @@ -38,10 +40,11 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter { static Result> Create( int32_t max_level, std::unique_ptr>&& lookup_levels, const std::shared_ptr& dv_maintainer, - MergeFunctionWrapperFactory merge_function_wrapper_factory, int32_t bucket, - const BinaryRow& partition, const std::shared_ptr& table_schema, + ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory, + int32_t bucket, const BinaryRow& partition, + const std::shared_ptr& table_schema, const std::shared_ptr& path_factory_cache, - const CoreOptions& options, + const CoreOptions& options, bool produce_changelog, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager, const std::shared_ptr& pool); @@ -50,16 +53,19 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter { return lookup_levels_->Close(); } - static std::shared_ptr> CreateFirstRowMergeFunctionWrapper( - std::unique_ptr&& merge_func, int32_t output_level, - LookupLevels* lookup_levels); + static std::shared_ptr> + CreateFirstRowMergeFunctionWrapper(std::unique_ptr&& merge_func, + int32_t output_level, LookupLevels* lookup_levels, + std::unique_ptr&& value_serializer); - static Result>> CreateLookupMergeFunctionWrapper( + static Result>> + CreateLookupMergeFunctionWrapper( std::unique_ptr&& merge_func, int32_t output_level, const std::shared_ptr& deletion_vectors_maintainer, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& user_defined_seq_comparator, - LookupLevels* lookup_levels); + LookupLevels* lookup_levels, std::unique_ptr&& value_serializer, + typename LookupChangelogMergeFunctionWrapper::ValueEqualizer value_equalizer); private: LookupMergeTreeCompactRewriter( @@ -72,6 +78,8 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter { const std::shared_ptr& path_factory_cache, std::unique_ptr&& merge_file_split_read, MergeFunctionWrapperFactory merge_function_wrapper_factory, + ChangelogMergeFunctionWrapperFactory changelog_merge_function_wrapper_factory, + bool produce_changelog, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager, const std::shared_ptr& pool); diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp index bbc1608c4..3ebeefbfc 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp @@ -24,6 +24,7 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/fields_comparator.h" @@ -35,6 +36,7 @@ #include "paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h" #include "paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h" #include "paimon/core/mergetree/compact/deduplicate_merge_function.h" +#include "paimon/core/mergetree/compact/internal_row_equalizer.h" #include "paimon/core/mergetree/compact/interval_partition.h" #include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" #include "paimon/core/mergetree/lookup/default_lookup_serializer_factory.h" @@ -148,16 +150,25 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam>> CreateCompactRewriterForFirstRow( const std::string& table_path, const std::shared_ptr& table_schema, - const CoreOptions& options, std::unique_ptr>&& lookup_levels) const { + const CoreOptions& options, std::unique_ptr>&& lookup_levels, + std::optional produce_changelog = std::nullopt) const { auto path_factory_cache = std::make_shared(table_path, table_schema, options, pool_); + bool should_produce_changelog = + produce_changelog.value_or(options.GetLookupStrategy().produce_changelog); auto merge_function_wrapper_factory = - [lookup_levels_ptr = lookup_levels.get()]( - int32_t output_level) -> Result>> { - std::shared_ptr> merge_function_wrapper = + [lookup_levels_ptr = lookup_levels.get(), data_schema = arrow_schema_, + should_produce_changelog, pool = pool_](int32_t output_level) + -> Result>> { + std::unique_ptr value_serializer; + if (should_produce_changelog) { + PAIMON_ASSIGN_OR_RAISE(value_serializer, + RowCompactedSerializer::Create(data_schema, pool)); + } + std::shared_ptr> merge_function_wrapper = LookupMergeTreeCompactRewriter::CreateFirstRowMergeFunctionWrapper( std::make_unique(/*ignore_delete=*/true), output_level, - lookup_levels_ptr); + lookup_levels_ptr, std::move(value_serializer)); return merge_function_wrapper; }; auto cancellation_controller = std::make_shared(); @@ -166,7 +177,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam>> @@ -177,18 +189,29 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam(table_path, table_schema, options, pool_); auto merge_function_wrapper_factory = [this, table_schema, options, lookup_levels_ptr = lookup_levels.get(), - lookup_strategy = options.GetLookupStrategy()]( - int32_t output_level) -> Result>> { - PAIMON_ASSIGN_OR_RAISE(auto merge_func, - AggregateMergeFunction::Create( - arrow_schema_, table_schema->TrimmedPrimaryKeys().value(), - options, GetDefaultPool())); + lookup_strategy = options.GetLookupStrategy()](int32_t output_level) + -> Result>> { PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, + auto merge_func, + AggregateMergeFunction::Create( + arrow_schema_, table_schema->TrimmedPrimaryKeys().value(), options, pool_)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr value_serializer, + RowCompactedSerializer::Create(arrow_schema_, pool_)); + InternalRowEqualizer::Equalizer value_equalizer; + if (options.ChangelogRowDeduplicate()) { + PAIMON_ASSIGN_OR_RAISE( + value_equalizer, + InternalRowEqualizer::Create(arrow_schema_, + options.GetChangelogRowDeduplicateIgnoreFields())); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr> merge_function_wrapper, LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( std::make_unique(std::move(merge_func)), output_level, /*deletion_vectors_maintainer=*/nullptr, lookup_strategy, - /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr)); + /*should_produce_changelog=*/lookup_strategy.produce_changelog, + /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr, + std::move(value_serializer), std::move(value_equalizer))); return merge_function_wrapper; }; auto cancellation_controller = std::make_shared(); @@ -196,7 +219,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam>> @@ -213,17 +237,22 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam(dv_index_file, deletion_vectors); - auto merge_function_wrapper_factory = - [lookup_levels_ptr = lookup_levels.get(), lookup_strategy = options.GetLookupStrategy(), - dv_maintainer_ptr = dv_maintainer]( - int32_t output_level) -> Result>> { + auto merge_function_wrapper_factory = [this, lookup_levels_ptr = lookup_levels.get(), + lookup_strategy = options.GetLookupStrategy(), + dv_maintainer_ptr = + dv_maintainer](int32_t output_level) + -> Result>> { auto merge_func = std::make_unique(false); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr value_serializer, + RowCompactedSerializer::Create(arrow_schema_, pool_)); PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, + std::shared_ptr> merge_function_wrapper, LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( std::make_unique(std::move(merge_func)), output_level, dv_maintainer_ptr, lookup_strategy, - /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr)); + /*should_produce_changelog=*/lookup_strategy.produce_changelog, + /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr, + std::move(value_serializer), /*value_equalizer=*/{})); return merge_function_wrapper; }; auto cancellation_controller = std::make_shared(); @@ -231,7 +260,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam>> @@ -253,19 +283,31 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam Result>> { - PAIMON_ASSIGN_OR_RAISE(auto merge_func, - AggregateMergeFunction::Create( - arrow_schema_, table_schema->TrimmedPrimaryKeys().value(), - options, GetDefaultPool())); + lookup_strategy = options.GetLookupStrategy(), + dv_maintainer_ptr = dv_maintainer](int32_t output_level) + -> Result>> { + PAIMON_ASSIGN_OR_RAISE( + auto merge_func, + AggregateMergeFunction::Create( + arrow_schema_, table_schema->TrimmedPrimaryKeys().value(), options, pool_)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr value_serializer, + RowCompactedSerializer::Create(arrow_schema_, pool_)); + InternalRowEqualizer::Equalizer value_equalizer; + if (options.ChangelogRowDeduplicate()) { + PAIMON_ASSIGN_OR_RAISE( + value_equalizer, + InternalRowEqualizer::Create(arrow_schema_, + options.GetChangelogRowDeduplicateIgnoreFields())); + } PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, + std::shared_ptr> merge_function_wrapper, LookupMergeTreeCompactRewriter:: CreateLookupMergeFunctionWrapper( std::make_unique(std::move(merge_func)), output_level, dv_maintainer_ptr, lookup_strategy, - /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr)); + /*should_produce_changelog=*/lookup_strategy.produce_changelog, + /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr, + std::move(value_serializer), std::move(value_equalizer))); return merge_function_wrapper; }; auto cancellation_controller = std::make_shared(); @@ -273,7 +315,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParamEquals(*result_array)) << result_array->ToString(); } + void CheckShreddingFileSchema(const std::string& file_name, + const std::shared_ptr& table_schema, + const std::string& file_format_name, + const std::shared_ptr& expected_physical_schema, + int32_t field_index, + const MapSharedShreddingFieldMeta& expected_meta) const { + ASSERT_OK_AND_ASSIGN(auto file_format, + FileFormatFactory::Get(file_format_name, table_schema->Options())); + ASSERT_OK_AND_ASSIGN(auto reader_builder, + file_format->CreateReaderBuilder(/*batch_size=*/10)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(auto file_batch_reader, reader_builder->Build(input_stream)); + ASSERT_OK_AND_ASSIGN(auto c_file_schema, file_batch_reader->GetFileSchema()); + std::shared_ptr file_schema = + arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + + ASSERT_TRUE(file_schema->Equals(*expected_physical_schema, /*check_metadata=*/false)) + << "Expected schema:\n" + << expected_physical_schema->ToString() << "\nActual schema:\n" + << file_schema->ToString(); + std::shared_ptr metadata = + file_schema->field(field_index)->metadata(); + ASSERT_NE(nullptr, metadata); + ASSERT_OK_AND_ASSIGN(MapSharedShreddingFieldMeta actual_meta, + MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy())); + ASSERT_EQ(expected_meta, actual_meta); + } + Result> CreateFileStorePathFactory( const std::string& table_path, const CoreOptions& options) const { PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, @@ -422,7 +493,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam options = {{Options::MERGE_ENGINE, "first-row"}, - {Options::FILE_FORMAT, "orc"}}; + {Options::FILE_FORMAT, "orc"}, + {Options::CHANGELOG_PRODUCER, "lookup"}}; ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options)); auto schema_manager = std::make_shared(fs_, table_path); @@ -447,6 +519,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowRewrite) { /*output_level=*/5, /*drop_delete=*/true, runs)); ASSERT_EQ(2, compact_result.Before().size()); ASSERT_EQ(1, compact_result.After().size()); + ASSERT_EQ(1, compact_result.Changelog().size()); // check compact result const auto& compact_file_meta = compact_result.After()[0]; @@ -482,6 +555,148 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowRewrite) { &expected_array); ASSERT_TRUE(array_status.ok()); CheckResult(compact_file_name, table_schema, "orc", expected_array); + std::string changelog_file_name = + table_path + "/bucket-0/" + compact_result.Changelog()[0]->file_name; + CheckResult(changelog_file_name, table_schema, "orc", expected_array); +} + +TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowLooksUpExistingKeys) { + std::map options = {{Options::MERGE_ENGINE, "first-row"}, + {Options::FILE_FORMAT, "orc"}, + {Options::CHANGELOG_PRODUCER, "lookup"}}; + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options)); + auto schema_manager = std::make_shared(fs_, table_path); + ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager->ReadSchema(0)); + + ASSERT_OK_AND_ASSIGN(auto level0_file, + NewFiles(/*level=*/0, /*last_sequence_number=*/0, table_path, core_options, + "[[1, 111], [2, 22]]")); + ASSERT_OK_AND_ASSIGN(auto high_level_file, NewFiles(/*level=*/2, /*last_sequence_number=*/-1, + table_path, core_options, "[[1, 11]]")); + auto processor_factory = std::make_shared(); + ASSERT_OK_AND_ASSIGN(auto lookup_levels, + CreateLookupLevels(table_path, table_schema, processor_factory, + std::vector>{ + level0_file, high_level_file})); + ASSERT_OK_AND_ASSIGN(auto rewriter, + CreateCompactRewriterForFirstRow(table_path, table_schema, core_options, + std::move(lookup_levels))); + ASSERT_OK_AND_ASSIGN( + auto runs, GenerateSortedRuns(std::vector>{level0_file})); + ASSERT_OK_AND_ASSIGN(auto compact_result, rewriter->Rewrite( + /*output_level=*/1, /*drop_delete=*/true, runs)); + + ASSERT_EQ(1, compact_result.After().size()); + ASSERT_EQ(1, compact_result.After()[0]->row_count); + ASSERT_EQ(1, compact_result.Changelog().size()); + ASSERT_EQ(1, compact_result.Changelog()[0]->row_count); + + auto type_with_special_fields = + arrow::struct_(SpecialFields::CompleteSequenceAndValueKindField(arrow_schema_)->fields()); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(type_with_special_fields, + {"[[2, 0, 2, 22]]"}, &expected) + .ok()); + CheckResult(table_path + "/bucket-0/" + compact_result.After()[0]->file_name, table_schema, + "orc", expected); + CheckResult(table_path + "/bucket-0/" + compact_result.Changelog()[0]->file_name, table_schema, + "orc", expected); +} + +TEST_F(LookupMergeTreeCompactRewriterTest, TestLookupChangelogCanBeDisabled) { + std::map options = {{Options::MERGE_ENGINE, "first-row"}, + {Options::FILE_FORMAT, "orc"}, + {Options::CHANGELOG_PRODUCER, "lookup"}}; + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options)); + auto schema_manager = std::make_shared(fs_, table_path); + ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager->ReadSchema(0)); + ASSERT_OK_AND_ASSIGN(auto file, NewFiles(/*level=*/0, /*last_sequence_number=*/-1, table_path, + core_options, "[[1, 11], [2, 22]]")); + auto processor_factory = std::make_shared(); + ASSERT_OK_AND_ASSIGN(auto lookup_levels, CreateLookupLevels(table_path, table_schema, + processor_factory, {file})); + ASSERT_OK_AND_ASSIGN(auto rewriter, + CreateCompactRewriterForFirstRow(table_path, table_schema, core_options, + std::move(lookup_levels), + /*produce_changelog=*/false)); + ASSERT_OK_AND_ASSIGN(auto runs, GenerateSortedRuns({file})); + ASSERT_OK_AND_ASSIGN(auto compact_result, rewriter->Rewrite( + /*output_level=*/1, /*drop_delete=*/true, runs)); + + ASSERT_EQ(1, compact_result.After().size()); + ASSERT_TRUE(compact_result.Changelog().empty()); +} + +TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowRewriteWithSharedShreddingChangelog) { + arrow::FieldVector fields = { + arrow::field("key", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + arrow_schema_ = arrow::schema(fields); + key_schema_ = arrow::schema({fields[0]}); + std::map options = { + {Options::MERGE_ENGINE, "first-row"}, + {Options::FILE_FORMAT, "orc"}, + {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::CHANGELOG_FILE_FORMAT, "parquet"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + }; + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options)); + auto schema_manager = std::make_shared(fs_, table_path); + ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager->ReadSchema(0)); + + ASSERT_OK_AND_ASSIGN( + auto file0, NewFiles(/*level=*/0, /*last_sequence_number=*/-1, table_path, core_options, + R"([[1, [["a", 11]]], [3, [["c", 33]]], [5, [["e", 55]]]])")); + ASSERT_OK_AND_ASSIGN(auto file1, + NewFiles(/*level=*/0, /*last_sequence_number=*/2, table_path, core_options, + R"([[2, [["b", 22]]], [5, [["f", 555]]]])")); + std::vector> files = {file0, file1}; + auto processor_factory = std::make_shared(); + ASSERT_OK_AND_ASSIGN(auto lookup_levels, CreateLookupLevels(table_path, table_schema, + processor_factory, files)); + ASSERT_OK_AND_ASSIGN(auto rewriter, + CreateCompactRewriterForFirstRow(table_path, table_schema, core_options, + std::move(lookup_levels))); + ASSERT_OK_AND_ASSIGN(auto runs, GenerateSortedRuns(files)); + ASSERT_OK_AND_ASSIGN(auto compact_result, rewriter->Rewrite( + /*output_level=*/5, /*drop_delete=*/true, runs)); + + ASSERT_EQ(1, compact_result.Changelog().size()); + const std::shared_ptr& changelog_file = compact_result.Changelog()[0]; + ASSERT_EQ(4, changelog_file->row_count); + ASSERT_EQ(FileSource::Append(), changelog_file->file_source); + ASSERT_TRUE(StringUtils::EndsWith(changelog_file->file_name, ".parquet")); + std::string changelog_file_name = table_path + "/bucket-0/" + changelog_file->file_name; + + std::shared_ptr write_schema = SpecialFields::CompleteSequenceAndValueKindField( + DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields())); + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + write_schema, {{"tags", 3}})); + std::shared_ptr expected_changelog; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(physical_schema->fields()), {R"([ + [0, 0, 1, [[0, -1, -1], 11, null, null, null]], + [3, 0, 2, [[1, -1, -1], 22, null, null, null]], + [1, 0, 3, [[2, -1, -1], 33, null, null, null]], + [2, 0, 5, [[3, -1, -1], 55, null, null, null]] + ])"}, + &expected_changelog) + .ok()); + CheckResult(changelog_file_name, table_schema, "parquet", expected_changelog); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"e", 3}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {0}}, {2, {0}}, {3, {0}}}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 1; + CheckShreddingFileSchema(changelog_file_name, table_schema, "parquet", physical_schema, + /*field_index=*/3, expected_meta); } TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowUpgrade) { @@ -639,7 +854,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithAllHighLevel) { TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithForceLookupAndSumAgg) { std::map options = {{Options::MERGE_ENGINE, "aggregation"}, {Options::FILE_FORMAT, "orc"}, - {Options::FORCE_LOOKUP, "true"}, + {Options::CHANGELOG_PRODUCER, "lookup"}, {Options::FIELDS_DEFAULT_AGG_FUNC, "sum"}}; ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options)); @@ -668,6 +883,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithForceLookupAndSumAgg) /*output_level=*/4, /*drop_delete=*/true, runs)); ASSERT_EQ(2, compact_result.Before().size()); ASSERT_EQ(1, compact_result.After().size()); + ASSERT_EQ(1, compact_result.Changelog().size()); const auto& compact_file_meta = compact_result.After()[0]; // check compact file exist @@ -688,6 +904,20 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithForceLookupAndSumAgg) &expected_array); ASSERT_TRUE(array_status.ok()); CheckResult(compact_file_name, table_schema, "orc", expected_array); + + const auto& changelog_file_meta = compact_result.Changelog()[0]; + std::string changelog_file_name = table_path + "/bucket-0/" + changelog_file_meta->file_name; + std::shared_ptr expected_changelog; + auto changelog_status = + arrow::ipc::internal::json::ChunkedArrayFromJSON(type_with_special_fields, {R"([ +[6, 0, 2, 244], +[4, 0, 4, 44], +[2, 1, 5, 55], +[7, 2, 5, 615] +])"}, + &expected_changelog); + ASSERT_TRUE(changelog_status.ok()); + CheckResult(changelog_file_name, table_schema, "orc", expected_changelog); } TEST_F(LookupMergeTreeCompactRewriterTest, TestRewriteWithDvAndDeduplicate) { @@ -1080,7 +1310,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + /*changelog_merge_function_wrapper_factory=*/nullptr, + /*produce_changelog=*/false, cancellation_controller, + /*remote_lookup_file_manager=*/nullptr, pool_); auto file = create_meta(/*level=*/1, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::NoChangelogNoRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file)); @@ -1095,7 +1327,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + /*changelog_merge_function_wrapper_factory=*/nullptr, + /*produce_changelog=*/false, cancellation_controller, + /*remote_lookup_file_manager=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/5, file)); @@ -1114,7 +1348,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + /*changelog_merge_function_wrapper_factory=*/nullptr, + /*produce_changelog=*/false, cancellation_controller, + /*remote_lookup_file_manager=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/1); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file)); @@ -1129,7 +1365,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + /*changelog_merge_function_wrapper_factory=*/nullptr, + /*produce_changelog=*/false, cancellation_controller, + /*remote_lookup_file_manager=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogNoRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/5, file)); @@ -1144,7 +1382,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + /*changelog_merge_function_wrapper_factory=*/nullptr, + /*produce_changelog=*/false, cancellation_controller, + /*remote_lookup_file_manager=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogNoRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file)); @@ -1160,7 +1400,9 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + /*changelog_merge_function_wrapper_factory=*/nullptr, + /*produce_changelog=*/false, cancellation_controller, + /*remote_lookup_file_manager=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file)); diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp index e5cb3abf6..4e6fd3854 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp @@ -25,6 +25,7 @@ #include "paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h" #include "paimon/core/mergetree/compact/early_full_compaction.h" #include "paimon/core/mergetree/compact/force_up_level0_compaction.h" +#include "paimon/core/mergetree/compact/internal_row_equalizer.h" #include "paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h" #include "paimon/core/mergetree/compact/merge_tree_compact_manager.h" #include "paimon/core/mergetree/compact/merge_tree_compact_rewriter.h" @@ -50,6 +51,24 @@ namespace paimon { namespace { +Result CreateChangelogValueEqualizer( + const std::shared_ptr& schema, const CoreOptions& options, + bool produce_changelog) { + if (!produce_changelog || !options.ChangelogRowDeduplicate()) { + return InternalRowEqualizer::Equalizer(); + } + return InternalRowEqualizer::Create(schema, options.GetChangelogRowDeduplicateIgnoreFields()); +} + +Result> CreateChangelogValueSerializer( + const std::shared_ptr& schema, bool produce_changelog, + const std::shared_ptr& pool) { + if (!produce_changelog) { + return std::unique_ptr(); + } + return RowCompactedSerializer::Create(schema, pool); +} + template Result>> CreateLookupLevelsInternal( const CoreOptions& options, const std::shared_ptr& schema_manager, @@ -174,6 +193,8 @@ Result> MergeTreeCompactManagerFactory::CreateL const LookupStrategy& lookup_strategy, const std::shared_ptr& path_factory_cache, const std::shared_ptr& cancellation_controller) const { + const bool should_produce_changelog = + lookup_strategy.produce_changelog && !ignore_previous_files_; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr remote_lookup_file_manager, CreateRemoteLookupFileManager(partition, bucket)); if (lookup_strategy.is_first_row) { @@ -191,39 +212,47 @@ Result> MergeTreeCompactManagerFactory::CreateL table_schema_, partition, bucket, levels, processor_factory, dv_maintainer, lookup_file_cache_, remote_lookup_file_manager, pool_)); auto merge_function_wrapper_factory = - [lookup_levels_ptr = lookup_levels.get(), ignore_delete = options_.IgnoreDelete()]( - int32_t output_level) -> Result>> { - std::shared_ptr> merge_function_wrapper = + [lookup_levels_ptr = lookup_levels.get(), data_schema = schema_, lookup_strategy, + should_produce_changelog, ignore_delete = options_.IgnoreDelete(), + pool = pool_](int32_t output_level) + -> Result>> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr value_serializer, + CreateChangelogValueSerializer(data_schema, should_produce_changelog, pool)); + std::shared_ptr> merge_function_wrapper = LookupMergeTreeCompactRewriter::CreateFirstRowMergeFunctionWrapper( std::make_unique(ignore_delete), output_level, - lookup_levels_ptr); + lookup_levels_ptr, std::move(value_serializer)); return merge_function_wrapper; }; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr> rewriter, - LookupMergeTreeCompactRewriter::Create( - max_level, std::move(lookup_levels), dv_maintainer, - std::move(merge_function_wrapper_factory), bucket, partition, - table_schema_, path_factory_cache, options_, - cancellation_controller, remote_lookup_file_manager, pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr> rewriter, + LookupMergeTreeCompactRewriter::Create( + max_level, std::move(lookup_levels), dv_maintainer, + std::move(merge_function_wrapper_factory), bucket, partition, table_schema_, + path_factory_cache, options_, should_produce_changelog, cancellation_controller, + remote_lookup_file_manager, pool_)); return std::shared_ptr(std::move(rewriter)); } if (lookup_strategy.deletion_vector) { return CreateLookupRewriterWithDeletionVector( partition, bucket, levels, dv_maintainer, max_level, lookup_strategy, - path_factory_cache, cancellation_controller, remote_lookup_file_manager); + should_produce_changelog, path_factory_cache, cancellation_controller, + remote_lookup_file_manager); } return CreateLookupRewriterWithoutDeletionVector( - partition, bucket, levels, dv_maintainer, max_level, lookup_strategy, path_factory_cache, - cancellation_controller, remote_lookup_file_manager); + partition, bucket, levels, dv_maintainer, max_level, lookup_strategy, + should_produce_changelog, path_factory_cache, cancellation_controller, + remote_lookup_file_manager); } Result> MergeTreeCompactManagerFactory::CreateLookupRewriterWithDeletionVector( const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, const std::shared_ptr& dv_maintainer, int32_t max_level, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& path_factory_cache, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager) const { @@ -241,20 +270,28 @@ MergeTreeCompactManagerFactory::CreateLookupRewriterWithDeletionVector( lookup_file_cache_, remote_lookup_file_manager, pool_)); auto merge_function_wrapper_factory = [data_schema = schema_, options = options_, trimmed_primary_keys, - lookup_levels_ptr = lookup_levels.get(), lookup_strategy, - dv_maintainer_ptr = dv_maintainer, pool = pool_, - user_defined_seq_comparator = user_defined_seq_comparator_]( - int32_t output_level) -> Result>> { + lookup_levels_ptr = lookup_levels.get(), lookup_strategy, should_produce_changelog, + dv_maintainer_ptr = dv_maintainer, + user_defined_seq_comparator = user_defined_seq_comparator_, + pool = pool_](int32_t output_level) + -> Result>> { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_func, PrimaryKeyTableUtils::CreateMergeFunction( data_schema, trimmed_primary_keys, options, pool)); PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, + std::unique_ptr value_serializer, + CreateChangelogValueSerializer(data_schema, should_produce_changelog, pool)); + PAIMON_ASSIGN_OR_RAISE( + InternalRowEqualizer::Equalizer value_equalizer, + CreateChangelogValueEqualizer(data_schema, options, should_produce_changelog)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr> merge_function_wrapper, LookupMergeTreeCompactRewriter:: CreateLookupMergeFunctionWrapper( std::make_unique(std::move(merge_func)), output_level, - dv_maintainer_ptr, lookup_strategy, user_defined_seq_comparator, - lookup_levels_ptr)); + dv_maintainer_ptr, lookup_strategy, should_produce_changelog, + user_defined_seq_comparator, lookup_levels_ptr, std::move(value_serializer), + std::move(value_equalizer))); return merge_function_wrapper; }; PAIMON_ASSIGN_OR_RAISE( @@ -262,8 +299,8 @@ MergeTreeCompactManagerFactory::CreateLookupRewriterWithDeletionVector( LookupMergeTreeCompactRewriter::Create( max_level, std::move(lookup_levels), dv_maintainer, std::move(merge_function_wrapper_factory), bucket, partition, table_schema_, - path_factory_cache, options_, cancellation_controller, remote_lookup_file_manager, - pool_)); + path_factory_cache, options_, should_produce_changelog, cancellation_controller, + remote_lookup_file_manager, pool_)); return std::shared_ptr(std::move(rewriter)); } auto processor_factory = std::make_shared(); @@ -275,27 +312,36 @@ MergeTreeCompactManagerFactory::CreateLookupRewriterWithDeletionVector( lookup_file_cache_, remote_lookup_file_manager, pool_)); auto merge_function_wrapper_factory = [data_schema = schema_, options = options_, trimmed_primary_keys, - lookup_levels_ptr = lookup_levels.get(), lookup_strategy, - dv_maintainer_ptr = dv_maintainer, pool = pool_, - user_defined_seq_comparator = user_defined_seq_comparator_]( - int32_t output_level) -> Result>> { + lookup_levels_ptr = lookup_levels.get(), lookup_strategy, should_produce_changelog, + dv_maintainer_ptr = dv_maintainer, + user_defined_seq_comparator = user_defined_seq_comparator_, + pool = pool_](int32_t output_level) + -> Result>> { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_func, PrimaryKeyTableUtils::CreateMergeFunction( data_schema, trimmed_primary_keys, options, pool)); PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, + std::unique_ptr value_serializer, + CreateChangelogValueSerializer(data_schema, should_produce_changelog, pool)); + PAIMON_ASSIGN_OR_RAISE( + InternalRowEqualizer::Equalizer value_equalizer, + CreateChangelogValueEqualizer(data_schema, options, should_produce_changelog)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr> merge_function_wrapper, LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( std::make_unique(std::move(merge_func)), output_level, - dv_maintainer_ptr, lookup_strategy, user_defined_seq_comparator, - lookup_levels_ptr)); + dv_maintainer_ptr, lookup_strategy, should_produce_changelog, + user_defined_seq_comparator, lookup_levels_ptr, std::move(value_serializer), + std::move(value_equalizer))); return merge_function_wrapper; }; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr> rewriter, - LookupMergeTreeCompactRewriter::Create( - max_level, std::move(lookup_levels), dv_maintainer, - std::move(merge_function_wrapper_factory), bucket, partition, - table_schema_, path_factory_cache, options_, cancellation_controller, - remote_lookup_file_manager, pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr> rewriter, + LookupMergeTreeCompactRewriter::Create( + max_level, std::move(lookup_levels), dv_maintainer, + std::move(merge_function_wrapper_factory), bucket, partition, table_schema_, + path_factory_cache, options_, should_produce_changelog, cancellation_controller, + remote_lookup_file_manager, pool_)); return std::shared_ptr(std::move(rewriter)); } @@ -303,7 +349,7 @@ Result> MergeTreeCompactManagerFactory::CreateLookupRewriterWithoutDeletionVector( const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, const std::shared_ptr& dv_maintainer, int32_t max_level, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& path_factory_cache, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager) const { @@ -318,28 +364,37 @@ MergeTreeCompactManagerFactory::CreateLookupRewriterWithoutDeletionVector( lookup_file_cache_, remote_lookup_file_manager, pool_)); auto merge_function_wrapper_factory = [data_schema = schema_, options = options_, trimmed_primary_keys, - lookup_levels_ptr = lookup_levels.get(), lookup_strategy, - dv_maintainer_ptr = dv_maintainer, pool = pool_, - user_defined_seq_comparator = user_defined_seq_comparator_]( - int32_t output_level) -> Result>> { + lookup_levels_ptr = lookup_levels.get(), lookup_strategy, should_produce_changelog, + dv_maintainer_ptr = dv_maintainer, + user_defined_seq_comparator = user_defined_seq_comparator_, + pool = pool_](int32_t output_level) + -> Result>> { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_func, PrimaryKeyTableUtils::CreateMergeFunction( data_schema, trimmed_primary_keys, options, pool)); PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, + std::unique_ptr value_serializer, + CreateChangelogValueSerializer(data_schema, should_produce_changelog, pool)); + PAIMON_ASSIGN_OR_RAISE( + InternalRowEqualizer::Equalizer value_equalizer, + CreateChangelogValueEqualizer(data_schema, options, should_produce_changelog)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr> merge_function_wrapper, LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( std::make_unique(std::move(merge_func)), output_level, - dv_maintainer_ptr, lookup_strategy, user_defined_seq_comparator, - lookup_levels_ptr)); + dv_maintainer_ptr, lookup_strategy, should_produce_changelog, + user_defined_seq_comparator, lookup_levels_ptr, std::move(value_serializer), + std::move(value_equalizer))); return merge_function_wrapper; }; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr> rewriter, - LookupMergeTreeCompactRewriter::Create( - max_level, std::move(lookup_levels), dv_maintainer, - std::move(merge_function_wrapper_factory), bucket, partition, - table_schema_, path_factory_cache, options_, cancellation_controller, - remote_lookup_file_manager, pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr> rewriter, + LookupMergeTreeCompactRewriter::Create( + max_level, std::move(lookup_levels), dv_maintainer, + std::move(merge_function_wrapper_factory), bucket, partition, table_schema_, + path_factory_cache, options_, should_produce_changelog, cancellation_controller, + remote_lookup_file_manager, pool_)); return std::shared_ptr(std::move(rewriter)); } diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h index 13a059620..e18acb631 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h @@ -71,7 +71,8 @@ class MergeTreeCompactManagerFactory { const std::shared_ptr& io_manager, const std::shared_ptr& cache_manager, const std::shared_ptr& file_store_path_factory, - const std::string& root_path, const std::shared_ptr& pool) + const std::string& root_path, bool ignore_previous_files, + const std::shared_ptr& pool) : options_(options), pool_(pool), key_comparator_(key_comparator), @@ -83,7 +84,8 @@ class MergeTreeCompactManagerFactory { io_manager_(io_manager), cache_manager_(cache_manager), file_store_path_factory_(file_store_path_factory), - root_path_(root_path) {} + root_path_(root_path), + ignore_previous_files_(ignore_previous_files) {} std::shared_ptr CreateCompactStrategy() const; @@ -118,7 +120,7 @@ class MergeTreeCompactManagerFactory { Result> CreateLookupRewriterWithDeletionVector( const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, const std::shared_ptr& dv_maintainer, int32_t max_level, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& path_factory_cache, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager) const; @@ -126,7 +128,7 @@ class MergeTreeCompactManagerFactory { Result> CreateLookupRewriterWithoutDeletionVector( const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, const std::shared_ptr& dv_maintainer, int32_t max_level, - const LookupStrategy& lookup_strategy, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& path_factory_cache, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager) const; @@ -146,6 +148,7 @@ class MergeTreeCompactManagerFactory { std::shared_ptr cache_manager_; std::shared_ptr file_store_path_factory_; std::string root_path_; + bool ignore_previous_files_; std::shared_ptr lookup_file_cache_; }; diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp index 7473b7c0b..f38caa8ed 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp @@ -87,6 +87,7 @@ class MergeTreeCompactManagerFactoryStrategyTest : public ::testing::Test { /*cache_manager=*/nullptr, /*file_store_path_factory=*/nullptr, /*root_path=*/"", + /*ignore_previous_files=*/false, /*pool=*/nullptr); } }; @@ -332,7 +333,8 @@ TEST_F(MergeTreeCompactManagerFactoryWriteTest, ASSERT_NOK_WITH_MSG(CreateSingleStringFileStoreWrite( {{"bucket", "1"}, {Options::CHANGELOG_PRODUCER, "full-compaction"}}, /*with_io_manager=*/false), - "C++ Paimon does not support changelog-producer yet"); + "C++ Paimon only supports 'none', 'input' and 'lookup' " + "changelog-producer now"); } TEST_F(MergeTreeCompactManagerFactoryWriteTest, @@ -377,13 +379,12 @@ TEST_F(MergeTreeCompactManagerFactoryWriteTest, } TEST_F(MergeTreeCompactManagerFactoryWriteTest, - TestCreateFileStoreWriteShouldFailWhenLookupChangelogConfigured) { - ASSERT_NOK_WITH_MSG( - CreateSingleStringFileStoreWrite({{"bucket", "1"}, - {Options::DELETION_VECTORS_ENABLED, "true"}, - {Options::CHANGELOG_PRODUCER, "lookup"}}, - /*with_io_manager=*/true), - "C++ Paimon does not support changelog-producer yet"); + TestCreateFileStoreWriteShouldSucceedWhenLookupChangelogConfigured) { + ASSERT_OK(CreateSingleStringFileStoreWrite({{"bucket", "1"}, + {Options::DELETION_VECTORS_ENABLED, "true"}, + {Options::CHANGELOG_PRODUCER, "lookup"}}, + /*with_io_manager=*/true) + .status()); } } // namespace paimon::test diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp index 1b64be2da..2fabd9dc1 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp @@ -146,11 +146,40 @@ MergeTreeCompactRewriter::CreateRollingRowWriter(int32_t level) { factory = std::make_shared( options_, schema_id_, write_schema_, level, FileSource::Compact(), trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true, - plan_factory, pool_); + plan_factory, /*is_changelog=*/false, pool_); } else { factory = std::make_shared( options_, schema_id_, write_schema_, level, FileSource::Compact(), - trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true, pool_); + trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true, + /*is_changelog=*/false, pool_); + } + return std::make_unique( + options_.GetTargetFileSize(/*has_primary_key=*/true), + /*target_file_row_num=*/std::numeric_limits::max(), factory); +} + +Result> +MergeTreeCompactRewriter::CreateRollingChangelogWriter(int32_t level) { + std::shared_ptr format = options_.GetChangelogFileFormat(); + if (!format) { + format = options_.GetWriteFileFormat(level); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, + CreateDataFilePathFactory(format->Identifier())); + std::shared_ptr>> factory; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr plan_factory, + ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_)); + if (plan_factory != nullptr) { + factory = std::make_shared( + options_, schema_id_, write_schema_, level, FileSource::Append(), trimmed_primary_keys_, + data_file_path_factory, /*create_stats_extractor=*/true, plan_factory, + /*is_changelog=*/true, pool_); + } else { + factory = std::make_shared( + options_, schema_id_, write_schema_, level, FileSource::Append(), trimmed_primary_keys_, + data_file_path_factory, /*create_stats_extractor=*/true, + /*is_changelog=*/true, pool_); } return std::make_unique( options_.GetTargetFileSize(/*has_primary_key=*/true), @@ -183,6 +212,19 @@ Result> MergeTreeCompactRewriter::CreateDat return path_factory->CreateDataFilePathFactory(partition_, bucket_); } +Result> +MergeTreeCompactRewriter::CreateRawSortMergeReaderForSection( + const std::vector& section) { + if (!merge_file_split_read_) { + return Status::Invalid( + "merge_file_split_read in MergeTreeCompactRewriter cannot be nullptr"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, + CreateDataFilePathFactory(options_.GetFileFormat()->Identifier())); + return merge_file_split_read_->CreateRawSortMergeReaderForSection( + section, partition_, dv_factory_, /*predicate=*/nullptr, data_file_path_factory); +} + Status MergeTreeCompactRewriter::MergeReadAndWrite( int32_t output_level, bool drop_delete, const std::vector& section, const MergeTreeCompactRewriter::KeyValueConsumerCreator& create_consumer, diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h index c9c624704..c15e16fc5 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h @@ -92,6 +92,8 @@ class MergeTreeCompactRewriter : public CompactRewriter { Result> CreateRollingRowWriter(int32_t level); + Result> CreateRollingChangelogWriter(int32_t level); + Result GenerateKeyValueConsumer() const; Status MergeReadAndWrite(int32_t output_level, bool drop_delete, @@ -100,6 +102,13 @@ class MergeTreeCompactRewriter : public CompactRewriter { KeyValueRollingFileWriter* rolling_writer, std::vector>* reader_holders_ptr); + Result> CreateRawSortMergeReaderForSection( + const std::vector& section); + + bool IsCancelled() const { + return cancellation_controller_->IsCancelled(); + } + protected: CoreOptions options_; std::unique_ptr merge_file_split_read_; @@ -108,7 +117,6 @@ class MergeTreeCompactRewriter : public CompactRewriter { Result> CreateDataFilePathFactory( const std::string& format); - private: std::shared_ptr pool_; BinaryRow partition_; int32_t bucket_; diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp index 8b5f69e77..a931fea10 100644 --- a/src/paimon/core/mergetree/lookup_levels.cpp +++ b/src/paimon/core/mergetree/lookup_levels.cpp @@ -162,8 +162,8 @@ LookupLevels::LookupLevels( lookup_store_factory_(lookup_store_factory), lookup_file_cache_(lookup_file_cache), remote_lookup_file_manager_(remote_lookup_file_manager) { - if constexpr (std::is_same_v) { - // if T is FilePosition, only read key fields to create sst file is enough + if constexpr (std::is_same_v || std::is_same_v) { + // FilePosition and first-row lookup do not persist values, so reading key fields is enough. value_schema_ = key_schema_; } else { value_schema_ = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); @@ -332,16 +332,6 @@ std::optional LookupLevels::TryToDownloadRemoteSst( template Status LookupLevels::CreateSstFileFromDataFile(const std::shared_ptr& file, const std::string& kv_file_path) { - if constexpr (std::is_same_v) { - // Short-circuit logic: if T is bool, just write empty lookup file. - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr bloom_filter, - LookupStoreFactory::BfGenerator(file->row_count, options_, pool_.get())); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr kv_writer, - lookup_store_factory_->CreateWriter(fs_, kv_file_path, bloom_filter, pool_)); - return kv_writer->Close(); - } // Prepare reader to iterate KeyValue PAIMON_ASSIGN_OR_RAISE( std::vector> raw_readers, diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp index 3b6806c73..84bba8d47 100644 --- a/src/paimon/core/mergetree/merge_tree_writer.cpp +++ b/src/paimon/core/mergetree/merge_tree_writer.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -44,6 +45,7 @@ #include "paimon/core/io/shredding_key_value_data_file_writer_factory.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/mergetree/compact/sort_merge_reader_with_loser_tree.h" +#include "paimon/core/mergetree/compact/sort_merge_reader_with_min_heap.h" #include "paimon/core/mergetree/write_buffer.h" #include "paimon/core/utils/commit_increment.h" @@ -104,8 +106,13 @@ Status MergeTreeWriter::DoClose() { // delete temporary files std::vector> delete_files; - delete_files.reserve(new_files_.size() + compact_after_.size()); + delete_files.reserve(new_files_.size() + new_changelog_files_.size() + compact_after_.size() + + compact_changelog_files_.size()); delete_files.insert(delete_files.end(), new_files_.begin(), new_files_.end()); + delete_files.insert(delete_files.end(), new_changelog_files_.begin(), + new_changelog_files_.end()); + delete_files.insert(delete_files.end(), compact_changelog_files_.begin(), + compact_changelog_files_.end()); for (const auto& file : compact_after_) { // Upgrade file is required by previous snapshot, so we should ensure that this file is // not the output of upgraded. @@ -125,9 +132,11 @@ Status MergeTreeWriter::DoClose() { write_buffer_->Clear(); new_files_.clear(); + new_changelog_files_.clear(); deleted_files_.clear(); compact_before_.clear(); compact_after_.clear(); + compact_changelog_files_.clear(); if (compact_deletion_file_) { compact_deletion_file_->Clean(); @@ -210,7 +219,9 @@ Status MergeTreeWriter::UpdateCompactResult(const std::shared_ptr compact_after_.insert(compact_after_.end(), compact_result->After().begin(), compact_result->After().end()); - // TODO(yonghao.fyh): support compact changelog + compact_changelog_files_.insert(compact_changelog_files_.end(), + compact_result->Changelog().begin(), + compact_result->Changelog().end()); return UpdateCompactDeletionFile(compact_result->DeletionFile()); } @@ -256,19 +267,54 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction, wait_for_latest_compaction = true; } auto cleanup_guard = ScopeGuard([&]() { write_buffer_->Clear(); }); - // 1. flush write buffer to get sorted readers + + auto create_consumer = [target_schema = write_schema_, pool = pool_]() + -> Result>> { + return KeyValueMetaProjectionConsumer::Create(target_schema, pool); + }; + + std::unique_ptr> + async_changelog_producer_consumer; + std::unique_ptr>> + changelog_writer; + std::vector> flushed_changelog_files; + ScopeGuard changelog_write_guard([&]() -> void { + if (changelog_writer) { + changelog_writer->Abort(); + } + if (async_changelog_producer_consumer) { + async_changelog_producer_consumer->Close(); + } + }); + if (options_.GetChangelogProducer() == ChangelogProducer::INPUT) { + PAIMON_ASSIGN_OR_RAISE(std::vector> raw_readers, + write_buffer_->CreateRawReaders()); + auto raw_sort_merge_reader = std::make_unique( + std::move(raw_readers), key_comparator_, user_defined_seq_comparator_, + /*merge_function_wrapper=*/nullptr); + async_changelog_producer_consumer = + std::make_unique>( + std::move(raw_sort_merge_reader), create_consumer, options_.GetWriteBatchSize(), + /*projection_thread_num=*/1, pool_); + PAIMON_ASSIGN_OR_RAISE(changelog_writer, CreateRollingChangelogWriter()); + while (true) { + PAIMON_ASSIGN_OR_RAISE(KeyValueBatch key_value_batch, + async_changelog_producer_consumer->NextBatch()); + if (key_value_batch.batch == nullptr) { + break; + } + PAIMON_RETURN_NOT_OK(changelog_writer->Write(std::move(key_value_batch))); + } + PAIMON_RETURN_NOT_OK(changelog_writer->Close()); + PAIMON_ASSIGN_OR_RAISE(flushed_changelog_files, changelog_writer->GetResult()); + } + + // Flush write buffer to get sorted and merged data readers. PAIMON_ASSIGN_OR_RAISE(std::vector> readers, write_buffer_->CreateReaders()); - // 2. prepare loser tree sort merge reader auto sort_merge_reader = std::make_unique( std::move(readers), key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_); - // 3. project key value to arrow array - auto create_consumer = [target_schema = write_schema_, pool = pool_]() - -> Result>> { - return KeyValueMetaProjectionConsumer::Create(target_schema, pool); - }; - // consumer batch size is WriteBatchSize auto async_key_value_producer_consumer = std::make_unique>( std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(), @@ -292,13 +338,24 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction, PAIMON_ASSIGN_OR_RAISE(std::vector> flushed_files, rolling_writer->GetResult()); async_key_value_producer_consumer->Close(); + if (async_changelog_producer_consumer) { + async_changelog_producer_consumer->Close(); + } + + new_changelog_files_.insert(new_changelog_files_.end(), flushed_changelog_files.begin(), + flushed_changelog_files.end()); + new_files_.insert(new_files_.end(), flushed_files.begin(), flushed_files.end()); + write_guard.Release(); + changelog_write_guard.Release(); for (const auto& flushed_file : flushed_files) { - new_files_.emplace_back(flushed_file); PAIMON_RETURN_NOT_OK(compact_manager_->AddNewFile(flushed_file)); } metrics_->Merge(rolling_writer->GetMetrics()); + if (changelog_writer) { + metrics_->Merge(changelog_writer->GetMetrics()); + } } PAIMON_RETURN_NOT_OK(TrySyncLatestCompaction(wait_for_latest_compaction)); PAIMON_RETURN_NOT_OK(compact_manager_->TriggerCompaction(forced_full_compaction)); @@ -306,14 +363,18 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction, } Result MergeTreeWriter::DrainIncrement() { - DataIncrement data_increment(std::move(new_files_), std::move(deleted_files_), {}); - CompactIncrement compact_increment(std::move(compact_before_), std::move(compact_after_), {}); + DataIncrement data_increment(std::move(new_files_), std::move(deleted_files_), + std::move(new_changelog_files_)); + CompactIncrement compact_increment(std::move(compact_before_), std::move(compact_after_), + std::move(compact_changelog_files_)); auto drain_deletion_file = compact_deletion_file_; new_files_.clear(); + new_changelog_files_.clear(); deleted_files_.clear(); compact_before_.clear(); compact_after_.clear(); + compact_changelog_files_.clear(); compact_deletion_file_ = nullptr; return CommitIncrement(data_increment, compact_increment, drain_deletion_file); @@ -329,15 +390,38 @@ MergeTreeWriter::CreateRollingRowWriter() const { factory = std::make_shared( options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, plan_factory, - pool_); + /*is_changelog=*/false, pool_); } else { factory = std::make_shared( options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), - trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, pool_); + trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, + /*is_changelog=*/false, pool_); } return std::make_unique>>( options_.GetTargetFileSize(/*has_primary_key=*/true), options_.GetTargetFileRowNum(), factory); } +Result>>> +MergeTreeWriter::CreateRollingChangelogWriter() const { + std::shared_ptr>> factory; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr plan_factory, + ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_)); + if (plan_factory != nullptr) { + factory = std::make_shared( + options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), + trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, plan_factory, + /*is_changelog=*/true, pool_); + } else { + factory = std::make_shared( + options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), + trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, + /*is_changelog=*/true, pool_); + } + return std::make_unique>>( + options_.GetTargetFileSize(/*has_primary_key=*/true), + /*target_file_row_num=*/std::numeric_limits::max(), factory); +} + } // namespace paimon diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h index febce2afb..10f35fbbe 100644 --- a/src/paimon/core/mergetree/merge_tree_writer.h +++ b/src/paimon/core/mergetree/merge_tree_writer.h @@ -100,6 +100,9 @@ class MergeTreeWriter : public BatchWriter { Result>>> CreateRollingRowWriter() const; + Result>>> + CreateRollingChangelogWriter() const; + Status TrySyncLatestCompaction(bool blocking); Status UpdateCompactResult(const std::shared_ptr& compact_result); Status UpdateCompactDeletionFile(const std::shared_ptr& new_deletion_file); @@ -134,9 +137,11 @@ class MergeTreeWriter : public BatchWriter { std::shared_ptr metrics_; std::vector> new_files_; + std::vector> new_changelog_files_; std::vector> deleted_files_; std::vector> compact_before_; std::vector> compact_after_; + std::vector> compact_changelog_files_; std::shared_ptr compact_deletion_file_; }; diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp index 2155647a1..9fd85cad6 100644 --- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp +++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp @@ -37,6 +37,7 @@ #include "paimon/common/types/data_field.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/compact/noop_compact_manager.h" #include "paimon/core/disk/io_manager.h" #include "paimon/core/io/compact_increment.h" @@ -144,11 +145,13 @@ class MergeTreeWriterTest : public ::testing::TestWithParam { } void CheckFileContent(const std::string& data_file_name, - const std::shared_ptr& expected_array) const { + const std::shared_ptr& expected_array, + const std::string& file_format_name = "orc") const { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system_->Open(data_file_name)); ASSERT_TRUE(input_stream); - ASSERT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get("orc", /*options=*/{})); + ASSERT_OK_AND_ASSIGN(auto file_format, + FileFormatFactory::Get(file_format_name, /*options=*/{})); ASSERT_OK_AND_ASSIGN(auto reader_builder, file_format->CreateReaderBuilder(/*batch_size=*/10)); ASSERT_OK_AND_ASSIGN(auto orc_batch_reader, reader_builder->Build(input_stream)); @@ -160,11 +163,13 @@ class MergeTreeWriterTest : public ::testing::TestWithParam { void CheckShreddingFileSchema(const std::string& data_file_name, const std::shared_ptr& expected_physical_schema, int32_t field_index, - const MapSharedShreddingFieldMeta& expected_meta) const { + const MapSharedShreddingFieldMeta& expected_meta, + const std::string& file_format_name = "orc") const { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system_->Open(data_file_name)); ASSERT_TRUE(input_stream); - ASSERT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get("orc", /*options=*/{})); + ASSERT_OK_AND_ASSIGN(auto file_format, + FileFormatFactory::Get(file_format_name, /*options=*/{})); ASSERT_OK_AND_ASSIGN(auto reader_builder, file_format->CreateReaderBuilder(/*batch_size=*/10)); ASSERT_OK_AND_ASSIGN(auto orc_batch_reader, reader_builder->Build(input_stream)); @@ -295,6 +300,182 @@ TEST_P(MergeTreeWriterTest, TestSimple) { ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement()); } +TEST_P(MergeTreeWriterTest, TestInputChangelog) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, + {Options::CHANGELOG_PRODUCER, "input"}, + {Options::CHANGELOG_FILE_PREFIX, "changes-"}, + {Options::CHANGELOG_FILE_FORMAT, "parquet"}})); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = std::make_shared(); + ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); + std::string uuid = path_factory->uuid_; + + ASSERT_OK_AND_ASSIGN(auto merge_writer, + CreateMergeWriter(/*last_sequence_number=*/-1, dir->Str(), path_factory, + /*schema_id=*/1, options)); + + std::shared_ptr array = + arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([ + ["Bob", 20, 0, 20.0], + ["Alice", 10, 0, 10.0], + ["Alice", 11, 0, 11.0], + ["Bob", 21, 0, 21.0] + ])") + .ValueOrDie(); + WriteBatch(array, + {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_BEFORE, + RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE}, + merge_writer.get()); + if (GetParam()) { + ASSERT_OK(merge_writer->FlushMemory()); + } + + ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, + merge_writer->PrepareCommit(/*wait_compaction=*/false)); + ASSERT_OK(merge_writer->Close()); + + const DataIncrement& data_increment = commit_increment.GetNewFilesIncrement(); + ASSERT_EQ(1, data_increment.NewFiles().size()); + ASSERT_EQ(1, data_increment.ChangelogFiles().size()); + ASSERT_EQ("data-" + uuid + "-1.orc", data_increment.NewFiles()[0]->file_name); + ASSERT_EQ("changes-" + uuid + "-0.parquet", data_increment.ChangelogFiles()[0]->file_name); + ASSERT_EQ(2, data_increment.NewFiles()[0]->row_count); + ASSERT_EQ(4, data_increment.ChangelogFiles()[0]->row_count); + + std::shared_ptr expected_data; + auto data_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(write_type_, {R"([ + [2, 2, "Alice", 11, 0, 11.0], + [3, 3, "Bob", 21, 0, 21.0] + ])"}, + &expected_data); + ASSERT_TRUE(data_status.ok()); + CheckFileContent(dir->Str() + "/" + data_increment.NewFiles()[0]->file_name, expected_data); + + std::shared_ptr expected_changelog; + auto changelog_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(write_type_, {R"([ + [1, 1, "Alice", 10, 0, 10.0], + [2, 2, "Alice", 11, 0, 11.0], + [0, 0, "Bob", 20, 0, 20.0], + [3, 3, "Bob", 21, 0, 21.0] + ])"}, + &expected_changelog); + ASSERT_TRUE(changelog_status.ok()); + CheckFileContent(dir->Str() + "/" + data_increment.ChangelogFiles()[0]->file_name, + expected_changelog, "parquet"); +} + +TEST_P(MergeTreeWriterTest, TestInputChangelogIgnoresTargetFileRowNum) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, + {Options::CHANGELOG_PRODUCER, "input"}, + {Options::TARGET_FILE_ROW_NUM, "1"}, + {Options::WRITE_BATCH_SIZE, "1"}})); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = std::make_shared(); + ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto merge_writer, + CreateMergeWriter(/*last_sequence_number=*/-1, dir->Str(), path_factory, + /*schema_id=*/1, options)); + + std::shared_ptr array = + arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([ + ["Bob", 20, 0, 20.0], + ["Alice", 10, 0, 10.0], + ["Alice", 11, 0, 11.0], + ["Bob", 21, 0, 21.0] + ])") + .ValueOrDie(); + WriteBatch(array, /*row_kinds=*/{}, merge_writer.get()); + if (GetParam()) { + ASSERT_OK(merge_writer->FlushMemory()); + } + + ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, + merge_writer->PrepareCommit(/*wait_compaction=*/false)); + ASSERT_OK(merge_writer->Close()); + + const std::vector>& changelog_files = + commit_increment.GetNewFilesIncrement().ChangelogFiles(); + ASSERT_EQ(1, changelog_files.size()); + ASSERT_EQ(4, changelog_files[0]->row_count); +} + +TEST_P(MergeTreeWriterTest, TestInputChangelogWithSharedShredding) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({ + {Options::FILE_FORMAT, "orc"}, + {Options::CHANGELOG_PRODUCER, "input"}, + {Options::CHANGELOG_FILE_PREFIX, "changes-"}, + {Options::CHANGELOG_FILE_FORMAT, "parquet"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + {Options::WRITE_ONLY, "true"}, + })); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = std::make_shared(); + ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); + + std::vector value_fields = { + DataField(0, arrow::field("id", arrow::int32())), + DataField(1, arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64()))), + }; + auto value_schema = DataField::ConvertDataFieldsToArrowSchema(value_fields); + auto value_type = DataField::ConvertDataFieldsToArrowStructType(value_fields); + auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema); + ASSERT_OK_AND_ASSIGN(std::shared_ptr key_comparator, + FieldsComparator::Create({value_fields[0]}, + /*is_ascending_order=*/true)); + ASSERT_OK_AND_ASSIGN( + auto merge_writer, + MergeTreeWriter::Create( + /*last_sequence_number=*/-1, /*trimmed_primary_keys=*/{"id"}, path_factory, + key_comparator, + /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/5, + value_schema, options, noop_compact_manager_, + GetParam() ? std::make_shared(dir->Str() + "/tmp", file_system_) : nullptr, + /*enable_multi_thread_spill=*/false, pool_)); + + auto array = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30]]], + [1, [["a", 11], ["c", 31]]] + ])") + .ValueOrDie(); + WriteBatch(array, /*row_kinds=*/{}, merge_writer.get()); + if (GetParam()) { + ASSERT_OK(merge_writer->FlushMemory()); + } + + ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, + merge_writer->PrepareCommit(/*wait_compaction=*/false)); + ASSERT_OK(merge_writer->Close()); + + const std::vector>& changelog_files = + commit_increment.GetNewFilesIncrement().ChangelogFiles(); + ASSERT_EQ(1, changelog_files.size()); + ASSERT_EQ(3, changelog_files[0]->row_count); + ASSERT_TRUE(StringUtils::EndsWith(changelog_files[0]->file_name, ".parquet")); + + std::map column_to_k = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + write_schema, column_to_k)); + MapSharedShreddingFieldMeta expected_shredding_meta; + expected_shredding_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_shredding_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0, 1}}}; + expected_shredding_meta.num_columns = 3; + expected_shredding_meta.max_row_width = 2; + CheckShreddingFileSchema(path_factory->ToPath(changelog_files[0]->file_name), physical_schema, + /*field_index=*/3, expected_shredding_meta, "parquet"); +} + TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) { ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); @@ -1193,6 +1374,31 @@ TEST_P(MergeTreeWriterTest, TestUpdateCompactResultDeleteIntermediateFile) { ASSERT_EQ(merge_writer->compact_after_, std::vector>({file_y})); } +TEST_P(MergeTreeWriterTest, TestUpdateCompactResultPropagatesChangelog) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = std::make_shared(); + ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); + + auto fake_compact_manager = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + auto merge_writer, + CreateMergeWriter(/*last_sequence_number=*/-1, dir->Str(), path_factory, /*schema_id=*/0, + options, /*user_defined_seq_comparator=*/nullptr, fake_compact_manager)); + + auto changelog = CreateMeta("changelog", /*level=*/0); + auto compact_result = std::make_shared( + std::vector>(), std::vector>(), + std::vector>({changelog})); + ASSERT_OK(merge_writer->UpdateCompactResult(compact_result)); + + ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, merge_writer->DrainIncrement()); + ASSERT_EQ(commit_increment.GetCompactIncrement().ChangelogFiles(), + std::vector>({changelog})); +} + TEST_P(MergeTreeWriterTest, TestUpdateCompactResultWithFileInCompactAfter) { ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); diff --git a/src/paimon/core/mergetree/write_buffer.cpp b/src/paimon/core/mergetree/write_buffer.cpp index 549975a33..d6c09a1cb 100644 --- a/src/paimon/core/mergetree/write_buffer.cpp +++ b/src/paimon/core/mergetree/write_buffer.cpp @@ -82,6 +82,10 @@ Result>> WriteBuffer::CreateRe return merged_readers; } +Result>> WriteBuffer::CreateRawReaders() { + return sort_buffer_->CreateReaders(); +} + Result WriteBuffer::FlushMemory() { return sort_buffer_->FlushMemory(); } diff --git a/src/paimon/core/mergetree/write_buffer.h b/src/paimon/core/mergetree/write_buffer.h index f4f231000..1c71f7faf 100644 --- a/src/paimon/core/mergetree/write_buffer.h +++ b/src/paimon/core/mergetree/write_buffer.h @@ -72,6 +72,10 @@ class WriteBuffer { /// @return list of KeyValueRecordReaders built from buffered data Result>> CreateReaders(); + /// Create KeyValueRecordReaders containing the raw input records without merging duplicate + /// keys. The caller should invoke Clear() after consuming the readers. + Result>> CreateRawReaders(); + /// Try to spill current buffered data. Return false when the call completed normally but the /// caller should fall back to FlushWriteBuffer before buffering more data. Result FlushMemory(); diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp index f21b0bb7f..edee4ed2c 100644 --- a/src/paimon/core/operation/file_store_scan.cpp +++ b/src/paimon/core/operation/file_store_scan.cpp @@ -245,6 +245,8 @@ Status FileStoreScan::ReadManifestsWithSnapshot(const Snapshot& snapshot, return manifest_list_->ReadDataManifests(snapshot, manifests); case ScanMode::DELTA: return manifest_list_->ReadDeltaManifests(snapshot, manifests); + case ScanMode::CHANGELOG: + return manifest_list_->ReadChangelogManifests(snapshot, manifests); default: return Status::NotImplemented("Unknown scan mode ", std::to_string(static_cast(scan_mode_))); diff --git a/src/paimon/core/operation/key_value_file_store_scan.cpp b/src/paimon/core/operation/key_value_file_store_scan.cpp index 54197f871..014de79c7 100644 --- a/src/paimon/core/operation/key_value_file_store_scan.cpp +++ b/src/paimon/core/operation/key_value_file_store_scan.cpp @@ -176,9 +176,10 @@ Result KeyValueFileStoreScan::IsValueFilterEnabled() const { case ScanMode::ALL: return value_filter_force_enabled_; case ScanMode::DELTA: + case ScanMode::CHANGELOG: return false; default: - return Status::NotImplemented("only support ALL and DELTA scan mode"); + return Status::NotImplemented("unknown scan mode"); } } diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp index 08c5ea0c3..09755984d 100644 --- a/src/paimon/core/operation/key_value_file_store_write.cpp +++ b/src/paimon/core/operation/key_value_file_store_write.cpp @@ -73,7 +73,7 @@ KeyValueFileStoreWrite::KeyValueFileStoreWrite( compact_manager_factory_(std::make_unique( options_, key_comparator_, user_defined_seq_comparator_, compaction_metrics_, table_schema_, schema_, schema_manager_, io_manager_, cache_manager_, - file_store_path_factory_, root_path_, pool_)), + file_store_path_factory_, root_path_, ignore_previous_files, pool_)), logger_(Logger::GetLogger("KeyValueFileStoreWrite")) {} Result> KeyValueFileStoreWrite::CreateFileStoreScan( diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp index b753ea431..2e517cdff 100644 --- a/src/paimon/core/operation/merge_file_split_read.cpp +++ b/src/paimon/core/operation/merge_file_split_read.cpp @@ -491,6 +491,23 @@ Result> MergeFileSplitRead::CreateSortMergeRead return sort_merge_reader; } +Result> MergeFileSplitRead::CreateRawSortMergeReaderForSection( + const std::vector& section, const BinaryRow& partition, + DeletionVector::Factory dv_factory, const std::shared_ptr& predicate, + const std::shared_ptr& data_file_path_factory) { + std::vector> record_readers; + record_readers.reserve(section.size()); + for (const auto& run : section) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr run_reader, + CreateReaderForRun(partition, run, dv_factory, predicate, data_file_path_factory)); + record_readers.emplace_back(std::move(run_reader)); + } + return std::make_unique(std::move(record_readers), key_comparator_, + user_defined_seq_comparator_, + /*merge_function_wrapper=*/nullptr); +} + Result> MergeFileSplitRead::CreateReaderForRun( const BinaryRow& partition, const SortedRun& sorted_run, DeletionVector::Factory dv_factory, const std::shared_ptr& predicate, diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h index 5003cb55a..45aa72edc 100644 --- a/src/paimon/core/operation/merge_file_split_read.h +++ b/src/paimon/core/operation/merge_file_split_read.h @@ -103,6 +103,12 @@ class MergeFileSplitRead : public AbstractSplitRead { DeletionVector::Factory dv_factory, const std::shared_ptr& predicate, const std::shared_ptr& data_file_path_factory, bool drop_delete); + /// Creates a min-heap reader which only sorts records and preserves duplicate keys. + Result> CreateRawSortMergeReaderForSection( + const std::vector& section, const BinaryRow& partition, + DeletionVector::Factory dv_factory, const std::shared_ptr& predicate, + const std::shared_ptr& data_file_path_factory); + std::shared_ptr GetPathFactory() const { return path_factory_; } diff --git a/src/paimon/core/postpone/postpone_bucket_writer.cpp b/src/paimon/core/postpone/postpone_bucket_writer.cpp index 47fd3bcb8..73f08f263 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer.cpp +++ b/src/paimon/core/postpone/postpone_bucket_writer.cpp @@ -266,11 +266,12 @@ PostponeBucketWriter::CreateRollingRowWriter() const { factory = std::make_shared( options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, plan_factory, - pool_); + /*is_changelog=*/false, pool_); } else { factory = std::make_shared( options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), - trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, pool_); + trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, + /*is_changelog=*/false, pool_); } return std::make_unique>>( options_.GetTargetFileSize(/*has_primary_key=*/true), options_.GetTargetFileRowNum(), diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 7342826d4..bcf058981 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -52,6 +52,7 @@ #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/bucket_mode.h" #include "paimon/defs.h" +#include "paimon/format/file_format.h" #include "paimon/result.h" namespace paimon { @@ -163,14 +164,8 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { PAIMON_RETURN_NOT_OK(ValidateFieldsPrefix(schema, options)); PAIMON_RETURN_NOT_OK(ValidateSequenceField(schema, options)); PAIMON_RETURN_NOT_OK(ValidateSequenceGroup(schema, options)); + PAIMON_RETURN_NOT_OK(ValidateChangelogProducer(schema, options)); - ChangelogProducer changelog_producer = options.GetChangelogProducer(); - if (schema.PrimaryKeys().empty() && changelog_producer != ChangelogProducer::NONE) { - return Status::Invalid( - fmt::format("Can not set {} on table without primary keys, please define primary keys.", - Options::CHANGELOG_PRODUCER)); - } - PAIMON_RETURN_NOT_OK(ValidateChangelogProducer(options)); PAIMON_RETURN_NOT_OK(Preconditions::CheckState( options.GetExpireConfig().GetSnapshotRetainMin() > 0, std::string(Options::SNAPSHOT_NUM_RETAINED_MIN) + " should be at least 1")); @@ -323,10 +318,41 @@ Status SchemaValidation::ValidateBucket(const TableSchema& schema, const CoreOpt return Status::OK(); } -Status SchemaValidation::ValidateChangelogProducer(const CoreOptions& options) { - return Preconditions::CheckState(options.GetChangelogProducer() == ChangelogProducer::NONE, - "C++ Paimon does not support changelog-producer yet. Please " - "keep changelog-producer as 'none'."); +Status SchemaValidation::ValidateChangelogProducer(const TableSchema& schema, + const CoreOptions& options) { + ChangelogProducer changelog_producer = options.GetChangelogProducer(); + if (schema.PrimaryKeys().empty() && changelog_producer != ChangelogProducer::NONE) { + return Status::Invalid( + fmt::format("Can not set {} on table without primary keys, please define primary keys.", + Options::CHANGELOG_PRODUCER)); + } + + bool row_deduplicate = options.ChangelogRowDeduplicate(); + const std::vector& ignore_fields = + options.GetChangelogRowDeduplicateIgnoreFields(); + PAIMON_RETURN_NOT_OK(Preconditions::CheckState( + ignore_fields.empty() || row_deduplicate, "'{}' is only valid when '{}' is true.", + Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, + Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE)); + PAIMON_RETURN_NOT_OK(Preconditions::CheckState( + ObjectUtils::ContainsAll(schema.FieldNames(), ignore_fields), + "Fields {} configured in '{}' can not be found in table schema.", ignore_fields, + Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS)); + PAIMON_RETURN_NOT_OK(Preconditions::CheckState( + !row_deduplicate || changelog_producer == ChangelogProducer::LOOKUP || + changelog_producer == ChangelogProducer::FULL_COMPACTION, + "'{}' is only valid for 'lookup' or 'full-compaction' changelog producer.", + Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE)); + PAIMON_RETURN_NOT_OK(Preconditions::CheckState( + changelog_producer == ChangelogProducer::NONE || + changelog_producer == ChangelogProducer::INPUT || + changelog_producer == ChangelogProducer::LOOKUP, + "C++ Paimon only supports 'none', 'input' and 'lookup' changelog-producer now.")); + return Preconditions::CheckState( + options.GetMergeEngine() != MergeEngine::FIRST_ROW || + changelog_producer == ChangelogProducer::NONE || + changelog_producer == ChangelogProducer::LOOKUP, + "Only support 'none' and 'lookup' changelog-producer on FIRST_ROW merge engine"); } Status SchemaValidation::ValidateForDeletionVectors(const CoreOptions& options) { @@ -654,10 +680,20 @@ Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema, options.GetFileFormat()->Identifier())); PAIMON_RETURN_NOT_OK(ValidatePerLevelOption(options_map, Options::FILE_FORMAT_PER_LEVEL, ValidateSharedShreddingFileFormat)); + std::shared_ptr changelog_format = options.GetChangelogFileFormat(); + if (changelog_format) { + PAIMON_RETURN_NOT_OK(ValidateSharedShreddingFileFormat(Options::CHANGELOG_FILE_FORMAT, + changelog_format->Identifier())); + } PAIMON_RETURN_NOT_OK(ValidateSharedShreddingCompression(Options::FILE_COMPRESSION, options.GetFileCompression())); PAIMON_RETURN_NOT_OK(ValidatePerLevelOption(options_map, Options::FILE_COMPRESSION_PER_LEVEL, ValidateSharedShreddingCompression)); + std::optional changelog_compression = options.GetChangelogFileCompression(); + if (changelog_compression) { + PAIMON_RETURN_NOT_OK(ValidateSharedShreddingCompression(Options::CHANGELOG_FILE_COMPRESSION, + changelog_compression.value())); + } return Status::OK(); } diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h index abf4d5b02..fe2f0fe9d 100644 --- a/src/paimon/core/schema/schema_validation.h +++ b/src/paimon/core/schema/schema_validation.h @@ -66,7 +66,7 @@ class SchemaValidation { static Status ValidateFieldsPrefix(const TableSchema& schema, const CoreOptions& options); static Status ValidateSequenceField(const TableSchema& schema, const CoreOptions& options); static Status ValidateSequenceGroup(const TableSchema& schema, const CoreOptions& options); - static Status ValidateChangelogProducer(const CoreOptions& options); + static Status ValidateChangelogProducer(const TableSchema& schema, const CoreOptions& options); static Status ValidateForDeletionVectors(const CoreOptions& options); static Status ValidateRowTracking(const TableSchema& table_schema, const CoreOptions& options); diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 47603497b..10c418ff0 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -638,7 +638,18 @@ TEST(SchemaValidationTest, ValidateDeletionVector) { std::shared_ptr table_schema, TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options)); ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), - "C++ Paimon does not support changelog-producer yet"); + "C++ Paimon only supports 'none', 'input' and 'lookup' " + "changelog-producer now"); + } + { + std::map options = {{Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {Options::DELETION_VECTORS_ENABLED, "true"}, + {Options::CHANGELOG_PRODUCER, "input"}}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options)); + ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema)); } { std::map options = {{Options::BUCKET, "2"}, @@ -790,6 +801,41 @@ TEST(SchemaValidationTest, ValidateInvalidConfiguration) { "Can not set changelog-producer on table without primary keys, please " "define primary keys."); } + { + std::map options = { + {Options::CHANGELOG_PRODUCER, "input"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0"}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "'changelog-producer.row-deduplicate' is only valid for 'lookup' or " + "'full-compaction' changelog producer"); + } + { + std::map options = { + {Options::CHANGELOG_PRODUCER, "input"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, "f1"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0"}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "'changelog-producer.row-deduplicate-ignore-fields' is only valid when " + "'changelog-producer.row-deduplicate' is true"); + } + { + std::map options = { + {Options::CHANGELOG_PRODUCER, "input"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, "missing"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0"}, options)); + ASSERT_NOK_WITH_MSG( + SchemaValidation::ValidateTableSchema(*table_schema), + "Fields [\"missing\"] configured in " + "'changelog-producer.row-deduplicate-ignore-fields' can not be found in table schema"); + } { auto invalid_field = arrow::field("_SEQUENCE_NUMBER", arrow::int64()); arrow::FieldVector invalid_fields = fields; @@ -821,15 +867,18 @@ TEST(SchemaValidationTest, ValidateInvalidConfiguration) { TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, /*primary_keys=*/{"f0"}, options)); ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), - "C++ Paimon does not support changelog-producer yet"); + "Only support 'none' and 'lookup' changelog-producer on FIRST_ROW " + "merge engine"); } { - std::map options = {{Options::CHANGELOG_PRODUCER, "lookup"}}; + std::map options = { + {Options::CHANGELOG_PRODUCER, "full-compaction"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, /*primary_keys=*/{"f0"}, options)); - ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), - "C++ Paimon does not support changelog-producer yet"); + ASSERT_NOK_WITH_MSG( + SchemaValidation::ValidateTableSchema(*table_schema), + "C++ Paimon only supports 'none', 'input' and 'lookup' changelog-producer now."); } // test for row tracking { @@ -1106,6 +1155,16 @@ TEST(SchemaValidationTest, TestMapSharedShreddingCompression) { "MAP shared-shredding only supports none/lz4/zstd compression, but " "file.compression.per.level.1 is snappy."); } + { + auto options = base_options; + options[Options::CHANGELOG_FILE_COMPRESSION] = "snappy"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "MAP shared-shredding only supports none/lz4/zstd compression, but " + "changelog-file.compression is snappy."); + } { auto options = base_options; options.erase("fields.f1.map.storage-layout"); @@ -1157,6 +1216,16 @@ TEST(SchemaValidationTest, TestMapSharedShreddingFileFormat) { "MAP shared-shredding only supports parquet/orc file formats, but " "file.format.per.level.1 is avro."); } + { + auto options = base_options; + options[Options::CHANGELOG_FILE_FORMAT] = "avro"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "MAP shared-shredding only supports parquet/orc file formats, but " + "changelog-file.format is avro."); + } } TEST(SchemaValidationTest, TestMapSharedShreddingRejectsPostponeBucketMode) { diff --git a/src/paimon/core/table/source/data_table_stream_scan.cpp b/src/paimon/core/table/source/data_table_stream_scan.cpp index 4c6d4df11..1657c9c7b 100644 --- a/src/paimon/core/table/source/data_table_stream_scan.cpp +++ b/src/paimon/core/table/source/data_table_stream_scan.cpp @@ -26,6 +26,7 @@ #include "paimon/core/options/changelog_producer.h" #include "paimon/core/table/bucket_mode.h" #include "paimon/core/table/source/plan_impl.h" +#include "paimon/core/table/source/snapshot/changelog_follow_up_scanner.h" #include "paimon/core/table/source/snapshot/delta_follow_up_scanner.h" #include "paimon/core/table/source/snapshot/follow_up_scanner.h" #include "paimon/core/table/source/snapshot/snapshot_reader.h" @@ -55,10 +56,15 @@ Result> DataTableStreamScan::CreatePlan() { Result> DataTableStreamScan::TryFirstPlan() { std::shared_ptr scan_result; - if (core_options_.GetChangelogProducer() == ChangelogProducer::LOOKUP) { - return Status::NotImplemented("do not support lookup changelog producer"); - } else if (core_options_.GetChangelogProducer() == ChangelogProducer::FULL_COMPACTION) { + if (core_options_.GetChangelogProducer() == ChangelogProducer::FULL_COMPACTION) { return Status::NotImplemented("do not support full compaction changelog producer"); + } else if (core_options_.GetChangelogProducer() == ChangelogProducer::LOOKUP) { + // Level-0 files will be compacted later to produce changelog records. Exclude them from + // the initial full scan so that the same changes are not emitted both in the full phase + // and again in the incremental changelog phase. + snapshot_reader_->WithLevelFilter([](int32_t level) -> bool { return level > 0; }); + PAIMON_ASSIGN_OR_RAISE(scan_result, starting_scanner_->Scan(snapshot_reader_)); + snapshot_reader_->WithLevelFilter([](int32_t) -> bool { return true; }); } else { PAIMON_ASSIGN_OR_RAISE(scan_result, starting_scanner_->Scan(snapshot_reader_)); } @@ -85,6 +91,9 @@ Result> DataTableStreamScan::NextPlan() { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, follow_up_scanner_->Scan(snapshot.value(), snapshot_reader_)); next_snapshot_id_.value()++; + if (plan->Splits().empty()) { + continue; + } return plan; } else { next_snapshot_id_.value()++; @@ -122,8 +131,19 @@ Result> DataTableStreamScan::GetNextSnapshot( Status DataTableStreamScan::InitScanner() { PAIMON_ASSIGN_OR_RAISE(starting_scanner_, CreateStartingScanner(/*is_streaming=*/true)); - follow_up_scanner_ = std::make_shared(); - return Status::OK(); + switch (core_options_.GetChangelogProducer()) { + case ChangelogProducer::NONE: + follow_up_scanner_ = std::make_shared(); + return Status::OK(); + case ChangelogProducer::INPUT: + case ChangelogProducer::LOOKUP: + follow_up_scanner_ = std::make_shared(); + return Status::OK(); + case ChangelogProducer::FULL_COMPACTION: + return Status::NotImplemented("do not support full compaction changelog producer"); + default: + return Status::NotImplemented("unknown changelog producer"); + } } } // namespace paimon diff --git a/src/paimon/core/table/source/scan_mode.h b/src/paimon/core/table/source/scan_mode.h index c236aad69..a01fde506 100644 --- a/src/paimon/core/table/source/scan_mode.h +++ b/src/paimon/core/table/source/scan_mode.h @@ -26,10 +26,10 @@ enum class ScanMode { ALL = 0, /// Only scan newly changed files of a snapshot. - DELTA = 1 + DELTA = 1, /// Only scan changelog files of a snapshot. - /* CHANGELOG = 2 */ + CHANGELOG = 2 }; } // namespace paimon diff --git a/src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h b/src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h new file mode 100644 index 000000000..fc7c1be6a --- /dev/null +++ b/src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h @@ -0,0 +1,50 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "paimon/core/table/source/snapshot/follow_up_scanner.h" +#include "paimon/logging.h" + +namespace paimon { + +/// Follow-up scanner for snapshots containing changelog manifests. +class ChangelogFollowUpScanner : public FollowUpScanner { + public: + ChangelogFollowUpScanner() : logger_(Logger::GetLogger("ChangelogFollowUpScanner")) {} + + bool NeedScanSnapshot(const Snapshot& snapshot) const override { + if (snapshot.ChangelogManifestList()) { + return true; + } + PAIMON_LOG_DEBUG(logger_, "Snapshot #%ld has no changelog, check the next snapshot.", + snapshot.Id()); + return false; + } + + Result> Scan( + const Snapshot& snapshot, + const std::shared_ptr& snapshot_reader) const override { + return snapshot_reader->WithMode(ScanMode::CHANGELOG)->WithSnapshot(snapshot)->Read(); + } + + private: + std::unique_ptr logger_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp index cebc644cf..cd704174d 100644 --- a/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp +++ b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp @@ -30,6 +30,8 @@ #include "paimon/core/index/index_file_handler.h" #include "paimon/core/index/index_file_meta.h" #include "paimon/core/io/data_file_meta.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/source/snapshot/changelog_follow_up_scanner.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/index_file_path_factories.h" #include "paimon/fs/local/local_file_system.h" @@ -112,4 +114,23 @@ TEST_F(SnapshotReaderTest, GetDeletionFilesOverwritesDuplicateDataFileName) { EXPECT_EQ(deletion_files[0]->cardinality, std::optional(4)); } +TEST_F(SnapshotReaderTest, ChangelogFollowUpScannerSkipsSnapshotsWithoutChangelog) { + auto create_snapshot = [](const std::optional& changelog_manifest_list) { + return Snapshot( + /*id=*/1, /*schema_id=*/0, /*base_manifest_list=*/"", + /*base_manifest_list_size=*/std::nullopt, /*delta_manifest_list=*/"", + /*delta_manifest_list_size=*/std::nullopt, changelog_manifest_list, + /*changelog_manifest_list_size=*/std::nullopt, + /*index_manifest=*/std::nullopt, /*commit_user=*/"user", /*commit_identifier=*/1, + Snapshot::CommitKind::Append(), /*time_millis=*/0, /*total_record_count=*/0, + /*delta_record_count=*/0, /*changelog_record_count=*/std::nullopt, + /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, + /*properties=*/std::nullopt, /*next_row_id=*/std::nullopt); + }; + + ChangelogFollowUpScanner scanner; + ASSERT_FALSE(scanner.NeedScanSnapshot(create_snapshot(std::nullopt))); + ASSERT_TRUE(scanner.NeedScanSnapshot(create_snapshot("changelog-manifest-list"))); +} + } // namespace paimon::test diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index fdc38ea9f..0e9f0ec62 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -1273,9 +1273,9 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { .value()); std::vector>> expected_data_splits = { - {}, {expected_data_split1_2}, {expected_data_split2_1}, {}, {expected_data_split4_1}}; + {}, {expected_data_split1_2}, {expected_data_split2_1}, {expected_data_split4_1}}; - std::vector> expected_snapshot_ids = {std::nullopt, 1, 2, 3, 4}; + std::vector> expected_snapshot_ids = {std::nullopt, 1, 2, 4}; CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, expected_data_splits); } diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 5916fc97c..99a40ee77 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -90,6 +90,26 @@ class WriteAndReadInteTest return new_options; } + std::string LookupTempDirectory() const { + return PathUtil::JoinPath(test_dir_, "tmp"); + } + + Result> CreateLookupTestHelper( + const std::shared_ptr& schema, + const std::map& options) const { + return TestHelper::Create(test_dir_, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"pk"}, options, + /*is_streaming_mode=*/true, /*ignore_if_exists=*/false, + LookupTempDirectory()); + } + + Result> CreateLookupTestHelper( + const std::string& table_path, + const std::map& options) const { + return TestHelper::Create(table_path, options, /*is_streaming_mode=*/true, + LookupTempDirectory()); + } + Status WriteNextSchema(const std::vector& fields, int32_t highest_field_id, const std::map& options) const { return TestHelper::WriteNextSchema(dir_->GetFileSystem(), @@ -129,6 +149,34 @@ class WriteAndReadInteTest return file_system->AtomicStore(schema_path, std::string(buffer.GetString())); } + Status CompactAndCommit(const std::string& table_path, + const std::map& options, + int64_t commit_identifier) const { + WriteContextBuilder write_context_builder(table_path, "commit_user"); + write_context_builder.WithTempDirectory(LookupTempDirectory()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr write_context, + write_context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_write, + FileStoreWrite::Create(std::move(write_context))); + PAIMON_RETURN_NOT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, + /*full_compaction=*/true)); + PAIMON_ASSIGN_OR_RAISE( + std::vector> compact_messages, + file_store_write->PrepareCommit(/*wait_compaction=*/true, commit_identifier)); + PAIMON_RETURN_NOT_OK(file_store_write->Close()); + if (compact_messages.empty()) { + return Status::Invalid("expected lookup compaction commit messages"); + } + CommitContextBuilder commit_context_builder(table_path, "commit_user"); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit_context, + commit_context_builder.SetOptions(options).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, + FileStoreCommit::Create(std::move(commit_context))); + return file_store_commit->Commit(compact_messages, commit_identifier); + } + Result ReadAndCheckProjectedResult(const std::map& options, const std::vector& read_fields, const std::shared_ptr& expected_type, @@ -630,6 +678,671 @@ TEST_P(WriteAndReadInteTest, TestPKSimple) { ASSERT_TRUE(success); } +TEST_P(WriteAndReadInteTest, TestInputChangelogStreamRead) { + arrow::FieldVector fields = { + arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32()), + }; + auto [file_format, file_system] = GetParam(); + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "input"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, + /*primary_keys=*/{"pk"}, options, /*is_streaming_mode=*/true)); + + ASSERT_OK_AND_ASSIGN(std::vector> initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch( + arrow::struct_(fields), R"([["Alice", 10], ["Bob", 20], ["Alice", 11], ["Bob", 21]])", + /*partition_map=*/{}, /*bucket=*/0, + {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_BEFORE, + RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(std::vector> changelog_splits, helper->Scan()); + ASSERT_TRUE(changelog_splits.empty()); + ASSERT_OK_AND_ASSIGN(changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + fields[0], + fields[1], + }); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[0, "Alice", 10], [2, "Alice", 11], + [1, "Bob", 20], [3, "Bob", 21]])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogStreamRead) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = { + arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32()), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + auto compact_and_commit = [this, &options, + &table_path](int64_t commit_identifier) -> Status { + WriteContextBuilder write_context_builder(table_path, "commit_user"); + write_context_builder.WithTempDirectory(LookupTempDirectory()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr write_context, + write_context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_write, + FileStoreWrite::Create(std::move(write_context))); + PAIMON_RETURN_NOT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, + /*full_compaction=*/true)); + PAIMON_ASSIGN_OR_RAISE( + std::vector> compact_messages, + file_store_write->PrepareCommit(/*wait_compaction=*/true, commit_identifier)); + PAIMON_RETURN_NOT_OK(file_store_write->Close()); + if (compact_messages.empty()) { + return Status::Invalid("expected lookup compaction commit messages"); + } + CommitContextBuilder commit_context_builder(table_path, "commit_user"); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit_context, + commit_context_builder.SetOptions(options).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, + FileStoreCommit::Create(std::move(commit_context))); + return file_store_commit->Commit(compact_messages, commit_identifier); + }; + + // Move the initial value to a high level so the next compaction must look it up. + ASSERT_OK(compact_and_commit(/*commit_identifier=*/1)); + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + + ASSERT_OK_AND_ASSIGN(std::vector> initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr update_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(compact_and_commit(/*commit_identifier=*/3)); + + ASSERT_OK_AND_ASSIGN(std::vector> changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + fields[0], + fields[1], + }); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 20]])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogInitialFullScanExcludesLevelZero) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = { + arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32()), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + WriteContextBuilder write_context_builder(table_path, "commit_user"); + write_context_builder.WithTempDirectory(LookupTempDirectory()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + write_context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, + FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, + /*full_compaction=*/true)); + ASSERT_OK_AND_ASSIGN( + std::vector> compact_messages, + file_store_write->PrepareCommit(/*wait_compaction=*/true, /*commit_identifier=*/1)); + ASSERT_OK(file_store_write->Close()); + ASSERT_FALSE(compact_messages.empty()); + CommitContextBuilder commit_context_builder(table_path, "commit_user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.SetOptions(options).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_commit, + FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK(file_store_commit->Commit(compact_messages, /*commit_identifier=*/1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr update_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN( + std::vector> initial_full_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(initial_full_splits.empty()); + for (const auto& split : initial_full_splits) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + for (const auto& file : data_split->DataFiles()) { + ASSERT_GT(file->level, 0); + } + } + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + fields[0], + fields[1], + }); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(expected_type, initial_full_splits, + R"([[0, "Alice", 10]])")); + ASSERT_TRUE(success); + + WriteContextBuilder changelog_write_context_builder(table_path, "commit_user"); + changelog_write_context_builder.WithTempDirectory(LookupTempDirectory()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr changelog_write_context, + changelog_write_context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr changelog_file_store_write, + FileStoreWrite::Create(std::move(changelog_write_context))); + ASSERT_OK(changelog_file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, + /*full_compaction=*/true)); + ASSERT_OK_AND_ASSIGN( + std::vector> changelog_compact_messages, + changelog_file_store_write->PrepareCommit(/*wait_compaction=*/true, + /*commit_identifier=*/3)); + ASSERT_OK(changelog_file_store_write->Close()); + ASSERT_FALSE(changelog_compact_messages.empty()); + ASSERT_OK(file_store_commit->Commit(changelog_compact_messages, /*commit_identifier=*/3)); + + ASSERT_OK_AND_ASSIGN(std::vector> changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + ASSERT_OK_AND_ASSIGN(bool changelog_success, + helper->ReadAndCheckResult( + expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 20]])")); + ASSERT_TRUE(changelog_success); +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogInsertUpdateDelete) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32())}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + ASSERT_OK_AND_ASSIGN( + auto initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([["Alice", 10], ["Bob", 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + ASSERT_OK_AND_ASSIGN( + auto change_batch, + TestHelper::MakeRecordBatch( + arrow::struct_(fields), + R"([["Alice", 11], ["Bob", 0], ["Carol", 30], ["Dave", 0]])", + /*partition_map=*/{}, /*bucket=*/0, + {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE, + RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE})); + ASSERT_OK(helper->WriteAndCommit(std::move(change_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + + ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], + fields[1]}); + ASSERT_OK_AND_ASSIGN( + bool success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 11], + [3, "Bob", 20], [0, "Carol", 30]])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogWithFirstRow) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32())}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::MERGE_ENGINE, "first-row"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + ASSERT_OK_AND_ASSIGN(auto initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + ASSERT_OK_AND_ASSIGN( + auto change_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([["Alice", 20], ["Bob", 30]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(change_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + + ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], + fields[1]}); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[0, "Bob", 30]])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogWithDeletionVector) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32())}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::DELETION_VECTORS_ENABLED, "true"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + ASSERT_OK_AND_ASSIGN(auto initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + ASSERT_OK_AND_ASSIGN(auto update_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + + ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], + fields[1]}); + ASSERT_OK_AND_ASSIGN( + bool changelog_success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 20]])")); + ASSERT_TRUE(changelog_success); + + ASSERT_OK_AND_ASSIGN(auto batch_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + ASSERT_OK_AND_ASSIGN(bool batch_success, + helper->ReadAndCheckResult(expected_type, batch_splits, + R"([[0, "Alice", 20]])")); + ASSERT_TRUE(batch_success); +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicate) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32())}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + ASSERT_OK_AND_ASSIGN(auto initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + ASSERT_OK_AND_ASSIGN(auto write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto unchanged_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(unchanged_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + ASSERT_OK_AND_ASSIGN(auto empty_splits, helper->Scan()); + ASSERT_TRUE(empty_splits.empty()); + + write_helper.reset(); + ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto changed_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(changed_batch), /*commit_identifier=*/4, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/5)); + ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], + fields[1]}); + ASSERT_OK_AND_ASSIGN( + bool success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 20]])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicateIgnoreFields) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32()), + arrow::field("ignored", arrow::int32())}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"}, + {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, "ignored"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + ASSERT_OK_AND_ASSIGN(auto initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([["Alice", 10, 100]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + ASSERT_OK_AND_ASSIGN(auto write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto ignored_change_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([["Alice", 10, 200]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(ignored_change_batch), + /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + ASSERT_OK_AND_ASSIGN(auto empty_splits, helper->Scan()); + ASSERT_TRUE(empty_splits.empty()); + + write_helper.reset(); + ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto real_change_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([["Alice", 20, 300]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(real_change_batch), /*commit_identifier=*/4, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/5)); + ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], + fields[1], fields[2]}); + ASSERT_OK_AND_ASSIGN( + bool success, + helper->ReadAndCheckResult( + expected_type, changelog_splits, + R"([[1, "Alice", 10, 200], [2, "Alice", 20, 300]])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestChangelogWithSchemaEvolution) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields_v0 = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32())}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto write_helper, + CreateLookupTestHelper(arrow::schema(fields_v0), options)); + ASSERT_OK_AND_ASSIGN(auto initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields_v0), + R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + ASSERT_OK_AND_ASSIGN(auto scan_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto initial_splits, + scan_helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + + write_helper.reset(); + ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields_v0), + R"([["Alice", 11]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + + arrow::FieldVector fields_v1 = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32()), + arrow::field("extra", arrow::utf8())}; + ASSERT_OK(WriteNextSchema({DataField(0, fields_v1[0]), DataField(1, fields_v1[1]), + DataField(2, fields_v1[2])}, + /*highest_field_id=*/2, options)); + write_helper.reset(); + ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, + TestHelper::MakeRecordBatch(arrow::struct_(fields_v1), + R"([["Alice", 12, "v1"]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/4, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/5)); + + arrow::FieldVector fields_v2 = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32()), + arrow::field("renamed_extra", arrow::utf8())}; + ASSERT_OK(WriteNextSchema({DataField(0, fields_v2[0]), DataField(1, fields_v2[1]), + DataField(2, fields_v2[2])}, + /*highest_field_id=*/2, options)); + write_helper.reset(); + ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto batch_v2, + TestHelper::MakeRecordBatch(arrow::struct_(fields_v2), + R"([["Alice", 13, "v2"]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v2), /*commit_identifier=*/6, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/7)); + + arrow::FieldVector fields_v3 = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int64()), + arrow::field("renamed_extra", arrow::utf8())}; + ASSERT_OK(WriteNextSchema({DataField(0, fields_v3[0]), DataField(1, fields_v3[1]), + DataField(2, fields_v3[2])}, + /*highest_field_id=*/2, options)); + write_helper.reset(); + ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto batch_v3, + TestHelper::MakeRecordBatch(arrow::struct_(fields_v3), + R"([["Alice", 14, "v3"]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v3), /*commit_identifier=*/8, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/9)); + + auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields_v3[0], + fields_v3[1], fields_v3[2]}); + std::vector expected_data = { + R"([[1, "Alice", 10, null], [2, "Alice", 11, null]])", + R"([[1, "Alice", 11, null], [2, "Alice", 12, "v1"]])", + R"([[1, "Alice", 12, "v1"], [2, "Alice", 13, "v2"]])", + R"([[1, "Alice", 13, "v2"], [2, "Alice", 14, "v3"]])"}; + for (int64_t schema_id = 0; schema_id < static_cast(expected_data.size()); + schema_id++) { + ASSERT_OK_AND_ASSIGN(auto changelog_splits, scan_helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + for (const auto& split : changelog_splits) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + for (const auto& file : data_split->DataFiles()) { + ASSERT_EQ(file->schema_id, schema_id); + } + } + ASSERT_OK_AND_ASSIGN( + bool success, + scan_helper->ReadAndCheckResult(expected_type, changelog_splits, + expected_data[static_cast(schema_id)])); + ASSERT_TRUE(success); + } +} + +TEST_P(WriteAndReadInteTest, TestLookupChangelogWithExternalPath) { + auto [file_format, file_system] = GetParam(); + if (file_system == "jindo") { + return; + } + std::unique_ptr external_dir = + UniqueTestDirectory::Create(file_system); + ASSERT_TRUE(external_dir); + arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), + arrow::field("value", arrow::int32())}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::DATA_FILE_EXTERNAL_PATHS, "FILE://" + external_dir->Str()}, + {Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY, "round-robin"}}; + ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); + ASSERT_OK_AND_ASSIGN(auto initial_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(auto initial_splits, + helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); + ASSERT_TRUE(initial_splits.empty()); + ASSERT_OK_AND_ASSIGN(auto update_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); + + ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); + ASSERT_FALSE(changelog_splits.empty()); + bool found_external_changelog = false; + for (const auto& split : changelog_splits) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + for (const auto& file : data_split->DataFiles()) { + ASSERT_TRUE(file->external_path.has_value()); + ASSERT_TRUE(StringUtils::StartsWith(PathUtil::GetName(file->file_name), "changelog-")); + ASSERT_OK_AND_ASSIGN( + bool exists, + external_dir->GetFileSystem()->Exists(file->external_path.value())); + ASSERT_TRUE(exists); + found_external_changelog = true; + } + } + ASSERT_TRUE(found_external_changelog); + + auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], + fields[1]}); + ASSERT_OK_AND_ASSIGN( + bool success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 20]])")); + ASSERT_TRUE(success); +} + TEST_P(WriteAndReadInteTest, TestNestedType) { arrow::FieldVector fields = { arrow::field("f1", arrow::map(arrow::int8(), arrow::int16())), From 31aeaebd27cb1fab90afd8ec38cdeddf332d697c Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Wed, 26 Aug 2026 17:54:00 +0800 Subject: [PATCH 2/9] fix pre-commit --- test/inte/write_and_read_inte_test.cpp | 245 +++++++++++-------------- 1 file changed, 112 insertions(+), 133 deletions(-) diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 99a40ee77..b5c14bc13 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -104,8 +104,7 @@ class WriteAndReadInteTest } Result> CreateLookupTestHelper( - const std::string& table_path, - const std::map& options) const { + const std::string& table_path, const std::map& options) const { return TestHelper::Create(table_path, options, /*is_streaming_mode=*/true, LookupTempDirectory()); } @@ -169,9 +168,8 @@ class WriteAndReadInteTest return Status::Invalid("expected lookup compaction commit messages"); } CommitContextBuilder commit_context_builder(table_path, "commit_user"); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, + commit_context_builder.SetOptions(options).Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, FileStoreCommit::Create(std::move(commit_context))); return file_store_commit->Commit(compact_messages, commit_identifier); @@ -747,8 +745,7 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogStreamRead) { /*expected_commit_messages=*/std::nullopt)); std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); - auto compact_and_commit = [this, &options, - &table_path](int64_t commit_identifier) -> Status { + auto compact_and_commit = [this, &options, &table_path](int64_t commit_identifier) -> Status { WriteContextBuilder write_context_builder(table_path, "commit_user"); write_context_builder.WithTempDirectory(LookupTempDirectory()); PAIMON_ASSIGN_OR_RAISE( @@ -766,9 +763,8 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogStreamRead) { return Status::Invalid("expected lookup compaction commit messages"); } CommitContextBuilder commit_context_builder(table_path, "commit_user"); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, + commit_context_builder.SetOptions(options).Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, FileStoreCommit::Create(std::move(commit_context))); return file_store_commit->Commit(compact_messages, commit_identifier); @@ -852,9 +848,8 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogInitialFullScanExcludesLevelZero ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2, /*expected_commit_messages=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN( - std::vector> initial_full_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::vector> initial_full_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); ASSERT_FALSE(initial_full_splits.empty()); for (const auto& split : initial_full_splits) { auto data_split = std::dynamic_pointer_cast(split); @@ -869,9 +864,9 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogInitialFullScanExcludesLevelZero fields[0], fields[1], }); - ASSERT_OK_AND_ASSIGN(bool success, - helper->ReadAndCheckResult(expected_type, initial_full_splits, - R"([[0, "Alice", 10]])")); + ASSERT_OK_AND_ASSIGN( + bool success, + helper->ReadAndCheckResult(expected_type, initial_full_splits, R"([[0, "Alice", 10]])")); ASSERT_TRUE(success); WriteContextBuilder changelog_write_context_builder(table_path, "commit_user"); @@ -883,10 +878,9 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogInitialFullScanExcludesLevelZero FileStoreWrite::Create(std::move(changelog_write_context))); ASSERT_OK(changelog_file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); - ASSERT_OK_AND_ASSIGN( - std::vector> changelog_compact_messages, - changelog_file_store_write->PrepareCommit(/*wait_compaction=*/true, - /*commit_identifier=*/3)); + ASSERT_OK_AND_ASSIGN(std::vector> changelog_compact_messages, + changelog_file_store_write->PrepareCommit(/*wait_compaction=*/true, + /*commit_identifier=*/3)); ASSERT_OK(changelog_file_store_write->Close()); ASSERT_FALSE(changelog_compact_messages.empty()); ASSERT_OK(file_store_commit->Commit(changelog_compact_messages, /*commit_identifier=*/3)); @@ -894,9 +888,8 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogInitialFullScanExcludesLevelZero ASSERT_OK_AND_ASSIGN(std::vector> changelog_splits, helper->Scan()); ASSERT_FALSE(changelog_splits.empty()); ASSERT_OK_AND_ASSIGN(bool changelog_success, - helper->ReadAndCheckResult( - expected_type, changelog_splits, - R"([[1, "Alice", 10], [2, "Alice", 20]])")); + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 20]])")); ASSERT_TRUE(changelog_success); } @@ -914,8 +907,7 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogInsertUpdateDelete) { ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); ASSERT_OK_AND_ASSIGN( auto initial_batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields), - R"([["Alice", 10], ["Bob", 20]])", + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10], ["Bob", 20]])", /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, /*expected_commit_messages=*/std::nullopt)); @@ -929,24 +921,22 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogInsertUpdateDelete) { ASSERT_TRUE(initial_splits.empty()); ASSERT_OK_AND_ASSIGN( auto change_batch, - TestHelper::MakeRecordBatch( - arrow::struct_(fields), - R"([["Alice", 11], ["Bob", 0], ["Carol", 30], ["Dave", 0]])", - /*partition_map=*/{}, /*bucket=*/0, - {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE, - RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE})); + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([["Alice", 11], ["Bob", 0], ["Carol", 30], ["Dave", 0]])", + /*partition_map=*/{}, /*bucket=*/0, + {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE, + RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE})); ASSERT_OK(helper->WriteAndCommit(std::move(change_batch), /*commit_identifier=*/2, /*expected_commit_messages=*/std::nullopt)); ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); ASSERT_FALSE(changelog_splits.empty()); - auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], - fields[1]}); - ASSERT_OK_AND_ASSIGN( - bool success, - helper->ReadAndCheckResult(expected_type, changelog_splits, - R"([[1, "Alice", 10], [2, "Alice", 11], + auto expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]}); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 11], [3, "Bob", 20], [0, "Carol", 30]])")); ASSERT_TRUE(success); } @@ -956,7 +946,7 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogWithFirstRow) { arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32())}; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, {Options::MERGE_ENGINE, "first-row"}}; @@ -979,8 +969,7 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogWithFirstRow) { ASSERT_TRUE(initial_splits.empty()); ASSERT_OK_AND_ASSIGN( auto change_batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields), - R"([["Alice", 20], ["Bob", 30]])", + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20], ["Bob", 30]])", /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(helper->WriteAndCommit(std::move(change_batch), /*commit_identifier=*/2, /*expected_commit_messages=*/std::nullopt)); @@ -988,11 +977,10 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogWithFirstRow) { ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); ASSERT_FALSE(changelog_splits.empty()); - auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], - fields[1]}); - ASSERT_OK_AND_ASSIGN(bool success, - helper->ReadAndCheckResult(expected_type, changelog_splits, - R"([[0, "Bob", 30]])")); + auto expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]}); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[0, "Bob", 30]])")); ASSERT_TRUE(success); } @@ -1001,9 +989,9 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogWithDeletionVector) { arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32())}; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, {Options::DELETION_VECTORS_ENABLED, "true"}}; if (file_system == "jindo") { options = AddOptionsForJindo(options); @@ -1031,20 +1019,18 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogWithDeletionVector) { ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); ASSERT_FALSE(changelog_splits.empty()); - auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], - fields[1]}); - ASSERT_OK_AND_ASSIGN( - bool changelog_success, - helper->ReadAndCheckResult(expected_type, changelog_splits, - R"([[1, "Alice", 10], [2, "Alice", 20]])")); + auto expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]}); + ASSERT_OK_AND_ASSIGN(bool changelog_success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 20]])")); ASSERT_TRUE(changelog_success); ASSERT_OK_AND_ASSIGN(auto batch_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, /*is_streaming=*/false)); - ASSERT_OK_AND_ASSIGN(bool batch_success, - helper->ReadAndCheckResult(expected_type, batch_splits, - R"([[0, "Alice", 20]])")); + ASSERT_OK_AND_ASSIGN(bool batch_success, helper->ReadAndCheckResult(expected_type, batch_splits, + R"([[0, "Alice", 20]])")); ASSERT_TRUE(batch_success); } @@ -1053,9 +1039,12 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicate) { arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32())}; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "lookup"}, {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"}}; if (file_system == "jindo") { options = AddOptionsForJindo(options); @@ -1094,12 +1083,11 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicate) { ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/5)); ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); ASSERT_FALSE(changelog_splits.empty()); - auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], - fields[1]}); - ASSERT_OK_AND_ASSIGN( - bool success, - helper->ReadAndCheckResult(expected_type, changelog_splits, - R"([[1, "Alice", 10], [2, "Alice", 20]])")); + auto expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]}); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 20]])")); ASSERT_TRUE(success); } @@ -1109,19 +1097,21 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicateIgnoreFields) { arrow::field("value", arrow::int32()), arrow::field("ignored", arrow::int32())}; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "lookup"}, {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE, "true"}, {Options::CHANGELOG_PRODUCER_ROW_DEDUPLICATE_IGNORE_FIELDS, "ignored"}}; if (file_system == "jindo") { options = AddOptionsForJindo(options); } ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); - ASSERT_OK_AND_ASSIGN(auto initial_batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields), - R"([["Alice", 10, 100]])", - /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto initial_batch, TestHelper::MakeRecordBatch( + arrow::struct_(fields), R"([["Alice", 10, 100]])", + /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, /*expected_commit_messages=*/std::nullopt)); std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); @@ -1133,10 +1123,10 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicateIgnoreFields) { helper->NewScan(StartupMode::Latest(), /*snapshot_id=*/std::nullopt)); ASSERT_TRUE(initial_splits.empty()); ASSERT_OK_AND_ASSIGN(auto write_helper, CreateLookupTestHelper(table_path, options)); - ASSERT_OK_AND_ASSIGN(auto ignored_change_batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields), - R"([["Alice", 10, 200]])", - /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN( + auto ignored_change_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 10, 200]])", + /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(write_helper->WriteAndCommit(std::move(ignored_change_batch), /*commit_identifier=*/2, /*expected_commit_messages=*/std::nullopt)); @@ -1146,22 +1136,20 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicateIgnoreFields) { write_helper.reset(); ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); - ASSERT_OK_AND_ASSIGN(auto real_change_batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields), - R"([["Alice", 20, 300]])", - /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN( + auto real_change_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["Alice", 20, 300]])", + /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(write_helper->WriteAndCommit(std::move(real_change_batch), /*commit_identifier=*/4, /*expected_commit_messages=*/std::nullopt)); ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/5)); ASSERT_OK_AND_ASSIGN(auto changelog_splits, helper->Scan()); ASSERT_FALSE(changelog_splits.empty()); - auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], - fields[1], fields[2]}); - ASSERT_OK_AND_ASSIGN( - bool success, - helper->ReadAndCheckResult( - expected_type, changelog_splits, - R"([[1, "Alice", 10, 200], [2, "Alice", 20, 300]])")); + auto expected_type = arrow::struct_( + {arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1], fields[2]}); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult( + expected_type, changelog_splits, + R"([[1, "Alice", 10, 200], [2, "Alice", 20, 300]])")); ASSERT_TRUE(success); } @@ -1178,10 +1166,9 @@ TEST_P(WriteAndReadInteTest, TestChangelogWithSchemaEvolution) { } ASSERT_OK_AND_ASSIGN(auto write_helper, CreateLookupTestHelper(arrow::schema(fields_v0), options)); - ASSERT_OK_AND_ASSIGN(auto initial_batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields_v0), - R"([["Alice", 10]])", - /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto initial_batch, TestHelper::MakeRecordBatch( + arrow::struct_(fields_v0), R"([["Alice", 10]])", + /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(write_helper->WriteAndCommit(std::move(initial_batch), /*commit_identifier=*/0, /*expected_commit_messages=*/std::nullopt)); std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); @@ -1194,10 +1181,9 @@ TEST_P(WriteAndReadInteTest, TestChangelogWithSchemaEvolution) { write_helper.reset(); ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); - ASSERT_OK_AND_ASSIGN(auto batch_v0, - TestHelper::MakeRecordBatch(arrow::struct_(fields_v0), - R"([["Alice", 11]])", - /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN( + auto batch_v0, TestHelper::MakeRecordBatch(arrow::struct_(fields_v0), R"([["Alice", 11]])", + /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/2, /*expected_commit_messages=*/std::nullopt)); ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); @@ -1205,15 +1191,14 @@ TEST_P(WriteAndReadInteTest, TestChangelogWithSchemaEvolution) { arrow::FieldVector fields_v1 = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32()), arrow::field("extra", arrow::utf8())}; - ASSERT_OK(WriteNextSchema({DataField(0, fields_v1[0]), DataField(1, fields_v1[1]), - DataField(2, fields_v1[2])}, - /*highest_field_id=*/2, options)); + ASSERT_OK(WriteNextSchema( + {DataField(0, fields_v1[0]), DataField(1, fields_v1[1]), DataField(2, fields_v1[2])}, + /*highest_field_id=*/2, options)); write_helper.reset(); ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); - ASSERT_OK_AND_ASSIGN(auto batch_v1, - TestHelper::MakeRecordBatch(arrow::struct_(fields_v1), - R"([["Alice", 12, "v1"]])", - /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto batch_v1, TestHelper::MakeRecordBatch( + arrow::struct_(fields_v1), R"([["Alice", 12, "v1"]])", + /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/4, /*expected_commit_messages=*/std::nullopt)); ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/5)); @@ -1221,15 +1206,14 @@ TEST_P(WriteAndReadInteTest, TestChangelogWithSchemaEvolution) { arrow::FieldVector fields_v2 = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32()), arrow::field("renamed_extra", arrow::utf8())}; - ASSERT_OK(WriteNextSchema({DataField(0, fields_v2[0]), DataField(1, fields_v2[1]), - DataField(2, fields_v2[2])}, - /*highest_field_id=*/2, options)); + ASSERT_OK(WriteNextSchema( + {DataField(0, fields_v2[0]), DataField(1, fields_v2[1]), DataField(2, fields_v2[2])}, + /*highest_field_id=*/2, options)); write_helper.reset(); ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); - ASSERT_OK_AND_ASSIGN(auto batch_v2, - TestHelper::MakeRecordBatch(arrow::struct_(fields_v2), - R"([["Alice", 13, "v2"]])", - /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto batch_v2, TestHelper::MakeRecordBatch( + arrow::struct_(fields_v2), R"([["Alice", 13, "v2"]])", + /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v2), /*commit_identifier=*/6, /*expected_commit_messages=*/std::nullopt)); ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/7)); @@ -1237,21 +1221,20 @@ TEST_P(WriteAndReadInteTest, TestChangelogWithSchemaEvolution) { arrow::FieldVector fields_v3 = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int64()), arrow::field("renamed_extra", arrow::utf8())}; - ASSERT_OK(WriteNextSchema({DataField(0, fields_v3[0]), DataField(1, fields_v3[1]), - DataField(2, fields_v3[2])}, - /*highest_field_id=*/2, options)); + ASSERT_OK(WriteNextSchema( + {DataField(0, fields_v3[0]), DataField(1, fields_v3[1]), DataField(2, fields_v3[2])}, + /*highest_field_id=*/2, options)); write_helper.reset(); ASSERT_OK_AND_ASSIGN(write_helper, CreateLookupTestHelper(table_path, options)); - ASSERT_OK_AND_ASSIGN(auto batch_v3, - TestHelper::MakeRecordBatch(arrow::struct_(fields_v3), - R"([["Alice", 14, "v3"]])", - /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto batch_v3, TestHelper::MakeRecordBatch( + arrow::struct_(fields_v3), R"([["Alice", 14, "v3"]])", + /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(write_helper->WriteAndCommit(std::move(batch_v3), /*commit_identifier=*/8, /*expected_commit_messages=*/std::nullopt)); ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/9)); - auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields_v3[0], - fields_v3[1], fields_v3[2]}); + auto expected_type = arrow::struct_( + {arrow::field("_VALUE_KIND", arrow::int8()), fields_v3[0], fields_v3[1], fields_v3[2]}); std::vector expected_data = { R"([[1, "Alice", 10, null], [2, "Alice", 11, null]])", R"([[1, "Alice", 11, null], [2, "Alice", 12, "v1"]])", @@ -1268,10 +1251,9 @@ TEST_P(WriteAndReadInteTest, TestChangelogWithSchemaEvolution) { ASSERT_EQ(file->schema_id, schema_id); } } - ASSERT_OK_AND_ASSIGN( - bool success, - scan_helper->ReadAndCheckResult(expected_type, changelog_splits, - expected_data[static_cast(schema_id)])); + ASSERT_OK_AND_ASSIGN(bool success, scan_helper->ReadAndCheckResult( + expected_type, changelog_splits, + expected_data[static_cast(schema_id)])); ASSERT_TRUE(success); } } @@ -1281,8 +1263,7 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogWithExternalPath) { if (file_system == "jindo") { return; } - std::unique_ptr external_dir = - UniqueTestDirectory::Create(file_system); + std::unique_ptr external_dir = UniqueTestDirectory::Create(file_system); ASSERT_TRUE(external_dir); arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32())}; @@ -1326,20 +1307,18 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogWithExternalPath) { ASSERT_TRUE(file->external_path.has_value()); ASSERT_TRUE(StringUtils::StartsWith(PathUtil::GetName(file->file_name), "changelog-")); ASSERT_OK_AND_ASSIGN( - bool exists, - external_dir->GetFileSystem()->Exists(file->external_path.value())); + bool exists, external_dir->GetFileSystem()->Exists(file->external_path.value())); ASSERT_TRUE(exists); found_external_changelog = true; } } ASSERT_TRUE(found_external_changelog); - auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], - fields[1]}); - ASSERT_OK_AND_ASSIGN( - bool success, - helper->ReadAndCheckResult(expected_type, changelog_splits, - R"([[1, "Alice", 10], [2, "Alice", 20]])")); + auto expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields[0], fields[1]}); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(expected_type, changelog_splits, + R"([[1, "Alice", 10], [2, "Alice", 20]])")); ASSERT_TRUE(success); } From d2e54be76ce68a1930ae5eb11fa5ed74e44e63c4 Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Fri, 28 Aug 2026 16:28:55 +0800 Subject: [PATCH 3/9] fix comment by xinyu --- include/paimon/defs.h | 2 +- src/paimon/CMakeLists.txt | 1 + src/paimon/common/utils/fields_comparator.cpp | 24 +-- src/paimon/common/utils/fields_comparator.h | 5 +- .../async_key_value_producer_and_consumer.cpp | 143 ++++++++-------- .../async_key_value_producer_and_consumer.h | 71 ++++---- .../io/async_key_value_projection_reader.h | 6 +- .../key_value_data_file_writer_factories.cpp | 51 ++++++ .../io/key_value_data_file_writer_factories.h | 56 ++++++ .../compact/changelog_merge_tree_rewriter.cpp | 20 +-- .../core/mergetree/compact/changelog_result.h | 23 +-- .../first_row_merge_function_wrapper.h | 1 + .../compact/internal_row_equalizer.h | 38 +++-- .../compact/internal_row_equalizer_test.cpp | 58 +++---- .../lookup_changelog_merge_function_wrapper.h | 15 +- ..._changelog_merge_function_wrapper_test.cpp | 4 +- .../lookup_merge_tree_compact_rewriter.cpp | 4 +- .../lookup_merge_tree_compact_rewriter.h | 7 +- ...ookup_merge_tree_compact_rewriter_test.cpp | 6 +- .../merge_tree_compact_manager_factory.cpp | 160 ++++++------------ .../merge_tree_compact_manager_factory.h | 10 ++ ...erge_tree_compact_manager_factory_test.cpp | 3 +- .../compact/merge_tree_compact_rewriter.cpp | 41 ++--- .../core/mergetree/external_sort_buffer.cpp | 5 +- .../core/mergetree/merge_tree_writer.cpp | 54 +++--- .../operation/key_value_file_store_scan.cpp | 6 +- .../core/postpone/postpone_bucket_writer.cpp | 23 +-- .../snapshot/changelog_follow_up_scanner.h | 23 +-- test/inte/write_and_read_inte_test.cpp | 86 +--------- 29 files changed, 448 insertions(+), 498 deletions(-) create mode 100644 src/paimon/core/io/key_value_data_file_writer_factories.cpp create mode 100644 src/paimon/core/io/key_value_data_file_writer_factories.h diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 450ce036c..fb281ee1b 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -382,7 +382,7 @@ struct PAIMON_EXPORT Options { /// keeps the details of data changes, it can be read directly during stream reads. This can be /// applied to tables with primary keys. Values can be "none", "input", "lookup", /// "full-compaction". Default value is "none". - /// @note C++ Paimon currently supports "none" and "input". + /// @note C++ Paimon currently supports "none", "input", and "lookup". static const char CHANGELOG_PRODUCER[]; /// "changelog-producer.row-deduplicate" - Whether to generate update-before and update-after diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 7f001216f..cbb92859d 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -286,6 +286,7 @@ set(PAIMON_CORE_SRCS core/io/vector_file_batch_reader.cpp core/io/file_index_evaluator.cpp core/io/key_value_data_file_record_reader.cpp + core/io/key_value_data_file_writer_factories.cpp core/io/key_value_data_file_writer_factory.cpp core/io/key_value_data_file_writer.cpp core/io/key_value_in_memory_record_reader.cpp diff --git a/src/paimon/common/utils/fields_comparator.cpp b/src/paimon/common/utils/fields_comparator.cpp index 523491184..f841b15b1 100644 --- a/src/paimon/common/utils/fields_comparator.cpp +++ b/src/paimon/common/utils/fields_comparator.cpp @@ -80,33 +80,33 @@ int32_t FieldsComparator::CompareTo(const InternalRow& lhs, const InternalRow& r return 0; } -Result FieldsComparator::CompareField( +Result FieldsComparator::CompareField( int32_t field_idx, const std::shared_ptr& input_type) { arrow::Type::type type = input_type->id(); switch (type) { case arrow::Type::type::BOOL: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { bool lvalue = lhs.GetBoolean(field_idx); bool rvalue = rhs.GetBoolean(field_idx); return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); }); case arrow::Type::type::INT8: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { int8_t lvalue = lhs.GetByte(field_idx); int8_t rvalue = rhs.GetByte(field_idx); return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); }); case arrow::Type::type::INT16: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { int16_t lvalue = lhs.GetShort(field_idx); int16_t rvalue = rhs.GetShort(field_idx); return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); }); case arrow::Type::type::DATE32: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { int32_t lvalue = lhs.GetDate(field_idx); int32_t rvalue = rhs.GetDate(field_idx); @@ -114,28 +114,28 @@ Result FieldsComparator::CompareField( }); case arrow::Type::type::INT32: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { int32_t lvalue = lhs.GetInt(field_idx); int32_t rvalue = rhs.GetInt(field_idx); return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); }); case arrow::Type::type::INT64: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { int64_t lvalue = lhs.GetLong(field_idx); int64_t rvalue = rhs.GetLong(field_idx); return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); }); case arrow::Type::type::FLOAT: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { float lvalue = lhs.GetFloat(field_idx); float rvalue = rhs.GetFloat(field_idx); return CompareFloatingPoint(lvalue, rvalue); }); case arrow::Type::type::DOUBLE: - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { double lvalue = lhs.GetDouble(field_idx); double rvalue = rhs.GetDouble(field_idx); @@ -143,7 +143,7 @@ Result FieldsComparator::CompareField( }); case arrow::Type::type::STRING: case arrow::Type::type::BINARY: { - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { auto lvalue = lhs.GetStringView(field_idx); auto rvalue = rhs.GetStringView(field_idx); @@ -154,7 +154,7 @@ Result FieldsComparator::CompareField( case arrow::Type::type::TIMESTAMP: { auto timestamp_type = checked_pointer_cast(input_type); int32_t precision = DateTimeUtils::GetPrecisionFromType(timestamp_type); - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx, precision](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { Timestamp lvalue = lhs.GetTimestamp(field_idx, precision); Timestamp rvalue = rhs.GetTimestamp(field_idx, precision); @@ -165,7 +165,7 @@ Result FieldsComparator::CompareField( auto* decimal_type = checked_cast(input_type.get()); auto precision = decimal_type->precision(); auto scale = decimal_type->scale(); - return FieldsComparator::FieldComparatorFunc( + return FieldComparatorFunc( [field_idx, precision, scale](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { Decimal lvalue = lhs.GetDecimal(field_idx, precision, scale); diff --git a/src/paimon/common/utils/fields_comparator.h b/src/paimon/common/utils/fields_comparator.h index 7dbeca15e..0b101ba35 100644 --- a/src/paimon/common/utils/fields_comparator.h +++ b/src/paimon/common/utils/fields_comparator.h @@ -38,6 +38,8 @@ class DataType; namespace paimon { class DataField; +using FieldComparatorFunc = std::function; + /// A `Comparator` that compares the file store key. class FieldsComparator { public: @@ -82,9 +84,6 @@ class FieldsComparator { } private: - using FieldComparatorFunc = - std::function; - FieldsComparator(bool is_ascending_order, const std::vector& sort_fields, std::vector&& comparators) : is_ascending_order_(is_ascending_order), diff --git a/src/paimon/core/io/async_key_value_producer_and_consumer.cpp b/src/paimon/core/io/async_key_value_producer_and_consumer.cpp index 96a0a9d45..b353aefc8 100644 --- a/src/paimon/core/io/async_key_value_producer_and_consumer.cpp +++ b/src/paimon/core/io/async_key_value_producer_and_consumer.cpp @@ -25,6 +25,7 @@ #include "arrow/c/abi.h" #include "arrow/c/helpers.h" +#include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/reader/reader_utils.h" #include "paimon/reader/batch_reader.h" @@ -47,6 +48,10 @@ class AsyncKeyValueQueueBatchSink : public AsyncKeyValueBatchSink { return Status::OK(); } + bool IsCancelled() const override { + return *consume_finished_; + } + private: std::atomic* consume_finished_; tbb::concurrent_bounded_queue* kv_queue_; @@ -54,27 +59,60 @@ class AsyncKeyValueQueueBatchSink : public AsyncKeyValueBatchSink { } // namespace -template -AsyncKeyValueProducerAndConsumer::AsyncKeyValueProducerAndConsumer( - std::unique_ptr&& sort_merge_reader, ConsumerCreator create_consumer, - int32_t batch_size, int32_t consumer_thread_num, const std::shared_ptr& pool) - : batch_size_(NormalizeProjectionBatchSize(batch_size)), - consumer_thread_num_(consumer_thread_num), - pool_(pool), - sort_merge_reader_(std::move(sort_merge_reader)), - create_consumer_(std::move(create_consumer)) { - kv_queue_.set_capacity(consumer_thread_num * 2); - result_queue_.set_capacity(RESULT_BATCH_COUNT); +std::shared_ptr AsyncKeyValueBatchProducer::GetReaderMetrics() const { + return std::make_shared(); +} + +SortMergeReaderBatchProducer::SortMergeReaderBatchProducer( + std::unique_ptr&& sort_merge_reader, int32_t batch_size) + : sort_merge_reader_(std::move(sort_merge_reader)), + batch_size_(NormalizeProjectionBatchSize(batch_size)) {} + +Status SortMergeReaderBatchProducer::Produce(AsyncKeyValueBatchSink* sink) { + std::vector batch; + batch.reserve(batch_size_); + while (!sink->IsCancelled()) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, + sort_merge_reader_->NextBatch()); + if (iterator == nullptr) { + break; + } + while (!sink->IsCancelled()) { + PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext()); + if (!has_next) { + break; + } + batch.push_back(std::move(iterator->Next())); + if (static_cast(batch.size()) >= batch_size_) { + PAIMON_RETURN_NOT_OK(sink->Write(AsyncKeyValueBatchType::DATA, std::move(batch))); + batch = std::vector(); + batch.reserve(batch_size_); + } + } + } + if (!batch.empty() && !sink->IsCancelled()) { + PAIMON_RETURN_NOT_OK(sink->Write(AsyncKeyValueBatchType::DATA, std::move(batch))); + } + return Status::OK(); +} + +std::shared_ptr SortMergeReaderBatchProducer::GetReaderMetrics() const { + return sort_merge_reader_->GetReaderMetrics(); +} + +void SortMergeReaderBatchProducer::Close() { + if (!closed_) { + sort_merge_reader_->Close(); + closed_ = true; + } } template AsyncKeyValueProducerAndConsumer::AsyncKeyValueProducerAndConsumer( - std::unique_ptr&& producer, ConsumerCreator data_consumer_creator, - ConsumerCreator changelog_consumer_creator) - : batch_size_(0), - consumer_thread_num_(1), - create_consumer_(std::move(data_consumer_creator)), - create_changelog_consumer_(std::move(changelog_consumer_creator)), + std::unique_ptr&& producer, ConsumerCreator create_consumer, + int32_t consumer_thread_num) + : consumer_thread_num_(consumer_thread_num), + create_consumer_(std::move(create_consumer)), producer_(std::move(producer)) { kv_queue_.set_capacity(consumer_thread_num_ * 2); result_queue_.set_capacity(RESULT_BATCH_COUNT); @@ -121,16 +159,10 @@ Result> AsyncKeyValueProducerAndConsumer::Next consumers_.reserve(consumer_thread_num_); for (int32_t i = 0; i < consumer_thread_num_; i++) { std::unique_ptr> consumer; - if (create_consumer_) { - PAIMON_ASSIGN_OR_RAISE(consumer, create_consumer_()); - } - std::unique_ptr> changelog_consumer; - if (create_changelog_consumer_) { - PAIMON_ASSIGN_OR_RAISE(changelog_consumer, create_changelog_consumer_()); - } + PAIMON_ASSIGN_OR_RAISE(consumer, create_consumer_()); auto async_consumer = std::make_unique>( - std::move(consumer), std::move(changelog_consumer), consume_finished_, - consumer_finished_count_, kv_queue_, result_queue_); + std::move(consumer), consume_finished_, consumer_finished_count_, kv_queue_, + result_queue_); consumers_.push_back(std::move(async_consumer)); } } @@ -157,40 +189,8 @@ Result> AsyncKeyValueProducerAndConsumer::Next template Status AsyncKeyValueProducerAndConsumer::ProduceLoop() { - if (producer_) { - AsyncKeyValueQueueBatchSink sink(&consume_finished_, &kv_queue_); - PAIMON_RETURN_NOT_OK(producer_->Produce(&sink)); - kv_queue_.push(AsyncKeyValueRowsBatch()); - return Status::OK(); - } - - std::vector batch; - batch.reserve(batch_size_); - while (!consume_finished_) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, - sort_merge_reader_->NextBatch()); - if (iterator == nullptr) { - break; - } - while (!consume_finished_) { - PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext()); - if (!has_next) { - break; - } - batch.push_back(std::move(iterator->Next())); - if (static_cast(batch.size()) >= batch_size_) { - kv_queue_.push( - AsyncKeyValueRowsBatch{AsyncKeyValueBatchType::DATA, std::move(batch)}); - batch = std::vector(); - batch.reserve(batch_size_); - } - } - } - // Push remaining rows - if (!batch.empty()) { - kv_queue_.push(AsyncKeyValueRowsBatch{AsyncKeyValueBatchType::DATA, std::move(batch)}); - } - // Push empty batch as EOF signal + AsyncKeyValueQueueBatchSink sink(&consume_finished_, &kv_queue_); + PAIMON_RETURN_NOT_OK(producer_->Produce(&sink)); kv_queue_.push(AsyncKeyValueRowsBatch()); return Status::OK(); } @@ -235,13 +235,11 @@ template class AsyncKeyValueProducerAndConsumer; template AsyncKeyValueConsumer::AsyncKeyValueConsumer( - std::unique_ptr>&& key_value_consumer, - std::unique_ptr>&& changelog_consumer, - std::atomic& consume_finished, std::atomic& consumer_finished_count, + std::unique_ptr>&& consumer, std::atomic& consume_finished, + std::atomic& consumer_finished_count, tbb::concurrent_bounded_queue& kv_queue, tbb::concurrent_bounded_queue>& result_queue) - : key_value_consumer_(std::move(key_value_consumer)), - changelog_consumer_(std::move(changelog_consumer)), + : consumer_(std::move(consumer)), consume_finished_(consume_finished), consumer_finished_count_(consumer_finished_count), kv_queue_(kv_queue), @@ -274,13 +272,7 @@ Status AsyncKeyValueConsumer::ConsumeLoop() { kv_queue_.push(std::move(rows_batch)); break; } - RowToArrowArrayConverter* consumer = - rows_batch.type == AsyncKeyValueBatchType::CHANGELOG ? changelog_consumer_.get() - : key_value_consumer_.get(); - if (consumer == nullptr) { - return Status::Invalid("Missing consumer for asynchronous key value batch"); - } - PAIMON_ASSIGN_OR_RAISE(R result, consumer->NextBatch(rows_batch.rows)); + PAIMON_ASSIGN_OR_RAISE(R result, consumer_->NextBatch(rows_batch.rows)); result_queue_.push(AsyncKeyValueResultBatch{rows_batch.type, std::move(result)}); } consumer_finished_count_++; @@ -292,11 +284,8 @@ void AsyncKeyValueConsumer::CleanUp() { if (consumer_future_.valid()) { [[maybe_unused]] Status status = consumer_future_.get(); } - if (key_value_consumer_) { - key_value_consumer_->CleanUp(); - } - if (changelog_consumer_) { - changelog_consumer_->CleanUp(); + if (consumer_) { + consumer_->CleanUp(); } } diff --git a/src/paimon/core/io/async_key_value_producer_and_consumer.h b/src/paimon/core/io/async_key_value_producer_and_consumer.h index bf6249c99..086ff1112 100644 --- a/src/paimon/core/io/async_key_value_producer_and_consumer.h +++ b/src/paimon/core/io/async_key_value_producer_and_consumer.h @@ -38,7 +38,6 @@ namespace paimon { template class AsyncKeyValueConsumer; -class MemoryPool; class Metrics; enum class AsyncKeyValueBatchType { @@ -56,6 +55,8 @@ class AsyncKeyValueBatchSink { virtual ~AsyncKeyValueBatchSink() = default; virtual Status Write(AsyncKeyValueBatchType type, std::vector&& rows) = 0; + + virtual bool IsCancelled() const = 0; }; class AsyncKeyValueBatchProducer { @@ -63,6 +64,36 @@ class AsyncKeyValueBatchProducer { virtual ~AsyncKeyValueBatchProducer() = default; virtual Status Produce(AsyncKeyValueBatchSink* sink) = 0; + + virtual std::shared_ptr GetReaderMetrics() const; + + virtual void Close() {} + + protected: + // Limits the number of rows sent to one Arrow projection call. + static int32_t NormalizeProjectionBatchSize(int32_t batch_size) { + return std::min(batch_size, MAX_PROJECTION_BATCH_SIZE); + } + + private: + static constexpr int32_t MAX_PROJECTION_BATCH_SIZE = 100000; +}; + +class SortMergeReaderBatchProducer : public AsyncKeyValueBatchProducer { + public: + SortMergeReaderBatchProducer(std::unique_ptr&& sort_merge_reader, + int32_t batch_size); + + Status Produce(AsyncKeyValueBatchSink* sink) override; + + std::shared_ptr GetReaderMetrics() const override; + + void Close() override; + + private: + std::unique_ptr sort_merge_reader_; + int32_t batch_size_; + bool closed_ = false; }; template @@ -71,29 +102,16 @@ struct AsyncKeyValueResultBatch { R result; }; -// Asynchronous iterate SortMergeReader (producer) and row-to-array conversion (consumer), support -// multi-threaded conversion, R can be BatchReader::ReadBatch, KeyValueBatch +// Asynchronous iterates AsyncKeyValueBatchProducer(producer) and row-to-array conversion +// (consumer), support multi-threaded conversion, R can be BatchReader::ReadBatch or KeyValueBatch. template class AsyncKeyValueProducerAndConsumer { public: using ConsumerCreator = std::function>>()>; - // Limits the number of rows sent to one Arrow projection call. - static int32_t NormalizeProjectionBatchSize(int32_t batch_size) { - return std::min(batch_size, MAX_PROJECTION_BATCH_SIZE); - } - - AsyncKeyValueProducerAndConsumer(std::unique_ptr&& sort_merge_reader, - ConsumerCreator create_consumer, int32_t batch_size, - int32_t consumer_thread_num, - const std::shared_ptr& pool); - - /// Creates a single-conversion-thread pipeline whose producer emits tagged data and changelog - /// row batches. Converted batches retain the producer order and are returned with their tag. AsyncKeyValueProducerAndConsumer(std::unique_ptr&& producer, - ConsumerCreator data_consumer_creator, - ConsumerCreator changelog_consumer_creator); + ConsumerCreator create_consumer, int32_t consumer_thread_num); ~AsyncKeyValueProducerAndConsumer() { CleanUp(); @@ -104,22 +122,17 @@ class AsyncKeyValueProducerAndConsumer { Result> NextBatchWithType(); std::shared_ptr GetReaderMetrics() const { - return sort_merge_reader_ ? sort_merge_reader_->GetReaderMetrics() : nullptr; + return producer_->GetReaderMetrics(); } void Close() { CleanUp(); - if (sort_merge_reader_) { - sort_merge_reader_->Close(); - } + producer_->Close(); } private: static constexpr int32_t RESULT_BATCH_COUNT = 3; - // in case write batch size is too large and overflow arrow array - static constexpr int32_t MAX_PROJECTION_BATCH_SIZE = 100000; - void CleanUpQueue(); Status ProduceLoop(); void CleanUp(); @@ -127,12 +140,8 @@ class AsyncKeyValueProducerAndConsumer { Status CheckStatusAndCleanUp(); private: - int32_t batch_size_; int32_t consumer_thread_num_; - std::shared_ptr pool_; - std::unique_ptr sort_merge_reader_; ConsumerCreator create_consumer_; - ConsumerCreator create_changelog_consumer_; std::unique_ptr producer_; // produce: merge sort KeyValue and push result KeyValue to kv_queue_, consume: project KeyValue @@ -149,8 +158,7 @@ class AsyncKeyValueProducerAndConsumer { template class AsyncKeyValueConsumer { public: - AsyncKeyValueConsumer(std::unique_ptr>&& key_value_consumer, - std::unique_ptr>&& changelog_consumer, + AsyncKeyValueConsumer(std::unique_ptr>&& consumer, std::atomic& consume_finished, std::atomic& consumer_finished_count, tbb::concurrent_bounded_queue& kv_queue, @@ -167,8 +175,7 @@ class AsyncKeyValueConsumer { Status ConsumeLoop(); private: - std::unique_ptr> key_value_consumer_; - std::unique_ptr> changelog_consumer_; + std::unique_ptr> consumer_; std::shared_future consumer_future_; std::atomic& consume_finished_; std::atomic& consumer_finished_count_; diff --git a/src/paimon/core/io/async_key_value_projection_reader.h b/src/paimon/core/io/async_key_value_projection_reader.h index a272a0edf..79c7aa4de 100644 --- a/src/paimon/core/io/async_key_value_projection_reader.h +++ b/src/paimon/core/io/async_key_value_projection_reader.h @@ -38,10 +38,12 @@ class AsyncKeyValueProjectionReader : public BatchReader { -> Result>> { return KeyValueProjectionConsumer::Create(target_schema, target_to_src_mapping, pool); }; + std::unique_ptr producer = + std::make_unique(std::move(sort_merge_reader), + batch_size); producer_and_consumer_ = std::make_unique>( - std::move(sort_merge_reader), create_consumer, batch_size, projection_thread_num, - pool); + std::move(producer), create_consumer, projection_thread_num); } Result NextBatch() override { diff --git a/src/paimon/core/io/key_value_data_file_writer_factories.cpp b/src/paimon/core/io/key_value_data_file_writer_factories.cpp new file mode 100644 index 000000000..5aa5a8caa --- /dev/null +++ b/src/paimon/core/io/key_value_data_file_writer_factories.cpp @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/key_value_data_file_writer_factories.h" + +#include "paimon/common/data/shredding/shredding_write_plan_factories.h" +#include "paimon/core/io/key_value_data_file_writer_factory.h" +#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h" + +namespace paimon { + +Result> +KeyValueDataFileWriterFactories::Create(const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + int32_t level, FileSource file_source, + const std::vector& primary_keys, + const std::shared_ptr& path_factory, + bool create_stats_extractor, bool is_changelog, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan_factory, + ShreddingWritePlanFactories::SelectActive(options, write_schema, pool)); + std::shared_ptr writer_factory; + if (plan_factory != nullptr) { + writer_factory = std::make_shared( + options, schema_id, write_schema, level, file_source, primary_keys, path_factory, + create_stats_extractor, plan_factory, is_changelog, pool); + } else { + writer_factory = std::make_shared( + options, schema_id, write_schema, level, file_source, primary_keys, path_factory, + create_stats_extractor, is_changelog, pool); + } + return writer_factory; +} + +} // namespace paimon diff --git a/src/paimon/core/io/key_value_data_file_writer_factories.h b/src/paimon/core/io/key_value_data_file_writer_factories.h new file mode 100644 index 000000000..335bb94d4 --- /dev/null +++ b/src/paimon/core/io/key_value_data_file_writer_factories.h @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/single_file_writer_factory.h" +#include "paimon/core/key_value.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class CoreOptions; +class DataFilePathFactory; +class MemoryPool; + +/// Creates the appropriate key-value data file writer factory for the configured write schema. +class KeyValueDataFileWriterFactories { + public: + using WriterFactory = SingleFileWriterFactory>; + + static Result> Create( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, int32_t level, FileSource file_source, + const std::vector& primary_keys, + const std::shared_ptr& path_factory, bool create_stats_extractor, + bool is_changelog, const std::shared_ptr& pool); +}; + +} // namespace paimon diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp index abf797dde..db59d7615 100644 --- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp @@ -46,7 +46,7 @@ class ChangelogCompactionBatchProducer : public AsyncKeyValueBatchProducer { bool produce_data, bool produce_changelog) : sections_(sections), reader_holders_(reader_holders), - write_batch_size_(write_batch_size), + write_batch_size_(NormalizeProjectionBatchSize(write_batch_size)), reader_factory_(std::move(reader_factory)), merge_function_wrapper_factory_(std::move(merge_function_wrapper_factory)), key_comparator_(std::move(key_comparator)), @@ -255,7 +255,6 @@ Result ChangelogMergeTreeRewriter::RewriteOrProduceChangelog( bool produce_data = compact_file_writer != nullptr; bool produce_changelog = changelog_file_writer != nullptr; - using AsyncPipeline = AsyncKeyValueProducerAndConsumer; SortMergeReaderFactory reader_factory = [this](const std::vector& section) { return CreateRawSortMergeReaderForSection(section); }; @@ -269,18 +268,13 @@ Result ChangelogMergeTreeRewriter::RewriteOrProduceChangelog( CancellationChecker cancellation_checker = [this]() { return IsCancelled(); }; std::unique_ptr producer = std::make_unique( - sections, reader_holders, - AsyncPipeline::NormalizeProjectionBatchSize(options_.GetWriteBatchSize()), - std::move(reader_factory), std::move(merge_function_wrapper_factory), - std::move(key_comparator), std::move(cancellation_checker), drop_delete, produce_data, - produce_changelog); + sections, reader_holders, options_.GetWriteBatchSize(), std::move(reader_factory), + std::move(merge_function_wrapper_factory), std::move(key_comparator), + std::move(cancellation_checker), drop_delete, produce_data, produce_changelog); - AsyncPipeline::ConsumerCreator data_consumer = - produce_data ? create_consumer : AsyncPipeline::ConsumerCreator(); - AsyncPipeline::ConsumerCreator changelog_consumer = - produce_changelog ? create_consumer : AsyncPipeline::ConsumerCreator(); - auto producer_and_consumer = std::make_unique( - std::move(producer), std::move(data_consumer), std::move(changelog_consumer)); + auto producer_and_consumer = + std::make_unique>( + std::move(producer), std::move(create_consumer), /*consumer_thread_num=*/1); while (true) { if (IsCancelled()) { return Status::Cancelled("Compaction is cancelled"); diff --git a/src/paimon/core/mergetree/compact/changelog_result.h b/src/paimon/core/mergetree/compact/changelog_result.h index 5b9b78008..778183ed7 100644 --- a/src/paimon/core/mergetree/compact/changelog_result.h +++ b/src/paimon/core/mergetree/compact/changelog_result.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once diff --git a/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h b/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h index 0baed203e..e0d7e6d7f 100644 --- a/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h +++ b/src/paimon/core/mergetree/compact/first_row_merge_function_wrapper.h @@ -71,6 +71,7 @@ class FirstRowMergeFunctionWrapper : public MergeFunctionWrapper(std::move(changelog_result)); } + // TODO(lisizhuo.lsz): avoid serialize & deserialize here. if (value_serializer_) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, value_serializer_->SerializeToBytes(*result->value)); diff --git a/src/paimon/core/mergetree/compact/internal_row_equalizer.h b/src/paimon/core/mergetree/compact/internal_row_equalizer.h index d5336999f..64e4b4de4 100644 --- a/src/paimon/core/mergetree/compact/internal_row_equalizer.h +++ b/src/paimon/core/mergetree/compact/internal_row_equalizer.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once @@ -38,12 +41,13 @@ namespace paimon { /// Creates equality functions for internal rows, including nested values. +/// Java's RecordEqualiser also compares the RowKind embedded in InternalRow. This comparator +/// currently compares field values only. This does not affect the current lookup changelog results +/// because changelog kinds are tracked separately by KeyValue::value_kind. class InternalRowEqualizer { public: - using Equalizer = std::function; - - static Result Create(const std::shared_ptr& schema, - const std::vector& ignore_fields) { + static Result Create(const std::shared_ptr& schema, + const std::vector& ignore_fields) { std::set ignored(ignore_fields.begin(), ignore_fields.end()); std::vector> equalizers; for (int32_t i = 0; i < schema->num_fields(); ++i) { @@ -54,14 +58,14 @@ class InternalRowEqualizer { CreateValueEqualizer(schema->field(i)->type())); equalizers.emplace_back(i, std::move(equalizer)); } - return Equalizer( + return FieldComparatorFunc( [equalizers = std::move(equalizers)](const InternalRow& lhs, const InternalRow& rhs) { for (const auto& [field_idx, equalizer] : equalizers) { if (!EqualAt(lhs, field_idx, rhs, field_idx, equalizer)) { - return false; + return 1; } } - return true; + return 0; }); } diff --git a/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp b/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp index d655513ec..e8f9e57a0 100644 --- a/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp +++ b/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/core/mergetree/compact/internal_row_equalizer.h" @@ -109,34 +112,33 @@ TEST(InternalRowEqualizerTest, PrimitiveTypesAndIgnoreFields) { std::unique_ptr left = GenericRow::Of(left_values); std::unique_ptr right = GenericRow::Of(right_values); - ASSERT_OK_AND_ASSIGN(InternalRowEqualizer::Equalizer equalizer, + ASSERT_OK_AND_ASSIGN(FieldComparatorFunc equalizer, InternalRowEqualizer::Create(schema, {"ignored"})); - ASSERT_TRUE(equalizer(*left, *right)); + ASSERT_EQ(0, equalizer(*left, *right)); right->SetField(/*pos=*/3, int32_t{30}); - ASSERT_FALSE(equalizer(*left, *right)); + ASSERT_NE(0, equalizer(*left, *right)); } TEST(InternalRowEqualizerTest, NullAndFloatingPointSemantics) { std::shared_ptr schema = arrow::schema( {arrow::field("value", arrow::float64()), arrow::field("nullable", arrow::int32())}); - ASSERT_OK_AND_ASSIGN(InternalRowEqualizer::Equalizer equalizer, - InternalRowEqualizer::Create(schema, {})); + ASSERT_OK_AND_ASSIGN(FieldComparatorFunc equalizer, InternalRowEqualizer::Create(schema, {})); std::unique_ptr negative_zero = GenericRow::Of({static_cast(-0.0), NullType()}); std::unique_ptr positive_zero = GenericRow::Of({static_cast(0.0), NullType()}); - ASSERT_FALSE(equalizer(*negative_zero, *positive_zero)); + ASSERT_NE(0, equalizer(*negative_zero, *positive_zero)); double nan1 = std::numeric_limits::quiet_NaN(); double nan2 = -std::numeric_limits::quiet_NaN(); std::unique_ptr left_nan = GenericRow::Of({nan1, NullType()}); std::unique_ptr right_nan = GenericRow::Of({nan2, NullType()}); - ASSERT_TRUE(equalizer(*left_nan, *right_nan)); + ASSERT_EQ(0, equalizer(*left_nan, *right_nan)); right_nan->SetField(/*pos=*/1, int32_t{1}); - ASSERT_FALSE(equalizer(*left_nan, *right_nan)); + ASSERT_NE(0, equalizer(*left_nan, *right_nan)); } TEST(InternalRowEqualizerTest, NestedTypes) { @@ -146,8 +148,7 @@ TEST(InternalRowEqualizerTest, NestedTypes) { arrow::field("map", arrow::map(arrow::int32(), arrow::int32())), arrow::field("row", arrow::struct_({arrow::field("value", arrow::int32()), arrow::field("floating", arrow::float64())}))}); - ASSERT_OK_AND_ASSIGN(InternalRowEqualizer::Equalizer equalizer, - InternalRowEqualizer::Create(schema, {})); + ASSERT_OK_AND_ASSIGN(FieldComparatorFunc equalizer, InternalRowEqualizer::Create(schema, {})); std::unique_ptr left = GenericRow::Of({CreateNullableIntArray({1, 0, 3}, /*null_pos=*/1, pool.get()), @@ -157,39 +158,38 @@ TEST(InternalRowEqualizerTest, NestedTypes) { GenericRow::Of({CreateNullableIntArray({1, 9, 3}, /*null_pos=*/1, pool.get()), CreateIntMap({1, 2}, {10, 20}, pool.get()), CreateNestedRow(100, -std::numeric_limits::quiet_NaN())}); - ASSERT_TRUE(equalizer(*left, *right)); + ASSERT_EQ(0, equalizer(*left, *right)); right->SetField(/*pos=*/0, CreateIntArray({1, 2}, pool.get())); - ASSERT_FALSE(equalizer(*left, *right)); + ASSERT_NE(0, equalizer(*left, *right)); right->SetField(/*pos=*/0, CreateNullableIntArray({1, 0, 3}, /*null_pos=*/1, pool.get())); right->SetField(/*pos=*/0, CreateNullableIntArray({1, 0, 4}, /*null_pos=*/1, pool.get())); - ASSERT_FALSE(equalizer(*left, *right)); + ASSERT_NE(0, equalizer(*left, *right)); right->SetField(/*pos=*/0, CreateNullableIntArray({1, 0, 3}, /*null_pos=*/1, pool.get())); right->SetField(/*pos=*/1, CreateIntMap({1, 3}, {10, 20}, pool.get())); - ASSERT_FALSE(equalizer(*left, *right)); + ASSERT_NE(0, equalizer(*left, *right)); right->SetField(/*pos=*/1, CreateIntMap({1, 2}, {10, 20}, pool.get())); right->SetField(/*pos=*/2, CreateNestedRow(101, 1.0)); - ASSERT_FALSE(equalizer(*left, *right)); + ASSERT_NE(0, equalizer(*left, *right)); } TEST(InternalRowEqualizerTest, MapEqualityDoesNotDependOnEntryOrder) { std::shared_ptr pool = GetDefaultPool(); std::shared_ptr schema = arrow::schema({arrow::field("map", arrow::map(arrow::int32(), arrow::int32()))}); - ASSERT_OK_AND_ASSIGN(InternalRowEqualizer::Equalizer equalizer, - InternalRowEqualizer::Create(schema, {})); + ASSERT_OK_AND_ASSIGN(FieldComparatorFunc equalizer, InternalRowEqualizer::Create(schema, {})); std::unique_ptr left = GenericRow::Of({CreateIntMap({1, 2, 3}, {10, 20, 30}, pool.get())}); std::unique_ptr reordered = GenericRow::Of({CreateIntMap({3, 1, 2}, {30, 10, 20}, pool.get())}); - ASSERT_TRUE(equalizer(*left, *reordered)); + ASSERT_EQ(0, equalizer(*left, *reordered)); reordered->SetField(/*pos=*/0, CreateIntMap({3, 1, 2}, {30, 10, 21}, pool.get())); - ASSERT_FALSE(equalizer(*left, *reordered)); + ASSERT_NE(0, equalizer(*left, *reordered)); } TEST(InternalRowEqualizerTest, UnsupportedType) { diff --git a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h index 51fa4aee9..3fc0fd4a8 100644 --- a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h +++ b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h @@ -52,8 +52,6 @@ namespace paimon { template class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper { public: - using ValueEqualizer = std::function; - static Result> Create( std::unique_ptr&& merge_function, std::function>(const std::shared_ptr&)> lookup, @@ -61,7 +59,7 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper& deletion_vectors_maintainer, const std::shared_ptr& comparator, std::unique_ptr&& value_serializer, - ValueEqualizer value_equalizer) { + FieldComparatorFunc value_equalizer) { if (lookup_strategy.deletion_vector && !deletion_vectors_maintainer) { return Status::Invalid("deletionVectorsMaintainer should not be null, there is a bug."); } @@ -149,7 +147,8 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper& deletion_vectors_maintainer, const std::shared_ptr& user_defined_seq_comparator, - std::unique_ptr&& value_serializer, ValueEqualizer value_equalizer) + std::unique_ptr&& value_serializer, + FieldComparatorFunc value_equalizer) : merge_function_(std::move(merge_function)), lookup_(std::move(lookup)), lookup_strategy_(lookup_strategy), @@ -160,11 +159,13 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper CloneKeyValue(const KeyValue& from, const RowKind* value_kind) { + // TODO(lisizhuo.lsz): avoid serialize & deserialize here. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, value_serializer_->SerializeToBytes(*from.value)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr value, value_serializer_->Deserialize(bytes)); - return KeyValue(value_kind, from.sequence_number, from.level, from.key, std::move(value)); + return KeyValue(value_kind, from.sequence_number, KeyValue::UNKNOWN_LEVEL, from.key, + std::move(value)); } Status SetChangelog(std::optional&& before, const std::optional& after, @@ -184,7 +185,7 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrappervalue, *after->value)) { + if (!value_equalizer_ || value_equalizer_(*before->value, *after->value) != 0) { before->value_kind = RowKind::UpdateBefore(); PAIMON_ASSIGN_OR_RAISE(KeyValue update_after, CloneKeyValue(after.value(), RowKind::UpdateAfter())); @@ -218,7 +219,7 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper deletion_vectors_maintainer_; std::function comparator_; std::unique_ptr value_serializer_; - ValueEqualizer value_equalizer_; + FieldComparatorFunc value_equalizer_; }; } // namespace paimon diff --git a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp index 15e2223b0..c4d255d9a 100644 --- a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp +++ b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper_test.cpp @@ -140,8 +140,10 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestWithLookup) { ASSERT_EQ(result->result->sequence_number, 3); ASSERT_EQ(result->changelogs.size(), 2); ASSERT_EQ(result->changelogs[0].value_kind, RowKind::UpdateBefore()); + ASSERT_EQ(result->changelogs[0].level, KeyValue::UNKNOWN_LEVEL); ASSERT_EQ(result->changelogs[0].value->GetInt(0), 1001); ASSERT_EQ(result->changelogs[1].value_kind, RowKind::UpdateAfter()); + ASSERT_EQ(result->changelogs[1].level, KeyValue::UNKNOWN_LEVEL); ASSERT_EQ(result->changelogs[1].value->GetInt(0), 300); } @@ -161,7 +163,7 @@ TEST(LookupChangelogMergeFunctionWrapperTest, TestRowDeduplicate) { LookupStrategy lookup_strategy(/*is_first_row=*/false, /*produce_changelog=*/true, /*deletion_vector=*/false, /*force_lookup=*/true); auto value_equalizer = [](const InternalRow& lhs, const InternalRow& rhs) { - return lhs.GetInt(0) == rhs.GetInt(0); + return lhs.GetInt(0) == rhs.GetInt(0) ? 0 : 1; }; ASSERT_OK_AND_ASSIGN(auto wrapper, LookupChangelogMergeFunctionWrapper::Create( diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp index 30b7b9ac4..608b36303 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp @@ -121,7 +121,7 @@ template std::shared_ptr> LookupMergeTreeCompactRewriter::CreateFirstRowMergeFunctionWrapper( std::unique_ptr&& merge_func, int32_t output_level, - LookupLevels* lookup_levels, std::unique_ptr&& value_serializer) { + std::unique_ptr&& value_serializer, LookupLevels* lookup_levels) { auto contains = [output_level, lookup_levels](const std::shared_ptr& key) -> Result { PAIMON_ASSIGN_OR_RAISE(std::optional contain, @@ -140,7 +140,7 @@ LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& user_defined_seq_comparator, LookupLevels* lookup_levels, std::unique_ptr&& value_serializer, - typename LookupChangelogMergeFunctionWrapper::ValueEqualizer value_equalizer) { + FieldComparatorFunc value_equalizer) { auto lookup = [output_level, lookup_levels]( const std::shared_ptr& key) -> Result> { return lookup_levels->Lookup(key, output_level + 1); diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h index 5870ee805..39745f5fd 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h @@ -55,8 +55,9 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter { static std::shared_ptr> CreateFirstRowMergeFunctionWrapper(std::unique_ptr&& merge_func, - int32_t output_level, LookupLevels* lookup_levels, - std::unique_ptr&& value_serializer); + int32_t output_level, + std::unique_ptr&& value_serializer, + LookupLevels* lookup_levels); static Result>> CreateLookupMergeFunctionWrapper( @@ -65,7 +66,7 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter { const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& user_defined_seq_comparator, LookupLevels* lookup_levels, std::unique_ptr&& value_serializer, - typename LookupChangelogMergeFunctionWrapper::ValueEqualizer value_equalizer); + FieldComparatorFunc value_equalizer); private: LookupMergeTreeCompactRewriter( diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp index 3ebeefbfc..fae403af9 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp @@ -168,7 +168,7 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam> merge_function_wrapper = LookupMergeTreeCompactRewriter::CreateFirstRowMergeFunctionWrapper( std::make_unique(/*ignore_delete=*/true), output_level, - lookup_levels_ptr, std::move(value_serializer)); + std::move(value_serializer), lookup_levels_ptr); return merge_function_wrapper; }; auto cancellation_controller = std::make_shared(); @@ -197,7 +197,7 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParamTrimmedPrimaryKeys().value(), options, pool_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr value_serializer, RowCompactedSerializer::Create(arrow_schema_, pool_)); - InternalRowEqualizer::Equalizer value_equalizer; + FieldComparatorFunc value_equalizer; if (options.ChangelogRowDeduplicate()) { PAIMON_ASSIGN_OR_RAISE( value_equalizer, @@ -292,7 +292,7 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParamTrimmedPrimaryKeys().value(), options, pool_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr value_serializer, RowCompactedSerializer::Create(arrow_schema_, pool_)); - InternalRowEqualizer::Equalizer value_equalizer; + FieldComparatorFunc value_equalizer; if (options.ChangelogRowDeduplicate()) { PAIMON_ASSIGN_OR_RAISE( value_equalizer, diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp index 4e6fd3854..66c806a19 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp @@ -51,11 +51,11 @@ namespace paimon { namespace { -Result CreateChangelogValueEqualizer( +Result CreateChangelogValueEqualizer( const std::shared_ptr& schema, const CoreOptions& options, bool produce_changelog) { if (!produce_changelog || !options.ChangelogRowDeduplicate()) { - return InternalRowEqualizer::Equalizer(); + return FieldComparatorFunc(); } return InternalRowEqualizer::Create(schema, options.GetChangelogRowDeduplicateIgnoreFields()); } @@ -222,7 +222,7 @@ Result> MergeTreeCompactManagerFactory::CreateL std::shared_ptr> merge_function_wrapper = LookupMergeTreeCompactRewriter::CreateFirstRowMergeFunctionWrapper( std::make_unique(ignore_delete), output_level, - lookup_levels_ptr, std::move(value_serializer)); + std::move(value_serializer), lookup_levels_ptr); return merge_function_wrapper; }; PAIMON_ASSIGN_OR_RAISE( @@ -248,68 +248,24 @@ Result> MergeTreeCompactManagerFactory::CreateL remote_lookup_file_manager); } +template Result> -MergeTreeCompactManagerFactory::CreateLookupRewriterWithDeletionVector( +MergeTreeCompactManagerFactory::CreateLookupRewriterInternal( const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, const std::shared_ptr& dv_maintainer, int32_t max_level, const LookupStrategy& lookup_strategy, bool should_produce_changelog, + const std::shared_ptr::Factory>& processor_factory, const std::shared_ptr& path_factory_cache, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager) const { - auto merge_engine = options_.GetMergeEngine(); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, table_schema_->TrimmedPrimaryKeys()); - if (lookup_strategy.produce_changelog || merge_engine != MergeEngine::DEDUPLICATE || - !options_.GetSequenceField().empty()) { - auto processor_factory = std::make_shared(schema_); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr> lookup_levels, - CreateLookupLevelsInternal( - options_, schema_manager_, io_manager_, cache_manager_, file_store_path_factory_, - table_schema_, partition, bucket, levels, processor_factory, dv_maintainer, - lookup_file_cache_, remote_lookup_file_manager, pool_)); - auto merge_function_wrapper_factory = - [data_schema = schema_, options = options_, trimmed_primary_keys, - lookup_levels_ptr = lookup_levels.get(), lookup_strategy, should_produce_changelog, - dv_maintainer_ptr = dv_maintainer, - user_defined_seq_comparator = user_defined_seq_comparator_, - pool = pool_](int32_t output_level) - -> Result>> { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_func, - PrimaryKeyTableUtils::CreateMergeFunction( - data_schema, trimmed_primary_keys, options, pool)); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr value_serializer, - CreateChangelogValueSerializer(data_schema, should_produce_changelog, pool)); - PAIMON_ASSIGN_OR_RAISE( - InternalRowEqualizer::Equalizer value_equalizer, - CreateChangelogValueEqualizer(data_schema, options, should_produce_changelog)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, - LookupMergeTreeCompactRewriter:: - CreateLookupMergeFunctionWrapper( - std::make_unique(std::move(merge_func)), output_level, - dv_maintainer_ptr, lookup_strategy, should_produce_changelog, - user_defined_seq_comparator, lookup_levels_ptr, std::move(value_serializer), - std::move(value_equalizer))); - return merge_function_wrapper; - }; - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr> rewriter, - LookupMergeTreeCompactRewriter::Create( - max_level, std::move(lookup_levels), dv_maintainer, - std::move(merge_function_wrapper_factory), bucket, partition, table_schema_, - path_factory_cache, options_, should_produce_changelog, cancellation_controller, - remote_lookup_file_manager, pool_)); - return std::shared_ptr(std::move(rewriter)); - } - auto processor_factory = std::make_shared(); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr> lookup_levels, - CreateLookupLevelsInternal( - options_, schema_manager_, io_manager_, cache_manager_, file_store_path_factory_, - table_schema_, partition, bucket, levels, processor_factory, dv_maintainer, - lookup_file_cache_, remote_lookup_file_manager, pool_)); + std::unique_ptr> lookup_levels, + CreateLookupLevelsInternal(options_, schema_manager_, io_manager_, cache_manager_, + file_store_path_factory_, table_schema_, partition, bucket, + levels, processor_factory, dv_maintainer, lookup_file_cache_, + remote_lookup_file_manager, pool_)); auto merge_function_wrapper_factory = [data_schema = schema_, options = options_, trimmed_primary_keys, lookup_levels_ptr = lookup_levels.get(), lookup_strategy, should_produce_changelog, @@ -324,20 +280,17 @@ MergeTreeCompactManagerFactory::CreateLookupRewriterWithDeletionVector( std::unique_ptr value_serializer, CreateChangelogValueSerializer(data_schema, should_produce_changelog, pool)); PAIMON_ASSIGN_OR_RAISE( - InternalRowEqualizer::Equalizer value_equalizer, + FieldComparatorFunc value_equalizer, CreateChangelogValueEqualizer(data_schema, options, should_produce_changelog)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, - LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( - std::make_unique(std::move(merge_func)), output_level, - dv_maintainer_ptr, lookup_strategy, should_produce_changelog, - user_defined_seq_comparator, lookup_levels_ptr, std::move(value_serializer), - std::move(value_equalizer))); - return merge_function_wrapper; + return LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( + std::make_unique(std::move(merge_func)), output_level, + dv_maintainer_ptr, lookup_strategy, should_produce_changelog, + user_defined_seq_comparator, lookup_levels_ptr, std::move(value_serializer), + std::move(value_equalizer)); }; PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr> rewriter, - LookupMergeTreeCompactRewriter::Create( + std::unique_ptr> rewriter, + LookupMergeTreeCompactRewriter::Create( max_level, std::move(lookup_levels), dv_maintainer, std::move(merge_function_wrapper_factory), bucket, partition, table_schema_, path_factory_cache, options_, should_produce_changelog, cancellation_controller, @@ -346,56 +299,45 @@ MergeTreeCompactManagerFactory::CreateLookupRewriterWithDeletionVector( } Result> -MergeTreeCompactManagerFactory::CreateLookupRewriterWithoutDeletionVector( +MergeTreeCompactManagerFactory::CreateLookupRewriterWithDeletionVector( const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, const std::shared_ptr& dv_maintainer, int32_t max_level, const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& path_factory_cache, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager) const { - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - table_schema_->TrimmedPrimaryKeys()); - auto processor_factory = std::make_shared(schema_); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr> lookup_levels, - CreateLookupLevelsInternal( - options_, schema_manager_, io_manager_, cache_manager_, file_store_path_factory_, - table_schema_, partition, bucket, levels, processor_factory, dv_maintainer, - lookup_file_cache_, remote_lookup_file_manager, pool_)); - auto merge_function_wrapper_factory = - [data_schema = schema_, options = options_, trimmed_primary_keys, - lookup_levels_ptr = lookup_levels.get(), lookup_strategy, should_produce_changelog, - dv_maintainer_ptr = dv_maintainer, - user_defined_seq_comparator = user_defined_seq_comparator_, - pool = pool_](int32_t output_level) - -> Result>> { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_func, - PrimaryKeyTableUtils::CreateMergeFunction( - data_schema, trimmed_primary_keys, options, pool)); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr value_serializer, - CreateChangelogValueSerializer(data_schema, should_produce_changelog, pool)); - PAIMON_ASSIGN_OR_RAISE( - InternalRowEqualizer::Equalizer value_equalizer, - CreateChangelogValueEqualizer(data_schema, options, should_produce_changelog)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr> merge_function_wrapper, - LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( - std::make_unique(std::move(merge_func)), output_level, - dv_maintainer_ptr, lookup_strategy, should_produce_changelog, - user_defined_seq_comparator, lookup_levels_ptr, std::move(value_serializer), - std::move(value_equalizer))); - return merge_function_wrapper; - }; + auto merge_engine = options_.GetMergeEngine(); + if (lookup_strategy.produce_changelog || merge_engine != MergeEngine::DEDUPLICATE || + !options_.GetSequenceField().empty()) { + std::shared_ptr::Factory> processor_factory = + std::make_shared(schema_); + return CreateLookupRewriterInternal( + partition, bucket, levels, dv_maintainer, max_level, lookup_strategy, + should_produce_changelog, processor_factory, path_factory_cache, + cancellation_controller, remote_lookup_file_manager); + } + std::shared_ptr::Factory> processor_factory = + std::make_shared(); + return CreateLookupRewriterInternal( + partition, bucket, levels, dv_maintainer, max_level, lookup_strategy, + should_produce_changelog, processor_factory, path_factory_cache, cancellation_controller, + remote_lookup_file_manager); +} - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr> rewriter, - LookupMergeTreeCompactRewriter::Create( - max_level, std::move(lookup_levels), dv_maintainer, - std::move(merge_function_wrapper_factory), bucket, partition, table_schema_, - path_factory_cache, options_, should_produce_changelog, cancellation_controller, - remote_lookup_file_manager, pool_)); - return std::shared_ptr(std::move(rewriter)); +Result> +MergeTreeCompactManagerFactory::CreateLookupRewriterWithoutDeletionVector( + const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, + const std::shared_ptr& dv_maintainer, int32_t max_level, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, + const std::shared_ptr& path_factory_cache, + const std::shared_ptr& cancellation_controller, + const std::shared_ptr& remote_lookup_file_manager) const { + std::shared_ptr::Factory> processor_factory = + std::make_shared(schema_); + return CreateLookupRewriterInternal( + partition, bucket, levels, dv_maintainer, max_level, lookup_strategy, + should_produce_changelog, processor_factory, path_factory_cache, cancellation_controller, + remote_lookup_file_manager); } Result> diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h index e18acb631..2eaa6ffd4 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.h @@ -117,6 +117,16 @@ class MergeTreeCompactManagerFactory { const std::shared_ptr& path_factory_cache, const std::shared_ptr& cancellation_controller) const; + template + Result> CreateLookupRewriterInternal( + const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, + const std::shared_ptr& dv_maintainer, int32_t max_level, + const LookupStrategy& lookup_strategy, bool should_produce_changelog, + const std::shared_ptr::Factory>& processor_factory, + const std::shared_ptr& path_factory_cache, + const std::shared_ptr& cancellation_controller, + const std::shared_ptr& remote_lookup_file_manager) const; + Result> CreateLookupRewriterWithDeletionVector( const BinaryRow& partition, int32_t bucket, const std::shared_ptr& levels, const std::shared_ptr& dv_maintainer, int32_t max_level, diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp index f38caa8ed..c0cff7cdf 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp @@ -383,8 +383,7 @@ TEST_F(MergeTreeCompactManagerFactoryWriteTest, ASSERT_OK(CreateSingleStringFileStoreWrite({{"bucket", "1"}, {Options::DELETION_VECTORS_ENABLED, "true"}, {Options::CHANGELOG_PRODUCER, "lookup"}}, - /*with_io_manager=*/true) - .status()); + /*with_io_manager=*/true)); } } // namespace paimon::test diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp index 2fabd9dc1..5e7578e4d 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp @@ -23,14 +23,12 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" -#include "paimon/common/data/shredding/shredding_write_plan_factories.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/scope_guard.h" -#include "paimon/core/io/key_value_data_file_writer_factory.h" +#include "paimon/core/io/key_value_data_file_writer_factories.h" #include "paimon/core/io/key_value_meta_projection_consumer.h" #include "paimon/core/io/key_value_record_reader.h" #include "paimon/core/io/row_to_arrow_array_converter.h" -#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/operation/internal_read_context.h" #include "paimon/format/file_format.h" @@ -138,21 +136,12 @@ MergeTreeCompactRewriter::CreateRollingRowWriter(int32_t level) { auto format = options_.GetWriteFileFormat(level); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, CreateDataFilePathFactory(format->Identifier())); - std::shared_ptr>> factory; PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr plan_factory, - ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_)); - if (plan_factory != nullptr) { - factory = std::make_shared( + std::shared_ptr factory, + KeyValueDataFileWriterFactories::Create( options_, schema_id_, write_schema_, level, FileSource::Compact(), trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true, - plan_factory, /*is_changelog=*/false, pool_); - } else { - factory = std::make_shared( - options_, schema_id_, write_schema_, level, FileSource::Compact(), - trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true, - /*is_changelog=*/false, pool_); - } + /*is_changelog=*/false, pool_)); return std::make_unique( options_.GetTargetFileSize(/*has_primary_key=*/true), /*target_file_row_num=*/std::numeric_limits::max(), factory); @@ -166,21 +155,12 @@ MergeTreeCompactRewriter::CreateRollingChangelogWriter(int32_t level) { } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, CreateDataFilePathFactory(format->Identifier())); - std::shared_ptr>> factory; PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr plan_factory, - ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_)); - if (plan_factory != nullptr) { - factory = std::make_shared( - options_, schema_id_, write_schema_, level, FileSource::Append(), trimmed_primary_keys_, - data_file_path_factory, /*create_stats_extractor=*/true, plan_factory, - /*is_changelog=*/true, pool_); - } else { - factory = std::make_shared( + std::shared_ptr factory, + KeyValueDataFileWriterFactories::Create( options_, schema_id_, write_schema_, level, FileSource::Append(), trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true, - /*is_changelog=*/true, pool_); - } + /*is_changelog=*/true, pool_)); return std::make_unique( options_.GetTargetFileSize(/*has_primary_key=*/true), /*target_file_row_num=*/std::numeric_limits::max(), factory); @@ -269,11 +249,12 @@ Status MergeTreeCompactRewriter::MergeReadAndWrite( return Status::OK(); } - // consumer batch size is WriteBatchSize + std::unique_ptr producer = + std::make_unique(std::move(sort_merge_reader), + options_.GetWriteBatchSize()); auto async_key_value_producer_consumer = std::make_shared>( - std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(), - /*projection_thread_num=*/1, pool_); + std::move(producer), create_consumer, /*projection_thread_num=*/1); reader_holders.push_back(async_key_value_producer_consumer); // read KeyValueBatch from SortMergeReader and write to RollingWriter while (true) { diff --git a/src/paimon/core/mergetree/external_sort_buffer.cpp b/src/paimon/core/mergetree/external_sort_buffer.cpp index 9bfeec899..718a2de62 100644 --- a/src/paimon/core/mergetree/external_sort_buffer.cpp +++ b/src/paimon/core/mergetree/external_sort_buffer.cpp @@ -222,10 +222,11 @@ Result ExternalSortBuffer::SpillToDisk( -> Result>> { return KeyValueMetaProjectionConsumer::Create(target_schema, pool); }; + std::unique_ptr producer = + std::make_unique(std::move(sorted_reader), write_batch_size); auto async_key_value_producer_consumer = std::make_unique>( - std::move(sorted_reader), create_consumer, write_batch_size, - /*projection_thread_num=*/1, pool_); + std::move(producer), create_consumer, /*projection_thread_num=*/1); auto close_guard = ScopeGuard([&]() { async_key_value_producer_consumer->Close(); }); while (true) { diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp index 84bba8d47..bb7a01054 100644 --- a/src/paimon/core/mergetree/merge_tree_writer.cpp +++ b/src/paimon/core/mergetree/merge_tree_writer.cpp @@ -28,7 +28,6 @@ #include "arrow/api.h" #include "arrow/c/abi.h" #include "arrow/c/helpers.h" -#include "paimon/common/data/shredding/shredding_write_plan_factories.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -38,11 +37,10 @@ #include "paimon/core/io/compact_increment.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/io/data_increment.h" -#include "paimon/core/io/key_value_data_file_writer_factory.h" +#include "paimon/core/io/key_value_data_file_writer_factories.h" #include "paimon/core/io/key_value_meta_projection_consumer.h" #include "paimon/core/io/key_value_record_reader.h" #include "paimon/core/io/row_to_arrow_array_converter.h" -#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/mergetree/compact/sort_merge_reader_with_loser_tree.h" #include "paimon/core/mergetree/compact/sort_merge_reader_with_min_heap.h" @@ -292,10 +290,12 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction, auto raw_sort_merge_reader = std::make_unique( std::move(raw_readers), key_comparator_, user_defined_seq_comparator_, /*merge_function_wrapper=*/nullptr); + std::unique_ptr producer = + std::make_unique(std::move(raw_sort_merge_reader), + options_.GetWriteBatchSize()); async_changelog_producer_consumer = std::make_unique>( - std::move(raw_sort_merge_reader), create_consumer, options_.GetWriteBatchSize(), - /*projection_thread_num=*/1, pool_); + std::move(producer), create_consumer, /*projection_thread_num=*/1); PAIMON_ASSIGN_OR_RAISE(changelog_writer, CreateRollingChangelogWriter()); while (true) { PAIMON_ASSIGN_OR_RAISE(KeyValueBatch key_value_batch, @@ -315,10 +315,12 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction, auto sort_merge_reader = std::make_unique( std::move(readers), key_comparator_, user_defined_seq_comparator_, merge_function_wrapper_); + std::unique_ptr producer = + std::make_unique(std::move(sort_merge_reader), + options_.GetWriteBatchSize()); auto async_key_value_producer_consumer = std::make_unique>( - std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(), - /*projection_thread_num=*/1, pool_); + std::move(producer), create_consumer, /*projection_thread_num=*/1); std::unique_ptr>> rolling_writer; PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter()); @@ -382,21 +384,12 @@ Result MergeTreeWriter::DrainIncrement() { Result>>> MergeTreeWriter::CreateRollingRowWriter() const { - std::shared_ptr>> factory; PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr plan_factory, - ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_)); - if (plan_factory != nullptr) { - factory = std::make_shared( - options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), - trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, plan_factory, - /*is_changelog=*/false, pool_); - } else { - factory = std::make_shared( - options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), - trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, - /*is_changelog=*/false, pool_); - } + std::shared_ptr factory, + KeyValueDataFileWriterFactories::Create(options_, schema_id_, write_schema_, /*level=*/0, + FileSource::Append(), trimmed_primary_keys_, + path_factory_, /*create_stats_extractor=*/true, + /*is_changelog=*/false, pool_)); return std::make_unique>>( options_.GetTargetFileSize(/*has_primary_key=*/true), options_.GetTargetFileRowNum(), factory); @@ -404,21 +397,12 @@ MergeTreeWriter::CreateRollingRowWriter() const { Result>>> MergeTreeWriter::CreateRollingChangelogWriter() const { - std::shared_ptr>> factory; PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr plan_factory, - ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_)); - if (plan_factory != nullptr) { - factory = std::make_shared( - options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), - trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, plan_factory, - /*is_changelog=*/true, pool_); - } else { - factory = std::make_shared( - options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), - trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, - /*is_changelog=*/true, pool_); - } + std::shared_ptr factory, + KeyValueDataFileWriterFactories::Create(options_, schema_id_, write_schema_, /*level=*/0, + FileSource::Append(), trimmed_primary_keys_, + path_factory_, /*create_stats_extractor=*/true, + /*is_changelog=*/true, pool_)); return std::make_unique>>( options_.GetTargetFileSize(/*has_primary_key=*/true), /*target_file_row_num=*/std::numeric_limits::max(), factory); diff --git a/src/paimon/core/operation/key_value_file_store_scan.cpp b/src/paimon/core/operation/key_value_file_store_scan.cpp index 014de79c7..54af98a0c 100644 --- a/src/paimon/core/operation/key_value_file_store_scan.cpp +++ b/src/paimon/core/operation/key_value_file_store_scan.cpp @@ -176,8 +176,12 @@ Result KeyValueFileStoreScan::IsValueFilterEnabled() const { case ScanMode::ALL: return value_filter_force_enabled_; case ScanMode::DELTA: - case ScanMode::CHANGELOG: return false; + case ScanMode::CHANGELOG: { + ChangelogProducer producer = core_options_.GetChangelogProducer(); + return producer == ChangelogProducer::LOOKUP || + producer == ChangelogProducer::FULL_COMPACTION; + } default: return Status::NotImplemented("unknown scan mode"); } diff --git a/src/paimon/core/postpone/postpone_bucket_writer.cpp b/src/paimon/core/postpone/postpone_bucket_writer.cpp index 73f08f263..555c8d1b3 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer.cpp +++ b/src/paimon/core/postpone/postpone_bucket_writer.cpp @@ -31,7 +31,6 @@ #include "arrow/c/helpers.h" #include "arrow/scalar.h" #include "fmt/format.h" -#include "paimon/common/data/shredding/shredding_write_plan_factories.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" @@ -43,8 +42,7 @@ #include "paimon/core/io/compact_increment.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/io/data_increment.h" -#include "paimon/core/io/key_value_data_file_writer_factory.h" -#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h" +#include "paimon/core/io/key_value_data_file_writer_factories.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/utils/commit_increment.h" #include "paimon/format/file_format.h" @@ -258,21 +256,12 @@ PostponeBucketWriter::PrepareMinMaxKey( Result>>> PostponeBucketWriter::CreateRollingRowWriter() const { - std::shared_ptr>> factory; PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr plan_factory, - ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_)); - if (plan_factory != nullptr) { - factory = std::make_shared( - options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), - trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, plan_factory, - /*is_changelog=*/false, pool_); - } else { - factory = std::make_shared( - options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), - trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, - /*is_changelog=*/false, pool_); - } + std::shared_ptr factory, + KeyValueDataFileWriterFactories::Create(options_, schema_id_, write_schema_, /*level=*/0, + FileSource::Append(), trimmed_primary_keys_, + path_factory_, /*create_stats_extractor=*/false, + /*is_changelog=*/false, pool_)); return std::make_unique>>( options_.GetTargetFileSize(/*has_primary_key=*/true), options_.GetTargetFileRowNum(), factory); diff --git a/src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h b/src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h index fc7c1be6a..ee2b73dfd 100644 --- a/src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h +++ b/src/paimon/core/table/source/snapshot/changelog_follow_up_scanner.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index b5c14bc13..56944d9c3 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -165,7 +165,7 @@ class WriteAndReadInteTest file_store_write->PrepareCommit(/*wait_compaction=*/true, commit_identifier)); PAIMON_RETURN_NOT_OK(file_store_write->Close()); if (compact_messages.empty()) { - return Status::Invalid("expected lookup compaction commit messages"); + return Status::Invalid("expected compaction commit messages"); } CommitContextBuilder commit_context_builder(table_path, "commit_user"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, @@ -745,33 +745,8 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogStreamRead) { /*expected_commit_messages=*/std::nullopt)); std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); - auto compact_and_commit = [this, &options, &table_path](int64_t commit_identifier) -> Status { - WriteContextBuilder write_context_builder(table_path, "commit_user"); - write_context_builder.WithTempDirectory(LookupTempDirectory()); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr write_context, - write_context_builder.SetOptions(options).WithStreamingMode(true).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); - PAIMON_RETURN_NOT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, - /*full_compaction=*/true)); - PAIMON_ASSIGN_OR_RAISE( - std::vector> compact_messages, - file_store_write->PrepareCommit(/*wait_compaction=*/true, commit_identifier)); - PAIMON_RETURN_NOT_OK(file_store_write->Close()); - if (compact_messages.empty()) { - return Status::Invalid("expected lookup compaction commit messages"); - } - CommitContextBuilder commit_context_builder(table_path, "commit_user"); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); - return file_store_commit->Commit(compact_messages, commit_identifier); - }; - // Move the initial value to a high level so the next compaction must look it up. - ASSERT_OK(compact_and_commit(/*commit_identifier=*/1)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); helper.reset(); ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); @@ -784,7 +759,7 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogStreamRead) { /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2, /*expected_commit_messages=*/std::nullopt)); - ASSERT_OK(compact_and_commit(/*commit_identifier=*/3)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); ASSERT_OK_AND_ASSIGN(std::vector> changelog_splits, helper->Scan()); ASSERT_FALSE(changelog_splits.empty()); @@ -819,26 +794,7 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogInitialFullScanExcludesLevelZero /*expected_commit_messages=*/std::nullopt)); std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); - WriteContextBuilder write_context_builder(table_path, "commit_user"); - write_context_builder.WithTempDirectory(LookupTempDirectory()); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr write_context, - write_context_builder.SetOptions(options).WithStreamingMode(true).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); - ASSERT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, - /*full_compaction=*/true)); - ASSERT_OK_AND_ASSIGN( - std::vector> compact_messages, - file_store_write->PrepareCommit(/*wait_compaction=*/true, /*commit_identifier=*/1)); - ASSERT_OK(file_store_write->Close()); - ASSERT_FALSE(compact_messages.empty()); - CommitContextBuilder commit_context_builder(table_path, "commit_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); - ASSERT_OK(file_store_commit->Commit(compact_messages, /*commit_identifier=*/1)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/1)); helper.reset(); ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); @@ -869,21 +825,7 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogInitialFullScanExcludesLevelZero helper->ReadAndCheckResult(expected_type, initial_full_splits, R"([[0, "Alice", 10]])")); ASSERT_TRUE(success); - WriteContextBuilder changelog_write_context_builder(table_path, "commit_user"); - changelog_write_context_builder.WithTempDirectory(LookupTempDirectory()); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr changelog_write_context, - changelog_write_context_builder.SetOptions(options).WithStreamingMode(true).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr changelog_file_store_write, - FileStoreWrite::Create(std::move(changelog_write_context))); - ASSERT_OK(changelog_file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, - /*full_compaction=*/true)); - ASSERT_OK_AND_ASSIGN(std::vector> changelog_compact_messages, - changelog_file_store_write->PrepareCommit(/*wait_compaction=*/true, - /*commit_identifier=*/3)); - ASSERT_OK(changelog_file_store_write->Close()); - ASSERT_FALSE(changelog_compact_messages.empty()); - ASSERT_OK(file_store_commit->Commit(changelog_compact_messages, /*commit_identifier=*/3)); + ASSERT_OK(CompactAndCommit(table_path, options, /*commit_identifier=*/3)); ASSERT_OK_AND_ASSIGN(std::vector> changelog_splits, helper->Scan()); ASSERT_FALSE(changelog_splits.empty()); @@ -3637,23 +3579,7 @@ TEST_P(WriteAndReadInteTest, TestAppendMapStorageLayoutSharedShreddingToDefaultC ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1_file3), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - WriteContextBuilder write_context_builder(table_path, "commit_user"); - ASSERT_OK_AND_ASSIGN( - auto write_context, - write_context_builder.SetOptions(options_v1).WithStreamingMode(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); - ASSERT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, - /*full_compaction=*/true)); - ASSERT_OK_AND_ASSIGN(auto compact_messages, file_store_write->PrepareCommit( - /*wait_compaction=*/true, commit_identifier)); - ASSERT_FALSE(compact_messages.empty()); - - CommitContextBuilder commit_context_builder(table_path, "commit_user"); - ASSERT_OK_AND_ASSIGN(auto commit_context, - commit_context_builder.SetOptions(options_v1).Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); - ASSERT_OK(file_store_commit->Commit(compact_messages, commit_identifier)); + ASSERT_OK(CompactAndCommit(table_path, options_v1, commit_identifier)); arrow::FieldVector expected_fields = fields; expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); From 14640bad8e5c651d388b4fe68191c6a983df81d1 Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Fri, 28 Aug 2026 17:02:38 +0800 Subject: [PATCH 4/9] fix ci --- .../compact/changelog_merge_tree_rewriter.cpp | 176 +++++++++--------- .../merge_tree_compact_manager_factory.cpp | 8 +- 2 files changed, 87 insertions(+), 97 deletions(-) diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp index db59d7615..eb5a30eb8 100644 --- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp @@ -28,29 +28,21 @@ namespace paimon { namespace { -using SortMergeReaderFactory = - std::function>(const std::vector& section)>; -using BoundChangelogMergeFunctionWrapperFactory = - std::function>>()>; using KeyComparator = std::function; using CancellationChecker = std::function; class ChangelogCompactionBatchProducer : public AsyncKeyValueBatchProducer { public: ChangelogCompactionBatchProducer( - const std::vector>& sections, - std::vector>& reader_holders, int32_t write_batch_size, - SortMergeReaderFactory reader_factory, - BoundChangelogMergeFunctionWrapperFactory merge_function_wrapper_factory, - KeyComparator key_comparator, CancellationChecker cancellation_checker, bool drop_delete, - bool produce_data, bool produce_changelog) - : sections_(sections), - reader_holders_(reader_holders), + std::unique_ptr&& sort_merge_reader, int32_t write_batch_size, + std::shared_ptr>&& merge_function_wrapper, + const KeyComparator& key_comparator, const CancellationChecker& cancellation_checker, + bool drop_delete, bool produce_data, bool produce_changelog) + : sort_merge_reader_(std::move(sort_merge_reader)), write_batch_size_(NormalizeProjectionBatchSize(write_batch_size)), - reader_factory_(std::move(reader_factory)), - merge_function_wrapper_factory_(std::move(merge_function_wrapper_factory)), - key_comparator_(std::move(key_comparator)), - cancellation_checker_(std::move(cancellation_checker)), + merge_function_wrapper_(std::move(merge_function_wrapper)), + key_comparator_(key_comparator), + cancellation_checker_(cancellation_checker), drop_delete_(drop_delete), produce_data_(produce_data), produce_changelog_(produce_changelog) {} @@ -91,74 +83,71 @@ class ChangelogCompactionBatchProducer : public AsyncKeyValueBatchProducer { return Status::OK(); }; - for (const auto& section : sections_) { + std::shared_ptr current_key; + auto finish_group = [&]() -> Status { + if (!current_key) { + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE(std::optional result, + merge_function_wrapper_->GetResult()); + current_key.reset(); + if (result) { + PAIMON_RETURN_NOT_OK(emit_result(std::move(result).value())); + } + return Status::OK(); + }; + + while (true) { if (cancellation_checker_()) { return Status::Cancelled("Compaction is cancelled"); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr sort_merge_reader, - reader_factory_(section)); - reader_holders_.emplace_back(std::move(sort_merge_reader)); - SortMergeReader* reader = reader_holders_.back().get(); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr> wrapper, - merge_function_wrapper_factory_()); - - std::shared_ptr current_key; - auto finish_group = [&]() -> Status { - if (!current_key) { - return Status::OK(); - } - PAIMON_ASSIGN_OR_RAISE(std::optional result, wrapper->GetResult()); - current_key.reset(); - if (result) { - PAIMON_RETURN_NOT_OK(emit_result(std::move(result).value())); - } - return Status::OK(); - }; - + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, + sort_merge_reader_->NextBatch()); + if (!iterator) { + break; + } while (true) { - if (cancellation_checker_()) { - return Status::Cancelled("Compaction is cancelled"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, - reader->NextBatch()); - if (!iterator) { + PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext()); + if (!has_next) { break; } - while (true) { - PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext()); - if (!has_next) { - break; - } - KeyValue key_value = iterator->Next(); - if (current_key && key_comparator_(*current_key, *key_value.key) != 0) { - PAIMON_RETURN_NOT_OK(finish_group()); - } - if (!current_key) { - wrapper->Reset(); - current_key = key_value.key; - } - PAIMON_RETURN_NOT_OK(wrapper->Add(std::move(key_value))); + KeyValue key_value = iterator->Next(); + if (current_key && key_comparator_(*current_key, *key_value.key) != 0) { + PAIMON_RETURN_NOT_OK(finish_group()); + } + if (!current_key) { + merge_function_wrapper_->Reset(); + current_key = key_value.key; } + PAIMON_RETURN_NOT_OK(merge_function_wrapper_->Add(std::move(key_value))); } - PAIMON_RETURN_NOT_OK(finish_group()); - reader->Close(); } + PAIMON_RETURN_NOT_OK(finish_group()); + sort_merge_reader_->Close(); + closed_ = true; PAIMON_RETURN_NOT_OK(flush(AsyncKeyValueBatchType::DATA, &compact_buffer)); PAIMON_RETURN_NOT_OK(flush(AsyncKeyValueBatchType::CHANGELOG, &changelog_buffer)); return Status::OK(); } + void Close() override { + if (closed_) { + return; + } + sort_merge_reader_->Close(); + closed_ = true; + } + private: - const std::vector>& sections_; - std::vector>& reader_holders_; + std::unique_ptr sort_merge_reader_; int32_t write_batch_size_; - SortMergeReaderFactory reader_factory_; - BoundChangelogMergeFunctionWrapperFactory merge_function_wrapper_factory_; + std::shared_ptr> merge_function_wrapper_; KeyComparator key_comparator_; CancellationChecker cancellation_checker_; bool drop_delete_; bool produce_data_; bool produce_changelog_; + bool closed_ = false; }; } // namespace @@ -237,7 +226,7 @@ Result ChangelogMergeTreeRewriter::RewriteOrProduceChangelog( PAIMON_ASSIGN_OR_RAISE(changelog_file_writer, CreateRollingChangelogWriter(output_level)); } - std::vector> reader_holders; + std::vector> reader_holders; ScopeGuard write_guard([&]() -> void { if (compact_file_writer) { compact_file_writer->Abort(); @@ -255,42 +244,43 @@ Result ChangelogMergeTreeRewriter::RewriteOrProduceChangelog( bool produce_data = compact_file_writer != nullptr; bool produce_changelog = changelog_file_writer != nullptr; - SortMergeReaderFactory reader_factory = [this](const std::vector& section) { - return CreateRawSortMergeReaderForSection(section); - }; - BoundChangelogMergeFunctionWrapperFactory merge_function_wrapper_factory = - [factory = changelog_merge_function_wrapper_factory_, output_level]() { - return factory(output_level); - }; KeyComparator key_comparator = [this](const InternalRow& lhs, const InternalRow& rhs) { return merge_file_split_read_->GetKeyComparator()->CompareTo(lhs, rhs); }; CancellationChecker cancellation_checker = [this]() { return IsCancelled(); }; - std::unique_ptr producer = - std::make_unique( - sections, reader_holders, options_.GetWriteBatchSize(), std::move(reader_factory), - std::move(merge_function_wrapper_factory), std::move(key_comparator), - std::move(cancellation_checker), drop_delete, produce_data, produce_changelog); - auto producer_and_consumer = - std::make_unique>( - std::move(producer), std::move(create_consumer), /*consumer_thread_num=*/1); - while (true) { - if (IsCancelled()) { - return Status::Cancelled("Compaction is cancelled"); - } - PAIMON_ASSIGN_OR_RAISE(AsyncKeyValueResultBatch output, - producer_and_consumer->NextBatchWithType()); - if (output.result.batch == nullptr) { - break; - } - if (output.type == AsyncKeyValueBatchType::DATA) { - PAIMON_RETURN_NOT_OK(compact_file_writer->Write(std::move(output.result))); - } else { - PAIMON_RETURN_NOT_OK(changelog_file_writer->Write(std::move(output.result))); + for (const auto& section : sections) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr sort_merge_reader, + CreateRawSortMergeReaderForSection(section)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr> merge_function_wrapper, + changelog_merge_function_wrapper_factory_(output_level)); + std::unique_ptr producer = + std::make_unique( + std::move(sort_merge_reader), options_.GetWriteBatchSize(), + std::move(merge_function_wrapper), key_comparator, cancellation_checker, + drop_delete, produce_data, produce_changelog); + auto producer_and_consumer = + std::make_shared>( + std::move(producer), create_consumer, /*consumer_thread_num=*/1); + reader_holders.emplace_back(producer_and_consumer); + + while (true) { + if (IsCancelled()) { + return Status::Cancelled("Compaction is cancelled"); + } + PAIMON_ASSIGN_OR_RAISE(AsyncKeyValueResultBatch output, + producer_and_consumer->NextBatchWithType()); + if (output.result.batch == nullptr) { + break; + } + if (output.type == AsyncKeyValueBatchType::DATA) { + PAIMON_RETURN_NOT_OK(compact_file_writer->Write(std::move(output.result))); + } else { + PAIMON_RETURN_NOT_OK(changelog_file_writer->Write(std::move(output.result))); + } } } - producer_and_consumer->Close(); if (compact_file_writer) { PAIMON_RETURN_NOT_OK(compact_file_writer->Close()); } diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp index 66c806a19..06ca83b4a 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp @@ -211,10 +211,10 @@ Result> MergeTreeCompactManagerFactory::CreateL options_, schema_manager_, io_manager_, cache_manager_, file_store_path_factory_, table_schema_, partition, bucket, levels, processor_factory, dv_maintainer, lookup_file_cache_, remote_lookup_file_manager, pool_)); - auto merge_function_wrapper_factory = - [lookup_levels_ptr = lookup_levels.get(), data_schema = schema_, lookup_strategy, - should_produce_changelog, ignore_delete = options_.IgnoreDelete(), - pool = pool_](int32_t output_level) + auto merge_function_wrapper_factory = [lookup_levels_ptr = lookup_levels.get(), + data_schema = schema_, should_produce_changelog, + ignore_delete = options_.IgnoreDelete(), + pool = pool_](int32_t output_level) -> Result>> { PAIMON_ASSIGN_OR_RAISE( std::unique_ptr value_serializer, From fdbaca246369d78b41295559d60b6c6e686bcd25 Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Fri, 28 Aug 2026 17:14:53 +0800 Subject: [PATCH 5/9] fix ci --- src/paimon/common/utils/fields_comparator.cpp | 2 +- src/paimon/common/utils/fields_comparator.h | 5 +++-- .../compact/changelog_merge_tree_rewriter.cpp | 12 +++++++----- .../core/mergetree/compact/internal_row_equalizer.h | 7 ++++--- .../compact/internal_row_equalizer_test.cpp | 11 +++++++---- .../lookup_changelog_merge_function_wrapper.h | 6 +++--- .../compact/lookup_merge_tree_compact_rewriter.cpp | 2 +- .../compact/lookup_merge_tree_compact_rewriter.h | 2 +- .../lookup_merge_tree_compact_rewriter_test.cpp | 4 ++-- .../compact/merge_tree_compact_manager_factory.cpp | 6 +++--- 10 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/paimon/common/utils/fields_comparator.cpp b/src/paimon/common/utils/fields_comparator.cpp index f841b15b1..2fee9aa15 100644 --- a/src/paimon/common/utils/fields_comparator.cpp +++ b/src/paimon/common/utils/fields_comparator.cpp @@ -80,7 +80,7 @@ int32_t FieldsComparator::CompareTo(const InternalRow& lhs, const InternalRow& r return 0; } -Result FieldsComparator::CompareField( +Result FieldsComparator::CompareField( int32_t field_idx, const std::shared_ptr& input_type) { arrow::Type::type type = input_type->id(); switch (type) { diff --git a/src/paimon/common/utils/fields_comparator.h b/src/paimon/common/utils/fields_comparator.h index 0b101ba35..50e0260d9 100644 --- a/src/paimon/common/utils/fields_comparator.h +++ b/src/paimon/common/utils/fields_comparator.h @@ -38,11 +38,12 @@ class DataType; namespace paimon { class DataField; -using FieldComparatorFunc = std::function; - /// A `Comparator` that compares the file store key. class FieldsComparator { public: + using FieldComparatorFunc = + std::function; + static Result> Create( const std::vector& input_data_field, bool is_ascending_order); diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp index eb5a30eb8..4f632425b 100644 --- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp @@ -20,6 +20,7 @@ #include +#include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/io/key_value_meta_projection_consumer.h" #include "paimon/core/io/row_to_arrow_array_converter.h" @@ -28,7 +29,6 @@ namespace paimon { namespace { -using KeyComparator = std::function; using CancellationChecker = std::function; class ChangelogCompactionBatchProducer : public AsyncKeyValueBatchProducer { @@ -36,8 +36,9 @@ class ChangelogCompactionBatchProducer : public AsyncKeyValueBatchProducer { ChangelogCompactionBatchProducer( std::unique_ptr&& sort_merge_reader, int32_t write_batch_size, std::shared_ptr>&& merge_function_wrapper, - const KeyComparator& key_comparator, const CancellationChecker& cancellation_checker, - bool drop_delete, bool produce_data, bool produce_changelog) + const FieldsComparator::FieldComparatorFunc& key_comparator, + const CancellationChecker& cancellation_checker, bool drop_delete, bool produce_data, + bool produce_changelog) : sort_merge_reader_(std::move(sort_merge_reader)), write_batch_size_(NormalizeProjectionBatchSize(write_batch_size)), merge_function_wrapper_(std::move(merge_function_wrapper)), @@ -142,7 +143,7 @@ class ChangelogCompactionBatchProducer : public AsyncKeyValueBatchProducer { std::unique_ptr sort_merge_reader_; int32_t write_batch_size_; std::shared_ptr> merge_function_wrapper_; - KeyComparator key_comparator_; + FieldsComparator::FieldComparatorFunc key_comparator_; CancellationChecker cancellation_checker_; bool drop_delete_; bool produce_data_; @@ -244,7 +245,8 @@ Result ChangelogMergeTreeRewriter::RewriteOrProduceChangelog( bool produce_data = compact_file_writer != nullptr; bool produce_changelog = changelog_file_writer != nullptr; - KeyComparator key_comparator = [this](const InternalRow& lhs, const InternalRow& rhs) { + FieldsComparator::FieldComparatorFunc key_comparator = [this](const InternalRow& lhs, + const InternalRow& rhs) { return merge_file_split_read_->GetKeyComparator()->CompareTo(lhs, rhs); }; CancellationChecker cancellation_checker = [this]() { return IsCancelled(); }; diff --git a/src/paimon/core/mergetree/compact/internal_row_equalizer.h b/src/paimon/core/mergetree/compact/internal_row_equalizer.h index 64e4b4de4..22f15d8bd 100644 --- a/src/paimon/core/mergetree/compact/internal_row_equalizer.h +++ b/src/paimon/core/mergetree/compact/internal_row_equalizer.h @@ -46,8 +46,9 @@ namespace paimon { /// because changelog kinds are tracked separately by KeyValue::value_kind. class InternalRowEqualizer { public: - static Result Create(const std::shared_ptr& schema, - const std::vector& ignore_fields) { + static Result Create( + const std::shared_ptr& schema, + const std::vector& ignore_fields) { std::set ignored(ignore_fields.begin(), ignore_fields.end()); std::vector> equalizers; for (int32_t i = 0; i < schema->num_fields(); ++i) { @@ -58,7 +59,7 @@ class InternalRowEqualizer { CreateValueEqualizer(schema->field(i)->type())); equalizers.emplace_back(i, std::move(equalizer)); } - return FieldComparatorFunc( + return FieldsComparator::FieldComparatorFunc( [equalizers = std::move(equalizers)](const InternalRow& lhs, const InternalRow& rhs) { for (const auto& [field_idx, equalizer] : equalizers) { if (!EqualAt(lhs, field_idx, rhs, field_idx, equalizer)) { diff --git a/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp b/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp index e8f9e57a0..80bc37b55 100644 --- a/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp +++ b/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp @@ -112,7 +112,7 @@ TEST(InternalRowEqualizerTest, PrimitiveTypesAndIgnoreFields) { std::unique_ptr left = GenericRow::Of(left_values); std::unique_ptr right = GenericRow::Of(right_values); - ASSERT_OK_AND_ASSIGN(FieldComparatorFunc equalizer, + ASSERT_OK_AND_ASSIGN(FieldsComparator::FieldComparatorFunc equalizer, InternalRowEqualizer::Create(schema, {"ignored"})); ASSERT_EQ(0, equalizer(*left, *right)); @@ -123,7 +123,8 @@ TEST(InternalRowEqualizerTest, PrimitiveTypesAndIgnoreFields) { TEST(InternalRowEqualizerTest, NullAndFloatingPointSemantics) { std::shared_ptr schema = arrow::schema( {arrow::field("value", arrow::float64()), arrow::field("nullable", arrow::int32())}); - ASSERT_OK_AND_ASSIGN(FieldComparatorFunc equalizer, InternalRowEqualizer::Create(schema, {})); + ASSERT_OK_AND_ASSIGN(FieldsComparator::FieldComparatorFunc equalizer, + InternalRowEqualizer::Create(schema, {})); std::unique_ptr negative_zero = GenericRow::Of({static_cast(-0.0), NullType()}); @@ -148,7 +149,8 @@ TEST(InternalRowEqualizerTest, NestedTypes) { arrow::field("map", arrow::map(arrow::int32(), arrow::int32())), arrow::field("row", arrow::struct_({arrow::field("value", arrow::int32()), arrow::field("floating", arrow::float64())}))}); - ASSERT_OK_AND_ASSIGN(FieldComparatorFunc equalizer, InternalRowEqualizer::Create(schema, {})); + ASSERT_OK_AND_ASSIGN(FieldsComparator::FieldComparatorFunc equalizer, + InternalRowEqualizer::Create(schema, {})); std::unique_ptr left = GenericRow::Of({CreateNullableIntArray({1, 0, 3}, /*null_pos=*/1, pool.get()), @@ -180,7 +182,8 @@ TEST(InternalRowEqualizerTest, MapEqualityDoesNotDependOnEntryOrder) { std::shared_ptr pool = GetDefaultPool(); std::shared_ptr schema = arrow::schema({arrow::field("map", arrow::map(arrow::int32(), arrow::int32()))}); - ASSERT_OK_AND_ASSIGN(FieldComparatorFunc equalizer, InternalRowEqualizer::Create(schema, {})); + ASSERT_OK_AND_ASSIGN(FieldsComparator::FieldComparatorFunc equalizer, + InternalRowEqualizer::Create(schema, {})); std::unique_ptr left = GenericRow::Of({CreateIntMap({1, 2, 3}, {10, 20, 30}, pool.get())}); diff --git a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h index 3fc0fd4a8..34f7cf970 100644 --- a/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h +++ b/src/paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h @@ -59,7 +59,7 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper& deletion_vectors_maintainer, const std::shared_ptr& comparator, std::unique_ptr&& value_serializer, - FieldComparatorFunc value_equalizer) { + FieldsComparator::FieldComparatorFunc value_equalizer) { if (lookup_strategy.deletion_vector && !deletion_vectors_maintainer) { return Status::Invalid("deletionVectorsMaintainer should not be null, there is a bug."); } @@ -148,7 +148,7 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper& deletion_vectors_maintainer, const std::shared_ptr& user_defined_seq_comparator, std::unique_ptr&& value_serializer, - FieldComparatorFunc value_equalizer) + FieldsComparator::FieldComparatorFunc value_equalizer) : merge_function_(std::move(merge_function)), lookup_(std::move(lookup)), lookup_strategy_(lookup_strategy), @@ -219,7 +219,7 @@ class LookupChangelogMergeFunctionWrapper : public MergeFunctionWrapper deletion_vectors_maintainer_; std::function comparator_; std::unique_ptr value_serializer_; - FieldComparatorFunc value_equalizer_; + FieldsComparator::FieldComparatorFunc value_equalizer_; }; } // namespace paimon diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp index 608b36303..0bcd91ff8 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp @@ -140,7 +140,7 @@ LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& user_defined_seq_comparator, LookupLevels* lookup_levels, std::unique_ptr&& value_serializer, - FieldComparatorFunc value_equalizer) { + FieldsComparator::FieldComparatorFunc value_equalizer) { auto lookup = [output_level, lookup_levels]( const std::shared_ptr& key) -> Result> { return lookup_levels->Lookup(key, output_level + 1); diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h index 39745f5fd..87a094c7a 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h @@ -66,7 +66,7 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter { const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& user_defined_seq_comparator, LookupLevels* lookup_levels, std::unique_ptr&& value_serializer, - FieldComparatorFunc value_equalizer); + FieldsComparator::FieldComparatorFunc value_equalizer); private: LookupMergeTreeCompactRewriter( diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp index fae403af9..d6701fe1a 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp @@ -197,7 +197,7 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParamTrimmedPrimaryKeys().value(), options, pool_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr value_serializer, RowCompactedSerializer::Create(arrow_schema_, pool_)); - FieldComparatorFunc value_equalizer; + FieldsComparator::FieldComparatorFunc value_equalizer; if (options.ChangelogRowDeduplicate()) { PAIMON_ASSIGN_OR_RAISE( value_equalizer, @@ -292,7 +292,7 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParamTrimmedPrimaryKeys().value(), options, pool_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr value_serializer, RowCompactedSerializer::Create(arrow_schema_, pool_)); - FieldComparatorFunc value_equalizer; + FieldsComparator::FieldComparatorFunc value_equalizer; if (options.ChangelogRowDeduplicate()) { PAIMON_ASSIGN_OR_RAISE( value_equalizer, diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp index 06ca83b4a..ad21a250f 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp @@ -51,11 +51,11 @@ namespace paimon { namespace { -Result CreateChangelogValueEqualizer( +Result CreateChangelogValueEqualizer( const std::shared_ptr& schema, const CoreOptions& options, bool produce_changelog) { if (!produce_changelog || !options.ChangelogRowDeduplicate()) { - return FieldComparatorFunc(); + return FieldsComparator::FieldComparatorFunc(); } return InternalRowEqualizer::Create(schema, options.GetChangelogRowDeduplicateIgnoreFields()); } @@ -280,7 +280,7 @@ MergeTreeCompactManagerFactory::CreateLookupRewriterInternal( std::unique_ptr value_serializer, CreateChangelogValueSerializer(data_schema, should_produce_changelog, pool)); PAIMON_ASSIGN_OR_RAISE( - FieldComparatorFunc value_equalizer, + FieldsComparator::FieldComparatorFunc value_equalizer, CreateChangelogValueEqualizer(data_schema, options, should_produce_changelog)); return LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( std::make_unique(std::move(merge_func)), output_level, From 044e984575751c4143118bdcecb467f3b44270d8 Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Fri, 28 Aug 2026 17:59:26 +0800 Subject: [PATCH 6/9] fix xinyu comment --- .../mergetree/compact/lookup_merge_tree_compact_rewriter.cpp | 4 ++-- .../mergetree/compact/lookup_merge_tree_compact_rewriter.h | 4 ++-- .../mergetree/compact/merge_tree_compact_manager_factory.cpp | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp index 6432acc00..1a818dd01 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp @@ -134,8 +134,8 @@ LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( const std::shared_ptr& deletion_vectors_maintainer, const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& user_defined_seq_comparator, - LookupLevels* lookup_levels, std::unique_ptr&& value_serializer, - FieldsComparator::FieldComparatorFunc value_equalizer) { + std::unique_ptr&& value_serializer, + FieldsComparator::FieldComparatorFunc value_equalizer, LookupLevels* lookup_levels) { auto lookup = [output_level, lookup_levels]( const std::shared_ptr& key) -> Result> { return lookup_levels->Lookup(key, output_level + 1); diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h index 87a094c7a..a461177ba 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h @@ -65,8 +65,8 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter { const std::shared_ptr& deletion_vectors_maintainer, const LookupStrategy& lookup_strategy, bool should_produce_changelog, const std::shared_ptr& user_defined_seq_comparator, - LookupLevels* lookup_levels, std::unique_ptr&& value_serializer, - FieldsComparator::FieldComparatorFunc value_equalizer); + std::unique_ptr&& value_serializer, + FieldsComparator::FieldComparatorFunc value_equalizer, LookupLevels* lookup_levels); private: LookupMergeTreeCompactRewriter( diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp index ad21a250f..258336698 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory.cpp @@ -285,8 +285,8 @@ MergeTreeCompactManagerFactory::CreateLookupRewriterInternal( return LookupMergeTreeCompactRewriter::CreateLookupMergeFunctionWrapper( std::make_unique(std::move(merge_func)), output_level, dv_maintainer_ptr, lookup_strategy, should_produce_changelog, - user_defined_seq_comparator, lookup_levels_ptr, std::move(value_serializer), - std::move(value_equalizer)); + user_defined_seq_comparator, std::move(value_serializer), std::move(value_equalizer), + lookup_levels_ptr); }; PAIMON_ASSIGN_OR_RAISE( std::unique_ptr> rewriter, From 6027c49f0462e5673aab6dac390713610cb904ce Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Fri, 28 Aug 2026 18:06:30 +0800 Subject: [PATCH 7/9] fix ci --- ...ookup_merge_tree_compact_rewriter_test.cpp | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp index b92371b69..29d4e34fd 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp @@ -700,45 +700,6 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowRewriteWithSharedShreddin /*field_index=*/3, expected_meta); } -TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowLooksUpExistingKeys) { - std::map options = {{Options::MERGE_ENGINE, "first-row"}, - {Options::FILE_FORMAT, "orc"}}; - ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); - ASSERT_OK_AND_ASSIGN(auto table_path, CreateTable(options)); - auto schema_manager = std::make_shared(fs_, table_path); - ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager->ReadSchema(0)); - - ASSERT_OK_AND_ASSIGN(auto level0_file, - NewFiles(/*level=*/0, /*last_sequence_number=*/0, table_path, core_options, - "[[1, 111], [2, 22]]")); - ASSERT_OK_AND_ASSIGN(auto high_level_file, NewFiles(/*level=*/2, /*last_sequence_number=*/-1, - table_path, core_options, "[[1, 11]]")); - auto processor_factory = std::make_shared(); - ASSERT_OK_AND_ASSIGN(auto lookup_levels, - CreateLookupLevels(table_path, table_schema, processor_factory, - std::vector>{ - level0_file, high_level_file})); - ASSERT_OK_AND_ASSIGN(auto rewriter, - CreateCompactRewriterForFirstRow(table_path, table_schema, core_options, - std::move(lookup_levels))); - ASSERT_OK_AND_ASSIGN( - auto runs, GenerateSortedRuns(std::vector>{level0_file})); - ASSERT_OK_AND_ASSIGN(auto compact_result, rewriter->Rewrite( - /*output_level=*/1, /*drop_delete=*/true, runs)); - - ASSERT_EQ(1, compact_result.After().size()); - ASSERT_EQ(1, compact_result.After()[0]->row_count); - - auto type_with_special_fields = - arrow::struct_(SpecialFields::CompleteSequenceAndValueKindField(arrow_schema_)->fields()); - std::shared_ptr expected; - ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(type_with_special_fields, - {"[[2, 0, 2, 22]]"}, &expected) - .ok()); - CheckResult(table_path + "/bucket-0/" + compact_result.After()[0]->file_name, table_schema, - "orc", expected); -} - TEST_F(LookupMergeTreeCompactRewriterTest, TestFirstRowUpgrade) { std::map options = {{Options::MERGE_ENGINE, "first-row"}, {Options::FILE_FORMAT, "orc"}}; From f1db29de4a59895d8046edd66ea7268955ea2138 Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Fri, 28 Aug 2026 18:35:19 +0800 Subject: [PATCH 8/9] fix ci --- .../lookup_merge_tree_compact_rewriter_test.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp index 29d4e34fd..c930c5ca7 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp @@ -210,8 +210,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam(std::move(merge_func)), output_level, /*deletion_vectors_maintainer=*/nullptr, lookup_strategy, /*should_produce_changelog=*/lookup_strategy.produce_changelog, - /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr, - std::move(value_serializer), std::move(value_equalizer))); + /*user_defined_seq_comparator=*/nullptr, std::move(value_serializer), + std::move(value_equalizer), lookup_levels_ptr)); return merge_function_wrapper; }; auto cancellation_controller = std::make_shared(); @@ -251,8 +251,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam(std::move(merge_func)), output_level, dv_maintainer_ptr, lookup_strategy, /*should_produce_changelog=*/lookup_strategy.produce_changelog, - /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr, - std::move(value_serializer), /*value_equalizer=*/{})); + /*user_defined_seq_comparator=*/nullptr, std::move(value_serializer), + /*value_equalizer=*/{}, lookup_levels_ptr)); return merge_function_wrapper; }; auto cancellation_controller = std::make_shared(); @@ -306,8 +306,8 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam(std::move(merge_func)), output_level, dv_maintainer_ptr, lookup_strategy, /*should_produce_changelog=*/lookup_strategy.produce_changelog, - /*user_defined_seq_comparator=*/nullptr, lookup_levels_ptr, - std::move(value_serializer), std::move(value_equalizer))); + /*user_defined_seq_comparator=*/nullptr, std::move(value_serializer), + std::move(value_equalizer), lookup_levels_ptr)); return merge_function_wrapper; }; auto cancellation_controller = std::make_shared(); From de7a51fd1091b21fa9266ae3de04be2d9d03d72e Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Sat, 29 Aug 2026 14:20:37 +0800 Subject: [PATCH 9/9] fix clang-tidy --- src/paimon/core/core_options.cpp | 89 +++++++++++++++----------------- 1 file changed, 41 insertions(+), 48 deletions(-) diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 3899dcdc9..71ba73deb 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -330,9 +330,7 @@ class ConfigParser { // storing various configurable fields and their default values. struct CoreOptions::Impl { int64_t page_size = 64 * 1024; - std::optional target_file_size; int64_t target_file_row_num = std::numeric_limits::max(); - std::optional blob_target_file_size; int64_t source_split_target_size = 128 * 1024 * 1024; int64_t source_split_open_file_cost = 4 * 1024 * 1024; int64_t manifest_target_file_size = 8 * 1024 * 1024; @@ -343,17 +341,29 @@ struct CoreOptions::Impl { int64_t commit_timeout = std::numeric_limits::max(); int64_t commit_min_retry_wait = 10; int64_t commit_max_retry_wait = 10 * 1000; - bool realtime_enabled = false; int64_t realtime_read_view_ttl_millis = 5 * 60 * 1000; - StatisticsMode realtime_store_statistics_mode = StatisticsMode::NONE; + int64_t write_buffer_spill_max_disk_size = std::numeric_limits::max(); + double variant_shredding_min_field_cardinality_ratio = 0.1; + double variant_shredding_adaptive_retention_ratio = 0.05; + double lookup_cache_bloom_filter_fpp = 0.05; + int64_t cache_page_size = 64 * 1024; // 64KB + int64_t lookup_cache_max_memory = 256 * 1024 * 1024; + double lookup_cache_high_prio_pool_ratio = 0.25; + int64_t lookup_cache_file_retention_ms = 1 * 3600 * 1000; // 1 hour + int64_t lookup_cache_max_disk_size = INT64_MAX; + std::optional target_file_size; + std::optional blob_target_file_size; std::shared_ptr file_format; std::shared_ptr file_system; std::shared_ptr manifest_file_format; std::shared_ptr cache; - std::optional scan_snapshot_id; std::optional scan_timestamp_millis; + std::optional optimized_compaction_interval; + std::optional compaction_total_size_threshold; + std::optional compaction_incremental_size_threshold; + std::shared_ptr changelog_file_format; ExpireConfig expire_config; std::vector sequence_field; std::vector remove_record_on_sequence_group; @@ -370,20 +380,23 @@ struct CoreOptions::Impl { std::string data_file_prefix = "data-"; std::string changelog_file_prefix = "changelog-"; std::string file_system_scheme_to_identifier_map_str; - std::optional field_default_func; std::optional scan_fallback_branch; std::optional data_file_external_paths; std::optional blob_view_upstream_warehouse; std::optional changelog_file_compression; - + std::optional global_index_external_path; + std::optional scan_tag_name; + CompressOptions lookup_compress_options{"zstd", 1}; + CompressOptions spill_compress_options{"zstd", 1}; std::map raw_options; + std::map> file_format_per_level; + std::map file_compression_per_level; + StatisticsMode realtime_store_statistics_mode = StatisticsMode::NONE; int32_t bucket = -1; - int32_t manifest_merge_min_count = 30; int32_t scan_manifest_entry_cache_max_snapshots = 0; - bool scan_manifest_entry_lazy_decode_enabled = true; int32_t read_batch_size = 1024; int32_t write_batch_size = 1024; int32_t local_sort_max_num_file_handles = 128; @@ -392,28 +405,38 @@ struct CoreOptions::Impl { int32_t compaction_max_size_amplification_percent = 200; int32_t compaction_size_ratio = 1; int32_t num_sorted_runs_compaction_trigger = 5; - std::optional num_sorted_runs_stop_trigger; - std::optional num_levels; - SortOrder sequence_field_sort_order = SortOrder::ASCENDING; MergeEngine merge_engine = MergeEngine::DEDUPLICATE; SortEngine sort_engine = SortEngine::LOSER_TREE; ChangelogProducer changelog_producer = ChangelogProducer::NONE; ExternalPathStrategy external_path_strategy = ExternalPathStrategy::NONE; LookupCompactMode lookup_compact_mode = LookupCompactMode::RADICAL; - std::optional lookup_compact_max_interval; BucketFunctionType bucket_function_type = BucketFunctionType::DEFAULT; - int32_t file_compression_zstd_level = 1; - int64_t write_buffer_spill_max_disk_size = std::numeric_limits::max(); + CoreOptions::SequenceNumberInitMode write_sequence_number_init_mode = + CoreOptions::SequenceNumberInitMode::SCAN; + VariantShreddingInferenceMode variant_shredding_inference_mode = + VariantShreddingInferenceMode::PER_FILE; + int32_t variant_shredding_max_schema_width = 300; + int32_t variant_shredding_max_schema_depth = 50; + int32_t variant_shredding_max_infer_buffer_row = 4096; + int32_t variant_shredding_adaptive_max_infer_buffer_row = 256; + int32_t compact_off_peak_start_hour = -1; + int32_t compact_off_peak_end_hour = -1; + int32_t compact_off_peak_ratio = 0; + int32_t lookup_remote_level_threshold = INT32_MIN; + std::optional num_sorted_runs_stop_trigger; + std::optional num_levels; + std::optional lookup_compact_max_interval; + std::optional global_index_thread_num; + bool realtime_enabled = false; + bool scan_manifest_entry_lazy_decode_enabled = true; bool ignore_delete = false; bool manifest_delete_file_drop_stats = false; bool write_buffer_spillable = true; bool write_only = false; bool bucket_append_ordered = false; - CoreOptions::SequenceNumberInitMode write_sequence_number_init_mode = - CoreOptions::SequenceNumberInitMode::SCAN; bool deletion_vectors_enabled = false; bool deletion_vectors_bitmap64 = false; bool force_lookup = false; @@ -431,49 +454,19 @@ struct CoreOptions::Impl { bool row_tracking_partition_group_on_commit = true; bool data_evolution_enabled = false; bool variant_infer_shredding_schema = false; - VariantShreddingInferenceMode variant_shredding_inference_mode = - VariantShreddingInferenceMode::PER_FILE; - int32_t variant_shredding_max_schema_width = 300; - int32_t variant_shredding_max_schema_depth = 50; - double variant_shredding_min_field_cardinality_ratio = 0.1; - int32_t variant_shredding_max_infer_buffer_row = 4096; - int32_t variant_shredding_adaptive_max_infer_buffer_row = 256; - double variant_shredding_adaptive_retention_ratio = 0.05; bool blob_view_resolve_enabled = true; bool blob_as_descriptor = false; - std::optional blob_split_by_file_size; bool legacy_partition_name_enabled = true; bool global_index_enabled = true; - std::optional global_index_thread_num; bool commit_force_compact = false; bool commit_discard_duplicate_files = false; bool dynamic_partition_overwrite = true; bool overwrite_upgrade = true; bool compaction_force_rewrite_all_files = false; bool compaction_force_up_level_0 = false; - std::optional global_index_external_path; - - std::optional scan_tag_name; - std::optional optimized_compaction_interval; - std::optional compaction_total_size_threshold; - std::optional compaction_incremental_size_threshold; - int32_t compact_off_peak_start_hour = -1; - int32_t compact_off_peak_end_hour = -1; - int32_t compact_off_peak_ratio = 0; bool lookup_cache_bloom_filter = true; - double lookup_cache_bloom_filter_fpp = 0.05; bool lookup_remote_file_enabled = false; - int32_t lookup_remote_level_threshold = INT32_MIN; - CompressOptions lookup_compress_options{"zstd", 1}; - CompressOptions spill_compress_options{"zstd", 1}; - int64_t cache_page_size = 64 * 1024; // 64KB - std::map> file_format_per_level; - std::map file_compression_per_level; - std::shared_ptr changelog_file_format; - int64_t lookup_cache_max_memory = 256 * 1024 * 1024; - double lookup_cache_high_prio_pool_ratio = 0.25; - int64_t lookup_cache_file_retention_ms = 1 * 3600 * 1000; // 1 hour - int64_t lookup_cache_max_disk_size = INT64_MAX; + std::optional blob_split_by_file_size; // Parse basic table options: bucket, partition, file sizes, batch sizes, file system, etc. Status ParseBasicOptions(