diff --git a/examples/read_write_demo.cpp b/examples/read_write_demo.cpp index 946b35039..bcf7db1e1 100644 --- a/examples/read_write_demo.cpp +++ b/examples/read_write_demo.cpp @@ -56,8 +56,7 @@ arrow::Result> PrepareData(const arrow::Fiel paimon::Status Run(const std::string& root_path, const std::string& db_name, const std::string& table_name) { - std::map options = {{paimon::Options::MANIFEST_FORMAT, "orc"}, - {paimon::Options::FILE_FORMAT, "parquet"}, + std::map options = {{paimon::Options::FILE_FORMAT, "parquet"}, {paimon::Options::FILE_SYSTEM, "local"}}; // create table diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 0069ded74..e819c85d0 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -164,10 +164,6 @@ struct PAIMON_EXPORT Options { /// Default value is 8MB. static const char MANIFEST_TARGET_FILE_SIZE[]; - /// "manifest.format" - Specify the message format of manifest files. - /// Default value is avro. - static const char MANIFEST_FORMAT[]; - /// "manifest.compression" - File compression for manifest, default value is zstd. static const char MANIFEST_COMPRESSION[]; diff --git a/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory_test.cpp index de8ce58a6..1763d6a22 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory_test.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory_test.cpp @@ -232,7 +232,6 @@ class MapSharedShreddingReadPlanFactoryTest : public ::testing::Test { std::map options_ = { {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "mock_format"}, - {Options::MANIFEST_FORMAT, "mock_format"}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "2"}, {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp b/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp index 6dbb8d79f..1546acade 100644 --- a/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp +++ b/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp @@ -47,8 +47,6 @@ class VariantShreddingWritePlanFactoryTest : public ::testing::Test { } Result MakeOptions(std::map options) const { - // Keep the manifest format resolvable in test binaries without the avro plugin. - options.emplace("manifest.format", "parquet"); return CoreOptions::FromMap(options); } diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index f0b0f5611..962216f95 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -48,7 +48,6 @@ const char Options::FILE_COMPRESSION[] = "file.compression"; const char Options::FILE_COMPRESSION_ZSTD_LEVEL[] = "file.compression.zstd-level"; const char Options::FILE_BLOCK_SIZE[] = "file.block-size"; const char Options::MANIFEST_TARGET_FILE_SIZE[] = "manifest.target-file-size"; -const char Options::MANIFEST_FORMAT[] = "manifest.format"; const char Options::MANIFEST_COMPRESSION[] = "manifest.compression"; const char Options::MANIFEST_MERGE_MIN_COUNT[] = "manifest.merge-min-count"; const char Options::MANIFEST_FULL_COMPACTION_FILE_SIZE[] = diff --git a/src/paimon/common/utils/binary_row_partition_computer_test.cpp b/src/paimon/common/utils/binary_row_partition_computer_test.cpp index 18c421780..2a9d6b2d5 100644 --- a/src/paimon/common/utils/binary_row_partition_computer_test.cpp +++ b/src/paimon/common/utils/binary_row_partition_computer_test.cpp @@ -251,20 +251,27 @@ TEST(BinaryRowPartitionComputerTest, TestNullOrWhitespaceOnlyStr) { arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::utf8()), arrow::field("f2", arrow::utf8()), + arrow::field("f3", arrow::utf8()), }; auto schema = arrow::schema(fields); - std::vector partition_keys = {"f0", "f1", "f2"}; + std::vector partition_keys = {"f0", "f1", "f2", "f3"}; ASSERT_OK_AND_ASSIGN( std::unique_ptr computer, BinaryRowPartitionComputer::Create(partition_keys, schema, "__DEFAULT_PARTITION__", /*legacy_partition_name_enabled=*/true, pool)); - ASSERT_OK_AND_ASSIGN(auto partition_key_values, - computer->GeneratePartitionVector(BinaryRowGenerator::GenerateRow( - {std::string(" "), std::string(""), std::string("ab ")}, pool.get()))); + ASSERT_OK_AND_ASSIGN( + auto partition_key_values, + computer->GeneratePartitionVector(BinaryRowGenerator::GenerateRow( + {std::string(" "), std::string(""), std::string("ab "), std::string(u8"\u3000\u2000")}, + pool.get()))); std::vector> expected = { - {"f0", "__DEFAULT_PARTITION__"}, {"f1", "__DEFAULT_PARTITION__"}, {"f2", "ab "}}; + {"f0", "__DEFAULT_PARTITION__"}, + {"f1", "__DEFAULT_PARTITION__"}, + {"f2", "ab "}, + {"f3", "__DEFAULT_PARTITION__"}, + }; ASSERT_EQ(partition_key_values, expected); } diff --git a/src/paimon/common/utils/string_utils.cpp b/src/paimon/common/utils/string_utils.cpp index 869e184d0..7b3ed96b8 100644 --- a/src/paimon/common/utils/string_utils.cpp +++ b/src/paimon/common/utils/string_utils.cpp @@ -37,6 +37,14 @@ bool IsTrimCharacter(unsigned char c) { return c <= 0x20; } +bool IsJavaWhitespace(uint32_t code_point) { + return (code_point >= 0x0009 && code_point <= 0x000d) || + (code_point >= 0x001c && code_point <= 0x0020) || code_point == 0x1680 || + (code_point >= 0x2000 && code_point <= 0x2006) || + (code_point >= 0x2008 && code_point <= 0x200a) || code_point == 0x2028 || + code_point == 0x2029 || code_point == 0x205f || code_point == 0x3000; +} + char ToAsciiLower(unsigned char c) { return c >= 'A' && c <= 'Z' ? static_cast(c + ('a' - 'A')) : static_cast(c); } @@ -85,18 +93,58 @@ bool StringUtils::EndsWith(const std::string& str, const std::string& suffix) { size_t s2 = suffix.size(); return (s1 >= s2) && (str.compare(s1 - s2, s2, suffix) == 0); } -bool StringUtils::IsNullOrWhitespaceOnly(const std::string& str) { - if (str.empty()) { - return true; - } - for (char c : str) { - if (!std::isspace(static_cast(c))) { + +bool StringUtils::IsBlank(std::string_view str) { + size_t offset = 0; + while (offset < str.size()) { + const auto first = static_cast(str[offset]); + uint32_t code_point = 0; + size_t length = 0; + if (first <= 0x7f) { + code_point = first; + length = 1; + } else if (first >= 0xc2 && first <= 0xdf) { + code_point = first & 0x1f; + length = 2; + } else if (first >= 0xe0 && first <= 0xef) { + code_point = first & 0x0f; + length = 3; + } else if (first >= 0xf0 && first <= 0xf4) { + code_point = first & 0x07; + length = 4; + } else { + return false; + } + if (offset + length > str.size()) { + return false; + } + for (size_t i = 1; i < length; ++i) { + const auto continuation = static_cast(str[offset + i]); + if ((continuation & 0xc0) != 0x80) { + return false; + } + code_point = (code_point << 6) | (continuation & 0x3f); + } + if ((length == 3 && code_point < 0x800) || (length == 4 && code_point < 0x10000) || + (code_point >= 0xd800 && code_point <= 0xdfff) || code_point > 0x10ffff) { return false; } + if (!IsJavaWhitespace(code_point)) { + return false; + } + offset += length; } return true; } +bool StringUtils::IsNullOrWhitespaceOnly(const std::string& str) { + return IsBlank(str); +} + +bool StringUtils::IsEmptyAfterTrim(std::string_view str) { + return std::all_of(str.begin(), str.end(), [](unsigned char c) { return IsTrimCharacter(c); }); +} + void StringUtils::Trim(std::string* str) { auto first = std::find_if_not(str->begin(), str->end(), [](unsigned char c) { return IsTrimCharacter(c); }); diff --git a/src/paimon/common/utils/string_utils.h b/src/paimon/common/utils/string_utils.h index 7681a8d9e..49c0aff2f 100644 --- a/src/paimon/common/utils/string_utils.h +++ b/src/paimon/common/utils/string_utils.h @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -96,8 +97,15 @@ class PAIMON_EXPORT StringUtils { static bool EndsWith(const std::string& str, const std::string& suffix); + /// Returns true if the string is empty or contains only characters recognized by Java + /// Character.isWhitespace. + static bool IsBlank(std::string_view str); + static bool IsNullOrWhitespaceOnly(const std::string& str); + /// Returns true if Java String::trim would produce an empty string. + static bool IsEmptyAfterTrim(std::string_view str); + static void Trim(std::string* str); static std::string ToLowerCase(const std::string& str); diff --git a/src/paimon/common/utils/string_utils_test.cpp b/src/paimon/common/utils/string_utils_test.cpp index a4f230781..89b0334f1 100644 --- a/src/paimon/common/utils/string_utils_test.cpp +++ b/src/paimon/common/utils/string_utils_test.cpp @@ -21,6 +21,7 @@ #include #include +#include #include "gtest/gtest.h" #include "paimon/status.h" @@ -209,6 +210,34 @@ TEST_F(StringUtilsTest, TestIsNullOrWhitespaceOnly) { auto ret = StringUtils::IsNullOrWhitespaceOnly(str); ASSERT_TRUE(ret); } + ASSERT_TRUE(StringUtils::IsNullOrWhitespaceOnly(u8"\u3000\u2000")); +} + +TEST_F(StringUtilsTest, TestIsBlank) { + const std::vector blank_strings = { + "", " ", " ", "\t", "\n", "\r", + "\r\n", " \t\n\r ", u8"\u1680", u8"\u2000", u8"\u3000", u8" \t\u3000\u2000\n"}; + for (const std::string& blank : blank_strings) { + ASSERT_TRUE(StringUtils::IsBlank(blank)) << blank; + } + + ASSERT_FALSE(StringUtils::IsBlank("user1")); + ASSERT_FALSE(StringUtils::IsBlank(" user1 ")); + ASSERT_FALSE(StringUtils::IsBlank(u8"\u00a0")); + ASSERT_FALSE(StringUtils::IsBlank(std::string("\xc0\x80", 2))); + + // Non-breaking or otherwise excluded by Character.isWhitespace. + ASSERT_FALSE(StringUtils::IsBlank(u8"\u2007")); // FIGURE SPACE + ASSERT_FALSE(StringUtils::IsBlank(u8"\u202f")); // NARROW NO-BREAK SPACE + ASSERT_FALSE(StringUtils::IsBlank(u8"\u0085")); // NEL + ASSERT_FALSE(StringUtils::IsBlank(u8"\u180e")); // Not whitespace since Java 8 +} + +TEST_F(StringUtilsTest, TestIsEmptyAfterTrim) { + ASSERT_TRUE(StringUtils::IsEmptyAfterTrim("")); + ASSERT_TRUE(StringUtils::IsEmptyAfterTrim(" \t\x1c")); + ASSERT_FALSE(StringUtils::IsEmptyAfterTrim(" a ")); + ASSERT_FALSE(StringUtils::IsEmptyAfterTrim(u8"\u3000")); } TEST_F(StringUtilsTest, TestToLowerCase) { diff --git a/src/paimon/core/append/append_only_writer_test.cpp b/src/paimon/core/append/append_only_writer_test.cpp index bf64420e1..4a4d5dc6f 100644 --- a/src/paimon/core/append/append_only_writer_test.cpp +++ b/src/paimon/core/append/append_only_writer_test.cpp @@ -190,7 +190,6 @@ class AppendOnlyWriterTest : public testing::Test { std::map raw_options = { {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "mock_format"}, - {Options::MANIFEST_FORMAT, "mock_format"}, }; for (const auto& [key, value] : overrides) { raw_options[key] = value; @@ -334,7 +333,6 @@ TEST_F(AppendOnlyWriterTest, TestEmptyCommits) { std::map raw_options; raw_options[Options::FILE_FORMAT] = "mock_format"; raw_options[Options::FILE_SYSTEM] = "local"; - raw_options[Options::MANIFEST_FORMAT] = "mock_format"; ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(raw_options)); arrow::FieldVector fields = { @@ -364,7 +362,6 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndPrepareCommit) { std::map raw_options; raw_options[Options::FILE_FORMAT] = "mock_format"; raw_options[Options::FILE_SYSTEM] = "local"; - raw_options[Options::MANIFEST_FORMAT] = "mock_format"; ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(raw_options)); arrow::FieldVector fields = { @@ -446,7 +443,6 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndClose) { std::map raw_options; raw_options[Options::FILE_FORMAT] = "orc"; raw_options[Options::FILE_SYSTEM] = "local"; - raw_options[Options::MANIFEST_FORMAT] = "orc"; ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(raw_options)); arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; @@ -491,7 +487,6 @@ TEST_F(AppendOnlyWriterTest, TestInvalidRowKind) { std::map raw_options; raw_options[Options::FILE_FORMAT] = "orc"; raw_options[Options::FILE_SYSTEM] = "local"; - raw_options[Options::MANIFEST_FORMAT] = "orc"; ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(raw_options)); arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; @@ -632,8 +627,7 @@ TEST_F(AppendOnlyWriterTest, } TEST_F(AppendOnlyWriterTest, TestCloseDeletesCompactAfterFiles) { - auto options = - CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}); + auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}}); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto path_factory = CreatePathFactory(dir->Str(), "orc", options); @@ -740,9 +734,8 @@ TEST_F(AppendOnlyWriterTest, TestCompactPassesFullCompactionFlag) { } TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) { - auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, - {Options::TARGET_FILE_ROW_NUM, "1"}}); + auto options = + CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_ROW_NUM, "1"}}); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto path_factory = CreatePathFactory(dir->Str(), "orc", options); @@ -780,8 +773,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) { } TEST_F(AppendOnlyWriterTest, TestWriteWithOnlyBlobField) { - auto options = - CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}); + auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}}); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto path_factory = CreatePathFactory(dir->Str(), "orc", options); @@ -828,8 +820,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithOnlyBlobField) { } TEST_F(AppendOnlyWriterTest, TestWriteWithMultipleBlobFields) { - auto options = - CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}); + auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}}); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto path_factory = CreatePathFactory(dir->Str(), "orc", options); @@ -896,9 +887,8 @@ TEST_F(AppendOnlyWriterTest, TestMultiplePrepareCommitSequenceContinuity) { } TEST_F(AppendOnlyWriterTest, TestWriteValidBlobViewField) { - auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, - {Options::BLOB_VIEW_FIELD, "view"}}); + auto options = + CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::BLOB_VIEW_FIELD, "view"}}); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto path_factory = CreatePathFactory(dir->Str(), "orc", options); @@ -933,9 +923,8 @@ TEST_F(AppendOnlyWriterTest, TestWriteValidBlobViewField) { } TEST_F(AppendOnlyWriterTest, TestWriteInvalidBlobViewFieldRejected) { - auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, - {Options::BLOB_VIEW_FIELD, "view"}}); + auto options = + CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::BLOB_VIEW_FIELD, "view"}}); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto path_factory = CreatePathFactory(dir->Str(), "orc", options); @@ -977,7 +966,7 @@ INSTANTIATE_TEST_SUITE_P(FileFormats, AppendOnlyWriterShreddingTest, TEST_F(AppendOnlyWriterTest, TestSharedShreddingMapRejectsAvroFormatOnCommit) { auto options = CreateOptions({ {Options::FILE_FORMAT, "avro"}, - {Options::MANIFEST_FORMAT, "avro"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "3"}, {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, @@ -1013,7 +1002,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapFieldContent) { // Configure with shared-shredding map on "tags" field, K=3. auto options = CreateOptions({ {Options::FILE_FORMAT, format}, - {Options::MANIFEST_FORMAT, format}, {Options::TARGET_FILE_ROW_NUM, "2"}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "3"}, @@ -1087,7 +1075,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllEmptyFirstFile) { std::string format = GetFormat(); auto options = CreateOptions({ {Options::FILE_FORMAT, format}, - {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "3"}, {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, @@ -1145,7 +1132,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyF std::string format = GetFormat(); auto options = CreateOptions({ {Options::FILE_FORMAT, format}, - {Options::MANIFEST_FORMAT, format}, {Options::TARGET_FILE_ROW_NUM, "2"}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "3"}, @@ -1258,7 +1244,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithOverflow) { // K=2, write rows with 3+ keys to trigger overflow. auto options = CreateOptions({ {Options::FILE_FORMAT, format}, - {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "2"}, {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, @@ -1330,7 +1315,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithLruPlacemen std::string format = GetFormat(); auto options = CreateOptions({ {Options::FILE_FORMAT, format}, - {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "3"}, {"fields.tags.map.shared-shredding.column-placement-policy", "lru"}, @@ -1397,7 +1381,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossRol std::string format = GetFormat(); auto options = CreateOptions({ {Options::FILE_FORMAT, format}, - {Options::MANIFEST_FORMAT, format}, {Options::TARGET_FILE_ROW_NUM, "1"}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "10"}, @@ -1504,7 +1487,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapFirstFileUsesConfigu std::string format = GetFormat(); auto options = CreateOptions({ {Options::FILE_FORMAT, format}, - {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "10"}, {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, @@ -1563,7 +1545,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsAdaptA // Two shared-shredding MAP fields with different initial K: tags(K=8), attrs(K=4). auto options = CreateOptions({ {Options::FILE_FORMAT, format}, - {Options::MANIFEST_FORMAT, format}, {Options::TARGET_FILE_ROW_NUM, "1"}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "8"}, @@ -1682,7 +1663,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapDataFileMetaInfo) { // Verify PrepareCommit returns correct DataFileMeta for shared-shredding map files. auto options = CreateOptions({ {Options::FILE_FORMAT, format}, - {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "3"}, {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, @@ -1762,7 +1742,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapWithBlobSeparation) // This tests that blob separation + shredding works correctly together. auto options = CreateOptions({ {Options::FILE_FORMAT, format}, - {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "3"}, {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 385b23002..aaa5b6457 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -610,9 +610,9 @@ struct CoreOptions::Impl { // Parse manifest file configurations: format, compression, merge, and compaction thresholds. Status ParseManifestOptions(const ConfigParser& parser) { - // Parse manifest.format - manifest file format, default "avro" + // Parse the legacy manifest.format option for reading existing tables. PAIMON_RETURN_NOT_OK(parser.ParseObject( - Options::MANIFEST_FORMAT, /*default_identifier=*/"avro", &manifest_file_format)); + "manifest.format", /*default_identifier=*/"avro", &manifest_file_format)); // Parse manifest.compression - manifest file compression, default "zstd" PAIMON_RETURN_NOT_OK(parser.Parse(Options::MANIFEST_COMPRESSION, &manifest_compression)); // Parse manifest.target-file-size - suggested manifest file size, default 8MB diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index 56ab466d5..7bd5b5e76 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -37,7 +37,8 @@ namespace paimon::test { TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); - ASSERT_EQ(core_options.GetManifestFormat()->Identifier(), "avro"); + std::shared_ptr manifest_format = core_options.GetManifestFormat(); + ASSERT_EQ(manifest_format->Identifier(), "avro"); ASSERT_EQ(core_options.GetFileFormat()->Identifier(), "parquet"); ASSERT_EQ(nullptr, core_options.GetChangelogFileFormat()); ASSERT_EQ(core_options.GetWriteFileFormat(0)->Identifier(), "parquet"); @@ -198,7 +199,7 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::FILE_SYSTEM, "Local"}, {Options::FILE_FORMAT, "ORC"}, {Options::CHANGELOG_FILE_FORMAT, "avro"}, - {Options::MANIFEST_FORMAT, "avRo"}, + {"manifest.format", "ORC"}, {Options::BUCKET, "3"}, {Options::PAGE_SIZE, "128 kb"}, {Options::TARGET_FILE_SIZE, "512MB"}, @@ -339,8 +340,8 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ(core_options.GetWriteFileFormat(1)->Identifier(), "orc"); ASSERT_EQ(core_options.GetWriteFileFormat(3)->Identifier(), "parquet"); - auto manifest_format = core_options.GetManifestFormat(); - ASSERT_EQ(manifest_format->Identifier(), "avro"); + std::shared_ptr manifest_format = core_options.GetManifestFormat(); + ASSERT_EQ(manifest_format->Identifier(), "orc"); ASSERT_EQ(3, core_options.GetBucket()); ASSERT_EQ(128 * 1024L, core_options.GetPageSize()); diff --git a/src/paimon/core/index/index_file_handler_test.cpp b/src/paimon/core/index/index_file_handler_test.cpp index a23591f94..a4b5295f5 100644 --- a/src/paimon/core/index/index_file_handler_test.cpp +++ b/src/paimon/core/index/index_file_handler_test.cpp @@ -50,10 +50,16 @@ class IndexFileHandlerTest : public testing::Test { } Result> CreateIndexFileHandler( - const std::string& table_path, const CoreOptions& core_options) const { - SchemaManager schema_manager(core_options.GetFileSystem(), table_path); + const std::string& table_path, const CoreOptions& override_options) const { + SchemaManager schema_manager(override_options.GetFileSystem(), table_path); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_schema, schema_manager.ReadSchema(/*schema_id=*/0)); + std::map options = table_schema->Options(); + for (const auto& [key, value] : override_options.ToMap()) { + options[key] = value; + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(options, override_options.GetFileSystem())); auto schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, core_options.CreateExternalPaths()); @@ -151,8 +157,7 @@ TEST_F(IndexFileHandlerTest, TestScan) { std::string table_path = paimon::test::GetDataDir() + "/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/"; - ASSERT_OK_AND_ASSIGN(CoreOptions core_options, - CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); ASSERT_OK_AND_ASSIGN(std::unique_ptr index_file_handler, CreateIndexFileHandler(table_path, core_options)); @@ -203,8 +208,7 @@ TEST_F(IndexFileHandlerTest, TestScan) { TEST_F(IndexFileHandlerTest, Test09VersionScan) { std::string table_path = paimon::test::GetDataDir() + "/orc/pk_09.db/pk_09/"; - ASSERT_OK_AND_ASSIGN(CoreOptions core_options, - CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); ASSERT_OK_AND_ASSIGN(std::unique_ptr index_file_handler, CreateIndexFileHandler(table_path, core_options)); @@ -254,8 +258,7 @@ TEST_F(IndexFileHandlerTest, Test09VersionScan) { TEST_F(IndexFileHandlerTest, TestScanWithNoIndexManifest) { std::string table_path = paimon::test::GetDataDir() + "/orc/pk_09.db/pk_09/"; - ASSERT_OK_AND_ASSIGN(CoreOptions core_options, - CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); ASSERT_OK_AND_ASSIGN(std::unique_ptr index_file_handler, CreateIndexFileHandler(table_path, core_options)); @@ -292,8 +295,7 @@ TEST_F(IndexFileHandlerTest, TestScanByPartitionBucketAndReadAllDeletionVectors) std::string table_path = paimon::test::GetDataDir() + "/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/"; - ASSERT_OK_AND_ASSIGN(CoreOptions core_options, - CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); ASSERT_OK_AND_ASSIGN(std::unique_ptr index_file_handler, CreateIndexFileHandler(table_path, core_options)); diff --git a/src/paimon/core/index/pk/primary_key_index_definitions.cpp b/src/paimon/core/index/pk/primary_key_index_definitions.cpp index 214338f4b..cc10d56ce 100644 --- a/src/paimon/core/index/pk/primary_key_index_definitions.cpp +++ b/src/paimon/core/index/pk/primary_key_index_definitions.cpp @@ -102,7 +102,10 @@ Result> SortedIndexOptions( std::string option_key = fmt::format("{}{}.{}.index.options", kFieldScopedPrefix, column, option_family); auto iter = table_options.find(option_key); - if (iter == table_options.end() || StringUtils::IsNullOrWhitespaceOnly(iter->second)) { + if (iter == table_options.end()) { + return resolved; + } + if (StringUtils::IsEmptyAfterTrim(iter->second)) { return resolved; } @@ -113,11 +116,13 @@ Result> SortedIndexOptions( fmt::format("{} must be a JSON object of option key-value pairs.", option_key)); } for (auto member = document.MemberBegin(); member != document.MemberEnd(); ++member) { - if (!member->name.IsString() || - StringUtils::IsNullOrWhitespaceOnly(member->name.GetString())) { + if (!member->name.IsString()) { return Status::Invalid(fmt::format("{} contains an empty option key.", option_key)); } std::string key = member->name.GetString(); + if (StringUtils::IsEmptyAfterTrim(key)) { + return Status::Invalid(fmt::format("{} contains an empty option key.", option_key)); + } if (member->value.IsNull()) { return Status::Invalid( fmt::format("{} value for key {} must not be null.", option_key, key)); diff --git a/src/paimon/core/io/infer_shredding_file_writer_test.cpp b/src/paimon/core/io/infer_shredding_file_writer_test.cpp index b0e827561..ff1cc4053 100644 --- a/src/paimon/core/io/infer_shredding_file_writer_test.cpp +++ b/src/paimon/core/io/infer_shredding_file_writer_test.cpp @@ -128,9 +128,7 @@ class InferShreddingFileWriterTest : public ::testing::Test { MakeWriter(int32_t buffer_rows) { std::map option_map = { {"variant.inferShreddingSchema", "true"}, - {"variant.shredding.maxInferBufferRow", std::to_string(buffer_rows)}, - // Keep the manifest format resolvable in test binaries without the avro plugin. - {"manifest.format", "parquet"}}; + {"variant.shredding.maxInferBufferRow", std::to_string(buffer_rows)}}; EXPECT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map)); auto plan_factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); auto create_inner = [this](const std::shared_ptr& converter) diff --git a/src/paimon/core/io/single_file_writer_test.cpp b/src/paimon/core/io/single_file_writer_test.cpp index 78fce54c7..121343bc7 100644 --- a/src/paimon/core/io/single_file_writer_test.cpp +++ b/src/paimon/core/io/single_file_writer_test.cpp @@ -58,9 +58,8 @@ TEST(SingleFileWriterTest, TestSimple) { return Status::OK(); }; SimpleSingleFileWriter writer("zstd", converter); - ASSERT_OK_AND_ASSIGN( - CoreOptions options, - CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); auto file_format = options.GetWriteFileFormat(/*level=*/0); auto file_system = options.GetFileSystem(); ArrowSchema arrow_schema; @@ -90,9 +89,8 @@ TEST(SingleFileWriterTest, TestInvalidConvert) { return Status::Invalid(""); }; SimpleSingleFileWriter writer("zstd", converter); - ASSERT_OK_AND_ASSIGN( - CoreOptions options, - CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); auto file_format = options.GetWriteFileFormat(/*level=*/0); auto file_system = options.GetFileSystem(); ArrowSchema arrow_schema; @@ -119,9 +117,8 @@ TEST(SingleFileWriterTest, CompletionCallbackFailureIsTerminal) { return Status::OK(); }; SimpleSingleFileWriter writer("zstd", converter); - ASSERT_OK_AND_ASSIGN( - CoreOptions options, - CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); auto file_format = options.GetWriteFileFormat(/*level=*/0); auto file_system = options.GetFileSystem(); ArrowSchema arrow_schema; diff --git a/src/paimon/core/manifest/index_manifest_file.cpp b/src/paimon/core/manifest/index_manifest_file.cpp index 648c7302d..ab8ed8c61 100644 --- a/src/paimon/core/manifest/index_manifest_file.cpp +++ b/src/paimon/core/manifest/index_manifest_file.cpp @@ -61,19 +61,21 @@ Result> IndexManifestFile::Create( std::shared_ptr index_manifest_file_factory = path_factory->CreateIndexManifestFileFactory(); - return std::unique_ptr( - new IndexManifestFile(file_system, reader_builder, writer_builder, compression, - index_manifest_file_factory, bucket_mode, options.GetCache(), pool)); + return std::unique_ptr(new IndexManifestFile( + file_system, reader_builder, writer_builder, file_format->Identifier(), compression, + index_manifest_file_factory, bucket_mode, options.GetCache(), pool)); } IndexManifestFile::IndexManifestFile(const std::shared_ptr& file_system, const std::shared_ptr& reader_builder, const std::shared_ptr& writer_builder, + const std::string& file_format_identifier, const std::string& compression, const std::shared_ptr& path_factory, int32_t bucket_mode, const std::shared_ptr& cache, const std::shared_ptr& pool) : ObjectsFile(file_system, reader_builder, writer_builder, + file_format_identifier, std::make_unique(pool), compression, path_factory, cache, pool), bucket_mode_(bucket_mode) {} diff --git a/src/paimon/core/manifest/index_manifest_file.h b/src/paimon/core/manifest/index_manifest_file.h index f6fd46f3f..99122327d 100644 --- a/src/paimon/core/manifest/index_manifest_file.h +++ b/src/paimon/core/manifest/index_manifest_file.h @@ -61,10 +61,9 @@ class IndexManifestFile : public ObjectsFile { IndexManifestFile(const std::shared_ptr& file_system, const std::shared_ptr& reader_builder, const std::shared_ptr& writer_builder, - const std::string& compression, + const std::string& file_format_identifier, const std::string& compression, const std::shared_ptr& path_factory, int32_t bucket_mode, const std::shared_ptr& cache, const std::shared_ptr& pool); - const int32_t bucket_mode_; }; } // namespace paimon diff --git a/src/paimon/core/manifest/index_manifest_file_handler_test.cpp b/src/paimon/core/manifest/index_manifest_file_handler_test.cpp index 8d20bc05e..9661697c0 100644 --- a/src/paimon/core/manifest/index_manifest_file_handler_test.cpp +++ b/src/paimon/core/manifest/index_manifest_file_handler_test.cpp @@ -49,9 +49,10 @@ class IndexManifestFileHandlerTest : public testing::Test { ASSERT_TRUE(dir_ != nullptr); } - Result> CreateManifestFile(int32_t bucket_mode) const { + Result> CreateManifestFile( + int32_t bucket_mode, const std::string& file_format_identifier = "avro") const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_format, - FileFormatFactory::Get("orc", {})); + FileFormatFactory::Get(file_format_identifier, {})); auto schema = arrow::schema({arrow::field("f0", arrow::int32())}); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, @@ -61,8 +62,7 @@ class IndexManifestFileHandlerTest : public testing::Test { /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, /*global_index_external_path=*/std::nullopt, /*index_file_in_data_file_dir=*/false, pool_)); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, - CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}})); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap({})); return IndexManifestFile::Create(dir_->GetFileSystem(), file_format, "zstd", path_factory, bucket_mode, pool_, options); } @@ -219,6 +219,17 @@ TEST_F(IndexManifestFileHandlerTest, GlobalCombinerOverwritesDuplicateAddedEntri ASSERT_EQ(written_entries[0].index_file->RowCount(), 20); } +TEST_F(IndexManifestFileHandlerTest, LegacyManifestFormatIsReadOnly) { + ASSERT_OK_AND_ASSIGN(auto index_manifest_file, + CreateManifestFile(/*bucket_mode=*/4, /*file_format_identifier=*/"orc")); + auto partition = BinaryRow::EmptyRow(); + std::vector entries = { + MakeEntry(FileKind::Add(), partition, /*bucket=*/0, /*index_type=*/"BTREE", "global-0", 1)}; + + ASSERT_NOK_WITH_MSG(index_manifest_file->WriteIndexFiles(std::nullopt, entries), + "manifest.format 'orc' is read-only"); +} + TEST_F(IndexManifestFileHandlerTest, GlobalDvCombinerReplacesIndexFileInBucketUnawareMode) { ASSERT_OK_AND_ASSIGN(auto index_manifest_file, CreateManifestFile(/*bucket_mode=*/-1)); diff --git a/src/paimon/core/manifest/manifest_file.cpp b/src/paimon/core/manifest/manifest_file.cpp index a556b82ea..0fdb78d70 100644 --- a/src/paimon/core/manifest/manifest_file.cpp +++ b/src/paimon/core/manifest/manifest_file.cpp @@ -51,12 +51,14 @@ class MemoryPool; ManifestFile::ManifestFile(const std::shared_ptr& file_system, const std::shared_ptr& reader_builder, const std::shared_ptr& writer_builder, + const std::string& file_format_identifier, const std::string& compression, const std::shared_ptr& path_factory, int64_t target_file_size, const std::shared_ptr& pool, const CoreOptions& options, const std::shared_ptr& partition_type) : ObjectsFile(file_system, reader_builder, writer_builder, + file_format_identifier, std::make_unique(pool), compression, path_factory, options.GetCache(), pool), target_file_size_(target_file_size), @@ -82,9 +84,9 @@ Result> ManifestFile::Create( writer_builder->WithMemoryPool(pool); std::shared_ptr manifest_file_factory = path_factory->CreateManifestFileFactory(); - return std::unique_ptr( - new ManifestFile(file_system, reader_builder, writer_builder, compression, - manifest_file_factory, target_file_size, pool, options, partition_type)); + return std::unique_ptr(new ManifestFile( + file_system, reader_builder, writer_builder, file_format->Identifier(), compression, + manifest_file_factory, target_file_size, pool, options, partition_type)); } Status ManifestFile::ReadBucketEntries(const std::string& file_name, int32_t bucket, @@ -112,6 +114,7 @@ Result> ManifestFile::Write( if (entries.empty()) { return std::vector(); } + PAIMON_RETURN_NOT_OK(ValidateWrite()); auto converter = [this](ManifestEntry entry, ::ArrowArray* dest) -> Status { if (!to_array_converter_) { PAIMON_ASSIGN_OR_RAISE(to_array_converter_, MetaToArrowArrayConverter::Create( diff --git a/src/paimon/core/manifest/manifest_file.h b/src/paimon/core/manifest/manifest_file.h index 0211e14d1..528bb7bf5 100644 --- a/src/paimon/core/manifest/manifest_file.h +++ b/src/paimon/core/manifest/manifest_file.h @@ -70,11 +70,11 @@ class ManifestFile : public ObjectsFile { ManifestFile(const std::shared_ptr& file_system, const std::shared_ptr& reader_builder, const std::shared_ptr& writer_builder, - const std::string& compression, const std::shared_ptr& path_factory, - int64_t target_file_size, const std::shared_ptr& pool, - const CoreOptions& options, const std::shared_ptr& partition_type); + const std::string& file_format_identifier, const std::string& compression, + const std::shared_ptr& path_factory, int64_t target_file_size, + const std::shared_ptr& pool, const CoreOptions& options, + const std::shared_ptr& partition_type); - private: int64_t target_file_size_; CoreOptions options_; std::shared_ptr partition_type_; diff --git a/src/paimon/core/manifest/manifest_file_test.cpp b/src/paimon/core/manifest/manifest_file_test.cpp index df78fc00d..5cd264aa3 100644 --- a/src/paimon/core/manifest/manifest_file_test.cpp +++ b/src/paimon/core/manifest/manifest_file_test.cpp @@ -32,6 +32,7 @@ #include "paimon/core/manifest/file_kind.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/data/decimal.h" @@ -116,8 +117,7 @@ class ManifestFileTest : public testing::Test { /*global_index_external_path=*/std::nullopt, /*index_file_in_data_file_dir=*/false, pool)); EXPECT_OK_AND_ASSIGN(CoreOptions options, - CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, file_format_str}})); + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); EXPECT_OK_AND_ASSIGN( std::unique_ptr manifest_file, ManifestFile::Create(file_system, file_format, "zstd", path_factory, @@ -254,9 +254,8 @@ TEST_F(ManifestFileTest, TestManifestCacheIsDisabledWithoutInjectedCache) { /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, /*global_index_external_path=*/std::nullopt, /*index_file_in_data_file_dir=*/false, pool)); - ASSERT_OK_AND_ASSIGN( - CoreOptions options, - CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); ASSERT_OK_AND_ASSIGN( std::unique_ptr manifest_file, ManifestFile::Create(counting_file_system, file_format, "zstd", path_factory, @@ -296,9 +295,8 @@ TEST_F(ManifestFileTest, TestManifestCacheReusesCachedBytes) { /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, /*global_index_external_path=*/std::nullopt, /*index_file_in_data_file_dir=*/false, pool)); - ASSERT_OK_AND_ASSIGN( - CoreOptions options, - CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); options.WithCache(manifest_cache); ASSERT_OK_AND_ASSIGN( std::unique_ptr manifest_file, @@ -337,9 +335,8 @@ TEST_F(ManifestFileTest, TestReadBucketEntriesMaterializesOnlySelectedBucket) { /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, /*global_index_external_path=*/std::nullopt, /*index_file_in_data_file_dir=*/false, pool)); - ASSERT_OK_AND_ASSIGN( - CoreOptions options, - CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); options.WithCache(manifest_cache); ASSERT_OK_AND_ASSIGN( std::unique_ptr manifest_file, @@ -381,7 +378,7 @@ TEST_F(ManifestFileTest, TestReadBucketEntriesSkipsDeserializingOtherBuckets) { std::shared_ptr file_system = test_dir->GetFileSystem(); ASSERT_OK(file_system->Mkdirs(FileStorePathFactory::ManifestPath(test_dir->Str()))); ASSERT_OK_AND_ASSIGN(std::shared_ptr file_format, - FileFormatFactory::Get("orc", {})); + FileFormatFactory::Get("avro", {})); auto unused_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", arrow::utf8())})); ASSERT_OK_AND_ASSIGN( std::shared_ptr path_factory, @@ -391,9 +388,8 @@ TEST_F(ManifestFileTest, TestReadBucketEntriesSkipsDeserializingOtherBuckets) { /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, /*global_index_external_path=*/std::nullopt, /*index_file_in_data_file_dir=*/false, pool)); - ASSERT_OK_AND_ASSIGN( - CoreOptions options, - CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); ASSERT_OK_AND_ASSIGN( std::unique_ptr manifest_file, ManifestFile::Create(file_system, file_format, "zstd", path_factory, @@ -418,6 +414,38 @@ TEST_F(ManifestFileTest, TestReadBucketEntriesSkipsDeserializingOtherBuckets) { ASSERT_EQ(std::vector({valid_target_bucket}), bucket_entries); } +TEST_F(ManifestFileTest, TestLegacyManifestFormatIsReadOnly) { + auto pool = GetDefaultPool(); + std::vector entries = + ReadManifestEntry("orc", paimon::test::GetDataDir() + "/orc/append_09.db/append_09", + "manifest-3a44a0da-1008-463c-914e-28d271375e24-0", pool); + ASSERT_FALSE(entries.empty()); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr file_format, + FileFormatFactory::Get("orc", {})); + auto unused_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", arrow::utf8())})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr path_factory, + FileStorePathFactory::Create(dir->Str(), unused_schema, /*partition_keys=*/{}, + /*default_part_value=*/"", file_format->Identifier(), + /*data_file_prefix=*/"data-", + /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, + /*global_index_external_path=*/std::nullopt, + /*index_file_in_data_file_dir=*/false, pool)); + ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr manifest_file, + ManifestFile::Create(dir->GetFileSystem(), file_format, "zstd", path_factory, + /*target_file_size=*/1024, pool, options, unused_schema)); + + ASSERT_NOK_WITH_MSG(manifest_file->Write({entries.front()}), + "manifest.format 'orc' is read-only"); + ASSERT_NOK_WITH_MSG(manifest_file->WriteWithoutRolling({entries.front()}), + "manifest.format 'orc' is read-only"); +} + TEST_F(ManifestFileTest, TestWithNullCount) { auto pool = GetDefaultPool(); auto manifest_entries = diff --git a/src/paimon/core/manifest/manifest_list.cpp b/src/paimon/core/manifest/manifest_list.cpp index 887b88455..83025a916 100644 --- a/src/paimon/core/manifest/manifest_list.cpp +++ b/src/paimon/core/manifest/manifest_list.cpp @@ -41,11 +41,13 @@ class MemoryPool; ManifestList::ManifestList(const std::shared_ptr& file_system, const std::shared_ptr& reader_builder, const std::shared_ptr& writer_builder, + const std::string& file_format_identifier, const std::string& compression, const std::shared_ptr& path_factory, const std::shared_ptr& cache, const std::shared_ptr& pool) : ObjectsFile(file_system, reader_builder, writer_builder, + file_format_identifier, std::make_unique(pool), compression, std::move(path_factory), cache, pool) {} @@ -71,8 +73,9 @@ Result> ManifestList::Create( // create manifest list std::shared_ptr manifest_list_path_factory = path_factory->CreateManifestListFactory(); - return std::unique_ptr(new ManifestList( - fs, reader_builder, writer_builder, compression, manifest_list_path_factory, cache, pool)); + return std::unique_ptr(new ManifestList(fs, reader_builder, writer_builder, + file_format->Identifier(), compression, + manifest_list_path_factory, cache, pool)); } Result> ManifestList::Write( diff --git a/src/paimon/core/manifest/manifest_list.h b/src/paimon/core/manifest/manifest_list.h index ff67e8080..6bf64e8ef 100644 --- a/src/paimon/core/manifest/manifest_list.h +++ b/src/paimon/core/manifest/manifest_list.h @@ -123,7 +123,8 @@ class ManifestList : public ObjectsFile { ManifestList(const std::shared_ptr& file_system, const std::shared_ptr& reader_builder, const std::shared_ptr& writer_builder, - const std::string& compression, const std::shared_ptr& path_factory, + const std::string& file_format_identifier, const std::string& compression, + const std::shared_ptr& path_factory, const std::shared_ptr& cache, const std::shared_ptr& pool); }; diff --git a/src/paimon/core/manifest/manifest_list_test.cpp b/src/paimon/core/manifest/manifest_list_test.cpp index 23949bfdd..97b420cf5 100644 --- a/src/paimon/core/manifest/manifest_list_test.cpp +++ b/src/paimon/core/manifest/manifest_list_test.cpp @@ -142,11 +142,20 @@ TEST_F(ManifestListTest, TestEmptyManifestList) { ASSERT_EQ(manifest_file_metas.size(), 0); } -TEST_F(ManifestListTest, TestReadChangelogManifests) { +TEST_F(ManifestListTest, TestLegacyManifestFormatIsReadOnly) { auto pool = GetDefaultPool(); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto manifest_list = CreateManifestList("orc", dir->Str(), pool); + + ASSERT_NOK_WITH_MSG(manifest_list->Write({}), "manifest.format 'orc' is read-only"); +} + +TEST_F(ManifestListTest, TestReadChangelogManifests) { + auto pool = GetDefaultPool(); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto manifest_list = CreateManifestList("avro", dir->Str(), pool); ManifestFileMeta expected_meta( "changelog-manifest", /*file_size=*/100, /*num_added_files=*/1, /*num_deleted_files=*/0, SimpleStats::EmptyStats(), /*schema_id=*/0, @@ -224,11 +233,11 @@ TEST_F(ManifestListTest, TestReadWithMinAndMaxRowId) { // test write meta auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - auto manifest_list = CreateManifestList("orc", dir->Str(), pool); + auto manifest_list = CreateManifestList("avro", dir->Str(), pool); std::pair file_meta; ASSERT_OK_AND_ASSIGN(file_meta, manifest_list->Write({expected_meta})); // test read meta from C++ - auto manifest_file_metas2 = ReadManifestFileMeta("orc", dir->Str(), file_meta.first, pool); + auto manifest_file_metas2 = ReadManifestFileMeta("avro", dir->Str(), file_meta.first, pool); ASSERT_EQ(manifest_file_metas2.size(), 1); ASSERT_EQ(manifest_file_metas2, expected_manifest_file_metas); } diff --git a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg.h b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg.h index a6d7ed5c4..a6dc04b7c 100644 --- a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg.h +++ b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg.h @@ -20,10 +20,12 @@ #include #include +#include #include #include #include "paimon/common/data/data_define.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/mergetree/compact/aggregate/field_aggregator.h" @@ -60,15 +62,18 @@ class FieldListaggAgg : public FieldAggregator { const VariantType& input_field) override { bool accumulator_null = DataDefine::IsVariantNull(accumulator); bool input_null = DataDefine::IsVariantNull(input_field); - if (accumulator_null || input_null) { - return accumulator_null ? input_field : accumulator; + if (input_null) { + return accumulator; } - std::string_view acc_str = DataDefine::GetStringView(accumulator); std::string_view in_str = DataDefine::GetStringView(input_field); - if (in_str.empty()) { + if (StringUtils::IsBlank(in_str)) { return accumulator; } - if (acc_str.empty()) { + if (accumulator_null) { + return input_field; + } + std::string_view acc_str = DataDefine::GetStringView(accumulator); + if (StringUtils::IsBlank(acc_str)) { return input_field; } @@ -95,9 +100,7 @@ class FieldListaggAgg : public FieldAggregator { size_t pos = remaining.find(delimiter_); std::string_view token = (pos == std::string_view::npos) ? remaining : remaining.substr(0, pos); - if (!token.empty()) { - seen.insert(token); - } + seen.insert(token); if (pos == std::string_view::npos) { break; } @@ -113,7 +116,7 @@ class FieldListaggAgg : public FieldAggregator { size_t pos = remaining.find(delimiter_); std::string_view token = (pos == std::string_view::npos) ? remaining : remaining.substr(0, pos); - if (!token.empty() && seen.insert(token).second) { + if (!StringUtils::IsBlank(token) && seen.insert(token).second) { result.append(delimiter_); result.append(token); } diff --git a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp index 3578cdaac..1efac34ed 100644 --- a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp +++ b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include "arrow/type_fwd.h" #include "gtest/gtest.h" @@ -88,13 +89,41 @@ TEST_F(FieldListaggAggTest, TestEmptyString) { auto ret = agg->Agg(std::string_view(""), std::string_view("world")).value(); ASSERT_EQ(DataDefine::GetVariantValue(ret), "world"); } - // both empty -> return input (which is empty) + // blank input -> return accumulator (which is empty) { auto ret = agg->Agg(std::string_view(""), std::string_view("")).value(); ASSERT_EQ(DataDefine::GetVariantValue(ret), ""); } } +TEST_F(FieldListaggAggTest, TestBlankStrings) { + ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg()); + + const std::vector blank_strings = {"", + " ", + " ", + "\t", + "\n", + "\r", + "\r\n", + " \t\n\r ", + u8"\u3000", + u8"\u2000", + u8" \t\u3000\u2000\n"}; + for (const std::string& blank : blank_strings) { + auto ret = agg->Agg(std::string_view("user1"), std::string_view(blank)).value(); + ASSERT_EQ(DataDefine::GetVariantValue(ret), "user1"); + } + + // A blank accumulator must not add a leading delimiter. + auto ret = agg->Agg(std::string_view(u8"\u3000\t"), std::string_view("user1")).value(); + ASSERT_EQ(DataDefine::GetVariantValue(ret), "user1"); + + // A blank input must not turn a null accumulator into a non-null value. + ret = agg->Agg(NullType(), std::string_view(u8" \t\u3000")).value(); + ASSERT_TRUE(DataDefine::IsVariantNull(ret)); +} + TEST_F(FieldListaggAggTest, TestMultipleAccumulation) { ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg()); @@ -113,6 +142,15 @@ TEST_F(FieldListaggAggTest, TestDistinct) { ASSERT_EQ(DataDefine::GetVariantValue(ret), "a;b;c"); } +TEST_F(FieldListaggAggTest, TestDistinctIgnoresBlankTokens) { + ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg(",", true)); + + auto ret = + agg->Agg(std::string_view("user1"), std::string_view(u8" ,user2,\t,\u3000,user1,\u2000")) + .value(); + ASSERT_EQ(DataDefine::GetVariantValue(ret), "user1,user2"); +} + TEST_F(FieldListaggAggTest, TestDistinctNoDuplicates) { ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg(" ", true)); diff --git a/src/paimon/core/operation/append_only_file_store_scan_test.cpp b/src/paimon/core/operation/append_only_file_store_scan_test.cpp index d72dcaf0d..fd16d47fe 100644 --- a/src/paimon/core/operation/append_only_file_store_scan_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_scan_test.cpp @@ -101,7 +101,6 @@ TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) { std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::MANIFEST_FORMAT, "orc") .AddOption(Options::SCAN_SNAPSHOT_ID, "5"); ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); @@ -148,7 +147,6 @@ TEST(AppendOnlyFileStoreScanTest, TestScanDurationMetric) { std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::MANIFEST_FORMAT, "orc") .AddOption(Options::SCAN_SNAPSHOT_ID, "5"); ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); @@ -204,7 +202,6 @@ std::shared_ptr BuildScan(const std::string& table_path, bool manifest_entry_lazy_decode_enabled = true) { ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::MANIFEST_FORMAT, "orc") .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "8") .AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, manifest_entry_lazy_decode_enabled ? "true" : "false") diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index 85d5c565d..f91e0f60d 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -78,7 +78,7 @@ namespace { // the table does not set them - ParquetWriterBuilder for the first, ParquetFileBatchReader for the // second. constexpr char kParquetFormat[] = "parquet"; -constexpr char kParquetEnableDictionary[] = "parquet.enable-dictionary"; +constexpr char kParquetEnableDictionary[] = "parquet.enable.dictionary"; constexpr char kParquetReadEnableDictionaryPassthrough[] = "parquet.read.enable-dictionary-passthrough"; constexpr bool kDefaultParquetEnableDictionary = true; @@ -328,7 +328,7 @@ Result> AppendOnlyFileStoreWrite::GetDictionaryPassth options_.ToMap(), kParquetEnableDictionary, kDefaultParquetEnableDictionary)); if (!enable_dictionary) { - return std::optional("parquet.enable-dictionary is false"); + return std::optional("parquet.enable.dictionary is false"); } if (plan_factory != nullptr) { return std::optional("the table is written through a shredding writer"); diff --git a/src/paimon/core/operation/append_only_file_store_write.h b/src/paimon/core/operation/append_only_file_store_write.h index 132118a72..d030a07cc 100644 --- a/src/paimon/core/operation/append_only_file_store_write.h +++ b/src/paimon/core/operation/append_only_file_store_write.h @@ -146,7 +146,7 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite { /// The veto stands unless all three of: /// /// - a Parquet output file, since no other writer takes a dictionary-encoded batch; - /// - `parquet.enable-dictionary`, or the writer densifies what the reader just handed it and + /// - `parquet.enable.dictionary`, or the writer densifies what the reader just handed it and /// the encoding is carried across the rewrite for nothing; /// - a rewrite that stays a passthrough, since a shredding writer reshapes each batch against /// a fixed physical schema and cannot take a dictionary-encoded one. diff --git a/src/paimon/core/operation/commit/commit_scanner_test.cpp b/src/paimon/core/operation/commit/commit_scanner_test.cpp index 3d5c62707..1e02849d0 100644 --- a/src/paimon/core/operation/commit/commit_scanner_test.cpp +++ b/src/paimon/core/operation/commit/commit_scanner_test.cpp @@ -119,7 +119,7 @@ class CommitScannerTest : public testing::Test { Result> CreateIndexManifestFile() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_format, - FileFormatFactory::Get("orc", {})); + FileFormatFactory::Get("avro", {})); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( diff --git a/src/paimon/core/operation/expire_snapshots_test.cpp b/src/paimon/core/operation/expire_snapshots_test.cpp index ffdd30feb..f3472d5b2 100644 --- a/src/paimon/core/operation/expire_snapshots_test.cpp +++ b/src/paimon/core/operation/expire_snapshots_test.cpp @@ -75,14 +75,15 @@ class ExpireSnapshotsTest : public testing::Test { test_data_path_ = "tmp"; path_factory_ = CreateFactory(test_data_path_); + std::shared_ptr manifest_format = options.GetManifestFormat(); ASSERT_OK_AND_ASSIGN( manifest_list_, - ManifestList::Create(fs_, options.GetManifestFormat(), options.GetManifestCompression(), + ManifestList::Create(fs_, manifest_format, options.GetManifestCompression(), path_factory_, options.GetCache(), mem_pool_)); ASSERT_OK_AND_ASSIGN( manifest_file_, - ManifestFile::Create(fs_, options.GetManifestFormat(), options.GetManifestCompression(), + ManifestFile::Create(fs_, manifest_format, options.GetManifestCompression(), path_factory_, options.GetManifestTargetFileSize(), mem_pool_, options, partition_schema_)); } @@ -113,7 +114,6 @@ class ExpireSnapshotsTest : public testing::Test { std::shared_ptr CreateFactory(const std::string& root) const { std::map raw_options; raw_options[Options::FILE_FORMAT] = "orc"; - raw_options[Options::MANIFEST_FORMAT] = "orc"; raw_options[Options::FILE_SYSTEM] = "local"; EXPECT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(raw_options)); EXPECT_OK_AND_ASSIGN(std::vector external_paths, diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index ad0942ade..6d9005603 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -127,7 +127,6 @@ Result> FileStoreCommit::Create( assert(options.GetFileSystem()); assert(options.GetFileFormat()); PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); - PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path)); if (is_object_store && !ctx->UseRESTCatalogCommit() && opts.find("enable-object-store-commit-in-inte-test") == opts.end()) { diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index b1b8677f8..a5079d7da 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -379,8 +379,7 @@ class FileStoreCommitImplTest : public testing::Test { TEST_F(FileStoreCommitImplTest, TestCommit) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -408,8 +407,7 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { TimezoneGuard guard("Asia/Shanghai"); CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .UseRESTCatalogCommit(true) .Finish()); @@ -466,8 +464,7 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { TEST_F(FileStoreCommitImplTest, TestSnapshotSequenceMaxPropertyMergedOnCommit) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") .Finish()); @@ -514,8 +511,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes) FileSystemFactory::Get("gmock_fs", table_path, {})); CommitContextBuilder context_builder(table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::COMMIT_MAX_RETRIES, "10") .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") @@ -548,7 +544,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes) } TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { - std::string test_data_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; + std::string test_data_path = paimon::test::GetDataDir() + "/parquet/append_09.db/append_09/"; auto dir = UniqueTestDirectory::Create(); std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); @@ -556,8 +552,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { FileSystemFactory::Get("gmock_fs", table_path, {})); CommitContextBuilder context_builder(table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") .WithFileSystem(fs) @@ -591,7 +586,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + - "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + "/parquet/append_09.db/append_09/commit_messages/commit_messages-01", /*version=*/3); ASSERT_GT(msgs.size(), 0); ASSERT_OK(commit->Commit(msgs)); @@ -606,7 +601,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { } TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActuallySucceed) { - std::string test_data_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; + std::string test_data_path = paimon::test::GetDataDir() + "/parquet/append_09.db/append_09/"; auto dir = UniqueTestDirectory::Create(); std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); @@ -614,8 +609,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua FileSystemFactory::Get("gmock_fs", table_path, {})); CommitContextBuilder context_builder(table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .WithFileSystem(fs) .Finish()); @@ -632,7 +626,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + - "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + "/parquet/append_09.db/append_09/commit_messages/commit_messages-01", /*version=*/3); ASSERT_GT(msgs.size(), 0); ASSERT_NOK(commit->Commit(msgs, /*commit_identifier=*/1)); @@ -642,8 +636,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua CommitContextBuilder context_builder_2(table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context_2, - context_builder_2.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder_2.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .WithFileSystem(fs) .Finish()); @@ -657,7 +650,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua })); std::vector> msgs_2 = GetCommitMessages(paimon::test::GetDataDir() + - "/orc/append_09.db/append_09/commit_messages/commit_messages-02", + "/parquet/append_09.db/append_09/commit_messages/commit_messages-02", /*version=*/3); ASSERT_OK(commit_2->Commit(msgs_2, /*commit_identifier=*/2)); ASSERT_OK_AND_ASSIGN(exist, file_system_->Exists(new_snapshot_7)); @@ -667,8 +660,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "5kb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "5kb") .AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -731,8 +723,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -803,8 +794,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -861,8 +851,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNoLatestSnapshotReturnsError) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -873,8 +862,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNoLatestSnapshotReturnsErr TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestTargetNotExistReturnsError) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -908,8 +896,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestDeletionVectorOnlyChange) CommitContextBuilder context_builder(dv_table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -978,8 +965,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF FileSystemFactory::Get("gmock_fs", table_path_, {})); CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .WithFileSystem(fs) .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -1021,8 +1007,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -1067,8 +1052,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -1125,8 +1109,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1243,8 +1226,7 @@ TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) .Finish()); @@ -1262,8 +1244,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) .Finish()); @@ -1293,8 +1274,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) .Finish()); @@ -1328,8 +1308,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) .Finish()); @@ -1356,8 +1335,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) .Finish()); @@ -1408,8 +1386,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { TEST_F(FileStoreCommitImplTest, TestDropPartitionAndExpireSnapshot) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") @@ -1456,8 +1433,7 @@ TEST_F(FileStoreCommitImplTest, TestDropPartitionAndExpireSnapshot) { TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") @@ -1504,8 +1480,7 @@ TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { TEST_F(FileStoreCommitImplTest, TestTruncateTable) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) .Finish()); @@ -1538,9 +1513,7 @@ TEST_F(FileStoreCommitImplTest, TestTruncateTable) { TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -1587,9 +1560,7 @@ TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { TEST_F(FileStoreCommitImplTest, AbortIgnoresMissingFilesAndFailsForNonImplMessage) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -1613,9 +1584,7 @@ TEST_F(FileStoreCommitImplTest, AbortIgnoresDeleteFailures) { // propagate the failure. CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -1654,8 +1623,7 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { // materialized. CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) .Finish()); @@ -1680,8 +1648,7 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) .Finish()); @@ -1701,8 +1668,7 @@ TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { TEST_F(FileStoreCommitImplTest, TestCollectChanges) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::BUCKET, "10") .Finish()); @@ -1745,8 +1711,7 @@ TEST_F(FileStoreCommitImplTest, TestCollectChanges) { TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1777,8 +1742,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1818,8 +1782,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1850,8 +1813,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) TEST_F(FileStoreCommitImplTest, FilterAndCommit) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1898,8 +1860,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommit) { TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1921,8 +1882,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1956,8 +1916,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -1990,8 +1949,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2030,8 +1988,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2061,8 +2018,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::ROW_TRACKING_ENABLED, "true") .AddOption(Options::DATA_EVOLUTION_ENABLED, "true") @@ -2094,8 +2050,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) .Finish()); @@ -2127,8 +2082,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2165,8 +2119,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .IgnoreEmptyCommit(true) .Finish()); @@ -2205,8 +2158,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2239,8 +2191,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2295,8 +2246,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2334,8 +2284,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2369,8 +2318,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2400,8 +2348,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::COMMIT_DISCARD_DUPLICATE_FILES, "true") .Finish()); @@ -2433,8 +2380,7 @@ TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -2466,8 +2412,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); CommitContextBuilder context_builder2(tmp_table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context2, - context_builder2.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder2.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); auto commit2 = FileStoreCommit::Create(std::move(commit_context2)); @@ -2495,9 +2440,7 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { // REST commit with local path should work without the object store flag CommitContextBuilder builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto ctx, - builder.AddOption(Options::MANIFEST_FORMAT, "orc").UseRESTCatalogCommit(true).Finish()); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.UseRESTCatalogCommit(true).Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx))); auto msgs = @@ -2563,8 +2506,7 @@ TEST_F(FileStoreCommitImplTest, ValidateCommitOptionsAllowsManifestDeleteFileDro TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::MANIFEST_DELETE_FILE_DROP_STATS, "true") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2593,8 +2535,7 @@ TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::MANIFEST_DELETE_FILE_DROP_STATS, "true") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); @@ -2669,9 +2610,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { TEST_F(FileStoreCommitImplTest, DropPartitionWithEmptyPartitionsFails) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); ASSERT_NOK_WITH_MSG(commit->DropPartition({}, /*commit_identifier=*/1), "partitions list cannot be empty"); @@ -2680,8 +2619,7 @@ TEST_F(FileStoreCommitImplTest, DropPartitionWithEmptyPartitionsFails) { TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -2720,9 +2658,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -2736,9 +2672,7 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -2765,8 +2699,7 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::DYNAMIC_PARTITION_OVERWRITE, "false") .Finish()); @@ -2794,8 +2727,7 @@ TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) TEST_F(FileStoreCommitImplTest, OverwriteWithChangelogFilesLogsWarning) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -2840,8 +2772,7 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) CommitContextBuilder builder(pk_table_path, "test_user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::FILE_SYSTEM, "local") + builder.AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::OVERWRITE_UPGRADE, "true") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -2891,8 +2822,7 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) TEST_F(FileStoreCommitImplTest, CommitWithAppendCommitCheckConflict) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AppendCommitCheckConflict(true) .Finish()); @@ -2915,8 +2845,7 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -2931,8 +2860,7 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { // the max sequence number is recomputed by scanning the base manifests. CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") .Finish()); @@ -2961,8 +2889,7 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -2988,8 +2915,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenOverwriteUpgradeDisabled) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::FILE_SYSTEM, "local") + context_builder.AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::OVERWRITE_UPGRADE, "false") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -3022,8 +2948,7 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { CommitContextBuilder builder(pk_table_path, "test_user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::FILE_SYSTEM, "local") + builder.AddOption(Options::FILE_SYSTEM, "local") .AddOption(Options::OVERWRITE_UPGRADE, "true") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -3043,8 +2968,7 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelta) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -3067,9 +2991,7 @@ TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelt TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3082,9 +3004,7 @@ TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests TEST_F(FileStoreCommitImplTest, RowIdCheckConflictSetsCheckSnapshotAndReturnsSelf) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index b43e32158..07d55eb82 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -77,8 +77,7 @@ TEST(FileStoreCommitTest, TestCreate) { CommitContextBuilder context_builder(table_path, "commit_user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); @@ -115,8 +114,7 @@ TEST(FileStoreCommitTest, TestAppendDvIndexShouldUseOverwriteCommitKind) { CommitContextBuilder context_builder(table_path, "commit_user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::FILE_SYSTEM, "local") .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); diff --git a/src/paimon/core/operation/key_value_file_store_scan_test.cpp b/src/paimon/core/operation/key_value_file_store_scan_test.cpp index f9fd2ea2f..f6c13cb3e 100644 --- a/src/paimon/core/operation/key_value_file_store_scan_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_scan_test.cpp @@ -70,12 +70,13 @@ class KeyValueFileStoreScanTest : public testing::Test { Result> CreateFileStoreScan( const std::string& table_path, const std::shared_ptr& scan_filter, int32_t table_schema_id, int32_t snapshot_id) const { - std::map options_map = {{Options::MANIFEST_FORMAT, "orc"}}; - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options_map)); - auto fs = core_options.GetFileSystem(); + PAIMON_ASSIGN_OR_RAISE(CoreOptions bootstrap_options, CoreOptions::FromMap({})); + auto fs = bootstrap_options.GetFileSystem(); auto schema_manager = std::make_shared(fs, table_path); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_schema, schema_manager->ReadSchema(table_schema_id)); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(table_schema->Options())); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp index 88b848eea..90cd793bb 100644 --- a/src/paimon/core/operation/key_value_file_store_write_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp @@ -743,7 +743,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) { std::map options = { {Options::BUCKET, "1"}, {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::MANIFEST_DELETE_FILE_DROP_STATS, "true"}}; ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir->Str(), options)); ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); diff --git a/src/paimon/core/operation/manifest_file_merger_test.cpp b/src/paimon/core/operation/manifest_file_merger_test.cpp index d2617a654..89a78a5a2 100644 --- a/src/paimon/core/operation/manifest_file_merger_test.cpp +++ b/src/paimon/core/operation/manifest_file_merger_test.cpp @@ -128,9 +128,8 @@ class ManifestFileMergerTest : public testing::Test { private: void CreateManifestFile(const std::string& path_str) { file_system_ = std::make_shared(); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr file_format, - FileFormatFactory::Get("parquet", std::map())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr file_format, + FileFormatFactory::Get("avro", std::map())); auto schema = arrow::schema(arrow::FieldVector({arrow::field("f0", partition_type_)})); ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); ASSERT_OK_AND_ASSIGN(std::vector external_paths, diff --git a/src/paimon/core/table/sink/commit_message_test.cpp b/src/paimon/core/table/sink/commit_message_test.cpp index f372361cb..8fd31c4c8 100644 --- a/src/paimon/core/table/sink/commit_message_test.cpp +++ b/src/paimon/core/table/sink/commit_message_test.cpp @@ -1222,7 +1222,6 @@ TEST(CommitMessageTest, TestSerialize) { ASSERT_TRUE(dir); std::map options = {{Options::FILE_FORMAT, "mock_format"}, - {Options::MANIFEST_FORMAT, "mock_format"}, {Options::TARGET_FILE_SIZE, "1024"}}; ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir->Str(), options)); ASSERT_OK(catalog->CreateDatabase("foo", options, /*ignore_if_exists=*/false)); @@ -1235,7 +1234,6 @@ TEST(CommitMessageTest, TestSerialize) { "commit_user1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.AddOption(Options::FILE_FORMAT, "mock_format") - .AddOption(Options::MANIFEST_FORMAT, "mock_format") .AddOption(Options::TARGET_FILE_SIZE, "1024") .Finish()); diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 6523588e8..68ec9ac05 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -211,7 +211,6 @@ Result AggregateFileStats(const std::shared_ptr core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), pool)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_list, ManifestList::Create(fs, core_options.GetManifestFormat(), core_options.GetManifestCompression(), path_factory, diff --git a/src/paimon/core/utils/file_store_path_factory_test.cpp b/src/paimon/core/utils/file_store_path_factory_test.cpp index 41a9635e9..c78db99df 100644 --- a/src/paimon/core/utils/file_store_path_factory_test.cpp +++ b/src/paimon/core/utils/file_store_path_factory_test.cpp @@ -65,7 +65,6 @@ class FileStorePathFactoryTest : public ::testing::Test { std::map raw_options; raw_options[Options::FILE_FORMAT] = "mock_format"; - raw_options[Options::MANIFEST_FORMAT] = "mock_format"; raw_options[Options::FILE_SYSTEM] = "local"; EXPECT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(raw_options)); EXPECT_OK_AND_ASSIGN(std::vector external_paths, diff --git a/src/paimon/core/utils/objects_file.h b/src/paimon/core/utils/objects_file.h index b79450433..56ba6e319 100644 --- a/src/paimon/core/utils/objects_file.h +++ b/src/paimon/core/utils/objects_file.h @@ -56,6 +56,7 @@ class ObjectsFile { ObjectsFile(const std::shared_ptr& file_system, const std::shared_ptr& reader_builder, const std::shared_ptr& writer_builder, + const std::string& file_format_identifier, std::unique_ptr>&& serializer, const std::string& compression, const std::shared_ptr& path_factory, const std::shared_ptr& cache, const std::shared_ptr& pool); @@ -78,6 +79,14 @@ class ObjectsFile { Result> WriteWithoutRolling(const std::vector& records); protected: + Status ValidateWrite() const { + if (file_format_identifier_ != "avro") { + return Status::Invalid("manifest.format '", file_format_identifier_, + "' is read-only; only 'avro' can be used for writing manifests"); + } + return Status::OK(); + } + Status ReadArrowBatches( const std::string& file_name, const std::function&)>& consumer) const; @@ -92,6 +101,7 @@ class ObjectsFile { private: std::shared_ptr file_system_; std::shared_ptr reader_builder_; + const std::string file_format_identifier_; std::string compression_; std::shared_ptr cache_; @@ -102,6 +112,7 @@ template ObjectsFile::ObjectsFile(const std::shared_ptr& file_system, const std::shared_ptr& reader_builder, const std::shared_ptr& writer_builder, + const std::string& file_format_identifier, std::unique_ptr>&& serializer, const std::string& compression, const std::shared_ptr& path_factory, @@ -114,6 +125,7 @@ ObjectsFile::ObjectsFile(const std::shared_ptr& file_system, writer_builder_(std::move(writer_builder)), file_system_(file_system), reader_builder_(std::move(reader_builder)), + file_format_identifier_(file_format_identifier), compression_(compression), cache_(cache) {} @@ -231,6 +243,7 @@ Result ObjectsFile::ReadFileSegment(const std::string& file_pa template Result> ObjectsFile::WriteWithoutRolling( const std::vector& records) { + PAIMON_RETURN_NOT_OK(ValidateWrite()); std::string file_path = path_factory_->NewPath(); std::vector rows; rows.reserve(records.size()); diff --git a/src/paimon/format/avro/avro_stats_extractor_test.cpp b/src/paimon/format/avro/avro_stats_extractor_test.cpp index c4d554d67..76f19ff1a 100644 --- a/src/paimon/format/avro/avro_stats_extractor_test.cpp +++ b/src/paimon/format/avro/avro_stats_extractor_test.cpp @@ -74,8 +74,7 @@ class AvroStatsExtractorTest : public ::testing::Test { } private: - std::map options_ = {{Options::FILE_FORMAT, "avro"}, - {Options::MANIFEST_FORMAT, "avro"}}; + std::map options_ = {{Options::FILE_FORMAT, "avro"}}; }; TEST_F(AvroStatsExtractorTest, TestPrimitiveStatsExtractor) { diff --git a/src/paimon/format/parquet/CMakeLists.txt b/src/paimon/format/parquet/CMakeLists.txt index 968546581..60d898511 100644 --- a/src/paimon/format/parquet/CMakeLists.txt +++ b/src/paimon/format/parquet/CMakeLists.txt @@ -69,6 +69,7 @@ if(PAIMON_BUILD_TESTS) paimon_shared test_utils_static ${PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS} + ${PAIMON_AVRO_FILE_FORMAT_STATIC_LINK_LIBS} ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} parquet ${GTEST_LINK_TOOLCHAIN}) diff --git a/src/paimon/format/parquet/parquet_format_defs.h b/src/paimon/format/parquet/parquet_format_defs.h index d22506c45..6bc811e5b 100644 --- a/src/paimon/format/parquet/parquet_format_defs.h +++ b/src/paimon/format/parquet/parquet_format_defs.h @@ -41,7 +41,7 @@ namespace paimon::parquet { static inline const char PARQUET_BLOCK_SIZE[] = "parquet.block.size"; static inline const char PARQUET_PAGE_SIZE[] = "parquet.page.size"; static inline const char PARQUET_DICTIONARY_PAGE_SIZE[] = "parquet.dictionary.page.size"; -static inline const char PARQUET_ENABLE_DICTIONARY[] = "parquet.enable-dictionary"; +static inline const char PARQUET_ENABLE_DICTIONARY[] = "parquet.enable.dictionary"; static inline const char PARQUET_WRITER_VERSION[] = "parquet.writer.version"; static inline const char PARQUET_WRITE_MAX_ROW_GROUP_LENGTH[] = "parquet.write.max-row-group-length"; diff --git a/src/paimon/format/parquet/parquet_format_writer_test.cpp b/src/paimon/format/parquet/parquet_format_writer_test.cpp index 3d1c00872..9ba276466 100644 --- a/src/paimon/format/parquet/parquet_format_writer_test.cpp +++ b/src/paimon/format/parquet/parquet_format_writer_test.cpp @@ -471,12 +471,10 @@ TEST_F(ParquetFormatWriterTest, TestGetEstimateLength) { TEST_F(ParquetFormatWriterTest, TestMemoryControl) { auto run = [&](bool all_null_value, uint64_t max_memory_use) { - ASSERT_OK_AND_ASSIGN( - std::unique_ptr file_format, - FileFormatFactory::Get( - "parquet", {{Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "parquet"}, - {"parquet.writer.max.memory.use", std::to_string(max_memory_use)}})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr file_format, + FileFormatFactory::Get("parquet", {{Options::FILE_FORMAT, "parquet"}, + {"parquet.writer.max.memory.use", + std::to_string(max_memory_use)}})); std::shared_ptr pool = GetMemoryPool(); auto schema_pair = PrepareArrowSchema(); @@ -527,7 +525,6 @@ TEST_F(ParquetFormatWriterTest, TestMemoryControlForCheckRowGroupCount) { ASSERT_OK_AND_ASSIGN( std::unique_ptr file_format, FileFormatFactory::Get("parquet", {{Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "parquet"}, {"parquet.writer.max.memory.use", "1"}})); auto schema_pair = PrepareArrowSchema(); diff --git a/src/paimon/format/parquet/parquet_writer_builder_test.cpp b/src/paimon/format/parquet/parquet_writer_builder_test.cpp index 3a95a8de3..796e002e8 100644 --- a/src/paimon/format/parquet/parquet_writer_builder_test.cpp +++ b/src/paimon/format/parquet/parquet_writer_builder_test.cpp @@ -41,7 +41,6 @@ TEST(ParquetWriterBuilderTest, DefaultPrepareWriterProperties) { std::shared_ptr schema = arrow::schema(fields); std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("zstd")); ASSERT_EQ(::parquet::ParquetVersion::PARQUET_2_6, properties->version()); @@ -64,7 +63,6 @@ TEST(ParquetWriterBuilderTest, PrepareWriterProperties) { options[PARQUET_COMPRESSION_CODEC_ZSTD_LEVEL] = "3"; options[PARQUET_BLOCK_SIZE] = "2048"; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024 * 1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("zstd")); // Version numbering differs between C++ and Java Parquet implementations. Java's PARQUET_2_0 @@ -77,12 +75,24 @@ TEST(ParquetWriterBuilderTest, PrepareWriterProperties) { ASSERT_EQ(3, properties->default_column_properties().compression_level()); } +TEST(ParquetWriterBuilderTest, PrepareWriterPropertiesWithDictionaryDisabled) { + arrow::FieldVector fields; + std::shared_ptr schema = arrow::schema(fields); + std::map options = { + {Options::FILE_FORMAT, "parquet"}, + {"parquet.enable.dictionary", "false"}, + }; + ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); + ASSERT_OK_AND_ASSIGN(std::shared_ptr<::parquet::WriterProperties> properties, + builder.PrepareWriterProperties("zstd")); + ASSERT_FALSE(properties->default_column_properties().dictionary_enabled()); +} + TEST(ParquetWriterBuilderTest, PrepareWriterPropertiesWithFileBlockSize) { arrow::FieldVector fields; std::shared_ptr schema = arrow::schema(fields); std::map options = { {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "parquet"}, {PARQUET_BLOCK_SIZE, "2048"}, {Options::FILE_BLOCK_SIZE, "8 KB"}, }; @@ -101,7 +111,6 @@ TEST(ParquetWriterBuilderTest, PrepareWriterPropertiesWithZstdLevelPriority) { options[PARQUET_COMPRESSION_CODEC_ZSTD_LEVEL] = "3"; options[PARQUET_WRITER_VERSION] = "PARQUET_1_0"; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024 * 1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("zstd")); ASSERT_EQ(3, properties->default_column_properties().compression_level()); @@ -111,7 +120,6 @@ TEST(ParquetWriterBuilderTest, PrepareWriterPropertiesWithZstdLevelPriority) { std::map options; options[Options::FILE_COMPRESSION_ZSTD_LEVEL] = "4"; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024 * 1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("zstd")); ASSERT_EQ(4, properties->default_column_properties().compression_level()); @@ -124,7 +132,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) { { std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("lz4")); ASSERT_EQ(properties->default_column_properties().compression(), @@ -133,7 +140,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) { { std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("lz4_raW")); ASSERT_EQ(properties->default_column_properties().compression(), arrow::Compression::LZ4); @@ -141,7 +147,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) { { std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("zstd")); ASSERT_EQ(properties->default_column_properties().compression(), arrow::Compression::ZSTD); @@ -149,7 +154,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) { { std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("LZ4")); ASSERT_EQ(properties->default_column_properties().compression(), @@ -158,7 +162,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) { { std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("ZSTd")); ASSERT_EQ(properties->default_column_properties().compression(), arrow::Compression::ZSTD); @@ -166,7 +169,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) { { std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; options[PARQUET_COMPRESSION_CODEC_ZLIB_LEVEL] = "3"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("gzip")); @@ -176,7 +178,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) { { std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("snappy")); ASSERT_EQ(properties->default_column_properties().compression(), @@ -185,7 +186,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) { { std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("lzo")); ASSERT_EQ(properties->default_column_properties().compression(), arrow::Compression::LZO); @@ -193,14 +193,12 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) { { std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_NOK(builder.PrepareWriterProperties("unknown")); } { std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("uncompressed")); ASSERT_EQ(properties->default_column_properties().compression(), @@ -209,7 +207,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) { { std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("lz4_hadoop")); ASSERT_EQ(properties->default_column_properties().compression(), @@ -218,7 +215,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) { { std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; options[PARQUET_COMPRESSION_CODEC_BROTLI_LEVEL] = "2"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("brotli")); @@ -229,7 +225,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) { { std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("None")); ASSERT_EQ(properties->default_column_properties().compression(), @@ -242,7 +237,6 @@ TEST(ParquetWriterBuilderTest, TestInvalidWriterVersion) { std::shared_ptr schema = arrow::schema(fields); std::map options; options[Options::FILE_FORMAT] = "parquet"; - options[Options::MANIFEST_FORMAT] = "parquet"; options[PARQUET_WRITER_VERSION] = "PARQUET_3_0"; ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options); ASSERT_NOK_WITH_MSG(builder.PrepareWriterProperties("zstd"), diff --git a/src/paimon/format/parquet/variant_parquet_test.cpp b/src/paimon/format/parquet/variant_parquet_test.cpp index a1481dcfb..46a63e38d 100644 --- a/src/paimon/format/parquet/variant_parquet_test.cpp +++ b/src/paimon/format/parquet/variant_parquet_test.cpp @@ -711,7 +711,6 @@ TEST_F(VariantParquetTest, AdaptiveInferenceUntypedPhysicalWriteAndReadRoundTrip auto logical = BuildArray(jsons); ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({ - {Options::MANIFEST_FORMAT, "parquet"}, {Options::VARIANT_INFER_SHREDDING_SCHEMA, "true"}, {Options::VARIANT_SHREDDING_INFERENCE_MODE, "adaptive"}, })); diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index ed61d859a..c0223af43 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -321,7 +321,6 @@ class RestCatalogTest : public ::testing::Test { // registered in the binary's own registry even when the real format plugin // dylibs register into a different one (macOS two-level namespace) {Options::FILE_FORMAT, "mock_format"}, - {Options::MANIFEST_FORMAT, "mock_format"}, {"header.x-client-header", "from-client"}, {"header.x-shared-header", "from-client"}, }; diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index b26c278e4..4b76cef16 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -124,8 +124,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter } void CreateTable(const std::vector& partition_keys) const { - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + std::map options = {{Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -207,7 +206,6 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -603,11 +601,13 @@ TEST_P(BlobTableInteTest, TestAppendTableWriteWithBlobAsDescriptorTrue) { arrow::field("f1", arrow::int32()), BlobUtils::ToArrowField("blob", true)}; - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::FILE_SYSTEM, "local"}}; + std::map options = {{Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::FILE_SYSTEM, "local"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -644,11 +644,13 @@ TEST_P(BlobTableInteTest, TestAppendTableWriteWithBlobAsDescriptorFalse) { arrow::field("f1", arrow::int32()), BlobUtils::ToArrowField("blob", true)}; - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_AS_DESCRIPTOR, "false"}, {Options::FILE_SYSTEM, "local"}}; + std::map options = {{Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "false"}, + {Options::FILE_SYSTEM, "local"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -679,10 +681,13 @@ TEST_P(BlobTableInteTest, TestWriteNullOnMissingFile) { BlobUtils::ToArrowField("blob", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::BLOB_WRITE_NULL_ON_MISSING_FILE, "true"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::BLOB_WRITE_NULL_ON_MISSING_FILE, "true"}, {Options::FILE_SYSTEM, "local"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -739,11 +744,13 @@ TEST_P(BlobTableInteTest, TestMissingFileFailsWriteWhenWriteNullDisabled) { arrow::field("f1", arrow::int32()), BlobUtils::ToArrowField("blob", true)}; - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::FILE_SYSTEM, "local"}}; + std::map options = {{Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::FILE_SYSTEM, "local"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -771,7 +778,6 @@ TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailure) { BlobUtils::ToArrowField("blob", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, @@ -838,7 +844,6 @@ TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailureCoversMissingFile) { BlobUtils::ToArrowField("blob", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, @@ -940,8 +945,7 @@ TEST_P(BlobTableInteTest, TestBasic) { } TEST_P(BlobTableInteTest, TestBlobFilesAcrossSchemaIds) { - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + std::map options = {{Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1116,8 +1120,7 @@ TEST_P(BlobTableInteTest, TestMultipleAppends) { TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyWriteWithFirstRowId) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + std::map options = {{Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1198,8 +1201,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateFallback) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0", /*nullable=*/true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + std::map options = {{Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1282,8 +1284,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateFallback) { TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateMultipleLayers) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + std::map options = {{Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1353,10 +1354,11 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateMultipleLayers) { TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithDeletionVectors) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, - {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::DELETION_VECTORS_ENABLED, "true"}}; + std::map options = {{Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::DELETION_VECTORS_ENABLED, "true"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); auto schema = arrow::schema(fields); @@ -1437,8 +1439,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithDeletionVectors) TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateCompactedLayers) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + std::map options = {{Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1504,8 +1505,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateCompactedLayers) { TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithRowRanges) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + std::map options = {{Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1564,8 +1564,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithRowRanges) { TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateRowTrackingWithSubrangeLayer) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", /*nullable=*/true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + std::map options = {{Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1650,8 +1649,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateRowTrackingWithSubra TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateAllPlaceholderRowTracking) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", /*nullable=*/true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + std::map options = {{Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1706,8 +1704,7 @@ TEST_P(BlobTableInteTest, TestBlobValueEqualToPlaceholderSentinelBytes) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0", /*nullable=*/true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + std::map options = {{Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1773,8 +1770,7 @@ TEST_P(BlobTableInteTest, TestBlobSentinelValueInBaseLayerDegradesToNull) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0", /*nullable=*/true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + std::map options = {{Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1817,13 +1813,10 @@ TEST_P(BlobTableInteTest, TestUserSuppliedInternalPlaceholderOptionsIgnored) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0", /*nullable=*/true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, - {Options::FILE_SYSTEM, "local"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {BlobDefs::kWritePlaceholderKey, "true"}, - {BlobDefs::kEmitPlaceholderSentinelKey, "true"}}; + std::map options = { + {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {BlobDefs::kWritePlaceholderKey, "true"}, {BlobDefs::kEmitPlaceholderSentinelKey, "true"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); auto schema = arrow::schema(fields); @@ -1844,8 +1837,7 @@ TEST_P(BlobTableInteTest, TestUserSuppliedInternalPlaceholderOptionsIgnored) { TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyFirstCommitFailsWithoutFirstRowId) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + std::map options = {{Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -2002,10 +1994,9 @@ TEST_P(BlobTableInteTest, TestMoreDataWithDataEvolution) { TEST_P(BlobTableInteTest, TestBlobWriteMultiRound) { std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, - {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::BLOB_TARGET_FILE_SIZE, "1000"}, {Options::TARGET_FILE_SIZE, "100"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}}; + {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::BLOB_TARGET_FILE_SIZE, "1000"}, + {Options::TARGET_FILE_SIZE, "100"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; CreateTable(/*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); auto schema = arrow::schema(fields_); @@ -2045,7 +2036,6 @@ TEST_P(BlobTableInteTest, TestExternalPath) { std::string external_test_dir = external_dir->Str(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -2106,10 +2096,11 @@ TEST_P(BlobTableInteTest, TestExternalPath) { TEST_P(BlobTableInteTest, TestPartitionWithPredicate) { auto file_format = GetParam(); std::vector partition_keys = {"f0"}; - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, - {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, {"parquet.write.max-row-group-length", "1"}}; + std::map options = {{Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {"parquet.write.max-row-group-length", "1"}}; CreateTable(partition_keys, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -2227,8 +2218,7 @@ TEST_P(BlobTableInteTest, TestPredicate) { return; } if (GetParam() == "mosaic") { - CreateTable(/*partition_keys=*/{}, {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + CreateTable(/*partition_keys=*/{}, {{Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, @@ -2567,14 +2557,11 @@ TEST_P(BlobTableInteTest, TestWithRowIdsSimple) { TEST_P(BlobTableInteTest, TestWithRowIdsForMultipleBlobFiles) { auto file_format = GetParam(); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1000"}, - {Options::BLOB_TARGET_FILE_SIZE, "80"}, - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::FILE_SYSTEM, "local"}}; + std::map options = { + {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1000"}, + {Options::BLOB_TARGET_FILE_SIZE, "80"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::FILE_SYSTEM, "local"}}; CreateTable(/*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); auto schema = arrow::schema(fields_); @@ -2677,11 +2664,12 @@ TEST_P(BlobTableInteTest, TestAppendTableWriteWithMultipleBlobFields) { arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), BlobUtils::ToArrowField("blob1", true), BlobUtils::ToArrowField("blob2", true)}; - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::FILE_SYSTEM, "local"}}; + std::map options = {{Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::FILE_SYSTEM, "local"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -2704,8 +2692,7 @@ TEST_P(BlobTableInteTest, TestAppendWriteWithNullBlob) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("blob", true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + std::map options = {{Options::FILE_FORMAT, GetParam()}, {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -2811,11 +2798,13 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorField) { BlobUtils::ToArrowField("b0", true), BlobUtils::ToArrowField("b1", true)}; - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; + std::map options = {{Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, + {Options::FILE_SYSTEM, "local"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -2869,11 +2858,13 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialInline) { BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), BlobUtils::ToArrowField("b3", true)}; - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; + std::map options = {{Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, + {Options::FILE_SYSTEM, "local"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -2932,11 +2923,13 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorMultiCommitAndShuffledReadSchema) { BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), BlobUtils::ToArrowField("b3", true)}; - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; + std::map options = {{Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, + {Options::FILE_SYSTEM, "local"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); auto schema = arrow::schema(fields); @@ -3061,7 +3054,6 @@ TEST_P(BlobTableInteTest, TestSharedShreddingWithBlobDataEvolution) { BlobUtils::ToArrowField("payload"), }; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3121,7 +3113,6 @@ TEST_P(BlobTableInteTest, TestMultipleSharedShreddingMapsWithBlobDataEvolution) BlobUtils::ToArrowField("payload"), }; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3182,7 +3173,6 @@ TEST_P(BlobTableInteTest, TestSharedShreddingMapOverrideWithBlobDataEvolution) { BlobUtils::ToArrowField("payload"), }; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3241,7 +3231,6 @@ TEST_P(BlobTableInteTest, TestOrcMapStorageLayoutEvolutionWithBlobDataEvolution) BlobUtils::ToArrowField("payload"), }; std::map options_v0 = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3312,11 +3301,13 @@ TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) { BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), BlobUtils::ToArrowField("b3", true)}; - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; + std::map options = {{Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, + {Options::FILE_SYSTEM, "local"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -3429,11 +3420,13 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWriteRawBytesDirectly) { BlobUtils::ToArrowField("b0", true), BlobUtils::ToArrowField("b1", true)}; - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; + std::map options = {{Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, + {Options::FILE_SYSTEM, "local"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -3468,7 +3461,6 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3637,13 +3629,10 @@ TEST_P(BlobTableInteTest, TestForwardBlobViewReference) { // dynamically disabled can succeed on it. arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}; - std::map source_options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format}, - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_VIEW_FIELD, "view"}, - {Options::FILE_SYSTEM, "local"}}; + std::map source_options = { + {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_VIEW_FIELD, "view"}, {Options::FILE_SYSTEM, "local"}}; CreateTable(fields, /*partition_keys=*/{}, source_options); std::string source_table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -3788,13 +3777,9 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamDescriptorBlob) { BlobUtils::ToArrowField("b1", true)}; auto upstream_schema = arrow::schema(upstream_fields); std::map upstream_options = { - {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format}, - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, - {Options::FILE_SYSTEM, "local"}}; + {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; ::ArrowSchema upstream_c_schema; ASSERT_TRUE(arrow::ExportSchema(*upstream_schema, &upstream_c_schema).ok()); @@ -3830,7 +3815,6 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamDescriptorBlob) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -3936,7 +3920,6 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithMultipleUpstreamTables) { BlobUtils::ToArrowField("view1", true), BlobUtils::ToArrowField("view2", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -4214,13 +4197,9 @@ TEST_P(BlobTableInteTest, TestBlobViewWithFallbackPath) { BlobUtils::ToArrowField("blob", true)}; auto upstream_schema = arrow::schema(upstream_fields); std::map upstream_options = { - {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format}, - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_AS_DESCRIPTOR, "true"}, - {Options::FILE_SYSTEM, "local"}}; + {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::FILE_SYSTEM, "local"}}; // Create the upstream table at the fallback path: /db/table (no .db). auto upstream_dir = UniqueTestDirectory::Create("local"); @@ -4261,7 +4240,6 @@ TEST_P(BlobTableInteTest, TestBlobViewWithFallbackPath) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, diff --git a/test/inte/clean_inte_test.cpp b/test/inte/clean_inte_test.cpp index 89cadd4f7..3f7f353ef 100644 --- a/test/inte/clean_inte_test.cpp +++ b/test/inte/clean_inte_test.cpp @@ -190,7 +190,7 @@ class CleanInteTest : public testing::Test { }; TEST_F(CleanInteTest, TestExpireSnapshotFailover) { - std::string test_data_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; + std::string test_data_path = paimon::test::GetDataDir() + "/parquet/append_09.db/append_09/"; std::map clean_options = { {Options::MANIFEST_TARGET_FILE_SIZE, "8mb"}, {Options::FILE_SYSTEM, "local"}, @@ -218,7 +218,7 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath( - table_path, "manifest/manifest-list-616d1847-a02c-495f-9cca-2c8b7def0fec-1"))); + table_path, "manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1"))); CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit_context, @@ -233,7 +233,7 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath( - table_path, "f1=10/bucket-1/data-10b9eea8-241d-4e4b-8ab8-2a82d72d79a2-0.orc"))); + table_path, "f1=10/bucket-1/data-7a912f84-04b7-4bbb-8dc6-53f4a292ea25-0.parquet"))); CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit_context, @@ -272,13 +272,9 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { ASSERT_TRUE(dir); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, "orc"}, - {Options::TARGET_FILE_SIZE, "1024"}, - {Options::FILE_SYSTEM, "local"}, - {Options::BUCKET, "2"}, - {Options::BUCKET_KEY, "f2"}, - {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}, + {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"}, + {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f2"}, {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}, }; ASSERT_OK_AND_ASSIGN( @@ -416,13 +412,9 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { auto schema = arrow::schema(arrow::FieldVector({string_field, int_field, int_field1, double_field})); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, "orc"}, - {Options::TARGET_FILE_SIZE, "1024"}, - {Options::FILE_SYSTEM, "local"}, - {Options::BUCKET, "2"}, - {Options::BUCKET_KEY, "f2"}, - {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}, + {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"}, + {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f2"}, {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}, }; bool drop_partition_scanned_all_io_hook = false; @@ -583,13 +575,9 @@ TEST_F(CleanInteTest, TestOrphanFilesClean) { ASSERT_TRUE(arrow::ExportSchema(*schema, &arrow_schema).ok()); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, "orc"}, - {Options::TARGET_FILE_SIZE, "1024"}, - {Options::FILE_SYSTEM, "local"}, - {Options::BUCKET, "2"}, - {Options::BUCKET_KEY, "f3"}, - {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}, + {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"}, + {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f3"}, {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}, }; auto dir = UniqueTestDirectory::Create(); @@ -701,13 +689,9 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithFileRetainCondition) { ASSERT_TRUE(arrow::ExportSchema(*schema, &arrow_schema).ok()); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, "orc"}, - {Options::TARGET_FILE_SIZE, "1024"}, - {Options::FILE_SYSTEM, "local"}, - {Options::BUCKET, "2"}, - {Options::BUCKET_KEY, "f3"}, - {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}, + {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"}, + {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f3"}, {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}, }; auto dir = UniqueTestDirectory::Create(); @@ -817,13 +801,9 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithIOException) { ASSERT_TRUE(dir); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, "orc"}, - {Options::TARGET_FILE_SIZE, "1024"}, - {Options::FILE_SYSTEM, "local"}, - {Options::BUCKET, "2"}, - {Options::BUCKET_KEY, "f2"}, - {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}}; + {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"}, + {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f2"}, {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}}; ASSERT_OK_AND_ASSIGN(std::string table_path, CreateTestTable(dir->Str(), /*db_name=*/"foo", diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index 4fb26b6ed..0d8425851 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -79,8 +79,7 @@ class DataEvolutionTableTest : public ::testing::Test, } void CreateTable(const std::vector& partition_keys) const { - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, FileFormat()}, + std::map options = {{Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -160,8 +159,7 @@ class DataEvolutionTableTest : public ::testing::Test, std::map CreateDataEvolutionTable( bool deletion_vectors_enabled, const std::map& extra_options = {}) const { - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, FileFormat()}, + std::map options = {{Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -861,7 +859,6 @@ TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) arrow::field("map2", map_type), }; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1190,7 +1187,6 @@ TEST_P(DataEvolutionTableTest, TestMoreData) { TEST_P(DataEvolutionTableTest, TestOnlyRowTrackingEnabled) { std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1234,7 +1230,6 @@ TEST_P(DataEvolutionTableTest, TestExternalPath) { std::string external_test_dir = external_dir->Str(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -1485,10 +1480,11 @@ TEST_P(DataEvolutionTableTest, TestPartitionWithPredicate) { return; } std::vector partition_keys = {"f1"}; - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, FileFormat()}, - {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, {"parquet.write.max-row-group-length", "1"}}; + std::map options = {{Options::FILE_FORMAT, FileFormat()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {"parquet.write.max-row-group-length", "1"}}; if (file_format == "mosaic") { options.emplace(mosaic::MOSAIC_STATS_COLUMNS, "f0"); } @@ -2134,7 +2130,7 @@ TEST_P(DataEvolutionTableTest, TestFormatPredicatePushDownWithoutFileIndex) { std::map options = {{Options::FILE_INDEX_READ_ENABLED, "false"}, {Options::WRITE_BATCH_SIZE, "1"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, {"parquet.read.enable-page-index-filter", "true"}, {"orc.stripe.size", "1"}, @@ -2185,8 +2181,7 @@ TEST_P(DataEvolutionTableTest, TestPredicate) { return; } if (FileFormat() == "mosaic") { - CreateTable(/*partition_keys=*/{}, {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, FileFormat()}, + CreateTable(/*partition_keys=*/{}, {{Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, @@ -2329,8 +2324,7 @@ TEST_P(DataEvolutionTableTest, TestIOException) { } TEST_P(DataEvolutionTableTest, TestWithRowIds) { - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, FileFormat()}, + std::map options = {{Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index e05dc4b23..2e410513e 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -85,8 +85,7 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter } void CreateTable(const std::vector& partition_keys) const { - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format_}, + std::map options = {{Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -264,8 +263,7 @@ TEST_P(GlobalIndexTest, TestWriteLuminaIndex) { {"lumina.encoding.type", "rawf32"}, {"lumina.search.parallel_number", "10"}}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format_}, + std::map options = {{Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -322,10 +320,11 @@ TEST_P(GlobalIndexTest, TestWriteLuminaIndexWithMismatchedDimension) { {"lumina.encoding.type", "rawf32"}, {"lumina.search.parallel_number", "10"}}; - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format_}, - {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::READ_BATCH_SIZE, "1"}}; + std::map options = {{Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::READ_BATCH_SIZE, "1"}}; CreateTable(/*partition_keys=*/{}, schema, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -366,10 +365,9 @@ TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithOrcDictionaryStringTags) R"([{"key_name":"color","type":"enum","value_type":"string"},)" R"({"key_name":"labels","type":"enum","value_type":"string"}])"}}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format_}, - {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, {"orc.dictionary-key-size-threshold", "1"}, - {"orc.read.enable-lazy-decoding", "true"}}; + {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {"orc.dictionary-key-size-threshold", "1"}, {"orc.read.enable-lazy-decoding", "true"}}; CreateTable(/*partition_keys=*/{}, schema, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -961,8 +959,7 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndexWithPartition) { {"lumina.encoding.type", "rawf32"}, {"lumina.search.parallel_number", "10"}}; auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format_}, + std::map options = {{Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1102,8 +1099,7 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndexWithScore) { {"lumina.encoding.type", "rawf32"}, {"lumina.search.parallel_number", "10"}}; auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format_}, + std::map options = {{Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1248,8 +1244,7 @@ TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithTagNullAndEmptyValues) { R"({"key_name":"scores","type":"range","value_type":"float"},)" R"({"key_name":"category","type":"enum","value_type":"int32"},)" R"({"key_name":"category_ids","type":"enum","value_type":"int32"}])"}}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format_}, + std::map options = {{Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1871,8 +1866,7 @@ TEST_P(GlobalIndexTest, TestScanIndexWithTwoIndexes) { {"lumina.encoding.type", "rawf32"}, {"lumina.search.parallel_number", "10"}}; auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format_}, + std::map options = {{Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1939,8 +1933,7 @@ TEST_P(GlobalIndexTest, TestDataEvolutionBatchScanWithExternalPath) { arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::list(arrow::float32())), arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}; auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format_}, + std::map options = {{Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -2006,8 +1999,7 @@ TEST_P(GlobalIndexTest, TestIOException) { ])") .ValueOrDie(); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format_}, + std::map options = {{Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -2332,8 +2324,7 @@ TEST_P(GlobalIndexTest, TestLuceneWriteCommitScanReadIndexWithScore) { {"lucene-fts.write.omit-term-freq-and-position", "false"}, {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}; auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format_}, + std::map options = {{Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -2418,8 +2409,7 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadLuceneIndexWithPartition) { {"lucene-fts.write.omit-term-freq-and-position", "false"}, {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}; auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format_}, + std::map options = {{Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -2781,8 +2771,7 @@ TEST_P(GlobalIndexTest, TestBTreeEmptyStringKeyPredicates) { TEST_P(GlobalIndexTest, TestBTreeWriteCommitScanReadIndexWithPartition) { // BTree index with partitioned table. Each partition's data is sorted by f0 independently. auto schema = arrow::schema(fields_); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format_}, + std::map options = {{Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -2926,8 +2915,7 @@ TEST_P(GlobalIndexTest, TestBTreeWithPartitionAndCustomExecutor) { // Test that UnionGlobalIndexReader uses a custom 8-thread executor to read // btree indexes from two partitions in parallel. auto schema = arrow::schema(fields_); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format_}, + std::map options = {{Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -3347,8 +3335,7 @@ TEST_P(GlobalIndexTest, TestBTreeWithLumina) { {"lumina.encoding.type", "rawf32"}, {"lumina.search.parallel_number", "10"}}; auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format_}, + std::map options = {{Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; diff --git a/test/inte/nested_column_pruning_inte_test.cpp b/test/inte/nested_column_pruning_inte_test.cpp index 119517af6..0f113cb2f 100644 --- a/test/inte/nested_column_pruning_inte_test.cpp +++ b/test/inte/nested_column_pruning_inte_test.cpp @@ -145,7 +145,6 @@ TEST_P(NestedColumnPruningInteTest, PruneStructSubFields) { auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -202,7 +201,6 @@ TEST_P(NestedColumnPruningInteTest, ProjectStructColumnAsEmptyStructReturnsNullC auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -259,7 +257,6 @@ TEST_P(NestedColumnPruningInteTest, PruneSameNestedFieldNameFromDifferentStructC auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -314,7 +311,6 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsAllNonExistent) { auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -376,7 +372,6 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithNonExistentField) { auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -439,7 +434,6 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithTypeMismatchShouldFa auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -523,7 +517,6 @@ TEST_P(NestedColumnPruningInteTest, auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -615,7 +608,6 @@ TEST_P(NestedColumnPruningInteTest, PruneEntireStructField) { auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -670,7 +662,6 @@ TEST_P(NestedColumnPruningInteTest, PruneDeepNestedStruct) { auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -731,7 +722,6 @@ TEST_P(NestedColumnPruningInteTest, PruneNestedStructWithSpecialFields) { auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -819,7 +809,6 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeys) { auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -874,7 +863,6 @@ TEST_P(NestedColumnPruningInteTest, NestedMapSelectedKeysInStruct) { auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -930,7 +918,6 @@ TEST_P(NestedColumnPruningInteTest, PruneStructSubFieldsWithNestedMapSelectedKey auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -988,7 +975,6 @@ TEST_P(NestedColumnPruningInteTest, PruneStructSubFieldsWithNestedMapSelectedKey auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -1045,7 +1031,6 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysEmptyStringKey) { auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -1096,7 +1081,6 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysPreserveOrder) { auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -1160,13 +1144,12 @@ TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1048576"}, {Options::BUCKET, "-1"}, {Options::WRITE_BATCH_SIZE, "1"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, {"parquet.write.max-row-group-length", "1"}, {"parquet.read.enable-page-index-filter", "true"}, @@ -1264,7 +1247,6 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysWithOrcDictionaryEncodedMap) auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -1367,7 +1349,6 @@ TEST_P(NestedColumnPruningInteTest, PruneDeeperNestedStruct) { auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -1428,7 +1409,6 @@ TEST_P(NestedColumnPruningInteTest, PruneListStructSubFields) { auto table_schema = arrow::schema(table_fields); std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp index 130535171..b312e6859 100644 --- a/test/inte/pk_compaction_inte_test.cpp +++ b/test/inte/pk_compaction_inte_test.cpp @@ -538,7 +538,7 @@ TEST_P(PkCompactionInteTest, TestKeyValueTableDvCompactionWithMapSharedShredding {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "2"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, {"parquet.write.max-row-group-length", "1"}, {"parquet.read.enable-page-index-filter", "true"}, diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index a22e3a104..de93a0350 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -634,7 +634,6 @@ TEST_P(ReadInteTest, TestReadOnlyPartitionField) { TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, {"custom.option", "custom-value"}}; auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -666,10 +665,8 @@ TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) { ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); ASSERT_TRUE(result); - std::map expected = {{"custom.option", "custom-value"}, - {"file-system", "local"}, - {"file.format", "orc"}, - {"manifest.format", "orc"}}; + std::map expected = { + {"custom.option", "custom-value"}, {"file-system", "local"}, {"file.format", "orc"}}; ASSERT_EQ(CollectStringMap(result), expected) << result->ToString(); } @@ -683,8 +680,7 @@ TEST(SystemTableReadInteTest, TestReadBranchOptionsSystemTable) { std::filesystem::path(table_path))); std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}}; + {Options::FILE_FORMAT, "parquet"}}; std::string system_table_path = table_path + "$branch_rt$options"; ScanContextBuilder scan_context_builder(system_table_path); scan_context_builder.SetOptions(options); @@ -713,7 +709,7 @@ TEST(SystemTableReadInteTest, TestReadMetadataSystemTables) { auto schema = arrow::schema(fields); std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}}; auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -879,7 +875,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTable) { auto schema = arrow::schema(fields); std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, {Options::NUM_LEVELS, "3"}}; auto dir = UniqueTestDirectory::Create(); @@ -938,7 +934,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedAppendOnlySystemTableWithStreamin auto schema = arrow::schema(fields); std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "k"}}; auto dir = UniqueTestDirectory::Create(); @@ -979,12 +975,12 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP std::map options = { {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "k"}, {Options::WRITE_BATCH_SIZE, "1"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, {"parquet.write.max-row-group-length", "1"}, {"parquet.read.enable-page-index-filter", "true"}}; @@ -1048,7 +1044,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { auto schema = arrow::schema(fields); std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "k"}}; auto dir = UniqueTestDirectory::Create(); @@ -1109,7 +1105,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableWithBranch) { auto schema = arrow::schema(fields); std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, {Options::NUM_LEVELS, "3"}}; auto dir = UniqueTestDirectory::Create(); @@ -1166,10 +1162,9 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableWithFirstRowMergeEngin }; auto schema = arrow::schema(fields); std::map options = { - {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "1"}, - {Options::BUCKET_KEY, "k"}, {Options::NUM_LEVELS, "5"}, - {Options::MERGE_ENGINE, "first-row"}}; + {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "k"}, + {Options::NUM_LEVELS, "5"}, {Options::MERGE_ENGINE, "first-row"}}; auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN( @@ -1207,8 +1202,8 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedTable) { }; auto schema = arrow::schema(fields); std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "1"}}; auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -1262,9 +1257,11 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite }; auto schema = arrow::schema(fields); std::map options = { - {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, }; auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -1331,8 +1328,8 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForDatePartition) { }; auto schema = arrow::schema(fields); std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "v"}}; auto dir = UniqueTestDirectory::Create(); @@ -1433,8 +1430,7 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableWithSchemaEvolutionStats) TEST(SystemTableReadInteTest, TestReadManifestAndFilesSystemTablesForEmptyTable) { std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}}; + {Options::FILE_FORMAT, "orc"}}; auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); @@ -1498,8 +1494,7 @@ TEST(SystemTableReadInteTest, TestReadTagBranchAndConsumerSystemTables) { /*overwrite=*/true)); std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}}; + {Options::FILE_FORMAT, "parquet"}}; ASSERT_OK_AND_ASSIGN(auto branches_result, ReadSystemTable(table_path + "$branches", options)); auto branches_array = SingleStructChunk(branches_result); @@ -1568,7 +1563,7 @@ TEST(SystemTableReadInteTest, TestReadAuditLogSystemTable) { std::map options = { {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, }; auto dir = UniqueTestDirectory::Create(); @@ -1605,7 +1600,7 @@ TEST(SystemTableReadInteTest, TestReadAuditLogSystemTableWithSequenceNumber) { std::map options = { {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, }; auto dir = UniqueTestDirectory::Create(); @@ -1643,7 +1638,7 @@ TEST(SystemTableReadInteTest, TestReadBinlogSystemTable) { std::map options = { {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, }; auto dir = UniqueTestDirectory::Create(); @@ -1679,7 +1674,7 @@ TEST(SystemTableReadInteTest, TestReadAuditLogAndBinlogSystemTableWithChangelogR std::map options = { {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, }; auto dir = UniqueTestDirectory::Create(); @@ -1734,9 +1729,11 @@ TEST(SystemTableReadInteTest, TestStreamingBinlogPacksUpdateBeforeAndAfter) { }; auto schema = arrow::schema(fields); std::map options = { - {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "1"}, - {Options::WRITE_BUFFER_SIZE, "1"}, {Options::WRITE_BUFFER_SPILLABLE, "false"}, + {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "1"}, + {Options::WRITE_BUFFER_SIZE, "1"}, + {Options::WRITE_BUFFER_SPILLABLE, "false"}, }; auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -1797,7 +1794,7 @@ TEST(SystemTableReadInteTest, TestReadBinlogSystemTableWithNullValue) { std::map options = { {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, }; auto dir = UniqueTestDirectory::Create(); @@ -1830,7 +1827,7 @@ TEST(SystemTableReadInteTest, TestReadAuditLogAndBinlogSystemTableWithBranch) { auto schema = arrow::schema(fields); std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}}; auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -1885,7 +1882,7 @@ TEST(SystemTableReadInteTest, TestReadAuditLogAndBinlogSystemTableWithNonPrimary std::map options = { {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "-1"}, }; auto dir = UniqueTestDirectory::Create(); @@ -4197,7 +4194,6 @@ TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) { TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, {"table.option.custom", "my-value"}}; auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -4274,15 +4270,12 @@ TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { } TEST(SystemTableReadInteTest, TestReadGlobalTables) { - std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, - {Options::BUCKET, "1"}, - {"owner", "alice"}, - {"createdAt", "1000"}, - {"createdBy", "creator"}, - {"updatedAt", "2000"}, - {"updatedBy", "updater"}}; + std::map options = { + {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, + + {Options::BUCKET, "1"}, {"owner", "alice"}, + {"createdAt", "1000"}, {"createdBy", "creator"}, + {"updatedAt", "2000"}, {"updatedBy", "updater"}}; auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); @@ -4393,8 +4386,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalTables) { TEST(SystemTableReadInteTest, TestReadGlobalPartitions) { std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "v"}}; auto dir = UniqueTestDirectory::Create(); @@ -4450,7 +4443,6 @@ TEST(SystemTableReadInteTest, TestReadGlobalPartitions) { TEST(SystemTableReadInteTest, TestGlobalSystemTablesPropagateCorruptSchema) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "v"}}; auto dir = UniqueTestDirectory::Create(); @@ -4480,8 +4472,8 @@ TEST(SystemTableReadInteTest, TestGlobalSystemTablesPropagateCorruptSchema) { TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptSnapshot) { std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "v"}}; auto dir = UniqueTestDirectory::Create(); @@ -4515,8 +4507,8 @@ TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptSnapshot TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest) { std::map options = {{Options::FILE_SYSTEM, "local"}, - {Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "v"}}; auto dir = UniqueTestDirectory::Create(); diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 5bc7bcb6c..6a58a97f7 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -324,9 +324,11 @@ class RealtimeWriteInteTest : public ::testing::Test { arrow::field("pt", arrow::utf8())}; schema_ = arrow::schema(fields_); options_ = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"}, - {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "1"}, - {Options::BUCKET_KEY, "id"}, {Options::TARGET_FILE_SIZE, "1048576"}, + {Options::FILE_FORMAT, "orc"}, + {Options::FILE_SYSTEM, "local"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::TARGET_FILE_SIZE, "1048576"}, {Options::REALTIME_ENABLED, "true"}, }; } @@ -2541,7 +2543,7 @@ TEST_F(RealtimeWriteInteTest, TestDiskPredicatePushdownWithoutMemoryFiltering) { options_[Options::FILE_FORMAT] = "parquet"; options_[Options::WRITE_BATCH_SIZE] = "1"; options_["parquet.page.size"] = "1"; - options_["parquet.enable-dictionary"] = "false"; + options_["parquet.enable.dictionary"] = "false"; options_["parquet.write.enable-page-index"] = "true"; options_["parquet.read.enable-page-index-filter"] = "true"; CreateTable(/*partition_keys=*/{}); @@ -2676,7 +2678,7 @@ TEST_F(RealtimeWriteInteTest, TestMemoryBatchStatisticsPredicatePushdownWithDisk options_[Options::WRITE_BATCH_SIZE] = "1"; options_[Options::REALTIME_STORE_STATS_MODE] = "full"; options_["parquet.page.size"] = "1"; - options_["parquet.enable-dictionary"] = "false"; + options_["parquet.enable.dictionary"] = "false"; options_["parquet.write.enable-page-index"] = "true"; options_["parquet.read.enable-page-index-filter"] = "true"; CreateTable(/*partition_keys=*/{}); @@ -2730,7 +2732,7 @@ TEST_F(RealtimeWriteInteTest, TestNullPredicateForMemoryAndDisk) { options_[Options::WRITE_BATCH_SIZE] = "1"; options_[Options::REALTIME_STORE_STATS_MODE] = "full"; options_["parquet.page.size"] = "1"; - options_["parquet.enable-dictionary"] = "false"; + options_["parquet.enable.dictionary"] = "false"; options_["parquet.write.enable-page-index"] = "true"; options_["parquet.write.max-row-group-length"] = "1"; options_["parquet.read.enable-page-index-filter"] = "true"; diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index de2d347ff..c4c50cc43 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -2177,8 +2177,7 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { arrow::field("_VALUE_KIND", arrow::int8())); auto schema = arrow::schema(fields); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format}, + std::map options = {{Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-2"}, {Options::FILE_SYSTEM, "local"}}; diff --git a/test/inte/variant_table_inte_test.cpp b/test/inte/variant_table_inte_test.cpp index ba41c59cd..14f05899e 100644 --- a/test/inte/variant_table_inte_test.cpp +++ b/test/inte/variant_table_inte_test.cpp @@ -179,7 +179,6 @@ class VariantTableInteTest : public ::testing::Test { double admission_ratio, double retention_ratio) const { return { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "parquet"}, {Options::BUCKET, "-1"}, {Options::WRITE_ONLY, "true"}, @@ -206,7 +205,6 @@ class VariantTableInteTest : public ::testing::Test { TEST_F(VariantTableInteTest, TestAppendTable) { std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "parquet"}, {Options::BUCKET, "-1"}, }; @@ -597,7 +595,6 @@ TEST_F(VariantTableInteTest, TestAdaptiveInferenceWithNestedVariant) { TEST_F(VariantTableInteTest, TestPrimaryKeyTable) { std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "parquet"}, {Options::BUCKET, "1"}, }; @@ -623,7 +620,6 @@ TEST_F(VariantTableInteTest, TestPrimaryKeyTable) { TEST_F(VariantTableInteTest, TestVariantAccessRead) { std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "parquet"}, {Options::BUCKET, "-1"}, }; @@ -667,7 +663,6 @@ TEST_F(VariantTableInteTest, TestNestedRowVariantAccessRead) { arrow::field("t", arrow::utf8())})); auto table_schema = arrow::schema({fields_[0], struct_field}); std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "parquet"}, {Options::BUCKET, "-1"}, }; @@ -730,7 +725,6 @@ TEST_F(VariantTableInteTest, TestArrayVariantAccessRead) { auto list_field = arrow::field("arr", arrow::list(VariantTypeUtils::ToArrowField("element"))); auto table_schema = arrow::schema({fields_[0], list_field}); std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "parquet"}, {Options::BUCKET, "-1"}, }; @@ -788,7 +782,6 @@ TEST_F(VariantTableInteTest, TestReadWithIOException) { // Injects an IO error at every position of the scan+read path and verifies each failure // surfaces as a clean error status. std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "parquet"}, {Options::BUCKET, "-1"}, }; @@ -826,7 +819,6 @@ TEST_F(VariantTableInteTest, TestWriteWithIOException) { // Injects an IO error at every position of the create+write+commit path (on a fresh table // directory per attempt) and verifies each failure surfaces as a clean error status. std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "parquet"}, {Options::BUCKET, "-1"}, }; @@ -862,7 +854,6 @@ TEST_F(VariantTableInteTest, TestWriteWithIOException) { TEST_F(VariantTableInteTest, TestOrcFormatRejected) { std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, "orc"}, {Options::BUCKET, "-1"}, }; diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 72bc37659..43d616e8a 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -313,9 +313,8 @@ TEST_P(WriteAndReadInteTest, TestAppendSimple) { arrow::field("f1", arrow::int32())}; auto schema = arrow::schema(fields); auto [file_format, file_system] = GetParam(); - // manifest and file format are upper case + // file format is upper case std::map options = { - {Options::MANIFEST_FORMAT, "AVRO"}, {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format)}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -368,8 +367,9 @@ TEST_P(WriteAndReadInteTest, TestAppendVector) { arrow::FieldVector fields = {arrow::field("id", arrow::int32()), arrow::field("embedding", vector_type)}; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, file_system}, }; if (file_system == "jindo") { @@ -427,8 +427,9 @@ TEST_P(WriteAndReadInteTest, TestAppendNestedVector) { arrow::field("by_name", arrow::map(arrow::utf8(), vector_type)), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, file_system}, }; if (file_system == "jindo") { @@ -476,7 +477,6 @@ TEST_P(WriteAndReadInteTest, TestAppendVectorWithPredicate) { arrow::FieldVector fields = {arrow::field("id", arrow::int32()), arrow::field("embedding", vector_type)}; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1048576"}, {Options::BUCKET, "-1"}, @@ -540,7 +540,6 @@ TEST_P(WriteAndReadInteTest, TestAppendWithExternalBitmapAndRangeBitmapIndexes) arrow::field("score", arrow::int32())}; auto [file_format, file_system] = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1MB"}, {Options::BUCKET, "-1"}, @@ -626,9 +625,11 @@ TEST_P(WriteAndReadInteTest, TestPKSimple) { auto schema = arrow::schema(fields); auto [file_format, file_system] = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {"orc.read.enable-lazy-decoding", "true"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {"orc.read.enable-lazy-decoding", "true"}, {"orc.dictionary-key-size-threshold", "1"}, }; if (file_system == "jindo") { @@ -685,9 +686,11 @@ TEST_P(WriteAndReadInteTest, TestInputChangelogStreamRead) { }; auto [file_format, file_system] = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "input"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "input"}, }; if (file_system == "jindo") { options = AddOptionsForJindo(options); @@ -731,10 +734,11 @@ TEST_P(WriteAndReadInteTest, TestFullCompactionChangelogStreamRead) { auto [file_format, file_system] = GetParam(); arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32())}; - std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "full-compaction"}}; + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "full-compaction"}}; if (file_system == "jindo") { options = AddOptionsForJindo(options); } @@ -793,10 +797,11 @@ TEST_P(WriteAndReadInteTest, TestFullCompactionChangelogInitialScanOnlyReadsMaxL auto [file_format, file_system] = GetParam(); arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32())}; - std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "full-compaction"}}; + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "full-compaction"}}; if (file_system == "jindo") { options = AddOptionsForJindo(options); } @@ -846,7 +851,6 @@ TEST_P(WriteAndReadInteTest, TestFullCompactionChangelogRowDeduplicate) { arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32())}; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, @@ -907,7 +911,6 @@ TEST_P(WriteAndReadInteTest, TestFullCompactionChangelogWithSharedShredding) { arrow::FieldVector fields = {arrow::field("pk", arrow::int32()), arrow::field("tags", map_type)}; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, @@ -979,9 +982,11 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogStreamRead) { arrow::field("value", arrow::int32()), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "lookup"}, }; ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); @@ -1028,9 +1033,11 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogInitialFullScanExcludesLevelZero arrow::field("value", arrow::int32()), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "lookup"}, }; ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options)); @@ -1086,10 +1093,11 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogInsertUpdateDelete) { auto [file_format, file_system] = GetParam(); arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32())}; - std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}}; + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "lookup"}}; if (file_system == "jindo") { options = AddOptionsForJindo(options); } @@ -1134,11 +1142,12 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogWithFirstRow) { auto [file_format, file_system] = GetParam(); arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32())}; - std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, - {Options::MERGE_ENGINE, "first-row"}}; + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::MERGE_ENGINE, "first-row"}}; if (file_system == "jindo") { options = AddOptionsForJindo(options); } @@ -1177,11 +1186,12 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogWithDeletionVector) { auto [file_format, file_system] = GetParam(); arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32())}; - std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}, - {Options::DELETION_VECTORS_ENABLED, "true"}}; + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "lookup"}, + {Options::DELETION_VECTORS_ENABLED, "true"}}; if (file_system == "jindo") { options = AddOptionsForJindo(options); } @@ -1228,7 +1238,6 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicate) { arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32())}; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, @@ -1286,7 +1295,6 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicateIgnoreFields) { arrow::field("value", arrow::int32()), arrow::field("ignored", arrow::int32())}; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, @@ -1346,10 +1354,11 @@ TEST_P(WriteAndReadInteTest, TestChangelogWithSchemaEvolution) { auto [file_format, file_system] = GetParam(); arrow::FieldVector fields_v0 = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32())}; - std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}}; + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::CHANGELOG_PRODUCER, "lookup"}}; if (file_system == "jindo") { options = AddOptionsForJindo(options); } @@ -1457,7 +1466,6 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogWithExternalPath) { arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), arrow::field("value", arrow::int32())}; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, @@ -1526,8 +1534,9 @@ TEST_P(WriteAndReadInteTest, TestNestedType) { return; } std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, file_system}, }; if (file_system == "jindo") { @@ -1586,7 +1595,6 @@ TEST_P(WriteAndReadInteTest, TestSchemaEvolutionAddFieldInsideListAndMap) { arrow::field("props", arrow::map(arrow::utf8(), map_value)), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -1863,10 +1871,11 @@ TEST_P(WriteAndReadInteTest, TestAppendTimestampType) { if (file_format == "mosaic") { return; } - std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, - {Options::FILE_SYSTEM, file_system}, {"orc.timestamp-ltz.legacy.type", "false"}}; + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"orc.timestamp-ltz.legacy.type", "false"}}; if (file_system == "jindo") { options = AddOptionsForJindo(options); } @@ -1919,10 +1928,11 @@ TEST_P(WriteAndReadInteTest, TestPkTimestampType) { if (file_format == "mosaic") { return; } - std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {"orc.timestamp-ltz.legacy.type", "false"}}; + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {"orc.timestamp-ltz.legacy.type", "false"}}; if (file_system == "jindo") { options = AddOptionsForJindo(options); } @@ -1979,10 +1989,11 @@ TEST_P(WriteAndReadInteTest, TestAppendNestedTimestampSecondPrecision) { arrow::field("events", arrow::list(arrow::field("element", event_type))), arrow::field("marks", arrow::map(arrow::utf8(), arrow::timestamp(arrow::TimeUnit::SECOND))), }; - std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, - {Options::FILE_SYSTEM, file_system}, {"orc.timestamp-ltz.legacy.type", "false"}}; + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"orc.timestamp-ltz.legacy.type", "false"}}; if (file_system == "jindo") { options = AddOptionsForJindo(options); } @@ -2038,10 +2049,11 @@ TEST_P(WriteAndReadInteTest, TestAppendNestedTimestampLtzMicroTimezoneOnly) { arrow::field("marks", arrow::map(arrow::utf8(), arrow::timestamp(arrow::TimeUnit::MICRO, timezone))), }; - std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, - {Options::FILE_SYSTEM, file_system}, {"orc.timestamp-ltz.legacy.type", "false"}}; + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"orc.timestamp-ltz.legacy.type", "false"}}; if (file_system == "jindo") { options = AddOptionsForJindo(options); } @@ -2085,10 +2097,13 @@ TEST_P(WriteAndReadInteTest, TestPKWithSequenceFieldInPKField) { auto schema = arrow::schema(fields); auto [file_format, file_system] = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {Options::SEQUENCE_FIELD, "p2"}, - {"orc.read.enable-lazy-decoding", "true"}, {"orc.dictionary-key-size-threshold", "1"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::SEQUENCE_FIELD, "p2"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"orc.dictionary-key-size-threshold", "1"}, }; if (file_system == "jindo") { options = AddOptionsForJindo(options); @@ -2150,10 +2165,13 @@ TEST_P(WriteAndReadInteTest, TestPKWithSequenceFieldPartialInPKField) { auto schema = arrow::schema(fields); auto [file_format, file_system] = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::FILE_SYSTEM, file_system}, {Options::SEQUENCE_FIELD, "p2,f1"}, - {"orc.read.enable-lazy-decoding", "true"}, {"orc.dictionary-key-size-threshold", "1"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::SEQUENCE_FIELD, "p2,f1"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"orc.dictionary-key-size-threshold", "1"}, }; if (file_system == "jindo") { options = AddOptionsForJindo(options); @@ -2404,8 +2422,9 @@ TEST_P(WriteAndReadInteTest, TestCharVarcharBinaryVarbinaryTypes) { arrow::field("vb", arrow::binary()), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, file_system}, }; if (file_system == "jindo") { @@ -2484,7 +2503,6 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}; auto schema = arrow::schema(fields); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "parquet"}, {Options::TARGET_FILE_SIZE, "1048576"}, {Options::BUCKET, "1"}, @@ -2500,7 +2518,7 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { // filter is enabled below). {Options::WRITE_BATCH_SIZE, "1"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, }; ASSERT_OK_AND_ASSIGN(auto helper, @@ -2600,7 +2618,6 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}; auto schema = arrow::schema(fields); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "parquet"}, {Options::TARGET_FILE_SIZE, "1048576"}, {Options::BUCKET, "-1"}, @@ -2611,7 +2628,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { // without row-level filter the reader output is precisely that one row. {Options::WRITE_BATCH_SIZE, "1"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, }; ASSERT_OK_AND_ASSIGN(auto helper, @@ -2708,7 +2725,6 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { arrow::field("f1", arrow::utf8())}; auto schema = arrow::schema(fields); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "parquet"}, {Options::TARGET_FILE_SIZE, "1048576"}, {Options::BUCKET, "-1"}, @@ -2718,7 +2734,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { // in 4 row groups of 4 single-row pages. {Options::WRITE_BATCH_SIZE, "1"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, {"parquet.write.max-row-group-length", "4"}, {"parquet.read.enable-page-index-filter", "true"}, @@ -2796,8 +2812,9 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { arrow::field("f1", arrow::int32())}; auto schema = arrow::schema(fields); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "parquet"}, - {Options::TARGET_FILE_SIZE, "1048576"}, {Options::BUCKET, "-1"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::TARGET_FILE_SIZE, "1048576"}, + {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, "local"}, }; ASSERT_OK_AND_ASSIGN( @@ -2886,7 +2903,6 @@ TEST_P(WriteAndReadInteTest, TestAppendSharedShreddingMap) { }; auto schema = arrow::schema(fields); std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -2945,7 +2961,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingColumnPlacementPolicies) { arrow::field("lru_metrics", map_type), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -3028,7 +3043,6 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPartitionAndBucket) }; auto schema = arrow::schema(fields); std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, @@ -3135,14 +3149,13 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { }; auto schema = arrow::schema(fields); std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1048576"}, {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, file_system}, {Options::WRITE_BATCH_SIZE, "1"}, {"parquet.page.size", "1"}, - {"parquet.enable-dictionary", "false"}, + {"parquet.enable.dictionary", "false"}, {"parquet.write.enable-page-index", "true"}, {"parquet.write.max-row-group-length", "1"}, {"parquet.read.enable-page-index-filter", "true"}, @@ -3229,7 +3242,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingNewWriterStartsWithMaxColumnC arrow::field("metrics", map_type), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, @@ -3298,7 +3310,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingAdaptsAcrossRollingFiles) { arrow::field("metrics", map_type), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1048576"}, {Options::TARGET_FILE_ROW_NUM, "2"}, @@ -3372,7 +3383,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingSwitchMapLayoutAndUseMaxColum arrow::field("labels", map_type), }; std::map options_v0 = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, @@ -3454,7 +3464,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingReadAfterRenameColumn) { arrow::field("metrics", map_type), }; std::map options_v0 = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -3539,7 +3548,6 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingWithSchemaEvolution) { arrow::field("k1", arrow::utf8()), }; std::map options_v0 = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -3648,9 +3656,11 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShredding) { arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), }; std::map options_v0 = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, - {Options::FILE_SYSTEM, file_system}, {"fields.tags.map.storage-layout", "default"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "default"}, }; if (file_system == "jindo") { options_v0 = AddOptionsForJindo(options_v0); @@ -3726,7 +3736,6 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefault) { arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), }; std::map options_v0 = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -3790,7 +3799,6 @@ TEST_P(WriteAndReadInteTest, TestAppendMapStorageLayoutSharedShreddingToDefaultC arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), }; std::map options_v0 = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, @@ -3882,7 +3890,6 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingWithStructValue) { arrow::field("tags", arrow::map(arrow::utf8(), value_type)), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -3955,7 +3962,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingWithComplexValue) { arrow::field("tags", map_type), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -4094,7 +4100,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingWithAllSupportedComplexValueT arrow::field("metrics", map_type), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -4222,7 +4227,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingStructValueSchemaEvolutionRea arrow::field("profile", profile_type), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -4340,10 +4344,13 @@ TEST_P(WriteAndReadInteTest, TestOrcDictionaryLazyDecodingWithSharedShredding) { arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, - {Options::FILE_SYSTEM, file_system}, {"fields.tags.map.storage-layout", "default"}, - {"orc.read.enable-lazy-decoding", "true"}, {"orc.dictionary-key-size-threshold", "1"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "default"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"orc.dictionary-key-size-threshold", "1"}, }; if (file_system == "jindo") { options = AddOptionsForJindo(options); @@ -4426,7 +4433,6 @@ TEST_P(WriteAndReadInteTest, TestPkSharedShreddingMap) { arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, @@ -4497,7 +4503,6 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithOverflow) { arrow::field("tags", map_type), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -4632,7 +4637,6 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithNullOrMissin arrow::field("tags", map_type), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -4744,7 +4748,6 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallMultipleColumns) arrow::field("metrics", map_type), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -4855,9 +4858,11 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShreddingPartial arrow::field("tags", map_type), }; std::map options_v0 = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, - {Options::FILE_SYSTEM, file_system}, {"fields.tags.map.storage-layout", "default"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "default"}, }; if (file_system == "jindo") { options_v0 = AddOptionsForJindo(options_v0); @@ -4945,7 +4950,6 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefaultPartial arrow::field("tags", map_type), }; std::map options_v0 = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -5040,7 +5044,6 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingDuplicateSelectedKeys) { arrow::field("tags", map_type), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, @@ -5088,7 +5091,6 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingAllNullMapColumn) { arrow::field("tags", map_type), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index 63768416c..27bcea674 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -409,8 +409,7 @@ TEST_P(WriteInteTest, TestAppendTableBatchWrite) { auto schema = arrow::schema(fields); auto file_format = GetParam(); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format}, + std::map options = {{Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, "local"}}; @@ -514,9 +513,11 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithOneBucket) { auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::BUCKET_KEY, "f5"}, {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "f5"}, + {Options::FILE_SYSTEM, "local"}, }; ASSERT_OK_AND_ASSIGN( @@ -668,9 +669,11 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithPartitionAndMultiBuckets) { std::vector partition_keys = {"f2", "f1"}; auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, - {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {Options::FILE_SYSTEM, "local"}, }; ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, @@ -824,9 +827,11 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithComplexType) { auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::BUCKET_KEY, "f5"}, {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "f5"}, + {Options::FILE_SYSTEM, "local"}, }; ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, @@ -974,9 +979,11 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { std::vector partition_keys = {"f1"}; auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, - {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {Options::FILE_SYSTEM, "local"}, }; ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, @@ -1250,9 +1257,11 @@ TEST_P(WriteInteTest, TestPkTableBatchWrite) { std::vector partition_keys = {"f1"}; auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, - {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {Options::FILE_SYSTEM, "local"}, }; ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, @@ -1411,9 +1420,11 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) { std::vector partition_keys = {}; auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, - {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {Options::FILE_SYSTEM, "local"}, }; ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, @@ -1650,9 +1661,11 @@ TEST_P(WriteInteTest, TestPkTableWriteWithComplexType) { std::vector partition_keys = {}; auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::BUCKET_KEY, "f5"}, {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "f5"}, + {Options::FILE_SYSTEM, "local"}, }; ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, @@ -1821,11 +1834,13 @@ TEST_P(WriteInteTest, TestPkTableForceLookup) { std::vector primary_keys = {"f0", "f1"}; std::vector partition_keys = {}; auto file_format = GetParam(); - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, - {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"}, - {Options::FORCE_LOOKUP, "true"}, {Options::WRITE_ONLY, "true"}}; + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "f0"}, + {Options::FILE_SYSTEM, "local"}, + {Options::FORCE_LOOKUP, "true"}, + {Options::WRITE_ONLY, "true"}}; ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); @@ -1883,8 +1898,7 @@ TEST_P(WriteInteTest, TestPkTableEnableDeletionVector) { std::vector primary_keys = {"f0", "f1"}; std::vector partition_keys = {}; auto file_format = GetParam(); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format}, + std::map options = {{Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "f0"}, @@ -1943,9 +1957,11 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { std::vector primary_keys = {"f0", "f1"}; std::vector partition_keys = {"f1"}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, - {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {Options::FILE_SYSTEM, "local"}, }; bool run_complete = false; auto io_hook = IOHook::GetInstance(); @@ -2235,7 +2251,7 @@ TEST_F(WriteInteTest, TestAppendTableWriteWithAlterTable) { arrow::field("e", arrow::int32()), }; std::map options = {{Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}}; ASSERT_OK_AND_ASSIGN(auto helper, @@ -2314,7 +2330,7 @@ TEST_F(WriteInteTest, TestPKTableWriteWithAlterTable) { arrow::field("v2", arrow::int32()), }; std::map options = {{Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}}; ASSERT_OK_AND_ASSIGN(auto helper, @@ -2409,7 +2425,7 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) { auto file_format = GetParam(); std::map options = { {Options::FILE_FORMAT, file_format}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"}, @@ -2513,7 +2529,6 @@ TEST_P(WriteInteTest, TestWriteWithFieldId) { ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportType(*arrow_data_type, &c_schema).ok()); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, @@ -2745,7 +2760,6 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithExternalPath) { auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"}, @@ -3179,7 +3193,7 @@ TEST_P(WriteInteTest, TestWriteWithIOException) { auto file_format = GetParam(); std::map options = { {Options::FILE_FORMAT, file_format}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"}, @@ -3239,7 +3253,7 @@ TEST_P(WriteInteTest, TestCommitWithIOException) { auto file_format = GetParam(); std::map options = { {Options::FILE_FORMAT, file_format}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"}, @@ -3311,7 +3325,7 @@ TEST_P(WriteInteTest, TestWriteMemoryUse) { auto file_format = GetParam(); std::map options = { {Options::FILE_FORMAT, file_format}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"}, @@ -3377,8 +3391,9 @@ TEST_P(WriteInteTest, TestAppendTableWithAllNull) { auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, "local"}, }; @@ -3417,8 +3432,7 @@ TEST_P(WriteInteTest, TestPkTablePostponeBucket) { auto schema = arrow::schema(fields); std::vector primary_keys = {"f0", "f1"}; auto file_format = GetParam(); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format}, + std::map options = {{Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-2"}, {Options::FILE_SYSTEM, "local"}}; @@ -3512,7 +3526,7 @@ TEST_F(WriteInteTest, TestBranchWrite) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); std::map options = {{Options::FILE_FORMAT, "orc"}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_SYSTEM, "local"}}; WriteContextBuilder context_builder(table_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, @@ -3613,7 +3627,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { auto file_format = GetParam(); auto dir = UniqueTestDirectory::Create(); std::map options = {{Options::FILE_FORMAT, file_format}, - {Options::MANIFEST_FORMAT, "orc"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::FILE_SYSTEM, "local"}}; @@ -3804,11 +3818,13 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { auto schema = arrow::schema(fields); auto file_format = GetParam(); - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::FILE_SYSTEM, "local"}, {Options::BLOB_AS_DESCRIPTOR, "true"}}; + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::FILE_SYSTEM, "local"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}}; ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, @@ -3916,8 +3932,7 @@ TEST_P(WriteInteTest, TestAppendTableWithDateFieldAsPartitionField) { auto schema = arrow::schema(fields); auto file_format = GetParam(); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format}, + std::map options = {{Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, "local"}}; @@ -4039,7 +4054,6 @@ TEST_P(WriteInteTest, TestPkSpillableMapSharedShreddingReadWrite) { arrow::field("metrics", map_type), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}, @@ -4572,10 +4586,13 @@ TEST_P(WriteInteTest, TestPkSpillableWithIOException) { std::vector partition_keys = {"f1"}; auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, - {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"}, - {Options::WRITE_BUFFER_SIZE, "1"}, {Options::WRITE_BUFFER_SPILLABLE, "true"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {Options::FILE_SYSTEM, "local"}, + {Options::WRITE_BUFFER_SIZE, "1"}, + {Options::WRITE_BUFFER_SPILLABLE, "true"}, {Options::WRITE_ONLY, "true"}, }; bool run_complete = false; @@ -4683,14 +4700,11 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { auto schema = arrow::schema(fields); auto file_format = GetParam(); - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format}, - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::FILE_SYSTEM, "local"}, - {Options::BLOB_AS_DESCRIPTOR, "true"}, - {Options::BLOB_FIELD, "blob2,blob1"}}; + std::map options = { + {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::FILE_SYSTEM, "local"}, {Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::BLOB_FIELD, "blob2,blob1"}}; ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, @@ -4828,7 +4842,6 @@ TEST_P(WriteInteTest, TestRowTrackingPartitionGroupOnCommit) { auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -4943,7 +4956,6 @@ TEST_P(WriteInteTest, TestRowTrackingPartitionGroupOnCommitDisabled) { auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, @@ -5041,14 +5053,11 @@ TEST_P(WriteInteTest, TestMultipleBlobFieldsSplitByTargetSize) { auto file_format = GetParam(); // Set a very small blob target file size to force splitting - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format}, - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::FILE_SYSTEM, "local"}, - {Options::BLOB_AS_DESCRIPTOR, "false"}, - {Options::BLOB_TARGET_FILE_SIZE, "1"}}; + std::map options = { + {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::FILE_SYSTEM, "local"}, {Options::BLOB_AS_DESCRIPTOR, "false"}, + {Options::BLOB_TARGET_FILE_SIZE, "1"}}; ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-ed03e3fd-3ff4-4d88-9d5d-683700f78967-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-ed03e3fd-3ff4-4d88-9d5d-683700f78967-0 index 1a0ea1eba..427e68590 100644 Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-ed03e3fd-3ff4-4d88-9d5d-683700f78967-0 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-ed03e3fd-3ff4-4d88-9d5d-683700f78967-0 differ diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-f2299c3d-c3f1-400f-ad3d-124e3a342389-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-f2299c3d-c3f1-400f-ad3d-124e3a342389-0 index 8f99c331a..d8ca5da5d 100644 Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-f2299c3d-c3f1-400f-ad3d-124e3a342389-0 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-f2299c3d-c3f1-400f-ad3d-124e3a342389-0 differ diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-0 index eea939b22..f8d0e6d11 100644 Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-0 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-0 differ diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-1 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-1 index fbf9cda34..d025ffe7d 100644 Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-1 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-1 differ diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-0 index c7de35987..e4cfe141c 100644 Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-0 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-0 differ diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-1 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-1 index eea939b22..d12b180cb 100644 Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-1 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-1 differ diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-0 index a1bd3ded6..b631fc3a1 100644 --- a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-0 +++ b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-0 @@ -34,8 +34,8 @@ "partitionKeys" : [ "key0", "key1" ], "primaryKeys" : [ ], "options" : { - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc" }, "timeMillis" : 1730430023755 -} \ No newline at end of file +} diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-1 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-1 index cb81068fa..83b8cc9e2 100644 --- a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-1 +++ b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-1 @@ -35,8 +35,8 @@ "partitionKeys" : [ "key0", "key1" ], "primaryKeys" : [ ], "options" : { - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc" }, "timeMillis" : 1730430297413 -} \ No newline at end of file +} diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-0 index 55f747076..d9a092c87 100644 --- a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-0 +++ b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-0 @@ -18,8 +18,8 @@ "partitionKeys" : [ "dt" ], "primaryKeys" : [ ], "options" : { - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc" }, "timeMillis" : 1744885904315 -} \ No newline at end of file +} diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-1 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-1 index 9e0b0819e..724aef673 100644 --- a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-1 +++ b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-1 @@ -19,8 +19,8 @@ "primaryKeys" : [ "dt", "name" ], "options" : { "bucket" : "2", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc" }, "timeMillis" : 1744885904631 -} \ No newline at end of file +} diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-6090bcc3-a5a6-4c51-bc5e-7af6498e5b99-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-6090bcc3-a5a6-4c51-bc5e-7af6498e5b99-0 index 3469a68b1..4120d9e07 100644 Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-6090bcc3-a5a6-4c51-bc5e-7af6498e5b99-0 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-6090bcc3-a5a6-4c51-bc5e-7af6498e5b99-0 differ diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-b204b605-41df-407f-85d9-722bef2754f4-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-b204b605-41df-407f-85d9-722bef2754f4-0 index 6e7a8b14c..f7b10cd0b 100644 Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-b204b605-41df-407f-85d9-722bef2754f4-0 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-b204b605-41df-407f-85d9-722bef2754f4-0 differ diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-0 index c9c696013..5a21b96d1 100644 Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-0 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-0 differ diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-1 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-1 index 7c3e8f780..230e1e70d 100644 Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-1 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-1 differ diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-0 index c9c696013..8b4e5a62e 100644 Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-0 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-0 differ diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-1 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-1 index 2d8d64790..2070099f9 100644 Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-1 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-1 differ diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-0 index 55f747076..d9a092c87 100644 --- a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-0 +++ b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-0 @@ -18,8 +18,8 @@ "partitionKeys" : [ "dt" ], "primaryKeys" : [ ], "options" : { - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc" }, "timeMillis" : 1744885904315 -} \ No newline at end of file +} diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-1 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-1 index db5856dc5..499fccc5f 100644 --- a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-1 +++ b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-1 @@ -18,9 +18,9 @@ "partitionKeys" : [ "dt" ], "primaryKeys" : [ ], "options" : { - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc", "scan.fallback-branch" : "rt" }, "timeMillis" : 1744885904624 -} \ No newline at end of file +} diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-0 index e02a4bc7a..3da1ad5d1 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-0 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-0 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-1 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-1 index 8a2b97205..5dcc67f76 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-1 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-1 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-0 index 5eb55ba49..2897e9b71 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-0 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-0 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-1 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-1 index 1518ddb1a..2ee0906d9 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-1 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-1 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-0 index 1641b8d34..34248ccc8 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-0 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-0 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-1 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-1 index daea579f8..ee59a3182 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-1 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-1 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-2 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-2 index 42382569a..07723ec4b 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-2 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-2 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-3 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-3 index a69ce5f00..2b31bc2b9 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-3 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-3 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-0 index 04ebc0214..c6a9e75da 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-0 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-0 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-1 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-1 index 65664835a..4b5975760 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-1 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-1 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-2 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-2 index 65664835a..3b778efd0 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-2 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-2 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-3 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-3 index 04ebc0214..995dcac94 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-3 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-3 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-4 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-4 index 65664835a..0dbea5922 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-4 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-4 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-5 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-5 index 0336fb8d6..a47a14d7e 100644 Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-5 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-5 differ diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/schema/schema-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/schema/schema-0 index 96dd5b130..bbc18eece 100644 --- a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/schema/schema-0 +++ b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/schema/schema-0 @@ -32,9 +32,9 @@ "fields.default-aggregate-function" : "min", "lookup.remote-file.level-threshold" : "1", "merge-engine" : "aggregation", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc", "deletion-vectors.enabled" : "true" }, "timeMillis" : 1775650042347 -} \ No newline at end of file +} diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-0 index a95c97f7b..22f5a4a00 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-0 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-0 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-1 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-1 index 295cfe0b0..25e07f911 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-1 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-1 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-daf4fd15-fe55-4216-8668-bac0b6a93781-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-daf4fd15-fe55-4216-8668-bac0b6a93781-0 index 4ad23b28b..64916dfee 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-daf4fd15-fe55-4216-8668-bac0b6a93781-0 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-daf4fd15-fe55-4216-8668-bac0b6a93781-0 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-0 index 68eff1597..76da03aa9 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-0 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-0 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-1 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-1 index 2dddc4691..d6d5f0a0d 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-1 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-1 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-2 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-2 index 2dddc4691..93e23c94e 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-2 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-2 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-3 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-3 index 8a6e7ed47..587549334 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-3 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-3 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-0 index d2c7c01e0..266ffa1a7 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-0 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-0 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-1 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-1 index 65e68ab9f..3a0a5211b 100644 Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-1 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-1 differ diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-0 index 277d45f2f..82fd8df80 100644 --- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-0 +++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-0 @@ -39,9 +39,9 @@ "primaryKeys" : [ "k0", "k1", "p0", "p1" ], "options" : { "bucket" : "1", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc", "sequence.field" : "s0,s1" }, "timeMillis" : 1735120478343 -} \ No newline at end of file +} diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-1 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-1 index 0bf55dbfa..1d6e3cc8f 100644 --- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-1 +++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-1 @@ -39,9 +39,9 @@ "primaryKeys" : [ "k0", "tmp", "p0", "p1" ], "options" : { "bucket" : "1", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc", "sequence.field" : "s0,s1" }, "timeMillis" : 1735207491195 -} \ No newline at end of file +} diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-2 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-2 index 13db16689..bc157552e 100644 --- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-2 +++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-2 @@ -39,9 +39,9 @@ "primaryKeys" : [ "k1", "tmp", "p0", "p1" ], "options" : { "bucket" : "1", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc", "sequence.field" : "s0,s1" }, "timeMillis" : 1735207514342 -} \ No newline at end of file +} diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-3 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-3 index 8ff9465ab..e1037a153 100644 --- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-3 +++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-3 @@ -39,9 +39,9 @@ "primaryKeys" : [ "k1", "k0", "p0", "p1" ], "options" : { "bucket" : "1", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc", "sequence.field" : "s0,s1" }, "timeMillis" : 1735207576625 -} \ No newline at end of file +} diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-4 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-4 index a7e52509b..23f2e45cc 100644 --- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-4 +++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-4 @@ -43,9 +43,9 @@ "primaryKeys" : [ "k1", "k0", "p0", "p1" ], "options" : { "bucket" : "1", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "orc", "sequence.field" : "s0,s1" }, "timeMillis" : 1735207715028 -} \ No newline at end of file +} diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-0 index fb7dcc927..0713a2fc3 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-1 index 135345f6a..0ff37d234 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-1 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-97ddb443-4cd9-42e9-9871-4d51d4fa1b49-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-97ddb443-4cd9-42e9-9871-4d51d4fa1b49-0 index 0a5198776..1317128e2 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-97ddb443-4cd9-42e9-9871-4d51d4fa1b49-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-97ddb443-4cd9-42e9-9871-4d51d4fa1b49-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-eb34bdd2-2c23-49af-b5aa-537a596d8fe3-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-eb34bdd2-2c23-49af-b5aa-537a596d8fe3-0 index cffc81ea3..15c7d9913 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-eb34bdd2-2c23-49af-b5aa-537a596d8fe3-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-eb34bdd2-2c23-49af-b5aa-537a596d8fe3-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-ed67f349-7c6f-4f51-8723-274a060042d1-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-ed67f349-7c6f-4f51-8723-274a060042d1-0 index 0a6862a6f..22f88c68a 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-ed67f349-7c6f-4f51-8723-274a060042d1-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-ed67f349-7c6f-4f51-8723-274a060042d1-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-0 index 0e0eb50ee..89a934cac 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-1 index 22f85d5d3..65ae80123 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-1 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-0 index c7de35987..722a9a06a 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1 index 0e0eb50ee..2077b919d 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-0 index 04d7af1a5..f60aee5da 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-1 index a20428113..456e1eb2c 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-1 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-2 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-2 index b07ea33d1..e8d006991 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-2 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-2 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-3 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-3 index 7a0345807..15f30ebb7 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-3 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-3 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-0 index 3023df2de..77010b30e 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-0 differ diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-1 index 41ad2bdf9..0097ed46f 100644 Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-1 differ diff --git a/test/test_data/parquet/append_09.db/append_09/schema/schema-0 b/test/test_data/parquet/append_09.db/append_09/schema/schema-0 index 2314054c7..b992da387 100644 --- a/test/test_data/parquet/append_09.db/append_09/schema/schema-0 +++ b/test/test_data/parquet/append_09.db/append_09/schema/schema-0 @@ -24,7 +24,7 @@ "options" : { "bucket" : "2", "bucket-key" : "f2", - "manifest.format" : "orc", + "manifest.format" : "avro", "file.format" : "parquet" }, "timeMillis" : 1755758260435