feat(realtime): add primary-key in-memory support

.asf.yaml

@@ -54,39 +54,7 @@ github:
         dismiss_stale_reviews: true
         require_last_push_approval: true
         required_approving_review_count: 1
-      required_status_checks:
-        - name: "pre-commit"
-          app_slug: -1
-        - name: "rat-license-check"
-          app_slug: -1
-        - name: "script-tests"
-          app_slug: -1
-        - name: "asan-ubsan-x86_64"
-          app_slug: -1
-        - name: "tsan-x86_64"
-          app_slug: -1
-        - name: "clang-debug-x86_64"
-          app_slug: -1
-        - name: "clang-release-x86_64"
-          app_slug: -1
-        - name: "gcc-debug-x86_64"
-          app_slug: -1
-        - name: "gcc-release-x86_64"
-          app_slug: -1
-        - name: "gcc-debug-aarch64"
-          app_slug: -1
-        - name: "gcc-release-aarch64"
-          app_slug: -1
-        - name: "clang-debug-aarch64"
-          app_slug: -1
-        - name: "asan-ubsan-aarch64"
-          app_slug: -1
-        - name: "clang-release-aarch64"
-          app_slug: -1
-        - name: "tsan-aarch64"
-          app_slug: -1
-        - name: "gcc8-test"
-          app_slug: -1
+
   pull_requests:
     allow_auto_merge: false
     allow_update_branch: true

.github/workflows/build_and_test.yaml

@@ -75,7 +75,6 @@ jobs:
           - name: asan-ubsan-x86_64
             build_args: --enable_asan --enable_ubsan
           - name: tsan-x86_64
-            skip_rust: true
             build_args: --enable_tsan
           - name: gcc-debug-aarch64
             runner: ubuntu-24.04-arm
@@ -96,7 +95,6 @@ jobs:
             build_args: --build_type Release
           - name: tsan-aarch64
             runner: ubuntu-24.04-arm
-            skip_rust: true
             build_args: --enable_tsan
     steps:
       - name: Checkout paimon-cpp
@@ -108,8 +106,7 @@ jobs:
         uses: ./.github/actions/setup-ccache
         with:
           cache-key-prefix: ccache-${{ matrix.name }}
-      - name: Install Rust toolchain (tantivy-fts)
-        if: ${{ !matrix.skip_rust }}
+      - name: Install Rust toolchain (Mosaic and tantivy-fts)
         shell: bash
         run: ci/scripts/setup_rust.sh
       - name: Install HTTP and TLS development dependencies

.github/workflows/gcc8_test.yaml

@@ -60,6 +60,9 @@ jobs:
           ls -la
       - name: Checkout paimon-cpp
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - name: Install Rust toolchain (Mosaic)
+        shell: bash
+        run: ci/scripts/setup_rust.sh
       - name: Setup ccache
         uses: ./.github/actions/setup-ccache
         with:

.github/workflows/release_candidate.yaml

@@ -131,7 +131,7 @@ jobs:
           name: source-archive
           path: release/ci
 
-      - name: Install Rust toolchain (tantivy-fts)
+      - name: Install Rust toolchain (Mosaic and tantivy-fts)
         shell: bash
         run: ci/scripts/setup_rust.sh
 

CMakeLists.txt

@@ -60,31 +60,45 @@ option(PAIMON_USE_UBSAN "Use Undefined Behavior Sanitizer" OFF)
 option(PAIMON_USE_CXX11_ABI "Use C++11 ABI" ON)
 option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
 option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
+option(PAIMON_ENABLE_MOSAIC "Whether to enable mosaic file format (Rust FFI)" OFF)
 option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
+option(PAIMON_ENABLE_OSS "Whether to enable OSS SDK v2 file system" OFF)
 option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF)
 option(PAIMON_ENABLE_NETWORK_TESTS
        "Whether to enable tests that access real remote services over the network" OFF)
 option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF)
 option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF)
 option(PAIMON_ENABLE_TANTIVY
        "Whether to enable tantivy-fulltext global index (Rust FFI, experimental)" OFF)
-option(PAIMON_ENABLE_REST "Whether to enable the rest catalog (requires libcurl)" OFF)
+option(PAIMON_ENABLE_REST
+       "Whether to enable the rest catalog (requires libcurl and OpenSSL)" OFF)
 if(PAIMON_ENABLE_ORC)
     add_definitions(-DPAIMON_ENABLE_ORC)
 endif()
 if(PAIMON_ENABLE_REST)
     add_definitions(-DPAIMON_ENABLE_REST)
 endif()
-# libcurl backs the HTTP client shared by the S3 file system and the rest catalog.
-if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST)
+# libcurl supports the REST and S3 HTTP clients, the OSS SDK transport, and object store timestamps.
+if(PAIMON_ENABLE_OSS
+   OR PAIMON_ENABLE_S3
+   OR PAIMON_ENABLE_REST)
     find_package(CURL REQUIRED)
 endif()
+if(PAIMON_ENABLE_REST)
+    find_package(OpenSSL 1.1.0 REQUIRED)
+endif()
 if(PAIMON_ENABLE_AVRO)
     add_definitions(-DPAIMON_ENABLE_AVRO)
 endif()
+if(PAIMON_ENABLE_MOSAIC)
+    add_definitions(-DPAIMON_ENABLE_MOSAIC)
+endif()
 if(PAIMON_ENABLE_JINDO)
     add_definitions(-DPAIMON_ENABLE_JINDO)
 endif()
+if(PAIMON_ENABLE_OSS)
+    add_definitions(-DPAIMON_ENABLE_OSS)
+endif()
 if(PAIMON_ENABLE_S3)
     add_definitions(-DPAIMON_ENABLE_S3)
 endif()
@@ -384,6 +398,11 @@ if(PAIMON_BUILD_TESTS OR PAIMON_BUILD_BENCHMARKS)
     paimon_link_libraries_whole_archive(PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS
                                         paimon_parquet_file_format_static)
 
+    if(PAIMON_ENABLE_MOSAIC)
+        paimon_link_libraries_whole_archive(PAIMON_MOSAIC_FILE_FORMAT_STATIC_LINK_LIBS
+                                            paimon_mosaic_file_format_static)
+    endif()
+
     if(PAIMON_ENABLE_ORC)
         paimon_link_libraries_whole_archive(PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS
                                             paimon_orc_file_format_static)
@@ -435,6 +454,14 @@ if(PAIMON_BUILD_TESTS)
     paimon_link_libraries_whole_archive(PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS
                                         paimon_parquet_file_format_static)
 
+    if(PAIMON_ENABLE_MOSAIC)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_mosaic_file_format_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
+        paimon_link_libraries_whole_archive(PAIMON_MOSAIC_FILE_FORMAT_STATIC_LINK_LIBS
+                                            paimon_mosaic_file_format_static)
+    endif()
+
     if(PAIMON_ENABLE_ORC)
         paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
                                            paimon_orc_file_format_shared)
@@ -452,6 +479,13 @@ if(PAIMON_BUILD_TESTS)
                                            paimon_jindo_file_system_shared)
         list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
     endif()
+    if(PAIMON_ENABLE_OSS)
+        paimon_link_libraries_whole_archive(PAIMON_OSS_FILE_SYSTEM_STATIC_LINK_LIBS
+                                            paimon_oss_file_system_static)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_oss_file_system_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
+    endif()
     if(PAIMON_ENABLE_S3)
         paimon_link_libraries_whole_archive(PAIMON_S3_FILE_SYSTEM_STATIC_LINK_LIBS
                                             paimon_s3_file_system_static)
@@ -511,11 +545,15 @@ add_subdirectory(src/paimon/fs/local)
 if(PAIMON_ENABLE_JINDO)
     add_subdirectory(src/paimon/fs/jindo)
 endif()
+add_subdirectory(src/paimon/fs/oss)
 add_subdirectory(src/paimon/fs/s3)
 add_subdirectory(src/paimon/format/blob)
 add_subdirectory(src/paimon/format/orc)
 add_subdirectory(src/paimon/format/parquet)
 add_subdirectory(src/paimon/format/avro)
+if(PAIMON_ENABLE_MOSAIC)
+    add_subdirectory(src/paimon/format/mosaic)
+endif()
 if(PAIMON_ENABLE_LUMINA)
     add_subdirectory(src/paimon/global_index/lumina)
 endif()

README.md

@@ -35,7 +35,7 @@ Paimon C++ currently provides:
 - **Scan**: batch and stream scan for append tables and primary key tables without changelog.
 - **Read**: append table read, primary key table read with deletion vector, and primary key table merge-on-read.
 - **Arrow integration**: batch read and write interfaces based on the [Arrow Columnar In-Memory Format](https://arrow.apache.org).
-- **File systems**: file system abstraction with built-in local and Jindo file system support.
+- **File systems**: file system abstraction with built-in local, Jindo, OSS, and S3 file system support.
 - **File formats**: file format abstraction with built-in ORC, Parquet, and Avro support.
 - **Runtime utilities**: memory pool and thread pool abstractions with default implementations.
 - **AI-Oriented Features**: supports RowTracking and DataEvolution mode and provides Global Index

benchmark/CMakeLists.txt

@@ -63,6 +63,20 @@ if(PAIMON_BUILD_BENCHMARKS)
                          ${PAIMON_BENCHMARK_LINK_TOOLCHAIN}
                          EXTRA_INCLUDES
                          ${CMAKE_SOURCE_DIR})
+
+    add_paimon_benchmark(parquet_format_benchmark
+                         SOURCES
+                         parquet_format_benchmark.cpp
+                         STATIC_LINK_LIBS
+                         arrow
+                         parquet
+                         ${PAIMON_BENCHMARK_STATIC_LINK_LIBS}
+                         test_utils_static
+                         Threads::Threads
+                         ${PAIMON_BENCHMARK_PLATFORM_LINK_LIBS}
+                         ${PAIMON_BENCHMARK_LINK_TOOLCHAIN}
+                         EXTRA_INCLUDES
+                         ${CMAKE_SOURCE_DIR})
 endif()
 
 if(PAIMON_BUILD_TESTS)
@@ -74,4 +88,21 @@ if(PAIMON_BUILD_TESTS)
                     STATIC_LINK_LIBS
                     paimon_shared
                     ${GTEST_LINK_TOOLCHAIN})
+
+    # Guards the format-layer assumptions parquet_format_benchmark.cpp is built on. The benchmark
+    # itself is only compiled under PAIMON_BUILD_BENCHMARKS, which CI does not set, so this test is
+    # what keeps those assumptions covered.
+    add_paimon_test(parquet_format_benchmark_test
+                    SOURCES
+                    parquet_format_benchmark_test.cpp
+                    EXTRA_INCLUDES
+                    ${CMAKE_SOURCE_DIR}
+                    STATIC_LINK_LIBS
+                    arrow
+                    parquet
+                    paimon_shared
+                    ${PAIMON_LOCAL_FILE_SYSTEM_SHARED_LINK_LIBS}
+                    ${PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS}
+                    test_utils_static
+                    ${GTEST_LINK_TOOLCHAIN})
 endif()