test(file_index): verify Rust reads of Java-generated indexed tables

crates/paimon/testdata/file_index/README.md

@@ -0,0 +1,38 @@
+# Java FileIndex table fixtures
+
+These four committed tables were produced through Apache Paimon Java's normal
+catalog, append writer, commit, snapshot, and manifest paths. No index bytes or
+metadata were assembled or edited by hand.
+
+Provenance:
+
+- Apache Paimon Java commit: `1d368b4a5932f8221fd28e2555001abdb8fb12ee`
+  (`2.2-SNAPSHOT`).
+- Generator: `java/JavaFileIndexTableFixtureGeneratorTest.java`.
+- Tables: Bitmap and Bloom Filter, each with an embedded (`1 MB` threshold) and
+  `.index` sidecar (`1 B` threshold) form.
+- Common options: `bucket=1`, `bucket-key=id`, `file.format=parquet`, and
+  `file-index.read.enabled=true`.
+- Bloom options: `items=16`, `fpp=0.01`.
+- Rows, in physical order: `(1, "keep")`, `(1, "drop")`,
+  `(NULL, "null-id")`, `(3, "three")`.
+
+The Rust test expects `id = 1` to return both duplicate rows, `id IS NULL` to
+return the null row, `id = 1 AND payload = 'keep'` to retain only the residual
+match, and `id = 2` to return no rows. The last predicate lies inside the data
+file's `[1, 3]` min/max range, so manifest statistics cannot prune it; it is also
+the observable FileIndex skip case.
+
+To regenerate into an empty temporary directory:
+
+```bash
+git clone https://github.com/apache/paimon.git /tmp/paimon-java
+git -C /tmp/paimon-java checkout 1d368b4a5932f8221fd28e2555001abdb8fb12ee
+crates/paimon/testdata/file_index/regenerate.sh \
+  /tmp/paimon-java /tmp/paimon-file-index-fixtures
+```
+
+After reviewing the generated metadata and running the Rust compatibility test,
+replace `default.db/` with the generated `default.db/` directory. UUID-bearing
+file names may change between runs; the rows, table options, storage shape, and
+query expectations are deterministic.

crates/paimon/testdata/file_index/default.db/bitmap_embedded/schema/schema-0

@@ -0,0 +1,25 @@
+{
+  "version" : 3,
+  "id" : 0,
+  "fields" : [ {
+    "id" : 0,
+    "name" : "id",
+    "type" : "INT"
+  }, {
+    "id" : 1,
+    "name" : "payload",
+    "type" : "STRING"
+  } ],
+  "highestFieldId" : 1,
+  "partitionKeys" : [ ],
+  "primaryKeys" : [ ],
+  "options" : {
+    "bucket" : "1",
+    "file-index.bitmap.columns" : "id",
+    "bucket-key" : "id",
+    "file-index.read.enabled" : "true",
+    "file.format" : "parquet",
+    "file-index.in-manifest-threshold" : "1 MB"
+  },
+  "timeMillis" : 1789814325908
+}

crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/EARLIEST

@@ -0,0 +1 @@
+1

crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/LATEST

@@ -0,0 +1 @@
+1

crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/snapshot-1

@@ -0,0 +1,18 @@
+{
+  "version" : 3,
+  "uuid" : "c566080a-5959-4fba-b8ec-ed10fea2d95f",
+  "id" : 1,
+  "schemaId" : 0,
+  "baseManifestList" : "manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-0",
+  "baseManifestListSize" : 1158,
+  "deltaManifestList" : "manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-1",
+  "deltaManifestListSize" : 1263,
+  "commitUser" : "9fa4d883-e8c7-4e38-8f41-40006e74ac3c",
+  "writerVersion" : "java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee",
+  "commitIdentifier" : 9223372036854775807,
+  "commitKind" : "APPEND",
+  "timeMillis" : 1789814327145,
+  "totalRecordCount" : 4,
+  "deltaRecordCount" : 4,
+  "nextRowId" : 0
+}

crates/paimon/testdata/file_index/default.db/bitmap_sidecar/schema/schema-0

@@ -0,0 +1,25 @@
+{
+  "version" : 3,
+  "id" : 0,
+  "fields" : [ {
+    "id" : 0,
+    "name" : "id",
+    "type" : "INT"
+  }, {
+    "id" : 1,
+    "name" : "payload",
+    "type" : "STRING"
+  } ],
+  "highestFieldId" : 1,
+  "partitionKeys" : [ ],
+  "primaryKeys" : [ ],
+  "options" : {
+    "bucket" : "1",
+    "file-index.bitmap.columns" : "id",
+    "bucket-key" : "id",
+    "file-index.read.enabled" : "true",
+    "file.format" : "parquet",
+    "file-index.in-manifest-threshold" : "1 B"
+  },
+  "timeMillis" : 1789814327169
+}

crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/EARLIEST

@@ -0,0 +1 @@
+1

crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/LATEST

@@ -0,0 +1 @@
+1

crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/snapshot-1

@@ -0,0 +1,18 @@
+{
+  "version" : 3,
+  "uuid" : "6476f620-ef94-47ba-8250-c49ab25124f2",
+  "id" : 1,
+  "schemaId" : 0,
+  "baseManifestList" : "manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0",
+  "baseManifestListSize" : 1158,
+  "deltaManifestList" : "manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-1",
+  "deltaManifestListSize" : 1263,
+  "commitUser" : "71026591-a529-45f5-9b77-71d993270c4b",
+  "writerVersion" : "java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee",
+  "commitIdentifier" : 9223372036854775807,
+  "commitKind" : "APPEND",
+  "timeMillis" : 1789814327179,
+  "totalRecordCount" : 4,
+  "deltaRecordCount" : 4,
+  "nextRowId" : 0
+}

crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/schema/schema-0

@@ -0,0 +1,27 @@
+{
+  "version" : 3,
+  "id" : 0,
+  "fields" : [ {
+    "id" : 0,
+    "name" : "id",
+    "type" : "INT"
+  }, {
+    "id" : 1,
+    "name" : "payload",
+    "type" : "STRING"
+  } ],
+  "highestFieldId" : 1,
+  "partitionKeys" : [ ],
+  "primaryKeys" : [ ],
+  "options" : {
+    "bucket" : "1",
+    "file-index.bloom-filter.id.items" : "16",
+    "file-index.in-manifest-threshold" : "1 MB",
+    "file-index.bloom-filter.columns" : "id",
+    "file-index.bloom-filter.id.fpp" : "0.01",
+    "bucket-key" : "id",
+    "file-index.read.enabled" : "true",
+    "file.format" : "parquet"
+  },
+  "timeMillis" : 1789814327180
+}

crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/EARLIEST

@@ -0,0 +1 @@
+1

crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/LATEST

@@ -0,0 +1 @@
+1

crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/snapshot-1

@@ -0,0 +1,18 @@
+{
+  "version" : 3,
+  "uuid" : "e21872d6-c6a1-4ef4-9a26-fe324ccd15cf",
+  "id" : 1,
+  "schemaId" : 0,
+  "baseManifestList" : "manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-0",
+  "baseManifestListSize" : 1158,
+  "deltaManifestList" : "manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-1",
+  "deltaManifestListSize" : 1263,
+  "commitUser" : "1930fc30-d2c7-493c-b2cc-554a5b504d22",
+  "writerVersion" : "java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee",
+  "commitIdentifier" : 9223372036854775807,
+  "commitKind" : "APPEND",
+  "timeMillis" : 1789814327189,
+  "totalRecordCount" : 4,
+  "deltaRecordCount" : 4,
+  "nextRowId" : 0
+}

crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/schema/schema-0

@@ -0,0 +1,27 @@
+{
+  "version" : 3,
+  "id" : 0,
+  "fields" : [ {
+    "id" : 0,
+    "name" : "id",
+    "type" : "INT"
+  }, {
+    "id" : 1,
+    "name" : "payload",
+    "type" : "STRING"
+  } ],
+  "highestFieldId" : 1,
+  "partitionKeys" : [ ],
+  "primaryKeys" : [ ],
+  "options" : {
+    "bucket" : "1",
+    "file-index.bloom-filter.id.items" : "16",
+    "file-index.in-manifest-threshold" : "1 B",
+    "file-index.bloom-filter.columns" : "id",
+    "file-index.bloom-filter.id.fpp" : "0.01",
+    "bucket-key" : "id",
+    "file-index.read.enabled" : "true",
+    "file.format" : "parquet"
+  },
+  "timeMillis" : 1789814327190
+}

crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/EARLIEST

@@ -0,0 +1 @@
+1

crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/LATEST

@@ -0,0 +1 @@
+1

crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/snapshot-1

@@ -0,0 +1,18 @@
+{
+  "version" : 3,
+  "uuid" : "4775d461-aa56-46ac-b12f-7aa8976a8b7f",
+  "id" : 1,
+  "schemaId" : 0,
+  "baseManifestList" : "manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-0",
+  "baseManifestListSize" : 1158,
+  "deltaManifestList" : "manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-1",
+  "deltaManifestListSize" : 1260,
+  "commitUser" : "04f8ab29-9471-4204-8e7c-6340649da117",
+  "writerVersion" : "java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee",
+  "commitIdentifier" : 9223372036854775807,
+  "commitKind" : "APPEND",
+  "timeMillis" : 1789814327197,
+  "totalRecordCount" : 4,
+  "deltaRecordCount" : 4,
+  "nextRowId" : 0
+}

crates/paimon/testdata/file_index/java/JavaFileIndexTableFixtureGeneratorTest.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fixture;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.CatalogFactory;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.paimon.options.CatalogOptions.WAREHOUSE;
+
+/** Generates committed Java tables used by paimon-rust FileIndex compatibility tests. */
+public class JavaFileIndexTableFixtureGeneratorTest {
+
+    private static final String DATABASE = "default";
+
+    @Test
+    public void generate() throws Exception {
+        String output = System.getProperty("generate.file-index.fixture.output");
+        if (output == null || output.trim().isEmpty()) {
+            throw new IllegalArgumentException(
+                    "Set -Dgenerate.file-index.fixture.output=<empty-output-directory>");
+        }
+
+        Path warehouse = Paths.get(output).toAbsolutePath().normalize();
+        Options catalogOptions = new Options();
+        catalogOptions.set(WAREHOUSE, warehouse.toUri().toString());
+        Catalog catalog = CatalogFactory.createCatalog(CatalogContext.create(catalogOptions));
+        catalog.createDatabase(DATABASE, false);
+
+        writeTable(catalog, "bitmap_embedded", "bitmap", "1 MB");
+        writeTable(catalog, "bitmap_sidecar", "bitmap", "1 B");
+        writeTable(catalog, "bloom_filter_embedded", "bloom-filter", "1 MB");
+        writeTable(catalog, "bloom_filter_sidecar", "bloom-filter", "1 B");
+    }
+
+    private static void writeTable(
+            Catalog catalog, String tableName, String indexType, String manifestThreshold)
+            throws Exception {
+        Map<String, String> options = new HashMap<>();
+        options.put("bucket", "1");
+        options.put("bucket-key", "id");
+        options.put("file.format", "parquet");
+        options.put("file-index.read.enabled", "true");
+        options.put("file-index.in-manifest-threshold", manifestThreshold);
+        options.put("file-index." + indexType + ".columns", "id");
+        if ("bloom-filter".equals(indexType)) {
+            options.put("file-index.bloom-filter.id.items", "16");
+            options.put("file-index.bloom-filter.id.fpp", "0.01");
+        }
+
+        Schema schema =
+                Schema.newBuilder()
+                        .column("id", DataTypes.INT())
+                        .column("payload", DataTypes.STRING())
+                        .options(options)
+                        .build();
+        Identifier identifier = Identifier.create(DATABASE, tableName);
+        catalog.createTable(identifier, schema, false);
+        Table table = catalog.getTable(identifier);
+
+        BatchWriteBuilder builder = table.newBatchWriteBuilder();
+        try (BatchTableWrite write = builder.newWrite();
+                BatchTableCommit commit = builder.newCommit()) {
+            write.write(row(1, "keep"));
+            write.write(row(1, "drop"));
+            write.write(row(null, "null-id"));
+            write.write(row(3, "three"));
+            commit.commit(write.prepareCommit());
+        }
+    }
+
+    private static GenericRow row(Integer id, String payload) {
+        return GenericRow.of(id, BinaryString.fromString(payload));
+    }
+}