diff --git a/crates/paimon/testdata/file_index/README.md b/crates/paimon/testdata/file_index/README.md new file mode 100644 index 000000000..dee46286a --- /dev/null +++ b/crates/paimon/testdata/file_index/README.md @@ -0,0 +1,38 @@ +# Java FileIndex table fixtures + +These four committed tables were produced through Apache Paimon Java's normal +catalog, append writer, commit, snapshot, and manifest paths. No index bytes or +metadata were assembled or edited by hand. + +Provenance: + +- Apache Paimon Java commit: `1d368b4a5932f8221fd28e2555001abdb8fb12ee` + (`2.2-SNAPSHOT`). +- Generator: `java/JavaFileIndexTableFixtureGeneratorTest.java`. +- Tables: Bitmap and Bloom Filter, each with an embedded (`1 MB` threshold) and + `.index` sidecar (`1 B` threshold) form. +- Common options: `bucket=1`, `bucket-key=id`, `file.format=parquet`, and + `file-index.read.enabled=true`. +- Bloom options: `items=16`, `fpp=0.01`. +- Rows, in physical order: `(1, "keep")`, `(1, "drop")`, + `(NULL, "null-id")`, `(3, "three")`. + +The Rust test expects `id = 1` to return both duplicate rows, `id IS NULL` to +return the null row, `id = 1 AND payload = 'keep'` to retain only the residual +match, and `id = 2` to return no rows. The last predicate lies inside the data +file's `[1, 3]` min/max range, so manifest statistics cannot prune it; it is also +the observable FileIndex skip case. + +To regenerate into an empty temporary directory: + +```bash +git clone https://github.com/apache/paimon.git /tmp/paimon-java +git -C /tmp/paimon-java checkout 1d368b4a5932f8221fd28e2555001abdb8fb12ee +crates/paimon/testdata/file_index/regenerate.sh \ + /tmp/paimon-java /tmp/paimon-file-index-fixtures +``` + +After reviewing the generated metadata and running the Rust compatibility test, +replace `default.db/` with the generated `default.db/` directory. UUID-bearing +file names may change between runs; the rows, table options, storage shape, and +query expectations are deterministic. diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/bucket-0/data-c6f0f8ca-a74f-42ce-af35-fc4f4ef2d003-0.parquet b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/bucket-0/data-c6f0f8ca-a74f-42ce-af35-fc4f4ef2d003-0.parquet new file mode 100644 index 000000000..d411cc375 Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/bucket-0/data-c6f0f8ca-a74f-42ce-af35-fc4f4ef2d003-0.parquet differ diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-982c31e0-7caa-4462-a270-3b130005e913-0 b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-982c31e0-7caa-4462-a270-3b130005e913-0 new file mode 100644 index 000000000..004f091ec Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-982c31e0-7caa-4462-a270-3b130005e913-0 differ diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-0 b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-0 new file mode 100644 index 000000000..d61cedd47 Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-0 differ diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-1 b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-1 new file mode 100644 index 000000000..ea7c30f59 Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-1 differ diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/schema/schema-0 b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/schema/schema-0 new file mode 100644 index 000000000..a0325d02d --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/schema/schema-0 @@ -0,0 +1,25 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "payload", + "type" : "STRING" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "1", + "file-index.bitmap.columns" : "id", + "bucket-key" : "id", + "file-index.read.enabled" : "true", + "file.format" : "parquet", + "file-index.in-manifest-threshold" : "1 MB" + }, + "timeMillis" : 1789814325908 +} \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/EARLIEST b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/LATEST b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/snapshot-1 b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/snapshot-1 new file mode 100644 index 000000000..df0c91ae1 --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "c566080a-5959-4fba-b8ec-ed10fea2d95f", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-0", + "baseManifestListSize" : 1158, + "deltaManifestList" : "manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-1", + "deltaManifestListSize" : 1263, + "commitUser" : "9fa4d883-e8c7-4e38-8f41-40006e74ac3c", + "writerVersion" : "java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1789814327145, + "totalRecordCount" : 4, + "deltaRecordCount" : 4, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet new file mode 100644 index 000000000..d411cc375 Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet differ diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet.index b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet.index new file mode 100644 index 000000000..c4886674c Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet.index differ diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-392c52de-fdae-4262-81fb-4049e95c37de-0 b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-392c52de-fdae-4262-81fb-4049e95c37de-0 new file mode 100644 index 000000000..205ac3295 Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-392c52de-fdae-4262-81fb-4049e95c37de-0 differ diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0 b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0 new file mode 100644 index 000000000..e1ea4553f Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0 differ diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-1 b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-1 new file mode 100644 index 000000000..0f1da8f85 Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-1 differ diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/schema/schema-0 b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/schema/schema-0 new file mode 100644 index 000000000..4fc7be2fd --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/schema/schema-0 @@ -0,0 +1,25 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "payload", + "type" : "STRING" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "1", + "file-index.bitmap.columns" : "id", + "bucket-key" : "id", + "file-index.read.enabled" : "true", + "file.format" : "parquet", + "file-index.in-manifest-threshold" : "1 B" + }, + "timeMillis" : 1789814327169 +} \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/EARLIEST b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/LATEST b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/snapshot-1 b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/snapshot-1 new file mode 100644 index 000000000..24b6b600d --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "6476f620-ef94-47ba-8250-c49ab25124f2", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0", + "baseManifestListSize" : 1158, + "deltaManifestList" : "manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-1", + "deltaManifestListSize" : 1263, + "commitUser" : "71026591-a529-45f5-9b77-71d993270c4b", + "writerVersion" : "java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1789814327179, + "totalRecordCount" : 4, + "deltaRecordCount" : 4, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/bucket-0/data-9ce3b83d-6c45-4dc1-a34e-819b5fb2e331-0.parquet b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/bucket-0/data-9ce3b83d-6c45-4dc1-a34e-819b5fb2e331-0.parquet new file mode 100644 index 000000000..d411cc375 Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/bucket-0/data-9ce3b83d-6c45-4dc1-a34e-819b5fb2e331-0.parquet differ diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-8dd937f3-747e-4f87-a296-489d11ef050c-0 b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-8dd937f3-747e-4f87-a296-489d11ef050c-0 new file mode 100644 index 000000000..c56a063b9 Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-8dd937f3-747e-4f87-a296-489d11ef050c-0 differ diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-0 b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-0 new file mode 100644 index 000000000..715016aed Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-0 differ diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-1 b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-1 new file mode 100644 index 000000000..b592f52e8 Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-1 differ diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/schema/schema-0 b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/schema/schema-0 new file mode 100644 index 000000000..4aa8eb0b5 --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/schema/schema-0 @@ -0,0 +1,27 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "payload", + "type" : "STRING" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "1", + "file-index.bloom-filter.id.items" : "16", + "file-index.in-manifest-threshold" : "1 MB", + "file-index.bloom-filter.columns" : "id", + "file-index.bloom-filter.id.fpp" : "0.01", + "bucket-key" : "id", + "file-index.read.enabled" : "true", + "file.format" : "parquet" + }, + "timeMillis" : 1789814327180 +} \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/EARLIEST b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/LATEST b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/snapshot-1 b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/snapshot-1 new file mode 100644 index 000000000..35573b11a --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "e21872d6-c6a1-4ef4-9a26-fe324ccd15cf", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-0", + "baseManifestListSize" : 1158, + "deltaManifestList" : "manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-1", + "deltaManifestListSize" : 1263, + "commitUser" : "1930fc30-d2c7-493c-b2cc-554a5b504d22", + "writerVersion" : "java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1789814327189, + "totalRecordCount" : 4, + "deltaRecordCount" : 4, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet new file mode 100644 index 000000000..d411cc375 Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet differ diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet.index b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet.index new file mode 100644 index 000000000..5a5fcc6a2 Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet.index differ diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-214eaa6e-a8e8-4903-a41a-3fa04732e5e4-0 b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-214eaa6e-a8e8-4903-a41a-3fa04732e5e4-0 new file mode 100644 index 000000000..a3525f302 Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-214eaa6e-a8e8-4903-a41a-3fa04732e5e4-0 differ diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-0 b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-0 new file mode 100644 index 000000000..d8d82bf6e Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-0 differ diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-1 b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-1 new file mode 100644 index 000000000..84f9fd476 Binary files /dev/null and b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-1 differ diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/schema/schema-0 b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/schema/schema-0 new file mode 100644 index 000000000..c37d6b228 --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/schema/schema-0 @@ -0,0 +1,27 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "payload", + "type" : "STRING" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "1", + "file-index.bloom-filter.id.items" : "16", + "file-index.in-manifest-threshold" : "1 B", + "file-index.bloom-filter.columns" : "id", + "file-index.bloom-filter.id.fpp" : "0.01", + "bucket-key" : "id", + "file-index.read.enabled" : "true", + "file.format" : "parquet" + }, + "timeMillis" : 1789814327190 +} \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/EARLIEST b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/LATEST b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/snapshot-1 b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/snapshot-1 new file mode 100644 index 000000000..d6a12ad59 --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "4775d461-aa56-46ac-b12f-7aa8976a8b7f", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-0", + "baseManifestListSize" : 1158, + "deltaManifestList" : "manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-1", + "deltaManifestListSize" : 1260, + "commitUser" : "04f8ab29-9471-4204-8e7c-6340649da117", + "writerVersion" : "java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1789814327197, + "totalRecordCount" : 4, + "deltaRecordCount" : 4, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/java/JavaFileIndexTableFixtureGeneratorTest.java b/crates/paimon/testdata/file_index/java/JavaFileIndexTableFixtureGeneratorTest.java new file mode 100644 index 000000000..6c592f7be --- /dev/null +++ b/crates/paimon/testdata/file_index/java/JavaFileIndexTableFixtureGeneratorTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fixture; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.paimon.options.CatalogOptions.WAREHOUSE; + +/** Generates committed Java tables used by paimon-rust FileIndex compatibility tests. */ +public class JavaFileIndexTableFixtureGeneratorTest { + + private static final String DATABASE = "default"; + + @Test + public void generate() throws Exception { + String output = System.getProperty("generate.file-index.fixture.output"); + if (output == null || output.trim().isEmpty()) { + throw new IllegalArgumentException( + "Set -Dgenerate.file-index.fixture.output="); + } + + Path warehouse = Paths.get(output).toAbsolutePath().normalize(); + Options catalogOptions = new Options(); + catalogOptions.set(WAREHOUSE, warehouse.toUri().toString()); + Catalog catalog = CatalogFactory.createCatalog(CatalogContext.create(catalogOptions)); + catalog.createDatabase(DATABASE, false); + + writeTable(catalog, "bitmap_embedded", "bitmap", "1 MB"); + writeTable(catalog, "bitmap_sidecar", "bitmap", "1 B"); + writeTable(catalog, "bloom_filter_embedded", "bloom-filter", "1 MB"); + writeTable(catalog, "bloom_filter_sidecar", "bloom-filter", "1 B"); + } + + private static void writeTable( + Catalog catalog, String tableName, String indexType, String manifestThreshold) + throws Exception { + Map options = new HashMap<>(); + options.put("bucket", "1"); + options.put("bucket-key", "id"); + options.put("file.format", "parquet"); + options.put("file-index.read.enabled", "true"); + options.put("file-index.in-manifest-threshold", manifestThreshold); + options.put("file-index." + indexType + ".columns", "id"); + if ("bloom-filter".equals(indexType)) { + options.put("file-index.bloom-filter.id.items", "16"); + options.put("file-index.bloom-filter.id.fpp", "0.01"); + } + + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("payload", DataTypes.STRING()) + .options(options) + .build(); + Identifier identifier = Identifier.create(DATABASE, tableName); + catalog.createTable(identifier, schema, false); + Table table = catalog.getTable(identifier); + + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + write.write(row(1, "keep")); + write.write(row(1, "drop")); + write.write(row(null, "null-id")); + write.write(row(3, "three")); + commit.commit(write.prepareCommit()); + } + } + + private static GenericRow row(Integer id, String payload) { + return GenericRow.of(id, BinaryString.fromString(payload)); + } +} diff --git a/crates/paimon/testdata/file_index/regenerate.sh b/crates/paimon/testdata/file_index/regenerate.sh new file mode 100755 index 000000000..2905c723f --- /dev/null +++ b/crates/paimon/testdata/file_index/regenerate.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +readonly EXPECTED_COMMIT="1d368b4a5932f8221fd28e2555001abdb8fb12ee" +readonly JAVA_CHECKOUT="$1" +readonly OUTPUT="$2" +readonly SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +readonly SOURCE="$SCRIPT_DIR/java/JavaFileIndexTableFixtureGeneratorTest.java" +readonly TARGET="$JAVA_CHECKOUT/paimon-core/src/test/java/org/apache/paimon/fixture/JavaFileIndexTableFixtureGeneratorTest.java" + +actual_commit="$(git -C "$JAVA_CHECKOUT" rev-parse HEAD)" +if [[ "$actual_commit" != "$EXPECTED_COMMIT" ]]; then + echo "expected Apache Paimon Java $EXPECTED_COMMIT, got $actual_commit" >&2 + exit 1 +fi +if [[ -e "$TARGET" ]]; then + echo "refusing to overwrite $TARGET" >&2 + exit 1 +fi +if [[ -e "$OUTPUT" ]] && [[ -n "$(find "$OUTPUT" -mindepth 1 -print -quit)" ]]; then + echo "output directory must be empty: $OUTPUT" >&2 + exit 1 +fi + +mkdir -p "$(dirname "$TARGET")" "$OUTPUT" +cp "$SOURCE" "$TARGET" +trap 'rm -f "$TARGET"' EXIT + +mvn -f "$JAVA_CHECKOUT/pom.xml" \ + -pl paimon-codegen-loader -am -Pfast-build \ + -DskipTests \ + package + +mvn -f "$JAVA_CHECKOUT/pom.xml" \ + -pl paimon-core -am -Pfast-build \ + -DfailIfNoTests=false \ + -DwildcardSuites=none \ + -Dtest=JavaFileIndexTableFixtureGeneratorTest \ + -Dgenerate.file-index.fixture.output="$OUTPUT" \ + test + +echo "generated fixtures under $OUTPUT/default.db" diff --git a/crates/paimon/tests/file_index_java_fixture_test.rs b/crates/paimon/tests/file_index_java_fixture_test.rs new file mode 100644 index 000000000..87a86fed9 --- /dev/null +++ b/crates/paimon/tests/file_index_java_fixture_test.rs @@ -0,0 +1,280 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! End-to-end compatibility coverage for FileIndex tables written and committed +//! by Apache Paimon Java. See `testdata/file_index/README.md` for provenance and +//! regeneration instructions. + +#![cfg(all(not(target_os = "windows"), feature = "storage-fs"))] + +use std::collections::HashMap; +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use arrow_array::{Array, Int32Array, RecordBatch, StringArray}; +use futures::TryStreamExt; +use opendal::Operator; +use paimon::catalog::Identifier; +use paimon::io::{FileIO, FileIOBuilder, FileIOProvider}; +use paimon::spec::{Datum, Predicate, PredicateBuilder}; +use paimon::table::{SchemaManager, Table}; + +const FIXTURE: &str = "testdata/file_index/default.db"; + +#[derive(Clone, Copy)] +struct FixtureCase { + table: &'static str, + embedded: bool, +} + +const CASES: &[FixtureCase] = &[ + FixtureCase { + table: "bitmap_embedded", + embedded: true, + }, + FixtureCase { + table: "bitmap_sidecar", + embedded: false, + }, + FixtureCase { + table: "bloom_filter_embedded", + embedded: true, + }, + FixtureCase { + table: "bloom_filter_sidecar", + embedded: false, + }, +]; + +#[derive(Debug)] +struct FsProbe { + operator: Operator, + data_file_opens: AtomicUsize, +} + +impl FsProbe { + fn new() -> Arc { + let mut config = opendal_service_fs::FsConfig::default(); + config.root = Some("/".to_string()); + Arc::new(Self { + operator: Operator::from_config(config).expect("build filesystem operator"), + data_file_opens: AtomicUsize::new(0), + }) + } + + fn reset(&self) { + self.data_file_opens.store(0, Ordering::SeqCst); + } + + fn data_file_opens(&self) -> usize { + self.data_file_opens.load(Ordering::SeqCst) + } +} + +#[async_trait::async_trait] +impl FileIOProvider for FsProbe { + async fn create(&self, path: &str) -> paimon::Result<(Operator, String)> { + if path.ends_with(".parquet") { + self.data_file_opens.fetch_add(1, Ordering::SeqCst); + } + let relative = path + .strip_prefix("file://") + .expect("fixture paths must be file URLs") + .trim_start_matches('/') + .to_string(); + Ok((self.operator.clone(), relative)) + } +} + +fn copy_dir(src: &Path, dst: &Path) { + std::fs::create_dir_all(dst).expect("create staged fixture directory"); + for entry in std::fs::read_dir(src).expect("read fixture directory") { + let entry = entry.expect("read fixture entry"); + let from = entry.path(); + let to = dst.join(entry.file_name()); + if from.is_dir() { + copy_dir(&from, &to); + } else { + std::fs::copy(&from, &to).expect("copy fixture file"); + } + } +} + +async fn open_fixture(case: FixtureCase) -> (tempfile::TempDir, Table, Arc) { + let src = Path::new(env!("CARGO_MANIFEST_DIR")) + .join(FIXTURE) + .join(case.table); + let temp = tempfile::tempdir().expect("create temp dir"); + let staged = temp.path().join(case.table); + copy_dir(&src, &staged); + + let probe = FsProbe::new(); + let file_io: FileIO = FileIOBuilder::new("file") + .with_provider(probe.clone()) + .build() + .expect("build probed FileIO"); + let location = format!("file://{}", staged.display()); + let schema = SchemaManager::new(file_io.clone(), location.clone()) + .latest() + .await + .expect("load fixture schemas") + .expect("fixture table has no schema"); + let table = Table::new( + file_io, + Identifier::new("default", case.table), + location, + (*schema).clone(), + None, + ); + (temp, table, probe) +} + +fn rows(batches: &[RecordBatch]) -> Vec<(Option, String)> { + let mut rows = Vec::new(); + for batch in batches { + let ids = batch + .column_by_name("id") + .expect("id column") + .as_any() + .downcast_ref::() + .expect("id is int32"); + let payloads = batch + .column_by_name("payload") + .expect("payload column") + .as_any() + .downcast_ref::() + .expect("payload is utf8"); + rows.extend((0..batch.num_rows()).map(|row| { + let id = (!ids.is_null(row)).then(|| ids.value(row)); + (id, payloads.value(row).to_string()) + })); + } + rows.sort_unstable(); + rows +} + +async fn query( + table: &Table, + index_enabled: bool, + predicate: Predicate, +) -> Vec<(Option, String)> { + let table = table.copy_with_options(HashMap::from([( + "file-index.read.enabled".to_string(), + index_enabled.to_string(), + )])); + let mut builder = table.new_read_builder(); + builder.with_filter(predicate); + let plan = builder.new_scan().plan().await.expect("plan fixture query"); + let batches = builder + .new_read() + .expect("create fixture reader") + .to_arrow(plan.splits()) + .expect("create fixture Arrow stream") + .try_collect::>() + .await + .expect("read fixture rows"); + rows(&batches) +} + +#[tokio::test] +async fn reads_java_written_file_indexes_and_prunes_data_files() { + for case in CASES.iter().copied() { + let (_temp, table, probe) = open_fixture(case).await; + let predicates = PredicateBuilder::new(table.schema().fields()); + let id_one = predicates.equal("id", Datum::Int(1)).unwrap(); + let null_id = predicates.is_null("id").unwrap(); + let missing_id = predicates.equal("id", Datum::Int(2)).unwrap(); + let residual = Predicate::and(vec![ + id_one.clone(), + predicates + .equal("payload", Datum::String("keep".to_string())) + .unwrap(), + ]); + + let mut planning_builder = table.new_read_builder(); + planning_builder.with_filter(missing_id.clone()); + let (plan, trace) = planning_builder + .new_scan() + .plan_with_trace() + .await + .expect("plan missing-value query"); + assert_eq!( + trace.manifest_entries_pruned_by_data_stats, 0, + "{}", + case.table + ); + assert_eq!(trace.final_files, 1, "{}", case.table); + let file = &plan.splits()[0].data_files()[0]; + assert_eq!(file.row_count, 4, "{}", case.table); + assert_eq!( + file.embedded_index.is_some(), + case.embedded, + "{}", + case.table + ); + assert_eq!( + file.extra_files.iter().any(|name| name.ends_with(".index")), + !case.embedded, + "{}", + case.table + ); + + for enabled in [false, true] { + assert_eq!( + query(&table, enabled, id_one.clone()).await, + vec![(Some(1), "drop".to_string()), (Some(1), "keep".to_string())], + "{} with FileIndex enabled={enabled}", + case.table + ); + assert_eq!( + query(&table, enabled, null_id.clone()).await, + vec![(None, "null-id".to_string())], + "{} with FileIndex enabled={enabled}", + case.table + ); + assert_eq!( + query(&table, enabled, residual.clone()).await, + vec![(Some(1), "keep".to_string())], + "{} must retain row-level residual filtering with FileIndex enabled={enabled}", + case.table + ); + assert!( + query(&table, enabled, missing_id.clone()).await.is_empty(), + "{} with FileIndex enabled={enabled}", + case.table + ); + } + + probe.reset(); + assert!(query(&table, true, missing_id.clone()).await.is_empty()); + assert_eq!( + probe.data_file_opens(), + 0, + "{} must be skipped by its Java-written FileIndex", + case.table + ); + + probe.reset(); + assert!(query(&table, false, missing_id).await.is_empty()); + assert!( + probe.data_file_opens() > 0, + "{} must open the data file when FileIndex reads are disabled", + case.table + ); + } +}