From d983c86a30c0686f3d97b5d9eecf3b48e24c2e59 Mon Sep 17 00:00:00 2001 From: liwuhen Date: Sat, 19 Sep 2026 19:02:21 +0800 Subject: [PATCH] test(file_index): verify Java table read compatibility --- crates/paimon/testdata/file_index/README.md | 38 +++ ...f8ca-a74f-42ce-af35-fc4f4ef2d003-0.parquet | Bin 0 -> 564 bytes ...est-982c31e0-7caa-4462-a270-3b130005e913-0 | Bin 0 -> 2273 bytes ...ist-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-0 | Bin 0 -> 1158 bytes ...ist-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-1 | Bin 0 -> 1263 bytes .../bitmap_embedded/schema/schema-0 | 25 ++ .../bitmap_embedded/snapshot/EARLIEST | 1 + .../bitmap_embedded/snapshot/LATEST | 1 + .../bitmap_embedded/snapshot/snapshot-1 | 18 ++ ...5532-1aa9-4878-bb30-24862120212c-0.parquet | Bin 0 -> 564 bytes ...aa9-4878-bb30-24862120212c-0.parquet.index | Bin 0 -> 130 bytes ...est-392c52de-fdae-4262-81fb-4049e95c37de-0 | Bin 0 -> 2211 bytes ...ist-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0 | Bin 0 -> 1158 bytes ...ist-973ebadd-00a4-4672-a3b1-81e48cc9ec83-1 | Bin 0 -> 1263 bytes .../default.db/bitmap_sidecar/schema/schema-0 | 25 ++ .../bitmap_sidecar/snapshot/EARLIEST | 1 + .../default.db/bitmap_sidecar/snapshot/LATEST | 1 + .../bitmap_sidecar/snapshot/snapshot-1 | 18 ++ ...b83d-6c45-4dc1-a34e-819b5fb2e331-0.parquet | Bin 0 -> 564 bytes ...est-8dd937f3-747e-4f87-a296-489d11ef050c-0 | Bin 0 -> 2255 bytes ...ist-ac933951-c2f2-4af6-b3d4-d20e80e79189-0 | Bin 0 -> 1158 bytes ...ist-ac933951-c2f2-4af6-b3d4-d20e80e79189-1 | Bin 0 -> 1263 bytes .../bloom_filter_embedded/schema/schema-0 | 27 ++ .../bloom_filter_embedded/snapshot/EARLIEST | 1 + .../bloom_filter_embedded/snapshot/LATEST | 1 + .../bloom_filter_embedded/snapshot/snapshot-1 | 18 ++ ...926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet | Bin 0 -> 564 bytes ...cd0-4dd5-b576-80ddf6e55632-0.parquet.index | Bin 0 -> 78 bytes ...est-214eaa6e-a8e8-4903-a41a-3fa04732e5e4-0 | Bin 0 -> 2212 bytes ...ist-baa54195-a358-4f0b-9ca9-51503661a534-0 | Bin 0 -> 1158 bytes ...ist-baa54195-a358-4f0b-9ca9-51503661a534-1 | Bin 0 -> 1260 bytes .../bloom_filter_sidecar/schema/schema-0 | 27 ++ .../bloom_filter_sidecar/snapshot/EARLIEST | 1 + .../bloom_filter_sidecar/snapshot/LATEST | 1 + .../bloom_filter_sidecar/snapshot/snapshot-1 | 18 ++ ...avaFileIndexTableFixtureGeneratorTest.java | 108 +++++++ .../paimon/testdata/file_index/regenerate.sh | 64 ++++ .../tests/file_index_java_fixture_test.rs | 280 ++++++++++++++++++ 38 files changed, 674 insertions(+) create mode 100644 crates/paimon/testdata/file_index/README.md create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_embedded/bucket-0/data-c6f0f8ca-a74f-42ce-af35-fc4f4ef2d003-0.parquet create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-982c31e0-7caa-4462-a270-3b130005e913-0 create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-0 create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-1 create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_embedded/schema/schema-0 create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/EARLIEST create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/LATEST create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/snapshot-1 create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet.index create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-392c52de-fdae-4262-81fb-4049e95c37de-0 create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0 create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-1 create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_sidecar/schema/schema-0 create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/EARLIEST create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/LATEST create mode 100644 crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/snapshot-1 create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/bucket-0/data-9ce3b83d-6c45-4dc1-a34e-819b5fb2e331-0.parquet create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-8dd937f3-747e-4f87-a296-489d11ef050c-0 create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-0 create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-1 create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/schema/schema-0 create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/EARLIEST create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/LATEST create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/snapshot-1 create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet.index create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-214eaa6e-a8e8-4903-a41a-3fa04732e5e4-0 create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-0 create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-1 create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/schema/schema-0 create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/EARLIEST create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/LATEST create mode 100644 crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/snapshot-1 create mode 100644 crates/paimon/testdata/file_index/java/JavaFileIndexTableFixtureGeneratorTest.java create mode 100755 crates/paimon/testdata/file_index/regenerate.sh create mode 100644 crates/paimon/tests/file_index_java_fixture_test.rs diff --git a/crates/paimon/testdata/file_index/README.md b/crates/paimon/testdata/file_index/README.md new file mode 100644 index 000000000..dee46286a --- /dev/null +++ b/crates/paimon/testdata/file_index/README.md @@ -0,0 +1,38 @@ +# Java FileIndex table fixtures + +These four committed tables were produced through Apache Paimon Java's normal +catalog, append writer, commit, snapshot, and manifest paths. No index bytes or +metadata were assembled or edited by hand. + +Provenance: + +- Apache Paimon Java commit: `1d368b4a5932f8221fd28e2555001abdb8fb12ee` + (`2.2-SNAPSHOT`). +- Generator: `java/JavaFileIndexTableFixtureGeneratorTest.java`. +- Tables: Bitmap and Bloom Filter, each with an embedded (`1 MB` threshold) and + `.index` sidecar (`1 B` threshold) form. +- Common options: `bucket=1`, `bucket-key=id`, `file.format=parquet`, and + `file-index.read.enabled=true`. +- Bloom options: `items=16`, `fpp=0.01`. +- Rows, in physical order: `(1, "keep")`, `(1, "drop")`, + `(NULL, "null-id")`, `(3, "three")`. + +The Rust test expects `id = 1` to return both duplicate rows, `id IS NULL` to +return the null row, `id = 1 AND payload = 'keep'` to retain only the residual +match, and `id = 2` to return no rows. The last predicate lies inside the data +file's `[1, 3]` min/max range, so manifest statistics cannot prune it; it is also +the observable FileIndex skip case. + +To regenerate into an empty temporary directory: + +```bash +git clone https://github.com/apache/paimon.git /tmp/paimon-java +git -C /tmp/paimon-java checkout 1d368b4a5932f8221fd28e2555001abdb8fb12ee +crates/paimon/testdata/file_index/regenerate.sh \ + /tmp/paimon-java /tmp/paimon-file-index-fixtures +``` + +After reviewing the generated metadata and running the Rust compatibility test, +replace `default.db/` with the generated `default.db/` directory. UUID-bearing +file names may change between runs; the rows, table options, storage shape, and +query expectations are deterministic. diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/bucket-0/data-c6f0f8ca-a74f-42ce-af35-fc4f4ef2d003-0.parquet b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/bucket-0/data-c6f0f8ca-a74f-42ce-af35-fc4f4ef2d003-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d411cc375fd0fc49deff36d5cb3fb15dcdb081f2 GIT binary patch literal 564 zcmYjP!D`z;5S?8sj-U={OJ?zc2);z9#c{QEW2c})ax*=dl7`-jENzIWY^zEf=rxd^ zNDjH>RC4R3htgZ~5&esjkV~L5Rs%hVH#@v}``)ZZyYCD-_!I-Y{Bd?ukLch41m9jg zz;FjZ06>jCLIiw{FY(vq#qU)QT_V=eeWVTnBvYS=np1K2FvmY%Pj4l2;eYN(RNPg% z2tHX`Fs9|KSSFm#^L#T)m#92GEUhIm{Y>g6s{#R=L8+De*XB9hTv~9sPb5$6;|D6Z zx)yjYxTyjji-s#joK0rak4MR&oy3$C<%Vp^EPYS94Jwz5_)9*E(+${?o<}Om)t+wq z;L-e+TX!_Gv7uabqOOF$@4H@QtMaI1urOJjK(~3LD0zJh5eZ(^a82Dv>;S7aSe2qZ zUc8V#73y60*COgeP=#wc9F;PW5SfOE5`p?&!*z8hF$8rllzZ15C@jT1Z*6XmF^tMfy^akCu`=%fEtl2h$fr%44KCnqLFk^er SH!pOXw!n+np(hy8|MC|{He&by literal 0 HcmV?d00001 diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-982c31e0-7caa-4462-a270-3b130005e913-0 b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-982c31e0-7caa-4462-a270-3b130005e913-0 new file mode 100644 index 0000000000000000000000000000000000000000..004f091ec2650da61597e97dfa8339b6a85621b4 GIT binary patch literal 2273 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ>MRLzoNCAS47GBwJv38(3<^){21Z-{kz1)MTM5U~lTS8x$U#zY#m7 zzPRbK*do`1hQH>ky8r)mQC?AU#ex8-=c?zqT8riyKzUIUP!?!*hIZbbvN-ZQLQuuh-*qEC|B_=StpnDVmyHod) literal 0 HcmV?d00001 diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-0 b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-0 new file mode 100644 index 0000000000000000000000000000000000000000..d61cedd47f0bf44bd0d3ffb5d9ddb24c1d89df33 GIT binary patch literal 1158 zcmbW1K~94}6oxhN0-DCH$*da|CT_YL%NU6Rm@?GHhD@d~Z7~#@ftpmq3A}+`!*~Jr z-ou3#aHCKuAXM9Q2h9J!f8O_I&<8iWdxz{+1X5^^`Gh_0--BpA<)8%ww*%pTCh*y$ z_8v}|oqi0&NMgzJm`hXUP6A(A6dQ4$3l{OzK6JU~grL>GUsj@B)G-uYOBS;%*N;Hl zECgo?MpT0;DkNbfTz^zo8On7@AY@VQT zBR)wdm92vXBP+I=z~MtS^CHknov{^{hh?C|jrAd-vnlyvyDh|*z5 z8{OKQv02CLP`YrQ52CdWNkRc6kKC zW8Vj9-plwgrFfo3<1rFsom#GgMMlG9*Z?jW<71zJZpKW8NwHaG?*M8NQsn=Hn_1eo zdoQ512CY@H=2s*N!;l%>n49XVZ~)cKWH+Uzs(s^CF_N|0Kbe*jE!o|eY#ElMF4M}# zaxH#c)bP;iOQn;tBL8bz!0KdHdN@q(EdB;g23Y}4_BZ}f9 fve)Ve;RC4R3htgZ~5&esjkV~L5Rs%hVH#@v}``)ZZyYCD-_!I-Y{Bd?ukLch41m9jg zz;FjZ06>jCLIiw{FY(vq#qU)QT_V=eeWVTnBvYS=np1K2FvmY%Pj4l2;eYN(RNPg% z2tHX`Fs9|KSSFm#^L#T)m#92GEUhIm{Y>g6s{#R=L8+De*XB9hTv~9sPb5$6;|D6Z zx)yjYxTyjji-s#joK0rak4MR&oy3$C<%Vp^EPYS94Jwz5_)9*E(+${?o<}Om)t+wq z;L-e+TX!_Gv7uabqOOF$@4H@QtMaI1urOJjK(~3LD0zJh5eZ(^a82Dv>;S7aSe2qZ zUc8V#73y60*COgeP=#wc9F;PW5SfOE5`p?&!*z8hF$8rllzZ15C@jT1Z*6XmF^tMfy^akCu`=%fEtl2h$fr%44KCnqLFk^er SH!pOXw!n+np(hy8|MC|{He&by literal 0 HcmV?d00001 diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet.index b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet.index new file mode 100644 index 0000000000000000000000000000000000000000..c4886674c248b8d2b303e8e811fa7fe9ae940591 GIT binary patch literal 130 zcmZQz_4B(RWx9@mfq@Z-4Zt*0W(t^QOUf+CO)P+l2LWjYCLm@3(TxB9|NjeQ2mvuj jCkTUp3=o49!T2ITHZxG|A0Ym31vDQb!N?#0lmPJnwR{sb literal 0 HcmV?d00001 diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-392c52de-fdae-4262-81fb-4049e95c37de-0 b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-392c52de-fdae-4262-81fb-4049e95c37de-0 new file mode 100644 index 0000000000000000000000000000000000000000..205ac3295d9589d4fabeb7d0a8d073eae5e8722d GIT binary patch literal 2211 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ>MRLzoNCAS47GBwJv38(3<^){21Z-{kz1)MTM5U~t0Kzsf8$)8Z3r4Lh^&XV#fsW^#ujVi}L+D?XAg z`1f|s*|g266Jr78 e%)FeSTA&}$-rVT;>_9?-fk8wgqoo52x>Es*HQhG= literal 0 HcmV?d00001 diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0 b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0 new file mode 100644 index 0000000000000000000000000000000000000000..e1ea4553f017d63f67f9f5f3a6c8659c23da2abf GIT binary patch literal 1158 zcmbW1y-ve06oo-zU?77OwP$ttDH6lM%fv7^A1Q%>`Xif z1FyhaAd-?cp((WLmc-}WP4Ni4V*4y!6txh<*uXhox28iBHmWRlf1i+XF7GEx!VLfe9Etl+c-Vq%Dk zsvF=DQZ=r@qbWM+K-DU5w?7pu{?)w(yIZJqOI1n3u=LU{dmxtN2vPV5?R*?>-b0qU7Ev6J$!%tjMwkSOv$aI=;!t;RLNOVKhV8-69CbQrQm zclM^aD(WD0x7b~+X=>ZJDI-~S`@?jV=t{dgo3VkRn$s$dBSTk7;{G{4kE%i)K?AQARk;>7GvmHv#>l(^V%t zf!)u^)8zDR}Y`wI~ zBM_Yg0m$-R#!na{^DG*l(=eaZvJMs*jni=pxYQ#j0SEn@nU2%)u*|_e)MTtG{|Ps@ z^u!(G5>ld>ZJYqx;a$*j$AoNh1v22Si!=k}X~lgK5^6+VA&_9Jli9$oyp zIREkGrubp2`SJcwVLudwdyU4+kVL*m6E5xcI?i6lrIP0oDs?*#JJRC^-bCu|bbGY7 f=j=YpYCGS6DBR61QCQrfa6|Zn-hQ15&8pCE9k`VX literal 0 HcmV?d00001 diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/schema/schema-0 b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/schema/schema-0 new file mode 100644 index 000000000..4fc7be2fd --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/schema/schema-0 @@ -0,0 +1,25 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "payload", + "type" : "STRING" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "1", + "file-index.bitmap.columns" : "id", + "bucket-key" : "id", + "file-index.read.enabled" : "true", + "file.format" : "parquet", + "file-index.in-manifest-threshold" : "1 B" + }, + "timeMillis" : 1789814327169 +} \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/EARLIEST b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/LATEST b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/snapshot-1 b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/snapshot-1 new file mode 100644 index 000000000..24b6b600d --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "6476f620-ef94-47ba-8250-c49ab25124f2", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0", + "baseManifestListSize" : 1158, + "deltaManifestList" : "manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-1", + "deltaManifestListSize" : 1263, + "commitUser" : "71026591-a529-45f5-9b77-71d993270c4b", + "writerVersion" : "java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1789814327179, + "totalRecordCount" : 4, + "deltaRecordCount" : 4, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/bucket-0/data-9ce3b83d-6c45-4dc1-a34e-819b5fb2e331-0.parquet b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/bucket-0/data-9ce3b83d-6c45-4dc1-a34e-819b5fb2e331-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d411cc375fd0fc49deff36d5cb3fb15dcdb081f2 GIT binary patch literal 564 zcmYjP!D`z;5S?8sj-U={OJ?zc2);z9#c{QEW2c})ax*=dl7`-jENzIWY^zEf=rxd^ zNDjH>RC4R3htgZ~5&esjkV~L5Rs%hVH#@v}``)ZZyYCD-_!I-Y{Bd?ukLch41m9jg zz;FjZ06>jCLIiw{FY(vq#qU)QT_V=eeWVTnBvYS=np1K2FvmY%Pj4l2;eYN(RNPg% z2tHX`Fs9|KSSFm#^L#T)m#92GEUhIm{Y>g6s{#R=L8+De*XB9hTv~9sPb5$6;|D6Z zx)yjYxTyjji-s#joK0rak4MR&oy3$C<%Vp^EPYS94Jwz5_)9*E(+${?o<}Om)t+wq z;L-e+TX!_Gv7uabqOOF$@4H@QtMaI1urOJjK(~3LD0zJh5eZ(^a82Dv>;S7aSe2qZ zUc8V#73y60*COgeP=#wc9F;PW5SfOE5`p?&!*z8hF$8rllzZ15C@jT1Z*6XmF^tMfy^akCu`=%fEtl2h$fr%44KCnqLFk^er SH!pOXw!n+np(hy8|MC|{He&by literal 0 HcmV?d00001 diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-8dd937f3-747e-4f87-a296-489d11ef050c-0 b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-8dd937f3-747e-4f87-a296-489d11ef050c-0 new file mode 100644 index 0000000000000000000000000000000000000000..c56a063b9a401e758fac27e69bd2726135d1cb7c GIT binary patch literal 2255 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ>MRLzoNCAS47GBwJv38(3<^){21Z-{kz1)MTM5U~;SSrXxl-m2p1VBS`c&e`tlY47BLDYuvNj&z@L>2|e<4!atB7-2sB6Pb z{gre7o<0AMBW(dwlho$+T6Ol$PVSDsvb#^1{h3p;il^=WrWJGc?*Du*Z+rBlh$}l6 zI6MqZcrfMrGsh`2erWW~KHcos`Q*_I)vP%+el^BbGHU|7s--!38Ol;tOE8(NXgQ#E aLNB1bxzX|2frJDDgNQ~(O9vKoZvg;~&H2Cp literal 0 HcmV?d00001 diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-0 b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-0 new file mode 100644 index 0000000000000000000000000000000000000000..715016aede36f6d11cdf7b3047d3c2155021d6d1 GIT binary patch literal 1158 zcmbW1%}#?r6oobE3uufhSF>(-fbPaJMj}53hT60tlPOGF3_s04O=`ow&!9V(u3WkD z1$+#5`W8ZM0ZX+FJ7CVaXYO}r&<3}MjZ=EZLm{%q%%=}Wa}dua4731ecE}yj1OfFc z@6m+X*~gHN1e!jNnJ}TQ9|pp**oXy;)0k!Uq02lc0G zG4r!Yt=7Skk(F3Y;IJW`dNF8a&csT}!!p{JNP$EW|ACvebZ)k<5Y9z;N;dpTMCmYO zjn?+2xiab?xn68tYnrNSH)SMww?9lbiEgx8+l(~~GkXR-Cevqe)hfrvWIr= literal 0 HcmV?d00001 diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-1 b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-1 new file mode 100644 index 0000000000000000000000000000000000000000..b592f52e8b04f1191f3c6d20b0b16d34fbf874f2 GIT binary patch literal 1263 zcmbVMO>Wab7oKH*2-K^f(L}k$K2GdVc#9 z#M21_JrFW82rWUxX!?A}8WZYxf!~-F8#AAUG-esyc9?5Lptpaz zs6>Wvpv#(?F3z-@S4vFH>@yAt^F25?E+@OR}4TXr31SOKy)bn~* zA|;3*z5(`yIKVPaEf5`xIInt5c!X4mSHz>pDjAB%C~o&)8nfsx_lmf81C(kYMEaVc z7IwK0{3F)|Y2NerX&BOJ8jTY(FYDBN9n3NsC8Ij9m`#t|81yn`GD?cg()+tm5s57Q zC)~`^Go$|;;w5OQk~O~~Q5c5I=*rww7r`M!tI4iPO;rcl%VH!;w|_D%Ct9+*G8t=F zMDw)rv0RE@7d70&1F>{c7UX|T3s{`YN)JcL`s{CD1{O2#oJ2A8EgD+f`PsY8i!VPu zeERx*^WM)(kN>!~@%hm&Zr9_vhqc;kkNS?yqFCryR(HE?Zwu{Kn+Ywu(-!FC?o*-F k=~`P`%x*qunnLsa1D?B|UOYFudF~c>iQax2a~ox$-x=1F%m4rY literal 0 HcmV?d00001 diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/schema/schema-0 b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/schema/schema-0 new file mode 100644 index 000000000..4aa8eb0b5 --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/schema/schema-0 @@ -0,0 +1,27 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "payload", + "type" : "STRING" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "1", + "file-index.bloom-filter.id.items" : "16", + "file-index.in-manifest-threshold" : "1 MB", + "file-index.bloom-filter.columns" : "id", + "file-index.bloom-filter.id.fpp" : "0.01", + "bucket-key" : "id", + "file-index.read.enabled" : "true", + "file.format" : "parquet" + }, + "timeMillis" : 1789814327180 +} \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/EARLIEST b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/LATEST b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/snapshot-1 b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/snapshot-1 new file mode 100644 index 000000000..35573b11a --- /dev/null +++ b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "e21872d6-c6a1-4ef4-9a26-fe324ccd15cf", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-0", + "baseManifestListSize" : 1158, + "deltaManifestList" : "manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-1", + "deltaManifestListSize" : 1263, + "commitUser" : "1930fc30-d2c7-493c-b2cc-554a5b504d22", + "writerVersion" : "java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1789814327189, + "totalRecordCount" : 4, + "deltaRecordCount" : 4, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d411cc375fd0fc49deff36d5cb3fb15dcdb081f2 GIT binary patch literal 564 zcmYjP!D`z;5S?8sj-U={OJ?zc2);z9#c{QEW2c})ax*=dl7`-jENzIWY^zEf=rxd^ zNDjH>RC4R3htgZ~5&esjkV~L5Rs%hVH#@v}``)ZZyYCD-_!I-Y{Bd?ukLch41m9jg zz;FjZ06>jCLIiw{FY(vq#qU)QT_V=eeWVTnBvYS=np1K2FvmY%Pj4l2;eYN(RNPg% z2tHX`Fs9|KSSFm#^L#T)m#92GEUhIm{Y>g6s{#R=L8+De*XB9hTv~9sPb5$6;|D6Z zx)yjYxTyjji-s#joK0rak4MR&oy3$C<%Vp^EPYS94Jwz5_)9*E(+${?o<}Om)t+wq z;L-e+TX!_Gv7uabqOOF$@4H@QtMaI1urOJjK(~3LD0zJh5eZ(^a82Dv>;S7aSe2qZ zUc8V#73y60*COgeP=#wc9F;PW5SfOE5`p?&!*z8hF$8rllzZ15C@jT1Z*6XmF^tMfy^akCu`=%fEtl2h$fr%44KCnqLFk^er SH!pOXw!n+np(hy8|MC|{He&by literal 0 HcmV?d00001 diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet.index b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet.index new file mode 100644 index 0000000000000000000000000000000000000000..5a5fcc6a2276c56b26b9390afa06ecc3a60028af GIT binary patch literal 78 zcmZQz_4B(RWx9@mfq@Z-&A>EMW(t_*Ny^F3&(%%K%qdANg33w&X%Jus5sU&13``&f Livxq8#t{wx%7+R7 literal 0 HcmV?d00001 diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-214eaa6e-a8e8-4903-a41a-3fa04732e5e4-0 b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-214eaa6e-a8e8-4903-a41a-3fa04732e5e4-0 new file mode 100644 index 0000000000000000000000000000000000000000..a3525f3022c871e57fa10b9a2c55a0200cd0b6f0 GIT binary patch literal 2212 zcmdT_O=#0#7;d-jFedoJa4?6;r=mjbdhuXSX6-kYHBD-gZq-rJtjW4a`a_xtbrcAhd)}wi z@Kv|xFn>+S`3k8~F~k2pIu6ugUIYQ4h^d?+058b$8OQfRo=<5%=9ILLN1sPUU!IpT zxvWnoON&`i;Z;#9ACW{^D1gAwc!h}@fHZ?sBswhLC1q7m@>+l}PKIiqMlqVj*&PJK zMO7^9py-JO!;q~_!cr_oa0ZI04eP}TXr4CEhU3cUT}IYUI1-CkWT2obQZ~J#X&uGj zS8OObSB0UeK`uzeNZV`;Qa z5iw{H*J!XYii{D2A%{ga1hQkY43MA7_Tm+VFCsA{Rm^BkET02iw?rx71%f!jkI5= zFoJQoZSft91RQ{bbz^mGz&J}|`Vv}ZFkh<51m^nLpE^iGObZr{#ci8f@ry~N_vJp^np2) zm@sZiW3m{FU#}A*bEp^{v{J_`*A%W-Y}*aAg@h%0g~8OUmi+bD(R)IgL@^!3*gofM zcmLz}vj>j-XnNjsW~qD2)i>pFHLZ4?zPTdplY56J@7!BR(#lg| z^W8$*L20)2>+u^M*f&sU2azKilzV%6dPbmIAAX!(pPDQFY<|;n;??GSzw6i8w`UoX8NX_0yv3=7QplWVKuMwe&e$xO86(d6Q6o9o$wwdARM Ycl7qH3GDN7GU@lnC)^huB0fKR>J zdpM<5`Z3^RfhNxrCQPa01-`H-HfBEKG-9cJ|Wi!f4x1&FORQJr=Q=oi+7R$ literal 0 HcmV?d00001 diff --git a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-1 b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-1 new file mode 100644 index 0000000000000000000000000000000000000000..84f9fd476b53f17c6d65f7a90d1b7d22f788bec8 GIT binary patch literal 1260 zcmeZI%3@>@ODrqO*DFrWNX<>0C0MOgQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yJTw)@egtIfrb_$S;8$R6x6N}AwiC4 zIR&N}S87p$"); + } + + Path warehouse = Paths.get(output).toAbsolutePath().normalize(); + Options catalogOptions = new Options(); + catalogOptions.set(WAREHOUSE, warehouse.toUri().toString()); + Catalog catalog = CatalogFactory.createCatalog(CatalogContext.create(catalogOptions)); + catalog.createDatabase(DATABASE, false); + + writeTable(catalog, "bitmap_embedded", "bitmap", "1 MB"); + writeTable(catalog, "bitmap_sidecar", "bitmap", "1 B"); + writeTable(catalog, "bloom_filter_embedded", "bloom-filter", "1 MB"); + writeTable(catalog, "bloom_filter_sidecar", "bloom-filter", "1 B"); + } + + private static void writeTable( + Catalog catalog, String tableName, String indexType, String manifestThreshold) + throws Exception { + Map options = new HashMap<>(); + options.put("bucket", "1"); + options.put("bucket-key", "id"); + options.put("file.format", "parquet"); + options.put("file-index.read.enabled", "true"); + options.put("file-index.in-manifest-threshold", manifestThreshold); + options.put("file-index." + indexType + ".columns", "id"); + if ("bloom-filter".equals(indexType)) { + options.put("file-index.bloom-filter.id.items", "16"); + options.put("file-index.bloom-filter.id.fpp", "0.01"); + } + + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("payload", DataTypes.STRING()) + .options(options) + .build(); + Identifier identifier = Identifier.create(DATABASE, tableName); + catalog.createTable(identifier, schema, false); + Table table = catalog.getTable(identifier); + + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + write.write(row(1, "keep")); + write.write(row(1, "drop")); + write.write(row(null, "null-id")); + write.write(row(3, "three")); + commit.commit(write.prepareCommit()); + } + } + + private static GenericRow row(Integer id, String payload) { + return GenericRow.of(id, BinaryString.fromString(payload)); + } +} diff --git a/crates/paimon/testdata/file_index/regenerate.sh b/crates/paimon/testdata/file_index/regenerate.sh new file mode 100755 index 000000000..2905c723f --- /dev/null +++ b/crates/paimon/testdata/file_index/regenerate.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +readonly EXPECTED_COMMIT="1d368b4a5932f8221fd28e2555001abdb8fb12ee" +readonly JAVA_CHECKOUT="$1" +readonly OUTPUT="$2" +readonly SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +readonly SOURCE="$SCRIPT_DIR/java/JavaFileIndexTableFixtureGeneratorTest.java" +readonly TARGET="$JAVA_CHECKOUT/paimon-core/src/test/java/org/apache/paimon/fixture/JavaFileIndexTableFixtureGeneratorTest.java" + +actual_commit="$(git -C "$JAVA_CHECKOUT" rev-parse HEAD)" +if [[ "$actual_commit" != "$EXPECTED_COMMIT" ]]; then + echo "expected Apache Paimon Java $EXPECTED_COMMIT, got $actual_commit" >&2 + exit 1 +fi +if [[ -e "$TARGET" ]]; then + echo "refusing to overwrite $TARGET" >&2 + exit 1 +fi +if [[ -e "$OUTPUT" ]] && [[ -n "$(find "$OUTPUT" -mindepth 1 -print -quit)" ]]; then + echo "output directory must be empty: $OUTPUT" >&2 + exit 1 +fi + +mkdir -p "$(dirname "$TARGET")" "$OUTPUT" +cp "$SOURCE" "$TARGET" +trap 'rm -f "$TARGET"' EXIT + +mvn -f "$JAVA_CHECKOUT/pom.xml" \ + -pl paimon-codegen-loader -am -Pfast-build \ + -DskipTests \ + package + +mvn -f "$JAVA_CHECKOUT/pom.xml" \ + -pl paimon-core -am -Pfast-build \ + -DfailIfNoTests=false \ + -DwildcardSuites=none \ + -Dtest=JavaFileIndexTableFixtureGeneratorTest \ + -Dgenerate.file-index.fixture.output="$OUTPUT" \ + test + +echo "generated fixtures under $OUTPUT/default.db" diff --git a/crates/paimon/tests/file_index_java_fixture_test.rs b/crates/paimon/tests/file_index_java_fixture_test.rs new file mode 100644 index 000000000..87a86fed9 --- /dev/null +++ b/crates/paimon/tests/file_index_java_fixture_test.rs @@ -0,0 +1,280 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! End-to-end compatibility coverage for FileIndex tables written and committed +//! by Apache Paimon Java. See `testdata/file_index/README.md` for provenance and +//! regeneration instructions. + +#![cfg(all(not(target_os = "windows"), feature = "storage-fs"))] + +use std::collections::HashMap; +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use arrow_array::{Array, Int32Array, RecordBatch, StringArray}; +use futures::TryStreamExt; +use opendal::Operator; +use paimon::catalog::Identifier; +use paimon::io::{FileIO, FileIOBuilder, FileIOProvider}; +use paimon::spec::{Datum, Predicate, PredicateBuilder}; +use paimon::table::{SchemaManager, Table}; + +const FIXTURE: &str = "testdata/file_index/default.db"; + +#[derive(Clone, Copy)] +struct FixtureCase { + table: &'static str, + embedded: bool, +} + +const CASES: &[FixtureCase] = &[ + FixtureCase { + table: "bitmap_embedded", + embedded: true, + }, + FixtureCase { + table: "bitmap_sidecar", + embedded: false, + }, + FixtureCase { + table: "bloom_filter_embedded", + embedded: true, + }, + FixtureCase { + table: "bloom_filter_sidecar", + embedded: false, + }, +]; + +#[derive(Debug)] +struct FsProbe { + operator: Operator, + data_file_opens: AtomicUsize, +} + +impl FsProbe { + fn new() -> Arc { + let mut config = opendal_service_fs::FsConfig::default(); + config.root = Some("/".to_string()); + Arc::new(Self { + operator: Operator::from_config(config).expect("build filesystem operator"), + data_file_opens: AtomicUsize::new(0), + }) + } + + fn reset(&self) { + self.data_file_opens.store(0, Ordering::SeqCst); + } + + fn data_file_opens(&self) -> usize { + self.data_file_opens.load(Ordering::SeqCst) + } +} + +#[async_trait::async_trait] +impl FileIOProvider for FsProbe { + async fn create(&self, path: &str) -> paimon::Result<(Operator, String)> { + if path.ends_with(".parquet") { + self.data_file_opens.fetch_add(1, Ordering::SeqCst); + } + let relative = path + .strip_prefix("file://") + .expect("fixture paths must be file URLs") + .trim_start_matches('/') + .to_string(); + Ok((self.operator.clone(), relative)) + } +} + +fn copy_dir(src: &Path, dst: &Path) { + std::fs::create_dir_all(dst).expect("create staged fixture directory"); + for entry in std::fs::read_dir(src).expect("read fixture directory") { + let entry = entry.expect("read fixture entry"); + let from = entry.path(); + let to = dst.join(entry.file_name()); + if from.is_dir() { + copy_dir(&from, &to); + } else { + std::fs::copy(&from, &to).expect("copy fixture file"); + } + } +} + +async fn open_fixture(case: FixtureCase) -> (tempfile::TempDir, Table, Arc) { + let src = Path::new(env!("CARGO_MANIFEST_DIR")) + .join(FIXTURE) + .join(case.table); + let temp = tempfile::tempdir().expect("create temp dir"); + let staged = temp.path().join(case.table); + copy_dir(&src, &staged); + + let probe = FsProbe::new(); + let file_io: FileIO = FileIOBuilder::new("file") + .with_provider(probe.clone()) + .build() + .expect("build probed FileIO"); + let location = format!("file://{}", staged.display()); + let schema = SchemaManager::new(file_io.clone(), location.clone()) + .latest() + .await + .expect("load fixture schemas") + .expect("fixture table has no schema"); + let table = Table::new( + file_io, + Identifier::new("default", case.table), + location, + (*schema).clone(), + None, + ); + (temp, table, probe) +} + +fn rows(batches: &[RecordBatch]) -> Vec<(Option, String)> { + let mut rows = Vec::new(); + for batch in batches { + let ids = batch + .column_by_name("id") + .expect("id column") + .as_any() + .downcast_ref::() + .expect("id is int32"); + let payloads = batch + .column_by_name("payload") + .expect("payload column") + .as_any() + .downcast_ref::() + .expect("payload is utf8"); + rows.extend((0..batch.num_rows()).map(|row| { + let id = (!ids.is_null(row)).then(|| ids.value(row)); + (id, payloads.value(row).to_string()) + })); + } + rows.sort_unstable(); + rows +} + +async fn query( + table: &Table, + index_enabled: bool, + predicate: Predicate, +) -> Vec<(Option, String)> { + let table = table.copy_with_options(HashMap::from([( + "file-index.read.enabled".to_string(), + index_enabled.to_string(), + )])); + let mut builder = table.new_read_builder(); + builder.with_filter(predicate); + let plan = builder.new_scan().plan().await.expect("plan fixture query"); + let batches = builder + .new_read() + .expect("create fixture reader") + .to_arrow(plan.splits()) + .expect("create fixture Arrow stream") + .try_collect::>() + .await + .expect("read fixture rows"); + rows(&batches) +} + +#[tokio::test] +async fn reads_java_written_file_indexes_and_prunes_data_files() { + for case in CASES.iter().copied() { + let (_temp, table, probe) = open_fixture(case).await; + let predicates = PredicateBuilder::new(table.schema().fields()); + let id_one = predicates.equal("id", Datum::Int(1)).unwrap(); + let null_id = predicates.is_null("id").unwrap(); + let missing_id = predicates.equal("id", Datum::Int(2)).unwrap(); + let residual = Predicate::and(vec![ + id_one.clone(), + predicates + .equal("payload", Datum::String("keep".to_string())) + .unwrap(), + ]); + + let mut planning_builder = table.new_read_builder(); + planning_builder.with_filter(missing_id.clone()); + let (plan, trace) = planning_builder + .new_scan() + .plan_with_trace() + .await + .expect("plan missing-value query"); + assert_eq!( + trace.manifest_entries_pruned_by_data_stats, 0, + "{}", + case.table + ); + assert_eq!(trace.final_files, 1, "{}", case.table); + let file = &plan.splits()[0].data_files()[0]; + assert_eq!(file.row_count, 4, "{}", case.table); + assert_eq!( + file.embedded_index.is_some(), + case.embedded, + "{}", + case.table + ); + assert_eq!( + file.extra_files.iter().any(|name| name.ends_with(".index")), + !case.embedded, + "{}", + case.table + ); + + for enabled in [false, true] { + assert_eq!( + query(&table, enabled, id_one.clone()).await, + vec![(Some(1), "drop".to_string()), (Some(1), "keep".to_string())], + "{} with FileIndex enabled={enabled}", + case.table + ); + assert_eq!( + query(&table, enabled, null_id.clone()).await, + vec![(None, "null-id".to_string())], + "{} with FileIndex enabled={enabled}", + case.table + ); + assert_eq!( + query(&table, enabled, residual.clone()).await, + vec![(Some(1), "keep".to_string())], + "{} must retain row-level residual filtering with FileIndex enabled={enabled}", + case.table + ); + assert!( + query(&table, enabled, missing_id.clone()).await.is_empty(), + "{} with FileIndex enabled={enabled}", + case.table + ); + } + + probe.reset(); + assert!(query(&table, true, missing_id.clone()).await.is_empty()); + assert_eq!( + probe.data_file_opens(), + 0, + "{} must be skipped by its Java-written FileIndex", + case.table + ); + + probe.reset(); + assert!(query(&table, false, missing_id).await.is_empty()); + assert!( + probe.data_file_opens() > 0, + "{} must open the data file when FileIndex reads are disabled", + case.table + ); + } +}