diff --git a/Cargo.lock b/Cargo.lock index 15895ae19..d34862ac7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -99,9 +99,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b" dependencies = [ "arrow-arith", "arrow-array", @@ -121,9 +121,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff" dependencies = [ "arrow-array", "arrow-buffer", @@ -135,9 +135,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192" dependencies = [ "ahash", "arrow-buffer", @@ -154,9 +154,9 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376" +checksum = "2e4f9b23a0d7b613acb59fa20bdbe0f80ffdae6411498378340b3915e45f5b84" dependencies = [ "arrow-array", "arrow-buffer", @@ -178,9 +178,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434" dependencies = [ "bytes", "half", @@ -190,9 +190,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a" dependencies = [ "arrow-array", "arrow-buffer", @@ -201,7 +201,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.22.1", "chrono", "comfy-table", "half", @@ -212,9 +212,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +checksum = "8aa7bf96d6141a7bcca2eed57c7c9767d2a2175281857b8a7b68308992864784" dependencies = [ "arrow-array", "arrow-cast", @@ -227,9 +227,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66" dependencies = [ "arrow-buffer", "arrow-schema", @@ -240,9 +240,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "29dac499fcbc6ba74ee0324057821d381929a48526a3966bd9dffb44aa06d98c" dependencies = [ "arrow-array", "arrow-buffer", @@ -256,9 +256,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +checksum = "0fe05e916ddc50f4c7a363cd69c0ef5894fcee063517e9a0b8582f0c56746af6" dependencies = [ "arrow-array", "arrow-buffer", @@ -281,9 +281,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b" dependencies = [ "arrow-array", "arrow-buffer", @@ -294,9 +294,9 @@ dependencies = [ [[package]] name = "arrow-pyarrow" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29abdf672a81c1aeb57fd2661457f9918964d49aed0e9f18932535f2a9e49ce" +checksum = "bf8d967bdece4fa5a0199706730175b3df3448b87e350250a86eb6c22639e445" dependencies = [ "arrow-array", "arrow-data", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592" dependencies = [ "arrow-array", "arrow-buffer", @@ -319,9 +319,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5" dependencies = [ "bitflags", "serde_core", @@ -330,9 +330,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9" dependencies = [ "ahash", "arrow-array", @@ -344,9 +344,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3" dependencies = [ "arrow-array", "arrow-buffer", @@ -385,7 +385,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -396,7 +396,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -426,6 +426,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9" + [[package]] name = "bigdecimal" version = "0.4.10" @@ -513,12 +519,6 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.11.1" @@ -790,9 +790,8 @@ dependencies = [ [[package]] name = "datafusion" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "arrow-schema", @@ -828,7 +827,7 @@ dependencies = [ "flate2", "futures", "indexmap", - "itertools", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -844,9 +843,8 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "async-trait", @@ -860,7 +858,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -869,9 +867,8 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "async-trait", @@ -885,16 +882,16 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools", + "itertools 0.15.0", "log", "object_store", + "percent-encoding", ] [[package]] name = "datafusion-common" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "arrow-ipc", @@ -904,9 +901,10 @@ dependencies = [ "half", "hashbrown 0.17.1", "indexmap", - "itertools", + "itertools 0.15.0", "libc", "log", + "num-traits", "object_store", "parquet", "recursive", @@ -918,9 +916,8 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "futures", "log", @@ -929,9 +926,8 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "async-compression", @@ -947,11 +943,12 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "flate2", "futures", "glob", - "itertools", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -965,9 +962,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "arrow-ipc", @@ -982,16 +978,15 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools", + "itertools 0.15.0", "object_store", "tokio", ] [[package]] name = "datafusion-datasource-avro" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "arrow-avro", @@ -1008,9 +1003,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "async-trait", @@ -1031,9 +1025,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "async-trait", @@ -1054,11 +1047,11 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", + "arrow-schema", "async-trait", "bytes", "datafusion-common", @@ -1075,7 +1068,7 @@ dependencies = [ "datafusion-pruning", "datafusion-session", "futures", - "itertools", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -1085,19 +1078,18 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" [[package]] name = "datafusion-execution" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "arrow-buffer", "async-trait", + "bytes", "dashmap", "datafusion-common", "datafusion-expr", @@ -1106,16 +1098,18 @@ dependencies = [ "log", "object_store", "parking_lot", + "pin-project-lite", "rand 0.9.4", "tempfile", + "tokio", + "tokio-util", "url", ] [[package]] name = "datafusion-expr" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "arrow-schema", @@ -1128,7 +1122,7 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap", - "itertools", + "itertools 0.15.0", "recursive", "serde_json", "sqlparser", @@ -1136,21 +1130,19 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "datafusion-common", "indexmap", - "itertools", + "itertools 0.15.0", ] [[package]] name = "datafusion-ffi" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5660e8fa79fd51e29ce46f3026b67317ef738ebd633e106beb1a1907a406152" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "arrow-schema", @@ -1203,13 +1195,12 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "arrow-buffer", - "base64", + "base64 0.23.0", "blake2", "blake3", "chrono", @@ -1222,7 +1213,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hex", - "itertools", + "itertools 0.15.0", "log", "md-5 0.11.0", "memchr", @@ -1235,9 +1226,8 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "datafusion-common", @@ -1248,17 +1238,16 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", - "foldhash 0.2.0", "half", + "hashbrown 0.17.1", "log", "num-traits", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "datafusion-common", @@ -1268,9 +1257,8 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "arrow-ord", @@ -1285,7 +1273,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hashbrown 0.17.1", - "itertools", + "itertools 0.15.0", "itoa", "log", "memchr", @@ -1293,9 +1281,8 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "async-trait", @@ -1309,9 +1296,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "datafusion-common", @@ -1326,9 +1312,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1336,20 +1321,18 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "datafusion-doc", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "datafusion-optimizer" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "chrono", @@ -1358,7 +1341,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-physical-expr", "indexmap", - "itertools", + "itertools 0.15.0", "log", "recursive", "regex", @@ -1367,9 +1350,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "datafusion-common", @@ -1377,10 +1359,11 @@ dependencies = [ "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", + "datafusion-proto-models", "half", "hashbrown 0.17.1", "indexmap", - "itertools", + "itertools 0.15.0", "parking_lot", "petgraph", "recursive", @@ -1389,9 +1372,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "datafusion-common", @@ -1399,31 +1381,30 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr-common", + "datafusion-proto-models", "hashbrown 0.17.1", "indexmap", - "itertools", + "itertools 0.15.0", "parking_lot", "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "datafusion-common", @@ -1434,15 +1415,15 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools", + "datafusion-session", + "itertools 0.15.0", "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "arrow-data", @@ -1450,6 +1431,7 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", + "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -1459,23 +1441,25 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", "futures", "half", "hashbrown 0.17.1", "indexmap", - "itertools", + "itertools 0.15.0", "log", "num-traits", "parking_lot", "pin-project-lite", + "serde_json", "tokio", ] [[package]] name = "datafusion-proto" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "chrono", @@ -1494,26 +1478,34 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-proto-common", + "datafusion-proto-models", "object_store", "prost", ] [[package]] name = "datafusion-proto-common" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "datafusion-common", "prost", ] +[[package]] +name = "datafusion-proto-models" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" +dependencies = [ + "datafusion-proto-common", + "prost", +] + [[package]] name = "datafusion-pruning" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "datafusion-common", @@ -1572,10 +1564,10 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ + "arrow-schema", "async-trait", "datafusion-common", "datafusion-execution", @@ -1586,14 +1578,14 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "bigdecimal", "chrono", "crc32fast", + "datafusion", "datafusion-catalog", "datafusion-common", "datafusion-execution", @@ -1615,9 +1607,8 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "arrow", "bigdecimal", @@ -1630,20 +1621,20 @@ dependencies = [ "recursive", "regex", "sqlparser", + "stacker", ] [[package]] name = "datafusion-substrait" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b22c8f8c72d317e54fad6f85c0ef6d1e1da53cc7faadc7eea8daf0f8d86d4f2" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" dependencies = [ "async-recursion", "async-trait", "chrono", "datafusion", "half", - "itertools", + "itertools 0.15.0", "object_store", "pbjson-types", "prost", @@ -1682,7 +1673,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1835,7 +1826,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2099,7 +2090,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -2255,12 +2246,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "ipnet" version = "2.12.0" @@ -2276,6 +2261,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -2575,7 +2569,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "form_urlencoded", @@ -2587,7 +2581,7 @@ dependencies = [ "httparse", "humantime", "hyper", - "itertools", + "itertools 0.14.0", "md-5 0.10.6", "parking_lot", "percent-encoding", @@ -2620,15 +2614,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "parking_lot" version = "0.12.5" @@ -2654,9 +2639,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +checksum = "5302d4da74d6596a1f11f9928767995b53bca657cbeea1e4e8c5074f8a1157dd" dependencies = [ "ahash", "arrow-array", @@ -2665,7 +2650,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64", + "base64 0.22.1", "brotli", "bytes", "chrono", @@ -2682,7 +2667,6 @@ dependencies = [ "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", "zstd", @@ -2700,7 +2684,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3" dependencies = [ - "base64", + "base64 0.22.1", "serde", ] @@ -2711,7 +2695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af22d08a625a2213a78dbb0ffa253318c5c79ce3133d32d296655a7bdfb02095" dependencies = [ "heck", - "itertools", + "itertools 0.14.0", "prost", "prost-types", ] @@ -2784,7 +2768,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2830,7 +2814,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.118", ] [[package]] @@ -2868,7 +2852,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -2876,7 +2860,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn", + "syn 2.0.118", "tempfile", ] @@ -2887,10 +2871,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2923,9 +2907,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" dependencies = [ "libc", "once_cell", @@ -2937,9 +2921,9 @@ dependencies = [ [[package]] name = "pyo3-async-runtimes" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e7364a95bf00e8377bbf9b0f09d7ff9715a29d8fcf93b47d1a967363b973178" +checksum = "b3ef68daa7316a3fac65e5e18b2203f010346de1c1c53456811a2624673ab046" dependencies = [ "futures-channel", "futures-util", @@ -2951,19 +2935,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" dependencies = [ - "python3-dll-a", "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" dependencies = [ "libc", "pyo3-build-config", @@ -2982,36 +2965,26 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "pyo3-macros-backend" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", - "syn", -] - -[[package]] -name = "python3-dll-a" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d80ba7540edb18890d444c5aa8e1f1f99b1bdf26fb26ae383135325f4a36042b" -dependencies = [ - "cc", + "syn 2.0.118", ] [[package]] @@ -3163,7 +3136,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3220,7 +3193,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -3396,7 +3369,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.118", ] [[package]] @@ -3471,7 +3444,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3482,7 +3455,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3508,7 +3481,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn", + "syn 2.0.118", ] [[package]] @@ -3635,7 +3608,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3669,7 +3642,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3700,7 +3673,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3725,7 +3698,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn", + "syn 2.0.118", "typify", "walkdir", ] @@ -3747,6 +3720,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -3764,7 +3748,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3803,18 +3787,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", -] - -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float", + "syn 2.0.118", ] [[package]] @@ -3874,7 +3847,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4006,7 +3979,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4064,7 +4037,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn", + "syn 2.0.118", "thiserror", "unicode-ident", ] @@ -4082,7 +4055,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn", + "syn 2.0.118", "typify-impl", ] @@ -4227,7 +4200,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -4303,7 +4276,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4314,7 +4287,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4537,7 +4510,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -4558,7 +4531,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4578,7 +4551,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -4618,7 +4591,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b37d65331..362159913 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,24 +32,24 @@ resolver = "3" [workspace.dependencies] tokio = { version = "1.52" } -pyo3 = { version = "0.28" } -pyo3-async-runtimes = { version = "0.28" } +pyo3 = { version = "0.29" } +pyo3-async-runtimes = { version = "0.29" } pyo3-log = "0.13.3" chrono = { version = "0.4", default-features = false } -arrow = { version = "58" } -arrow-array = { version = "58" } -arrow-schema = { version = "58" } -arrow-select = { version = "58" } -datafusion = { version = "54" } -datafusion-substrait = { version = "54" } -datafusion-proto = { version = "54" } -datafusion-ffi = { version = "54" } -datafusion-catalog = { version = "54", default-features = false } -datafusion-common = { version = "54", default-features = false } -datafusion-functions-aggregate = { version = "54" } -datafusion-functions-window = { version = "54" } -datafusion-spark = { version = "54" } -datafusion-expr = { version = "54" } +arrow = { version = "59" } +arrow-array = { version = "59" } +arrow-schema = { version = "59" } +arrow-select = { version = "59" } +datafusion = { version = "54.1.0" } +datafusion-substrait = { version = "54.1.0" } +datafusion-proto = { version = "54.1.0" } +datafusion-ffi = { version = "54.1.0" } +datafusion-catalog = { version = "54.1.0", default-features = false } +datafusion-common = { version = "54.1.0", default-features = false } +datafusion-functions-aggregate = { version = "54.1.0" } +datafusion-functions-window = { version = "54.1.0" } +datafusion-spark = { version = "54.1.0" } +datafusion-expr = { version = "54.1.0" } prost = "0.14.3" serde_json = "1" uuid = { version = "1.23" } @@ -62,7 +62,7 @@ url = "2" log = "0.4.29" parking_lot = "0.12" prost-types = "0.14.3" # keep in line with `datafusion-substrait` -pyo3-build-config = "0.28" +pyo3-build-config = "0.29" datafusion-python-util = { path = "crates/util", version = "54.0.0" } [profile.release] @@ -72,3 +72,13 @@ codegen-units = 2 # We cannot publish to crates.io with any patches in the below section. Developers # must remove any entries in this section before creating a release candidate. [patch.crates-io] +datafusion = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" } +datafusion-substrait = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" } +datafusion-proto = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" } +datafusion-ffi = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" } +datafusion-catalog = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" } +datafusion-common = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" } +datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" } +datafusion-functions-window = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" } +datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" } +datafusion-expr = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" } diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index e2e922a82..c5f1e0167 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -53,7 +53,7 @@ datafusion = { workspace = true, features = ["avro", "unicode_expressions"] } datafusion-substrait = { workspace = true, optional = true } datafusion-proto = { workspace = true } datafusion-ffi = { workspace = true } -datafusion-spark = { workspace = true } +datafusion-spark = { workspace = true, features = ["core"] } prost = { workspace = true } # keep in line with `datafusion-substrait` serde_json = { workspace = true } uuid = { workspace = true, features = ["v4"] } diff --git a/crates/core/src/array.rs b/crates/core/src/array.rs index f284fa9de..dfe963183 100644 --- a/crates/core/src/array.rs +++ b/crates/core/src/array.rs @@ -63,10 +63,10 @@ impl PyArrowArrayExportable { }; let ffi_schema = FFI_ArrowSchema::try_from(&field)?; - let schema_capsule = PyCapsule::new(py, ffi_schema, Some(cr"arrow_schema".into()))?; + let schema_capsule = PyCapsule::new_with_value(py, ffi_schema, cr"arrow_schema")?; let ffi_array = FFI_ArrowArray::new(&self.array.to_data()); - let array_capsule = PyCapsule::new(py, ffi_array, Some(cr"arrow_array".into()))?; + let array_capsule = PyCapsule::new_with_value(py, ffi_array, cr"arrow_array")?; Ok((schema_capsule, array_capsule)) } diff --git a/crates/core/src/codec.rs b/crates/core/src/codec.rs index a6ea6671c..26853e69f 100644 --- a/crates/core/src/codec.rs +++ b/crates/core/src/codec.rs @@ -100,9 +100,13 @@ use datafusion::logical_expr::{ TypeSignature, Volatility, WindowUDF, WindowUDFImpl, }; use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx; +use datafusion::physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx; use datafusion::physical_plan::ExecutionPlan; use datafusion_proto::logical_plan::{DefaultLogicalExtensionCodec, LogicalExtensionCodec}; -use datafusion_proto::physical_plan::{DefaultPhysicalExtensionCodec, PhysicalExtensionCodec}; +use datafusion_proto::physical_plan::{ + DefaultPhysicalExtensionCodec, PhysicalExtensionCodec, PhysicalProtoConverterExtension, +}; use pyo3::prelude::*; use pyo3::sync::PyOnceLock; use pyo3::types::{PyBytes, PyTuple}; @@ -483,12 +487,18 @@ impl PhysicalExtensionCodec for PythonPhysicalCodec { buf: &[u8], inputs: &[Arc], ctx: &TaskContext, + proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { - self.inner.try_decode(buf, inputs, ctx) + self.inner.try_decode(buf, inputs, ctx, proto_converter) } - fn try_encode(&self, node: Arc, buf: &mut Vec) -> Result<()> { - self.inner.try_encode(node, buf) + fn try_encode( + &self, + node: Arc, + buf: &mut Vec, + proto_converter: &dyn PhysicalProtoConverterExtension, + ) -> Result<()> { + self.inner.try_encode(node, buf, proto_converter) } fn try_encode_udf(&self, node: &ScalarUDF, buf: &mut Vec) -> Result<()> { @@ -509,16 +519,22 @@ impl PhysicalExtensionCodec for PythonPhysicalCodec { self.inner.try_decode_udf(name, buf) } - fn try_encode_expr(&self, node: &Arc, buf: &mut Vec) -> Result<()> { - self.inner.try_encode_expr(node, buf) + fn try_encode_expr( + &self, + node: &Arc, + buf: &mut Vec, + ctx: &PhysicalExprEncodeCtx<'_>, + ) -> Result<()> { + self.inner.try_encode_expr(node, buf, ctx) } fn try_decode_expr( &self, buf: &[u8], inputs: &[Arc], + ctx: &PhysicalExprDecodeCtx<'_>, ) -> Result> { - self.inner.try_decode_expr(buf, inputs) + self.inner.try_decode_expr(buf, inputs, ctx) } fn try_encode_udaf(&self, node: &AggregateUDF, buf: &mut Vec) -> Result<()> { diff --git a/crates/core/src/context.rs b/crates/core/src/context.rs index 6184c7c74..7bbeed2f1 100644 --- a/crates/core/src/context.rs +++ b/crates/core/src/context.rs @@ -1371,12 +1371,10 @@ impl PySessionContext { &self, py: Python<'py>, ) -> PyResult> { - let name = cr"datafusion_task_context_provider".into(); - let ctx_provider = Arc::clone(&self.ctx) as Arc; let ffi_ctx_provider = FFI_TaskContextProvider::from(&ctx_provider); - PyCapsule::new(py, ffi_ctx_provider, Some(name)) + PyCapsule::new_with_value(py, ffi_ctx_provider, cr"datafusion_task_context_provider") } pub fn __datafusion_logical_extension_codec__<'py>( diff --git a/crates/core/src/dataframe.rs b/crates/core/src/dataframe.rs index 8f1a20d0d..b1f305551 100644 --- a/crates/core/src/dataframe.rs +++ b/crates/core/src/dataframe.rs @@ -16,7 +16,7 @@ // under the License. use std::collections::HashMap; -use std::ffi::{CStr, CString}; +use std::ffi::CStr; use std::ptr::NonNull; use std::str::FromStr; use std::sync::Arc; @@ -1237,8 +1237,7 @@ impl PyDataFrame { // destructor provided by PyO3 will drop the stream unless ownership is // transferred to PyArrow during import. let stream = FFI_ArrowArrayStream::new(reader); - let name = CString::new(ARROW_ARRAY_STREAM_NAME.to_bytes()).unwrap(); - let capsule = PyCapsule::new(py, stream, Some(name))?; + let capsule = PyCapsule::new_with_value(py, stream, ARROW_ARRAY_STREAM_NAME)?; Ok(capsule) } @@ -1317,7 +1316,8 @@ impl PyDataFrame { None => Vec::new(), // Empty vector means fill null for all columns }; - let df = self.df.as_ref().clone().fill_null(scalar_value.0, cols)?; + let cols = cols.iter().map(String::as_str).collect::>(); + let df = self.df.as_ref().fill_null(&scalar_value.0, &cols)?; Ok(Self::new(df)) } } diff --git a/crates/core/src/expr/create_external_table.rs b/crates/core/src/expr/create_external_table.rs index 980eea131..e78b836bf 100644 --- a/crates/core/src/expr/create_external_table.rs +++ b/crates/core/src/expr/create_external_table.rs @@ -88,7 +88,7 @@ impl PyCreateExternalTable { let create = CreateExternalTable { schema: Arc::new(schema.into()), name: name.into(), - location, + locations: vec![location], file_type, table_partition_cols, if_not_exists, @@ -118,8 +118,8 @@ impl PyCreateExternalTable { Ok(self.create.name.to_string()) } - pub fn location(&self) -> String { - self.create.location.clone() + pub fn locations(&self) -> Vec { + self.create.locations.clone() } pub fn file_type(&self) -> String { diff --git a/crates/core/src/expr/dml.rs b/crates/core/src/expr/dml.rs index 26f975820..5967d181e 100644 --- a/crates/core/src/expr/dml.rs +++ b/crates/core/src/expr/dml.rs @@ -18,6 +18,7 @@ use datafusion::logical_expr::dml::InsertOp; use datafusion::logical_expr::{DmlStatement, WriteOp}; use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyNotImplementedError; use pyo3::prelude::*; use super::logical_node::LogicalNode; @@ -71,8 +72,8 @@ impl PyDmlStatement { }) } - pub fn op(&self) -> PyWriteOp { - self.dml.op.clone().into() + pub fn op(&self) -> PyResult { + self.dml.op.clone().try_into() } pub fn input(&self) -> PyLogicalPlan { @@ -112,16 +113,21 @@ pub enum PyWriteOp { Truncate, } -impl From for PyWriteOp { - fn from(write_op: WriteOp) -> Self { +impl TryFrom for PyWriteOp { + type Error = PyErr; + + fn try_from(write_op: WriteOp) -> Result { match write_op { - WriteOp::Insert(InsertOp::Append) => PyWriteOp::Append, - WriteOp::Insert(InsertOp::Overwrite) => PyWriteOp::Overwrite, - WriteOp::Insert(InsertOp::Replace) => PyWriteOp::Replace, - WriteOp::Update => PyWriteOp::Update, - WriteOp::Delete => PyWriteOp::Delete, - WriteOp::Ctas => PyWriteOp::Ctas, - WriteOp::Truncate => PyWriteOp::Truncate, + WriteOp::Insert(InsertOp::Append) => Ok(PyWriteOp::Append), + WriteOp::Insert(InsertOp::Overwrite) => Ok(PyWriteOp::Overwrite), + WriteOp::Insert(InsertOp::Replace) => Ok(PyWriteOp::Replace), + WriteOp::Update => Ok(PyWriteOp::Update), + WriteOp::Delete => Ok(PyWriteOp::Delete), + WriteOp::Ctas => Ok(PyWriteOp::Ctas), + WriteOp::Truncate => Ok(PyWriteOp::Truncate), + unsupported => Err(PyNotImplementedError::new_err(format!( + "DataFusion write operation {unsupported:?} is not supported" + ))), } } } diff --git a/crates/core/src/expr/drop_catalog_schema.rs b/crates/core/src/expr/drop_catalog_schema.rs index fd5105332..f349098b7 100644 --- a/crates/core/src/expr/drop_catalog_schema.rs +++ b/crates/core/src/expr/drop_catalog_schema.rs @@ -18,9 +18,8 @@ use std::fmt::{self, Display, Formatter}; use std::sync::Arc; -use datafusion::common::SchemaReference; +use datafusion::common::{SchemaReference, TableReference}; use datafusion::logical_expr::DropCatalogSchema; -use datafusion::sql::TableReference; use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; diff --git a/crates/core/src/expr/recursive_query.rs b/crates/core/src/expr/recursive_query.rs index e03137b80..0b198a191 100644 --- a/crates/core/src/expr/recursive_query.rs +++ b/crates/core/src/expr/recursive_query.rs @@ -22,6 +22,7 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::*; use super::logical_node::LogicalNode; +use crate::errors::PyDataFusionResult; use crate::sql::logical::PyLogicalPlan; #[pyclass( @@ -67,15 +68,15 @@ impl PyRecursiveQuery { static_term: PyLogicalPlan, recursive_term: PyLogicalPlan, is_distinct: bool, - ) -> Self { - Self { - query: RecursiveQuery { + ) -> PyDataFusionResult { + Ok(Self { + query: RecursiveQuery::try_new( name, - static_term: static_term.plan(), - recursive_term: recursive_term.plan(), + static_term.plan(), + recursive_term.plan(), is_distinct, - }, - } + )?, + }) } fn name(&self) -> PyResult { diff --git a/crates/core/src/sql/logical.rs b/crates/core/src/sql/logical.rs index 647c3fa7e..12fc43bdc 100644 --- a/crates/core/src/sql/logical.rs +++ b/crates/core/src/sql/logical.rs @@ -135,7 +135,7 @@ impl PyLogicalPlan { LogicalPlan::Dml(plan) => PyDmlStatement::from(plan.clone()).to_variant(py), LogicalPlan::Ddl(plan) => match plan { DdlStatement::CreateExternalTable(plan) => { - PyCreateExternalTable::from(plan.clone()).to_variant(py) + PyCreateExternalTable::from(plan.as_ref().clone()).to_variant(py) } DdlStatement::CreateMemoryTable(plan) => { PyCreateMemoryTable::from(plan.clone()).to_variant(py) @@ -154,7 +154,7 @@ impl PyLogicalPlan { PyDropCatalogSchema::from(plan.clone()).to_variant(py) } DdlStatement::CreateFunction(plan) => { - PyCreateFunction::from(plan.clone()).to_variant(py) + PyCreateFunction::from(plan.as_ref().clone()).to_variant(py) } DdlStatement::DropFunction(plan) => { PyDropFunction::from(plan.clone()).to_variant(py) diff --git a/crates/core/src/substrait.rs b/crates/core/src/substrait.rs index 27e446f48..220f865da 100644 --- a/crates/core/src/substrait.rs +++ b/crates/core/src/substrait.rs @@ -131,8 +131,8 @@ impl PySubstraitSerializer { } #[staticmethod] - pub fn deserialize_bytes(proto_bytes: Vec, py: Python) -> PyDataFusionResult { - let plan = wait_for_future(py, serializer::deserialize_bytes(proto_bytes))??; + pub fn deserialize_bytes(proto_bytes: Vec, _py: Python) -> PyDataFusionResult { + let plan = serializer::deserialize_bytes(&proto_bytes)?; Ok(PyPlan { plan: *plan }) } } diff --git a/crates/util/src/lib.rs b/crates/util/src/lib.rs index 07aa0a2d5..9327d7f2f 100644 --- a/crates/util/src/lib.rs +++ b/crates/util/src/lib.rs @@ -164,7 +164,8 @@ pub fn validate_pycapsule(capsule: &Bound, name: &str) -> PyResult<() ))); } - let capsule_name = unsafe { capsule_name.unwrap().as_cstr().to_str()? }; + let capsule_name = unsafe { capsule_name.unwrap().as_cstr().to_str() } + .map_err(|err| PyValueError::new_err(err.to_string()))?; if capsule_name != name { return Err(PyValueError::new_err(format!( "Expected name '{name}' in PyCapsule, instead got '{capsule_name}'" @@ -208,10 +209,9 @@ pub fn create_logical_extension_capsule<'py>( py: Python<'py>, codec: &FFI_LogicalExtensionCodec, ) -> PyResult> { - let name = cr"datafusion_logical_extension_codec".into(); let codec = codec.clone(); - PyCapsule::new(py, codec, Some(name)) + PyCapsule::new_with_value(py, codec, cr"datafusion_logical_extension_codec") } pub fn ffi_logical_codec_from_pycapsule(obj: Bound) -> PyResult { @@ -235,10 +235,9 @@ pub fn create_physical_extension_capsule<'py>( py: Python<'py>, codec: &FFI_PhysicalExtensionCodec, ) -> PyResult> { - let name = cr"datafusion_physical_extension_codec".into(); let codec = codec.clone(); - PyCapsule::new(py, codec, Some(name)) + PyCapsule::new_with_value(py, codec, cr"datafusion_physical_extension_codec") } /// Define a `(obj) -> PyResult>` extractor that diff --git a/examples/datafusion-ffi-example/src/aggregate_udf.rs b/examples/datafusion-ffi-example/src/aggregate_udf.rs index 86737778f..ea1518365 100644 --- a/examples/datafusion-ffi-example/src/aggregate_udf.rs +++ b/examples/datafusion-ffi-example/src/aggregate_udf.rs @@ -50,12 +50,10 @@ impl MySumUDF { &self, py: Python<'py>, ) -> PyResult> { - let name = cr"datafusion_aggregate_udf".into(); - let func = Arc::new(AggregateUDF::from(self.clone())); let provider = FFI_AggregateUDF::from(func); - PyCapsule::new(py, provider, Some(name)) + PyCapsule::new_with_value(py, provider, cr"datafusion_aggregate_udf") } } diff --git a/examples/datafusion-ffi-example/src/catalog_provider.rs b/examples/datafusion-ffi-example/src/catalog_provider.rs index 6131ab0f0..a56b5855c 100644 --- a/examples/datafusion-ffi-example/src/catalog_provider.rs +++ b/examples/datafusion-ffi-example/src/catalog_provider.rs @@ -33,8 +33,8 @@ use pyo3::types::PyCapsule; use pyo3::{Bound, PyAny, PyResult, Python, pyclass, pymethods}; pub fn my_table() -> Arc { + use arrow::array::record_batch; use arrow::datatypes::{DataType, Field}; - use datafusion_common::record_batch; let schema = Arc::new(Schema::new(vec![ Field::new("units", DataType::Int32, true), @@ -92,14 +92,12 @@ impl FixedSchemaProvider { py: Python<'py>, session: Bound, ) -> PyResult> { - let name = cr"datafusion_schema_provider".into(); - let provider = Arc::clone(&self.inner) as Arc; let codec = ffi_logical_codec_from_pycapsule(session)?; let provider = FFI_SchemaProvider::new_with_ffi_codec(provider, None, codec); - PyCapsule::new(py, provider, Some(name)) + PyCapsule::new_with_value(py, provider, cr"datafusion_schema_provider") } } @@ -186,14 +184,12 @@ impl MyCatalogProvider { py: Python<'py>, session: Bound, ) -> PyResult> { - let name = cr"datafusion_catalog_provider".into(); - let provider = Arc::clone(&self.inner) as Arc; let codec = ffi_logical_codec_from_pycapsule(session)?; let provider = FFI_CatalogProvider::new_with_ffi_codec(provider, None, codec); - PyCapsule::new(py, provider, Some(name)) + PyCapsule::new_with_value(py, provider, cr"datafusion_catalog_provider") } } @@ -247,13 +243,11 @@ impl MyCatalogProviderList { py: Python<'py>, session: Bound, ) -> PyResult> { - let name = cr"datafusion_catalog_provider_list".into(); - let provider = Arc::clone(&self.inner) as Arc; let codec = ffi_logical_codec_from_pycapsule(session)?; let provider = FFI_CatalogProviderList::new_with_ffi_codec(provider, None, codec); - PyCapsule::new(py, provider, Some(name)) + PyCapsule::new_with_value(py, provider, cr"datafusion_catalog_provider_list") } } diff --git a/examples/datafusion-ffi-example/src/config.rs b/examples/datafusion-ffi-example/src/config.rs index 6cdb8aa83..9cf704b94 100644 --- a/examples/datafusion-ffi-example/src/config.rs +++ b/examples/datafusion-ffi-example/src/config.rs @@ -53,14 +53,12 @@ impl MyConfig { &self, py: Python<'py>, ) -> PyResult> { - let name = cr"datafusion_extension_options".into(); - let mut config = FFI_ExtensionOptions::default(); config .add_config(self) .map_err(|e| PyRuntimeError::new_err(e.to_string()))?; - PyCapsule::new(py, config, Some(name)) + PyCapsule::new_with_value(py, config, cr"datafusion_extension_options") } } diff --git a/examples/datafusion-ffi-example/src/logical_extension_codec.rs b/examples/datafusion-ffi-example/src/logical_extension_codec.rs index da9efb297..8c3976d37 100644 --- a/examples/datafusion-ffi-example/src/logical_extension_codec.rs +++ b/examples/datafusion-ffi-example/src/logical_extension_codec.rs @@ -147,7 +147,6 @@ impl MyLogicalExtensionCodec { let ctx_provider = bare_session as Arc; let ffi = FFI_LogicalExtensionCodec::new(inner, Some(runtime), &ctx_provider); - let name = cr"datafusion_logical_extension_codec".into(); - PyCapsule::new(py, ffi, Some(name)) + PyCapsule::new_with_value(py, ffi, cr"datafusion_logical_extension_codec") } } diff --git a/examples/datafusion-ffi-example/src/physical_extension_codec.rs b/examples/datafusion-ffi-example/src/physical_extension_codec.rs index b1a586d9e..35ef77f6b 100644 --- a/examples/datafusion-ffi-example/src/physical_extension_codec.rs +++ b/examples/datafusion-ffi-example/src/physical_extension_codec.rs @@ -24,7 +24,9 @@ use datafusion::logical_expr::ScalarUDF; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::SessionContext; use datafusion_ffi::proto::physical_extension_codec::FFI_PhysicalExtensionCodec; -use datafusion_proto::physical_plan::{DefaultPhysicalExtensionCodec, PhysicalExtensionCodec}; +use datafusion_proto::physical_plan::{ + DefaultPhysicalExtensionCodec, PhysicalExtensionCodec, PhysicalProtoConverterExtension, +}; use datafusion_python_util::get_tokio_runtime; use pyo3::prelude::*; use pyo3::types::PyCapsule; @@ -51,12 +53,18 @@ impl PhysicalExtensionCodec for CountingPhysicalExtensionCodec { buf: &[u8], inputs: &[Arc], ctx: &TaskContext, + proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { - self.inner.try_decode(buf, inputs, ctx) + self.inner.try_decode(buf, inputs, ctx, proto_converter) } - fn try_encode(&self, node: Arc, buf: &mut Vec) -> Result<()> { - self.inner.try_encode(node, buf) + fn try_encode( + &self, + node: Arc, + buf: &mut Vec, + proto_converter: &dyn PhysicalProtoConverterExtension, + ) -> Result<()> { + self.inner.try_encode(node, buf, proto_converter) } fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result> { @@ -113,7 +121,6 @@ impl MyPhysicalExtensionCodec { let ctx_provider = bare_session as Arc; let ffi = FFI_PhysicalExtensionCodec::new(inner, Some(runtime), &ctx_provider); - let name = cr"datafusion_physical_extension_codec".into(); - PyCapsule::new(py, ffi, Some(name)) + PyCapsule::new_with_value(py, ffi, cr"datafusion_physical_extension_codec") } } diff --git a/examples/datafusion-ffi-example/src/physical_optimizer.rs b/examples/datafusion-ffi-example/src/physical_optimizer.rs index 0acd1bb4a..e17510495 100644 --- a/examples/datafusion-ffi-example/src/physical_optimizer.rs +++ b/examples/datafusion-ffi-example/src/physical_optimizer.rs @@ -92,7 +92,6 @@ impl MyPhysicalOptimizerRule { let runtime = get_tokio_runtime().handle().clone(); let ffi = FFI_PhysicalOptimizerRule::new(rule, Some(runtime)); - let name = cr"datafusion_physical_optimizer_rule".into(); - PyCapsule::new(py, ffi, Some(name)) + PyCapsule::new_with_value(py, ffi, cr"datafusion_physical_optimizer_rule") } } diff --git a/examples/datafusion-ffi-example/src/scalar_udf.rs b/examples/datafusion-ffi-example/src/scalar_udf.rs index a3c65e875..85b884ec6 100644 --- a/examples/datafusion-ffi-example/src/scalar_udf.rs +++ b/examples/datafusion-ffi-example/src/scalar_udf.rs @@ -50,12 +50,10 @@ impl IsNullUDF { } fn __datafusion_scalar_udf__<'py>(&self, py: Python<'py>) -> PyResult> { - let name = cr"datafusion_scalar_udf".into(); - let func = Arc::new(ScalarUDF::from(self.clone())); let provider = FFI_ScalarUDF::from(func); - PyCapsule::new(py, provider, Some(name)) + PyCapsule::new_with_value(py, provider, cr"datafusion_scalar_udf") } } diff --git a/examples/datafusion-ffi-example/src/table_function.rs b/examples/datafusion-ffi-example/src/table_function.rs index ed3ef142b..55543cb59 100644 --- a/examples/datafusion-ffi-example/src/table_function.rs +++ b/examples/datafusion-ffi-example/src/table_function.rs @@ -47,13 +47,11 @@ impl MyTableFunction { py: Python<'py>, session: Bound, ) -> PyResult> { - let name = cr"datafusion_table_function".into(); - let func = self.clone(); let codec = ffi_logical_codec_from_pycapsule(session)?; let provider = FFI_TableFunction::new_with_ffi_codec(Arc::new(func), None, codec); - PyCapsule::new(py, provider, Some(name)) + PyCapsule::new_with_value(py, provider, cr"datafusion_table_function") } } diff --git a/examples/datafusion-ffi-example/src/table_provider.rs b/examples/datafusion-ffi-example/src/table_provider.rs index 358ef7402..5756e6d02 100644 --- a/examples/datafusion-ffi-example/src/table_provider.rs +++ b/examples/datafusion-ffi-example/src/table_provider.rs @@ -99,8 +99,6 @@ impl MyTableProvider { py: Python<'py>, session: Bound, ) -> PyResult> { - let name = cr"datafusion_table_provider".into(); - let provider = self .create_table() .map_err(|e: DataFusionError| PyRuntimeError::new_err(e.to_string()))?; @@ -109,6 +107,6 @@ impl MyTableProvider { let provider = FFI_TableProvider::new_with_ffi_codec(Arc::new(provider), false, None, codec); - PyCapsule::new(py, provider, Some(name)) + PyCapsule::new_with_value(py, provider, cr"datafusion_table_provider") } } diff --git a/examples/datafusion-ffi-example/src/table_provider_factory.rs b/examples/datafusion-ffi-example/src/table_provider_factory.rs index 53248a905..71dfd73ca 100644 --- a/examples/datafusion-ffi-example/src/table_provider_factory.rs +++ b/examples/datafusion-ffi-example/src/table_provider_factory.rs @@ -77,11 +77,10 @@ impl MyTableProviderFactory { py: Python<'py>, codec: Bound, ) -> PyResult> { - let name = cr"datafusion_table_provider_factory".into(); let codec = ffi_logical_codec_from_pycapsule(codec)?; let factory = Arc::clone(&self.inner) as Arc; let factory = FFI_TableProviderFactory::new_with_ffi_codec(factory, None, codec); - PyCapsule::new(py, factory, Some(name)) + PyCapsule::new_with_value(py, factory, cr"datafusion_table_provider_factory") } } diff --git a/examples/datafusion-ffi-example/src/window_udf.rs b/examples/datafusion-ffi-example/src/window_udf.rs index f33a166ed..2956ad64c 100644 --- a/examples/datafusion-ffi-example/src/window_udf.rs +++ b/examples/datafusion-ffi-example/src/window_udf.rs @@ -45,12 +45,10 @@ impl MyRankUDF { } fn __datafusion_window_udf__<'py>(&self, py: Python<'py>) -> PyResult> { - let name = cr"datafusion_window_udf".into(); - let func = Arc::new(WindowUDF::from(self.clone())); let provider = FFI_WindowUDF::from(func); - PyCapsule::new(py, provider, Some(name)) + PyCapsule::new_with_value(py, provider, cr"datafusion_window_udf") } } diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index d2560fbc4..6019f178e 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -49,6 +49,11 @@ from collections.abc import Callable, Iterable, Sequence from typing import TYPE_CHECKING, Any, ClassVar +try: + from warnings import deprecated # Python 3.13+ +except ImportError: + from typing_extensions import deprecated # Python 3.12 + import pyarrow as pa from ._internal import expr as expr_internal @@ -90,6 +95,25 @@ CreateCatalog = expr_internal.CreateCatalog CreateCatalogSchema = expr_internal.CreateCatalogSchema CreateExternalTable = expr_internal.CreateExternalTable + + +@deprecated("CreateExternalTable.location() is deprecated; use locations() instead.") +def _create_external_table_location(self: Any) -> str: + """Return the first external table location. + + Examples: + >>> class Command: + ... def locations(self) -> list[str]: + ... return ["data.csv"] + >>> _create_external_table_location(Command()) + 'data.csv' + """ + locations = self.locations() + return locations[0] if locations else "" + + +CreateExternalTable.location = _create_external_table_location + CreateFunction = expr_internal.CreateFunction CreateFunctionBody = expr_internal.CreateFunctionBody CreateIndex = expr_internal.CreateIndex diff --git a/python/datafusion/functions/spark.py b/python/datafusion/functions/spark.py index 5163a6701..0a0f41400 100644 --- a/python/datafusion/functions/spark.py +++ b/python/datafusion/functions/spark.py @@ -273,12 +273,10 @@ def slice(x: Expr, start: Expr | int, length: Expr | int) -> Expr: Examples: >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"x": [1]}) + >>> df = ctx.from_pydict({"x": [[1, 2, 3, 4]]}) >>> r = df.select( ... dfn.functions.spark.slice( - ... dfn.functions.spark.array( - ... dfn.lit(1), dfn.lit(2), dfn.lit(3), dfn.lit(4)), - ... 2, 2, + ... dfn.col("x"), 2, 2, ... ).alias("v") ... ) >>> r.collect_column("v")[0].as_py() diff --git a/python/tests/test_context.py b/python/tests/test_context.py index 2f068b05d..7d038c7a5 100644 --- a/python/tests/test_context.py +++ b/python/tests/test_context.py @@ -17,6 +17,7 @@ import datetime as dt import gzip import pathlib +import shutil import pyarrow as pa import pyarrow.dataset as ds @@ -799,16 +800,15 @@ def test_read_csv(ctx): csv_df.select(column("c1")).show() -def test_read_csv_list(ctx): - csv_df = ctx.read_csv(path=["testing/data/csv/aggregate_test_100.csv"]) +def test_read_csv_list(ctx, tmp_path): + source_path = pathlib.Path("testing/data/csv/aggregate_test_100.csv") + copied_path = tmp_path / source_path.name + shutil.copy(source_path, copied_path) + + csv_df = ctx.read_csv(path=[source_path]) expected = csv_df.count() * 2 - double_csv_df = ctx.read_csv( - path=[ - "testing/data/csv/aggregate_test_100.csv", - "testing/data/csv/aggregate_test_100.csv", - ] - ) + double_csv_df = ctx.read_csv(path=[source_path, copied_path]) actual = double_csv_df.count() double_csv_df.select(column("c1")).show() diff --git a/python/tests/test_expr.py b/python/tests/test_expr.py index 606c6a984..4fca10c02 100644 --- a/python/tests/test_expr.py +++ b/python/tests/test_expr.py @@ -19,6 +19,7 @@ from concurrent.futures import ThreadPoolExecutor from datetime import date, datetime, time, timezone from decimal import Decimal +from unittest.mock import MagicMock import arro3.core import nanoarrow @@ -39,6 +40,7 @@ BinaryExpr, Column, CopyTo, + CreateExternalTable, CreateIndex, DescribeTable, DmlStatement, @@ -68,6 +70,17 @@ def test_ctx(): return ctx +def test_create_external_table_location_is_deprecated(): + """The singular location accessor delegates to locations().""" + command = MagicMock() + command.locations.return_value = ["first.csv", "second.csv"] + + with pytest.warns(DeprecationWarning, match=r"location\(\).+deprecated"): + location = CreateExternalTable.location(command) + + assert location == "first.csv" + + def test_projection(test_ctx): df = test_ctx.sql("select c1, 123, c1 < 123 from test") plan = df.logical_plan() diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 894d81fcf..cb16fbc04 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -145,7 +145,7 @@ def test_math_functions(): f.pow(col_v, literal(pa.scalar(4))), f.round(col_v), f.round(col_v, literal(pa.scalar(3))), - f.sqrt(col_v), + f.sqrt(f.abs(col_v)), f.signum(col_v), f.trunc(col_v), f.asinh(col_v), @@ -189,7 +189,7 @@ def test_math_functions(): np.testing.assert_array_almost_equal(result.column(16), np.power(values, 4)) np.testing.assert_array_almost_equal(result.column(17), np.round(values)) np.testing.assert_array_almost_equal(result.column(18), np.round(values, 3)) - np.testing.assert_array_almost_equal(result.column(19), np.sqrt(values)) + np.testing.assert_array_almost_equal(result.column(19), np.sqrt(np.abs(values))) np.testing.assert_array_almost_equal(result.column(20), np.sign(values)) np.testing.assert_array_almost_equal(result.column(21), np.trunc(values)) np.testing.assert_array_almost_equal(result.column(22), np.arcsinh(values)) @@ -215,6 +215,14 @@ def test_math_functions(): ) +def test_sqrt_rejects_negative_input(): + ctx = SessionContext() + df = ctx.from_pydict({"value": [-1.0]}) + + with pytest.raises(Exception, match="cannot take square root of a negative number"): + df.select(f.sqrt(column("value"))).collect() + + def py_indexof(arr, v): try: return arr.index(v) + 1 diff --git a/python/tests/test_io.py b/python/tests/test_io.py index 9f56f74d7..f0a9c3aa1 100644 --- a/python/tests/test_io.py +++ b/python/tests/test_io.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import shutil from pathlib import Path import pyarrow as pa @@ -72,16 +73,15 @@ def test_read_csv(): csv_df.select(column("c1")).show() -def test_read_csv_list(): - csv_df = read_csv(path=["testing/data/csv/aggregate_test_100.csv"]) +def test_read_csv_list(tmp_path): + source_path = Path("testing/data/csv/aggregate_test_100.csv") + copied_path = tmp_path / source_path.name + shutil.copy(source_path, copied_path) + + csv_df = read_csv(path=[source_path]) expected = csv_df.count() * 2 - double_csv_df = read_csv( - path=[ - "testing/data/csv/aggregate_test_100.csv", - "testing/data/csv/aggregate_test_100.csv", - ] - ) + double_csv_df = read_csv(path=[source_path, copied_path]) actual = double_csv_df.count() double_csv_df.select(column("c1")).show() diff --git a/python/tests/test_lambda.py b/python/tests/test_lambda.py index 68be22a04..ce37546be 100644 --- a/python/tests/test_lambda.py +++ b/python/tests/test_lambda.py @@ -16,6 +16,8 @@ # under the License. """Tests for lambda expressions and higher-order array functions.""" +import pickle + import pytest from datafusion import SessionConfig, SessionContext, col, lit from datafusion import functions as f @@ -137,8 +139,9 @@ def test_sql_lambda_keyword_syntax(dialect): assert result.to_pylist() == [[2, 4, 6]] -def test_pickle_lambda_expr_not_supported(): - # v1 limitation: upstream proto serialization rejects lambda expressions. +def test_pickle_lambda_expr_round_trip(df): expr = f.array_transform(col("a"), lambda v: v * 2) - with pytest.raises(Exception, match="Lambda not implemented"): - expr.to_bytes() + decoded = pickle.loads(pickle.dumps(expr)) # noqa: S301 + + assert decoded.canonical_name() == expr.canonical_name() + assert _column(df, decoded, "r") == [[2, 4, 6], [8, 10]] diff --git a/python/tests/test_spark_functions.py b/python/tests/test_spark_functions.py index aed46d7fa..78b86c998 100644 --- a/python/tests/test_spark_functions.py +++ b/python/tests/test_spark_functions.py @@ -172,6 +172,14 @@ def test_array_and_size(df): def test_slice(df): + assert _val(df, spark.slice(col("a"), lit(2), lit(2))) == [2, 3] + + +@pytest.mark.xfail( + strict=True, + reason="Upstream array_slice does not preserve Spark array field names", +) +def test_slice_spark_array(df): arr = spark.array(lit(1), lit(2), lit(3), lit(4)) assert _val(df, spark.slice(arr, lit(2), lit(2))) == [2, 3] diff --git a/python/tests/test_sql.py b/python/tests/test_sql.py index 6089c377d..57d22bd31 100644 --- a/python/tests/test_sql.py +++ b/python/tests/test_sql.py @@ -107,6 +107,7 @@ def test_register_csv(ctx, tmp_path): def test_register_csv_list(ctx, tmp_path): path = tmp_path / "test.csv" + second_path = tmp_path / "test2.csv" int_values = [1, 2, 3, 4] table = pa.Table.from_arrays( @@ -118,6 +119,7 @@ def test_register_csv_list(ctx, tmp_path): names=["int", "str", "float"], ) write_csv(table, path) + write_csv(table, second_path) ctx.register_csv("csv", path) csv_df = ctx.table("csv") @@ -126,7 +128,7 @@ def test_register_csv_list(ctx, tmp_path): "double_csv", path=[ path, - path, + second_path, ], )