From e16a5382e5b17160545a44fa5b9c1b4fe917f0f9 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 15 Sep 2026 10:35:34 +0100 Subject: [PATCH 1/2] Upgrade to arrow 59 and DataFusion 55 Signed-off-by: Adam Gutglick --- rust-toolchain.toml | 2 +- spatialbench-arrow/Cargo.toml | 4 ++-- spatialbench-cli/Cargo.toml | 6 +++--- spatialbench-cli/src/zone/transform.rs | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 7697bc1c..c7d61a9e 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -19,5 +19,5 @@ # to compile this workspace and run CI jobs. [toolchain] -channel = "1.90.0" +channel = "1.94.0" components = ["rustfmt", "clippy"] diff --git a/spatialbench-arrow/Cargo.toml b/spatialbench-arrow/Cargo.toml index e9a1bddb..b4b0fef4 100644 --- a/spatialbench-arrow/Cargo.toml +++ b/spatialbench-arrow/Cargo.toml @@ -29,11 +29,11 @@ keywords = ["spatial", "geospatial", "data-generation", "apache-arrow"] categories = ["science::geo", "database", "data-structures"] [dependencies] -arrow = { version = "58", default-features = false, features = ["prettyprint"] } +arrow = { version = "59", default-features = false, features = ["prettyprint"] } spatialbench = { path = "../spatialbench", version = "0.3.0" } geo = { workspace = true } geozero = { workspace = true } [dev-dependencies] -arrow-csv = "58" +arrow-csv = "59" chrono = "0.4.39" diff --git a/spatialbench-cli/Cargo.toml b/spatialbench-cli/Cargo.toml index 51c49e82..7544ecc8 100644 --- a/spatialbench-cli/Cargo.toml +++ b/spatialbench-cli/Cargo.toml @@ -29,8 +29,8 @@ keywords = ["spatial", "geospatial", "benchmark", "cli", "data-generation"] categories = ["science::geo", "database", "command-line-utilities", "development-tools"] [dependencies] -arrow = "58" -parquet = "58" +arrow = "59" +parquet = "59" clap = { version = "4.5.32", features = ["derive"] } spatialbench = { path = "../spatialbench", version = "0.3.0"} spatialbench-arrow = { path = "../spatialbench-arrow", version = "0.3.0" } @@ -42,7 +42,7 @@ env_logger = "0.11.7" serde = { version = "1.0.219", features = ["derive"] } anyhow = "1.0.99" serde_yaml = "0.9.33" -datafusion = "54.1" +datafusion = "55" object_store = { version = "0.13.2", features = ["http", "aws"] } url = "2.5.7" bytes = "1.10.1" diff --git a/spatialbench-cli/src/zone/transform.rs b/spatialbench-cli/src/zone/transform.rs index 4633df89..1cead735 100644 --- a/spatialbench-cli/src/zone/transform.rs +++ b/spatialbench-cli/src/zone/transform.rs @@ -17,7 +17,8 @@ use anyhow::Result; use arrow::datatypes::Schema; -use datafusion::{prelude::*, sql::TableReference}; +use datafusion::common::TableReference; +use datafusion::prelude::*; use log::{debug, info}; pub struct ZoneTransformer { From c412c9e4bf7aecee0f8511e2b7434b00fc9a70d8 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 15 Sep 2026 12:12:17 +0100 Subject: [PATCH 2/2] Fix tests because the parquet writer changed its dict behavior Signed-off-by: Adam Gutglick --- spatialbench-cli/tests/cli_integration.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/spatialbench-cli/tests/cli_integration.rs b/spatialbench-cli/tests/cli_integration.rs index ae46c1a2..8be08315 100644 --- a/spatialbench-cli/tests/cli_integration.rs +++ b/spatialbench-cli/tests/cli_integration.rs @@ -168,7 +168,7 @@ async fn test_zone_parquet_no_overwrite() { run_command(); let original_metadata = fs::metadata(&expected_file).expect("Failed to get metadata of generated file"); - assert_eq!(original_metadata.len(), 25400203); + assert_eq!(original_metadata.len(), 25342328); // Run the spatialbench-cli command again with the same parameters and expect the // file to not be overwritten @@ -541,7 +541,7 @@ async fn test_write_parquet_row_group_size_default() { }, RowGroups { table: "trip", - row_group_bytes: vec![123493205, 123460055, 123449607, 123465483], + row_group_bytes: vec![123492787, 123459637, 123449189, 123465065], }, RowGroups { table: "driver", @@ -553,7 +553,7 @@ async fn test_write_parquet_row_group_size_default() { }, RowGroups { table: "building", - row_group_bytes: vec![2492359], + row_group_bytes: vec![2491050], }, ], ); @@ -584,7 +584,7 @@ async fn test_zone_write_parquet_row_group_size_default() { output_dir.path(), vec![RowGroups { table: "zone/zone.1", - row_group_bytes: vec![86288517], + row_group_bytes: vec![86288569], }], ); } @@ -616,9 +616,9 @@ async fn test_write_parquet_row_group_size_20mb() { RowGroups { table: "trip", row_group_bytes: vec![ - 24356144, 24356407, 24345650, 24343404, 24348327, 24330535, 24353663, 24337733, - 24340689, 24356034, 24332349, 24340694, 24343446, 24356122, 24356250, 24340986, - 24345859, 24333134, 24343026, 24356402, 24346155, + 24355726, 24355989, 24345232, 24342986, 24347909, 24330117, 24353245, 24337315, + 24340271, 24355616, 24331931, 24340276, 24343028, 24355704, 24355832, 24340568, + 24345441, 24332716, 24342608, 24355984, 24345737, ], }, RowGroups { @@ -631,7 +631,7 @@ async fn test_write_parquet_row_group_size_20mb() { }, RowGroups { table: "building", - row_group_bytes: vec![2492359], + row_group_bytes: vec![2491050], }, ], ); @@ -664,7 +664,7 @@ async fn test_zone_write_parquet_row_group_size_20mb() { output_dir.path(), vec![RowGroups { table: "zone/zone.1", - row_group_bytes: vec![15428592, 17250042, 19338201, 17046885, 17251978], + row_group_bytes: vec![15427740, 17249254, 19337347, 17046007, 17251120], }], ); }