From 12e83a2aa453e1f8908b7488529d36ed6bd20671 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:27:19 +0000 Subject: [PATCH 1/7] wip: plan lets Databricks and ClickHouse read a Snowflake S3 unload unless masks or a time column need SQLLoad --- .../dispatch/main/route/unload_blocked.rs | 23 ++++-- .../src/adapters/snowflake/tests/helpers.rs | 24 +++++- .../adapters/snowflake/tests/test_types.rs | 10 +++ .../snowflake/tests/test_unload_blocked.rs | 77 ++++++++++++++++++- .../sqlload/src/loader/_helpers/table_plan.rs | 2 +- 5 files changed, 124 insertions(+), 12 deletions(-) diff --git a/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs b/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs index ac6ff774..05d3003c 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs @@ -2,16 +2,17 @@ use crate::adapters::snowflake::main::copy::_stage_covers::stage_covers; use crate::adapters::snowflake::models::StageRoute; -use crate::column_types::models::Warehouse; +use crate::column_types::models::{ColumnType, Warehouse}; use crate::loader::constants::{UNLOADED_OUTSIDE_STAGE, UNLOADED_TIMES}; use crate::loader::models::WarehouseRoute; +use crate::postgres_source::models::SourceColumn; use crate::routing::models::Store; use crate::store_paths::main::resolve_location::resolve_location; -/// Why `warehouse` cannot load a table's unloaded files exactly by reading them itself, so SQLLoad reads them instead; `None` when it can (where it also reaches their store). Only DuckDB and Snowflake turn the files' whole microseconds back into times and timestamps; a table that `masks` columns needs SQLLoad to see the rows; Snowflake reads another session's unload only through the target's external `stage`, which must cover the S3 folder `route` unloads to, as a user or named stage of the source's is not its to read. +/// Why `warehouse` cannot load a table's unloaded files exactly by reading them itself, so SQLLoad reads them instead; `None` when it can (where it also reaches their store). DuckDB and Snowflake turn the files' whole microseconds back into times and timestamps, and Databricks and ClickHouse into timestamps, so a table with a time column among `columns` goes through SQLLoad into those; a table that `masks` columns needs SQLLoad to see the rows; Snowflake reads another session's unload only through the target's external `stage`, which must cover the S3 folder `route` unloads to, as a user or named stage of the source's is not its to read. pub fn unload_blocked( warehouse: Warehouse, - masks: bool, + (masks, columns): (bool, &[SourceColumn]), route: Option<&WarehouseRoute>, stage: Option<&StageRoute>, ) -> Option { @@ -26,12 +27,20 @@ pub fn unload_blocked( .is_ok_and(|unloaded| stage_covers(&stage.location, &unloaded)), _ => false, }; + let times = columns + .iter() + .any(|column| column.column_type == Ok(ColumnType::Time)); + let reads = matches!( + warehouse, + Warehouse::DuckDb | Warehouse::Snowflake | Warehouse::Databricks | Warehouse::ClickHouse + ); match (warehouse, masks) { - (Warehouse::DuckDb | Warehouse::Snowflake, true) => { - Some("the table masks columns".to_owned()) - } + (_, true) if reads => Some("the table masks columns".to_owned()), (Warehouse::Snowflake, false) if !covered => Some(UNLOADED_OUTSIDE_STAGE.to_owned()), - (Warehouse::DuckDb | Warehouse::Snowflake, false) => None, + (Warehouse::Databricks | Warehouse::ClickHouse, false) if times => Some(format!( + "the table has a time column, which {warehouse} doesn't store as a time" + )), + _ if reads => None, _ => Some(UNLOADED_TIMES.to_owned()), } } diff --git a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs index c0e1db5d..98967d73 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs @@ -452,12 +452,34 @@ pub(crate) fn unload_blocked_for(test_case: &UnloadBlockedTestCase) -> Option Option { + let route = WarehouseRoute::Unload { + store: Store::ObjectStore("s3://lake/raw/unload/".to_owned()), + stage: String::new(), + same_connection: false, + }; + use crate::snowflake_source::_helpers::catalog::source_column; + let column = match test_case.times { + true => source_column("at", "TIME(9)", true), + false => source_column("id", "NUMBER(38,0)", true), + }; + unload_blocked( + test_case.warehouse, + (test_case.masks, &[column]), + Some(&route), + None, + ) +} + /// How many requests the case's statements are sent as, each checked to fit `MAX_COPY_SQL_BYTES`, and whether the first commits only when every unloaded row loads; or the error up to its first comma. pub(crate) fn unloaded_requests_for( test_case: &UnloadedRequestsTestCase, diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs index 1d254396..d36a8265 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs @@ -133,6 +133,16 @@ pub(crate) struct UnloadBlockedTestCase { pub(crate) expected_blocked: Option<&'static str>, } +pub(crate) struct UnloadReadersTestCase { + pub(crate) description: &'static str, + pub(crate) warehouse: crate::column_types::models::Warehouse, + pub(crate) masks: bool, + /// The table has a time column. + pub(crate) times: bool, + /// How the reason SQLLoad reads the files instead starts, or `None` when the target reads the S3 unload itself. + pub(crate) expected_blocked: Option<&'static str>, +} + /// How the statements loading one unload inside Snowflake are sent. pub(crate) struct UnloadedRequestsTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs b/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs index 4c0eb423..7a40f193 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs @@ -1,6 +1,7 @@ -use crate::adapters::snowflake::tests::helpers::unload_blocked_for; -use crate::adapters::snowflake::tests::test_types::UnloadBlockedTestCase; -use crate::loader::constants::UNLOADED_OUTSIDE_STAGE; +use crate::adapters::snowflake::tests::helpers::{unload_blocked_for, unload_readers_for}; +use crate::adapters::snowflake::tests::test_types::{UnloadBlockedTestCase, UnloadReadersTestCase}; +use crate::column_types::models::Warehouse; +use crate::loader::constants::{UNLOADED_OUTSIDE_STAGE, UNLOADED_TIMES}; const MASKS: &str = "the table masks columns"; const OUTSIDE: &str = UNLOADED_OUTSIDE_STAGE; @@ -82,3 +83,73 @@ fn given_a_snowflake_sources_unload_when_routing_into_snowflake_then_snowflake_c ); } } + +#[test] +fn given_a_snowflake_sources_s3_unload_when_routing_into_another_warehouse_then_databricks_and_clickhouse_read_it_unless_times_or_masks_need_sqlload() + { + let test_cases = [ + UnloadReadersTestCase { + description: "Databricks reads the unload itself", + warehouse: Warehouse::Databricks, + masks: false, + times: false, + expected_blocked: None, + }, + UnloadReadersTestCase { + description: "ClickHouse reads the unload itself", + warehouse: Warehouse::ClickHouse, + masks: false, + times: false, + expected_blocked: None, + }, + UnloadReadersTestCase { + description: "a time column goes through SQLLoad into Databricks, which keeps times as text", + warehouse: Warehouse::Databricks, + masks: false, + times: true, + expected_blocked: Some("the table has a time column"), + }, + UnloadReadersTestCase { + description: "a time column goes through SQLLoad into ClickHouse", + warehouse: Warehouse::ClickHouse, + masks: false, + times: true, + expected_blocked: Some("the table has a time column"), + }, + UnloadReadersTestCase { + description: "masks need SQLLoad to see the rows", + warehouse: Warehouse::ClickHouse, + masks: true, + times: false, + expected_blocked: Some(MASKS), + }, + UnloadReadersTestCase { + description: "DuckDB turns times back itself", + warehouse: Warehouse::DuckDb, + masks: false, + times: true, + expected_blocked: None, + }, + UnloadReadersTestCase { + description: "Postgres never reads files itself", + warehouse: Warehouse::Postgres, + masks: false, + times: false, + expected_blocked: Some(UNLOADED_TIMES), + }, + ]; + for test_case in &test_cases { + let blocked = unload_readers_for(test_case); + assert_eq!( + blocked.as_deref().map(|reason| { + test_case + .expected_blocked + .filter(|expected| reason.starts_with(expected)) + .unwrap_or(reason) + }), + test_case.expected_blocked, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index 48f8fad6..10500805 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -1094,7 +1094,7 @@ pub(crate) fn reader_offer( true => direct_blocked, false => unload_blocked( spec.warehouse, - !spec.masked.columns.is_empty(), + (!spec.masked.columns.is_empty(), &spec.source_table.columns), route, stage, ), From ccf4a938a61eb7d5b1782ce4f258c7b7aeac6649 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:52:20 +0000 Subject: [PATCH 2/7] wip: ClickHouse statement reading one file of a Snowflake unload, timestamps from microseconds --- .../clickhouse/_helpers/dialect/mod.rs | 1 + .../clickhouse/_helpers/dialect/pull.rs | 35 ++++---- .../clickhouse/_helpers/dialect/unloaded.rs | 85 +++++++++++++++++++ .../sqlload/src/adapters/clickhouse/tests.rs | 2 + .../clickhouse/tests/test_unloaded.rs | 71 ++++++++++++++++ 5 files changed, 179 insertions(+), 15 deletions(-) create mode 100644 crates/sqlload/src/adapters/clickhouse/_helpers/dialect/unloaded.rs create mode 100644 crates/sqlload/src/adapters/clickhouse/tests/test_unloaded.rs diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/mod.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/mod.rs index ac656d3c..f98d52a0 100644 --- a/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/mod.rs +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/mod.rs @@ -3,4 +3,5 @@ pub(crate) mod publish; pub(crate) mod pull; pub(crate) mod sql_adapter; pub(crate) mod type_names; +pub(crate) mod unloaded; pub(crate) mod write; diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/pull.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/pull.rs index b1adcf67..0c101a12 100644 --- a/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/pull.rs +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/pull.rs @@ -81,7 +81,6 @@ pub(crate) fn pull_insert( (file, credentials): (&PulledFile, Option<&PullCredentials>), (compat, load_id, loaded_at): (Compat, &str, &str), ) -> String { - let literal = |text: &str| ClickHouseSql.render_loader_value_literal(text); let identifier = |name: &str| ClickHouseSql.render_identifier(name); let cast = |value: String, column: &DesiredColumn| { format!("CAST({value} AS {})", sql_type(&column.column_type)) @@ -116,8 +115,26 @@ pub(crate) fn pull_insert( .iter() .map(|column| identifier(&column.name)) .collect(); + let numbered = columns + .iter() + .any(|column| column.name == SOURCE_ROW_COLUMN); + let settings = match numbered { + true => format!(" SETTINGS {ORDERED_READ_SETTINGS}"), + false => String::new(), + }; + format!( + "INSERT INTO {into} ({}) SELECT {} FROM {}{settings}", + targets.join(", "), + values.join(", "), + s3_parquet(&file.url, credentials) + ) +} + +/// `s3()` reading the Parquet file at `url`, signed with `credentials` (named as [query parameters](credential_parameters)), or unsigned (`NOSIGN`) when there are none. +pub(crate) fn s3_parquet(url: &str, credentials: Option<&PullCredentials>) -> String { + let literal = |text: &str| ClickHouseSql.render_loader_value_literal(text); let parameter = |name: &str| format!("{{{name}:String}}"); - let mut signed = vec![literal(&file.url)]; + let mut signed = vec![literal(url)]; match credentials { Some(credentials) => { signed.extend([parameter(KEY_ID_PARAMETER), parameter(SECRET_PARAMETER)]); @@ -131,19 +148,7 @@ pub(crate) fn pull_insert( None => signed.push("NOSIGN".to_owned()), } signed.push(literal("Parquet")); - let numbered = columns - .iter() - .any(|column| column.name == SOURCE_ROW_COLUMN); - let settings = match numbered { - true => format!(" SETTINGS {ORDERED_READ_SETTINGS}"), - false => String::new(), - }; - format!( - "INSERT INTO {into} ({}) SELECT {} FROM s3({}){settings}", - targets.join(", "), - values.join(", "), - signed.join(", ") - ) + format!("s3({})", signed.join(", ")) } /// The HTTP URL ClickHouse reads `key` of `bucket` at: under `endpoint` by path when there is one (or with the bucket already in it, `in_endpoint`, for virtual-hosted requests to a custom endpoint), otherwise on AWS, in `region` when known, by virtual host unless the bucket's name holds a dot, which AWS's certificate does not cover. Each path segment is percent-encoded, which ClickHouse decodes before signing. diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/unloaded.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/unloaded.rs new file mode 100644 index 00000000..dc035263 --- /dev/null +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/unloaded.rs @@ -0,0 +1,85 @@ +//! The statement that has ClickHouse read one file of a warehouse source's unload from S3 itself. + +use crate::adapters::clickhouse::_helpers::dialect::pull::{ + PullCredentials, SqlTypeOf, s3_parquet, +}; +use crate::adapters::clickhouse::_helpers::dialect::write::typed_literal; +use crate::adapters::clickhouse::constants::ORDERED_READ_SETTINGS; +use crate::adapters::clickhouse::models::ClickHouseSql; +use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS}; +use crate::adapters::contract::types::SqlAdapter; +use crate::column_types::models::ColumnType; +use crate::load_naming::main::metadata_role::metadata_role; +use crate::load_naming::models::{Compat, MetadataRole}; +use crate::postgres_source::models::ReadColumn; +use crate::schema_plan::constants::MERGE_ROW_COLUMN; +use crate::schema_plan::models::DesiredColumn; + +/// One file of an unload that ClickHouse reads: its URL, and its place among the unload's files, from 0, which a merge's row number counts by. +pub(crate) struct UnloadedFile<'a> { + pub url: &'a str, + pub index: usize, +} + +/// `INSERT INTO into` every one of `columns` (each typed by `sql_type`) from one of a warehouse's unloaded Parquet `file`s, read by [`s3_parquet`] with `credentials`. A column of `read` comes from the file, turned back from whole microseconds where it is read as a timestamp (a time never loads this way); a column of `constants` from its value; a merge's row number from `batch`, the file and the row in it, read in stored order; the load id and load time are the batch's; any other column is NULL. +pub(crate) fn unloaded_insert( + into: &str, + (columns, sql_type): (&[DesiredColumn], SqlTypeOf<'_>), + (read, constants): (&[ReadColumn], &[(String, ColumnType, String)]), + (file, credentials): (&UnloadedFile<'_>, Option<&PullCredentials>), + (compat, batch, load_id, loaded_at): (Compat, u64, &str, &str), +) -> String { + let identifier = |name: &str| ClickHouseSql.render_identifier(name); + let values: Vec = columns + .iter() + .map(|column| { + let constant = constants.iter().find(|(name, _, _)| *name == column.name); + let value = match (metadata_role(compat, &column.name), column.name.as_str()) { + (Some(MetadataRole::LoadId), _) => typed_literal(load_id, &column.column_type), + (Some(MetadataRole::LoadedAt), _) => { + typed_literal(loaded_at, &column.column_type) + } + (None, MERGE_ROW_COLUMN) => format!( + "{batch} * {UNLOADED_BATCH_ROWS} + {} * {UNLOADED_FILE_ROWS} + rowNumberInAllBlocks()", + file.index + ), + _ if constant.is_some() => constant + .map(|(_, column_type, value)| typed_literal(value, column_type)) + .unwrap_or_default(), + (_, name) => read + .iter() + .find(|read| read.name == name) + .map_or_else( + || "NULL".to_owned(), + |read| unloaded_value(&read.output, &identifier(name)), + ), + }; + format!("CAST({value} AS {})", sql_type(&column.column_type)) + }) + .collect(); + let targets: Vec = columns + .iter() + .map(|column| identifier(&column.name)) + .collect(); + let settings = match columns.iter().any(|column| column.name == MERGE_ROW_COLUMN) { + true => format!(" SETTINGS {ORDERED_READ_SETTINGS}"), + false => String::new(), + }; + format!( + "INSERT INTO {into} ({}) SELECT {} FROM {}{settings}", + targets.join(", "), + values.join(", "), + s3_parquet(file.url, credentials) + ) +} + +/// File column `quoted` as the value it was unloaded from: a timestamp, with or without time zone, from its whole microseconds as the UTC instant SQLLoad's reader gives (ClickHouse stores both as an instant); a binary through `substring`, which keeps a fixed-size value's trailing zero bytes; anything else as it is. +fn unloaded_value(output: &ColumnType, quoted: &str) -> String { + match output { + ColumnType::TimestampTz | ColumnType::TimestampNtz => { + format!("fromUnixTimestamp64Micro(CAST({quoted} AS Int64), 'UTC')") + } + ColumnType::Binary => format!("substring({quoted}, 1)"), + _ => quoted.to_owned(), + } +} diff --git a/crates/sqlload/src/adapters/clickhouse/tests.rs b/crates/sqlload/src/adapters/clickhouse/tests.rs index 6d121a23..89c63d5a 100644 --- a/crates/sqlload/src/adapters/clickhouse/tests.rs +++ b/crates/sqlload/src/adapters/clickhouse/tests.rs @@ -12,3 +12,5 @@ mod pull; mod rows; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_unloaded.rs"] +mod unloaded; diff --git a/crates/sqlload/src/adapters/clickhouse/tests/test_unloaded.rs b/crates/sqlload/src/adapters/clickhouse/tests/test_unloaded.rs new file mode 100644 index 00000000..7069d66a --- /dev/null +++ b/crates/sqlload/src/adapters/clickhouse/tests/test_unloaded.rs @@ -0,0 +1,71 @@ +use crate::adapters::clickhouse::_helpers::dialect::pull::PullCredentials; +use crate::adapters::clickhouse::_helpers::dialect::unloaded::{UnloadedFile, unloaded_insert}; +use crate::adapters::clickhouse::tests::helpers::desired; +use crate::column_types::models::ColumnType; +use crate::load_naming::models::Compat; +use crate::postgres_source::models::{ReadColumn, WireType}; + +#[test] +fn given_a_snowflake_unload_when_clickhouse_reads_one_of_its_files_then_timestamps_come_back_from_microseconds_and_the_rest_as_unloaded() + { + let read: Vec = [ + ("id", ColumnType::Int), + ("at", ColumnType::TimestampTz), + ("wall", ColumnType::TimestampNtz), + ("blob", ColumnType::Binary), + ("day", ColumnType::Date), + ] + .into_iter() + .map(|(name, output)| ReadColumn { + name: name.to_owned(), + expression: format!("\"{name}\""), + wire: WireType::Text, + output, + }) + .collect(); + let columns = [ + desired("id", ColumnType::Int), + desired("at", ColumnType::TimestampTz), + desired("wall", ColumnType::TimestampNtz), + desired("blob", ColumnType::Binary), + desired("day", ColumnType::Date), + desired("region", ColumnType::Text), + desired("missing", ColumnType::Text), + desired("_sqlload_row", ColumnType::Int), + desired("_sqlload_load_id", ColumnType::Text), + desired("_sqlload_loaded_at", ColumnType::TimestampTz), + ]; + let constants = [("region".to_owned(), ColumnType::Text, "eu".to_owned())]; + let credentials = PullCredentials { + key_id: "AKIDEXAMPLE".to_owned(), + secret: "secret".to_owned(), + token: None, + }; + let file = UnloadedFile { + url: "https://unload.s3.eu-west-2.amazonaws.com/run/data_0_1_0.snappy.parquet", + index: 2, + }; + let insert = unloaded_insert( + "`raw`.`orders__staging`", + (&columns, &|_: &ColumnType| "T".to_owned()), + (&read, &constants), + (&file, Some(&credentials)), + (Compat::Sqlload, 3, "L1", "2026-11-18T00:00:00Z"), + ); + assert_eq!( + insert, + "INSERT INTO `raw`.`orders__staging` (`id`, `at`, `wall`, `blob`, `day`, `region`, `missing`, `_sqlload_row`, `_sqlload_load_id`, `_sqlload_loaded_at`) SELECT \ + CAST(`id` AS T), \ + CAST(fromUnixTimestamp64Micro(CAST(`at` AS Int64), 'UTC') AS T), \ + CAST(fromUnixTimestamp64Micro(CAST(`wall` AS Int64), 'UTC') AS T), \ + CAST(substring(`blob`, 1) AS T), \ + CAST(`day` AS T), \ + CAST(CAST('eu' AS String) AS T), \ + CAST(NULL AS T), \ + CAST(3 * 281474976710656 + 2 * 4294967296 + rowNumberInAllBlocks() AS T), \ + CAST(CAST('L1' AS String) AS T), \ + CAST(parseDateTime64BestEffort('2026-11-18T00:00:00Z', 6, 'UTC') AS T) \ + FROM s3('https://unload.s3.eu-west-2.amazonaws.com/run/data_0_1_0.snappy.parquet', {sqlload_key_id:String}, {sqlload_secret:String}, 'Parquet') \ + SETTINGS max_threads = 1, max_insert_threads = 1, input_format_parquet_preserve_order = 1" + ); +} From bc31cc7ad8555a7a8c2a6657fc4cb92a1f7ccaee Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:55:48 +0000 Subject: [PATCH 3/7] wip: ClickHouse loads a Snowflake S3 unload itself, one s3() insert per file, rows checked against the unload --- .../clickhouse/_helpers/write/direct_load.rs | 92 ++++++++++--------- .../adapters/clickhouse/_helpers/write/mod.rs | 1 + .../_helpers/write/unloaded_load.rs | 83 +++++++++++++++++ .../dispatch/main/write/load_unloaded.rs | 1 + 4 files changed, 135 insertions(+), 42 deletions(-) create mode 100644 crates/sqlload/src/adapters/clickhouse/_helpers/write/unloaded_load.rs diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/write/direct_load.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/write/direct_load.rs index 38ec802a..ca427ae9 100644 --- a/crates/sqlload/src/adapters/clickhouse/_helpers/write/direct_load.rs +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/write/direct_load.rs @@ -48,47 +48,7 @@ impl ClickHouseWriter { ))); } self.settle_rows().await?; - let builder = s3_builder( - bucket, - region.as_deref(), - endpoint.as_deref(), - profile.as_deref(), - ) - .map_err(|error| { - LoaderError::Invalid(format!( - "cannot sign ClickHouse's reads of s3://{bucket}/: {error}" - )) - })?; - let setting = |key: AmazonS3ConfigKey| builder.get_config_value(&key); - let region = setting(AmazonS3ConfigKey::Region); - let endpoint = - setting(AmazonS3ConfigKey::S3Endpoint).or(setting(AmazonS3ConfigKey::Endpoint)); - let in_endpoint = endpoint.is_some() - && switched_on(setting(AmazonS3ConfigKey::VirtualHostedStyleRequest)); - let unsigned = switched_on(setting(AmazonS3ConfigKey::SkipSignature)); - let unreadable = |error: String| { - LoaderError::Invalid(format!( - "cannot sign ClickHouse's reads of s3://{bucket}/: {error}" - )) - }; - let credentials = match unsigned { - true => None, - false => { - let store = builder - .build() - .map_err(|error| unreadable(error.to_string()))?; - let credential = store - .credentials() - .get_credential() - .await - .map_err(|error| unreadable(error.to_string()))?; - Some(PullCredentials { - key_id: credential.key_id.clone(), - secret: credential.secret_key.clone(), - token: credential.token.clone(), - }) - } - }; + let (region, endpoint, credentials) = s3_reads(bucket, (region, endpoint, profile)).await?; let parameters = credential_parameters(credentials.as_ref()); let target = &batch.target; let (into, columns) = match (target.staging, target.builds) { @@ -107,7 +67,7 @@ impl ClickHouseWriter { let url = object_url( (bucket, &key), region.as_deref(), - (endpoint.as_deref(), in_endpoint), + (endpoint.0.as_deref(), endpoint.1), ) .map_err(LoaderError::Invalid)?; let file = pulled_file(source, data, url, (batch, columns))?; @@ -131,6 +91,54 @@ impl ClickHouseWriter { } } +/// How ClickHouse reads objects of `bucket`, as SQLLoad's own S3 reads would (`region`, `endpoint` and `profile` from the connection): the region, the endpoint with whether the bucket is already in it (virtual-hosted requests to a custom endpoint), and the credentials SQLLoad resolves to sign `s3()` with, `None` for unsigned reads. +pub(in crate::adapters) async fn s3_reads( + bucket: &str, + (region, endpoint, profile): (&Option, &Option, &Option), +) -> Result< + ( + Option, + (Option, bool), + Option, + ), + LoaderError, +> { + let unreadable = |error: String| { + LoaderError::Invalid(format!( + "cannot sign ClickHouse's reads of s3://{bucket}/: {error}" + )) + }; + let builder = s3_builder( + bucket, + region.as_deref(), + endpoint.as_deref(), + profile.as_deref(), + ) + .map_err(|error| unreadable(error.to_string()))?; + let setting = |key: AmazonS3ConfigKey| builder.get_config_value(&key); + let region = setting(AmazonS3ConfigKey::Region); + let endpoint = setting(AmazonS3ConfigKey::S3Endpoint).or(setting(AmazonS3ConfigKey::Endpoint)); + let in_endpoint = + endpoint.is_some() && switched_on(setting(AmazonS3ConfigKey::VirtualHostedStyleRequest)); + if switched_on(setting(AmazonS3ConfigKey::SkipSignature)) { + return Ok((region, (endpoint, in_endpoint), None)); + } + let store = builder + .build() + .map_err(|error| unreadable(error.to_string()))?; + let credential = store + .credentials() + .get_credential() + .await + .map_err(|error| unreadable(error.to_string()))?; + let credentials = PullCredentials { + key_id: credential.key_id.clone(), + secret: credential.secret_key.clone(), + token: credential.token.clone(), + }; + Ok((region, (endpoint, in_endpoint), Some(credentials))) +} + /// Whether an S3 switch is on, read as object_store reads it from the environment (`true`, `1`, `yes`, `on` or `y`, in any case). fn switched_on(value: Option) -> bool { value.is_some_and(|value| { diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/write/mod.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/write/mod.rs index d2184caf..14ca8d42 100644 --- a/crates/sqlload/src/adapters/clickhouse/_helpers/write/mod.rs +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/write/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod arrow_batch; pub(crate) mod direct_load; pub(crate) mod table_writer; +pub(crate) mod unloaded_load; diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/write/unloaded_load.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/write/unloaded_load.rs new file mode 100644 index 00000000..104362c7 --- /dev/null +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/write/unloaded_load.rs @@ -0,0 +1,83 @@ +//! A warehouse source's unloaded files loaded by ClickHouse reading them from S3 itself. + +use crate::adapters::clickhouse::_helpers::dialect::pull::{credential_parameters, object_url}; +use crate::adapters::clickhouse::_helpers::dialect::unloaded::{UnloadedFile, unloaded_insert}; +use crate::adapters::clickhouse::_helpers::session::http::written_rows; +use crate::adapters::clickhouse::_helpers::write::direct_load::s3_reads; +use crate::adapters::clickhouse::models::{ClickHouseSql, ClickHouseWriter}; +use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch, UnloadedSource}; +use crate::adapters::contract::types::{SqlAdapter, TableWriter}; +use crate::column_types::models::ColumnType; +use crate::loader::errors::LoaderError; +use crate::store_paths::models::StoreKind; + +impl ClickHouseWriter { + /// Load `batch` by having ClickHouse read the unloaded files from S3 itself, one `INSERT ... SELECT FROM s3()` per file, into the staging or build table the load publishes from, signed with the credentials SQLLoad reads S3 with. ClickHouse must load as many rows as the unload reports, or the batch fails before anything publishes. + pub(in crate::adapters) async fn pull_unloaded( + &mut self, + batch: &UnloadedBatch<'_>, + ) -> Result { + let UnloadedSource::Files(files) = batch.source else { + return Err(LoaderError::Invalid( + "ClickHouse reads a warehouse source's rows only from its unloaded files" + .to_owned(), + )); + }; + let StoreKind::S3 { + bucket, + region, + endpoint, + profile, + } = &files.folder.store + else { + return Err(LoaderError::Invalid(format!( + "ClickHouse reads unloaded files only from S3, not {}", + files.folder + ))); + }; + if files.keys.is_empty() { + return Ok(DirectLoaded::Staged(0)); + } + self.settle_rows().await?; + let (region, endpoint, credentials) = s3_reads(bucket, (region, endpoint, profile)).await?; + let parameters = credential_parameters(credentials.as_ref()); + let target = &batch.target; + let (into, columns) = match (target.staging, target.builds) { + (Some(staging), false) => (staging, target.columns), + (staging, _) => (staging.unwrap_or(target.table), batch.write_columns), + }; + let builds = target.builds; + let sql_type = |column_type: &ColumnType| match builds { + true => ClickHouseSql.render_loader_logical_type(column_type).sql, + false => ClickHouseSql.render_staging_type(column_type), + }; + let into = ClickHouseSql.render_qualified_name(into); + let mut rows = 0; + for (index, key) in files.keys.iter().enumerate() { + let object = format!("{}{key}", files.folder.prefix); + let url = object_url( + (bucket, &object), + region.as_deref(), + (endpoint.0.as_deref(), endpoint.1), + ) + .map_err(LoaderError::Invalid)?; + let insert = unloaded_insert( + &into, + (columns, &sql_type), + (batch.read, target.constants), + (&UnloadedFile { url: &url, index }, credentials.as_ref()), + (target.compat, target.batch, target.load_id, batch.loaded_at), + ); + rows += written_rows(&self.destination, &insert, ¶meters) + .await + .map_err(|error| LoaderError::Invalid(format!("{}{key}: {error}", files.folder)))?; + } + if rows != files.rows { + return Err(LoaderError::Invalid(format!( + "ClickHouse loaded {rows} rows from the unloaded files in {}, but the unload reports {}; the batch is not committed", + files.folder, files.rows + ))); + } + Ok(DirectLoaded::Staged(rows)) + } +} diff --git a/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs b/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs index 6ac0235d..bb15d0d3 100644 --- a/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs +++ b/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs @@ -13,6 +13,7 @@ pub async fn load_unloaded( #[cfg(feature = "duckdb")] Writer::DuckDb(duckdb) => duckdb.pull_unloaded(batch).await.map(Some), Writer::Snowflake(snowflake) => snowflake.pull_unloaded(batch).await.map(Some), + Writer::ClickHouse(clickhouse) => clickhouse.pull_unloaded(batch).await.map(Some), _ => Ok(None), } } From d7f0d2979b12545cc84c32a196bfe2a38af84e0d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:16:30 +0000 Subject: [PATCH 4/7] wip: Databricks COPYs a Snowflake S3 unload through its external location into a staging table; an unload outside every location fails plan --- .../clickhouse/_helpers/dialect/unloaded.rs | 69 +++++----- .../_helpers/write/unloaded_load.rs | 7 +- .../adapters/clickhouse/tests/test_types.rs | 8 ++ .../clickhouse/tests/test_unloaded.rs | 108 ++++++++++------ .../databricks/_helpers/write/direct_load.rs | 2 +- .../adapters/databricks/_helpers/write/mod.rs | 2 + .../databricks/_helpers/write/s3_copy.rs | 2 +- .../_helpers/write/unloaded_copy.rs | 111 ++++++++++++++++ .../_helpers/write/unloaded_load.rs | 64 +++++++++ .../sqlload/src/adapters/databricks/tests.rs | 2 + .../adapters/databricks/tests/test_types.rs | 7 + .../databricks/tests/test_unloaded_copy.rs | 122 ++++++++++++++++++ .../dispatch/main/route/check_reach.rs | 16 ++- .../dispatch/main/write/load_unloaded.rs | 1 + .../src/adapters/dispatch/main/write/mod.rs | 1 + .../dispatch/main/write/unloads_directly.rs | 8 ++ .../src/adapters/snowflake/tests/helpers.rs | 5 +- .../adapters/snowflake/tests/test_types.rs | 4 +- .../snowflake/tests/test_unload_blocked.rs | 14 +- crates/sqlload/src/loader/_helpers/publish.rs | 8 +- .../sqlload/src/loader/_helpers/table_plan.rs | 8 +- .../src/routing/_helpers/files_route.rs | 2 +- .../src/routing/_helpers/warehouse_source.rs | 16 ++- .../src/routing/tests/test_plan_route.rs | 20 +++ 24 files changed, 511 insertions(+), 96 deletions(-) create mode 100644 crates/sqlload/src/adapters/databricks/_helpers/write/unloaded_copy.rs create mode 100644 crates/sqlload/src/adapters/databricks/_helpers/write/unloaded_load.rs create mode 100644 crates/sqlload/src/adapters/databricks/tests/test_unloaded_copy.rs create mode 100644 crates/sqlload/src/adapters/dispatch/main/write/unloads_directly.rs diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/unloaded.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/unloaded.rs index dc035263..b09190fd 100644 --- a/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/unloaded.rs +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/unloaded.rs @@ -7,10 +7,11 @@ use crate::adapters::clickhouse::_helpers::dialect::write::typed_literal; use crate::adapters::clickhouse::constants::ORDERED_READ_SETTINGS; use crate::adapters::clickhouse::models::ClickHouseSql; use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS}; +use crate::adapters::contract::models::WriteTarget; use crate::adapters::contract::types::SqlAdapter; use crate::column_types::models::ColumnType; use crate::load_naming::main::metadata_role::metadata_role; -use crate::load_naming::models::{Compat, MetadataRole}; +use crate::load_naming::models::MetadataRole; use crate::postgres_source::models::ReadColumn; use crate::schema_plan::constants::MERGE_ROW_COLUMN; use crate::schema_plan::models::DesiredColumn; @@ -21,45 +22,26 @@ pub(crate) struct UnloadedFile<'a> { pub index: usize, } -/// `INSERT INTO into` every one of `columns` (each typed by `sql_type`) from one of a warehouse's unloaded Parquet `file`s, read by [`s3_parquet`] with `credentials`. A column of `read` comes from the file, turned back from whole microseconds where it is read as a timestamp (a time never loads this way); a column of `constants` from its value; a merge's row number from `batch`, the file and the row in it, read in stored order; the load id and load time are the batch's; any other column is NULL. +/// `INSERT INTO into` every one of `columns` (each typed by `sql_type`) from one of a warehouse's unloaded Parquet `file`s, read by [`s3_parquet`] with `credentials`. A column of `read` comes from the file, turned back from whole microseconds where it is read as a timestamp (a time never loads this way); a column of `target`'s constants from its value; a merge's row number from the batch, the file and the row in it, read in stored order; the load id and load time (`loaded_at`) are the batch's; any other column is NULL. pub(crate) fn unloaded_insert( - into: &str, - (columns, sql_type): (&[DesiredColumn], SqlTypeOf<'_>), - (read, constants): (&[ReadColumn], &[(String, ColumnType, String)]), + (into, columns, sql_type): (&str, &[DesiredColumn], SqlTypeOf<'_>), + target: &WriteTarget<'_>, + (read, loaded_at): (&[ReadColumn], &str), (file, credentials): (&UnloadedFile<'_>, Option<&PullCredentials>), - (compat, batch, load_id, loaded_at): (Compat, u64, &str, &str), ) -> String { - let identifier = |name: &str| ClickHouseSql.render_identifier(name); let values: Vec = columns .iter() .map(|column| { - let constant = constants.iter().find(|(name, _, _)| *name == column.name); - let value = match (metadata_role(compat, &column.name), column.name.as_str()) { - (Some(MetadataRole::LoadId), _) => typed_literal(load_id, &column.column_type), - (Some(MetadataRole::LoadedAt), _) => { - typed_literal(loaded_at, &column.column_type) - } - (None, MERGE_ROW_COLUMN) => format!( - "{batch} * {UNLOADED_BATCH_ROWS} + {} * {UNLOADED_FILE_ROWS} + rowNumberInAllBlocks()", - file.index - ), - _ if constant.is_some() => constant - .map(|(_, column_type, value)| typed_literal(value, column_type)) - .unwrap_or_default(), - (_, name) => read - .iter() - .find(|read| read.name == name) - .map_or_else( - || "NULL".to_owned(), - |read| unloaded_value(&read.output, &identifier(name)), - ), - }; - format!("CAST({value} AS {})", sql_type(&column.column_type)) + format!( + "CAST({} AS {})", + column_value(column, target, (read, loaded_at), file), + sql_type(&column.column_type) + ) }) .collect(); let targets: Vec = columns .iter() - .map(|column| identifier(&column.name)) + .map(|column| ClickHouseSql.render_identifier(&column.name)) .collect(); let settings = match columns.iter().any(|column| column.name == MERGE_ROW_COLUMN) { true => format!(" SETTINGS {ORDERED_READ_SETTINGS}"), @@ -73,6 +55,33 @@ pub(crate) fn unloaded_insert( ) } +/// Where `column`'s value comes from, before its cast: the load id or load time, a merge's row number, a constant's value, the file's column of `read` it is, or NULL when the file lacks it. +fn column_value( + column: &DesiredColumn, + target: &WriteTarget<'_>, + (read, loaded_at): (&[ReadColumn], &str), + file: &UnloadedFile<'_>, +) -> String { + let constant = target + .constants + .iter() + .find(|(name, _, _)| *name == column.name); + let held = read.iter().find(|read| read.name == column.name); + match (metadata_role(target.compat, &column.name), constant, held) { + (Some(MetadataRole::LoadId), _, _) => typed_literal(target.load_id, &column.column_type), + (Some(MetadataRole::LoadedAt), _, _) => typed_literal(loaded_at, &column.column_type), + _ if column.name == MERGE_ROW_COLUMN => format!( + "{} * {UNLOADED_BATCH_ROWS} + {} * {UNLOADED_FILE_ROWS} + rowNumberInAllBlocks()", + target.batch, file.index + ), + (_, Some((_, column_type, value)), _) => typed_literal(value, column_type), + (_, None, Some(read)) => { + unloaded_value(&read.output, &ClickHouseSql.render_identifier(&read.name)) + } + (_, None, None) => "NULL".to_owned(), + } +} + /// File column `quoted` as the value it was unloaded from: a timestamp, with or without time zone, from its whole microseconds as the UTC instant SQLLoad's reader gives (ClickHouse stores both as an instant); a binary through `substring`, which keeps a fixed-size value's trailing zero bytes; anything else as it is. fn unloaded_value(output: &ColumnType, quoted: &str) -> String { match output { diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/write/unloaded_load.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/write/unloaded_load.rs index 104362c7..23d8eca2 100644 --- a/crates/sqlload/src/adapters/clickhouse/_helpers/write/unloaded_load.rs +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/write/unloaded_load.rs @@ -62,11 +62,10 @@ impl ClickHouseWriter { ) .map_err(LoaderError::Invalid)?; let insert = unloaded_insert( - &into, - (columns, &sql_type), - (batch.read, target.constants), + (&into, columns, &sql_type), + target, + (batch.read, batch.loaded_at), (&UnloadedFile { url: &url, index }, credentials.as_ref()), - (target.compat, target.batch, target.load_id, batch.loaded_at), ); rows += written_rows(&self.destination, &insert, ¶meters) .await diff --git a/crates/sqlload/src/adapters/clickhouse/tests/test_types.rs b/crates/sqlload/src/adapters/clickhouse/tests/test_types.rs index 4a6f0b56..86526a4d 100644 --- a/crates/sqlload/src/adapters/clickhouse/tests/test_types.rs +++ b/crates/sqlload/src/adapters/clickhouse/tests/test_types.rs @@ -78,3 +78,11 @@ pub(crate) struct SortedInsertTestCase { pub(crate) key_columns: &'static [&'static str], pub(crate) expected_insert: &'static str, } + +pub(crate) struct UnloadedInsertTestCase { + pub(crate) description: &'static str, + pub(crate) columns: Vec, + /// Whether the read is signed, or unsigned for a public bucket. + pub(crate) signed: bool, + pub(crate) expected_insert: &'static str, +} diff --git a/crates/sqlload/src/adapters/clickhouse/tests/test_unloaded.rs b/crates/sqlload/src/adapters/clickhouse/tests/test_unloaded.rs index 7069d66a..c58f8869 100644 --- a/crates/sqlload/src/adapters/clickhouse/tests/test_unloaded.rs +++ b/crates/sqlload/src/adapters/clickhouse/tests/test_unloaded.rs @@ -1,6 +1,8 @@ use crate::adapters::clickhouse::_helpers::dialect::pull::PullCredentials; use crate::adapters::clickhouse::_helpers::dialect::unloaded::{UnloadedFile, unloaded_insert}; use crate::adapters::clickhouse::tests::helpers::desired; +use crate::adapters::clickhouse::tests::test_types::UnloadedInsertTestCase; +use crate::adapters::contract::models::{TableName, WriteTarget}; use crate::column_types::models::ColumnType; use crate::load_naming::models::Compat; use crate::postgres_source::models::{ReadColumn, WireType}; @@ -8,12 +10,47 @@ use crate::postgres_source::models::{ReadColumn, WireType}; #[test] fn given_a_snowflake_unload_when_clickhouse_reads_one_of_its_files_then_timestamps_come_back_from_microseconds_and_the_rest_as_unloaded() { + let test_cases = [ + UnloadedInsertTestCase { + description: "a merge's staging table: timestamps from microseconds, the constant, a column the files lack as NULL, the merge row by batch, file and row read in stored order", + columns: vec![ + desired("id", ColumnType::Int), + desired("at", ColumnType::TimestampTz), + desired("wall", ColumnType::TimestampNtz), + desired("blob", ColumnType::Binary), + desired("region", ColumnType::Text), + desired("missing", ColumnType::Text), + desired("_sqlload_row", ColumnType::Int), + desired("_sqlload_load_id", ColumnType::Text), + desired("_sqlload_loaded_at", ColumnType::TimestampTz), + ], + signed: true, + expected_insert: "INSERT INTO `raw`.`orders__staging` (`id`, `at`, `wall`, `blob`, `region`, `missing`, `_sqlload_row`, `_sqlload_load_id`, `_sqlload_loaded_at`) SELECT \ + CAST(`id` AS T), \ + CAST(fromUnixTimestamp64Micro(CAST(`at` AS Int64), 'UTC') AS T), \ + CAST(fromUnixTimestamp64Micro(CAST(`wall` AS Int64), 'UTC') AS T), \ + CAST(substring(`blob`, 1) AS T), \ + CAST(CAST('eu' AS String) AS T), \ + CAST(NULL AS T), \ + CAST(3 * 281474976710656 + 2 * 4294967296 + rowNumberInAllBlocks() AS T), \ + CAST(CAST('L1' AS String) AS T), \ + CAST(parseDateTime64BestEffort('2026-11-18T00:00:00Z', 6, 'UTC') AS T) \ + FROM s3('https://unload.s3.eu-west-2.amazonaws.com/run/data_0_1_0.snappy.parquet', {sqlload_key_id:String}, {sqlload_secret:String}, 'Parquet') \ + SETTINGS max_threads = 1, max_insert_threads = 1, input_format_parquet_preserve_order = 1", + }, + UnloadedInsertTestCase { + description: "without a merge row the file is read unsigned from a public bucket, in any order", + columns: vec![desired("id", ColumnType::Int)], + signed: false, + expected_insert: "INSERT INTO `raw`.`orders__staging` (`id`) SELECT CAST(`id` AS T) \ + FROM s3('https://unload.s3.eu-west-2.amazonaws.com/run/data_0_1_0.snappy.parquet', NOSIGN, 'Parquet')", + }, + ]; let read: Vec = [ ("id", ColumnType::Int), ("at", ColumnType::TimestampTz), ("wall", ColumnType::TimestampNtz), ("blob", ColumnType::Binary), - ("day", ColumnType::Date), ] .into_iter() .map(|(name, output)| ReadColumn { @@ -23,18 +60,11 @@ fn given_a_snowflake_unload_when_clickhouse_reads_one_of_its_files_then_timestam output, }) .collect(); - let columns = [ - desired("id", ColumnType::Int), - desired("at", ColumnType::TimestampTz), - desired("wall", ColumnType::TimestampNtz), - desired("blob", ColumnType::Binary), - desired("day", ColumnType::Date), - desired("region", ColumnType::Text), - desired("missing", ColumnType::Text), - desired("_sqlload_row", ColumnType::Int), - desired("_sqlload_load_id", ColumnType::Text), - desired("_sqlload_loaded_at", ColumnType::TimestampTz), - ]; + let table = TableName { + database: None, + schema: "raw".to_owned(), + table: "orders".to_owned(), + }; let constants = [("region".to_owned(), ColumnType::Text, "eu".to_owned())]; let credentials = PullCredentials { key_id: "AKIDEXAMPLE".to_owned(), @@ -45,27 +75,33 @@ fn given_a_snowflake_unload_when_clickhouse_reads_one_of_its_files_then_timestam url: "https://unload.s3.eu-west-2.amazonaws.com/run/data_0_1_0.snappy.parquet", index: 2, }; - let insert = unloaded_insert( - "`raw`.`orders__staging`", - (&columns, &|_: &ColumnType| "T".to_owned()), - (&read, &constants), - (&file, Some(&credentials)), - (Compat::Sqlload, 3, "L1", "2026-11-18T00:00:00Z"), - ); - assert_eq!( - insert, - "INSERT INTO `raw`.`orders__staging` (`id`, `at`, `wall`, `blob`, `day`, `region`, `missing`, `_sqlload_row`, `_sqlload_load_id`, `_sqlload_loaded_at`) SELECT \ - CAST(`id` AS T), \ - CAST(fromUnixTimestamp64Micro(CAST(`at` AS Int64), 'UTC') AS T), \ - CAST(fromUnixTimestamp64Micro(CAST(`wall` AS Int64), 'UTC') AS T), \ - CAST(substring(`blob`, 1) AS T), \ - CAST(`day` AS T), \ - CAST(CAST('eu' AS String) AS T), \ - CAST(NULL AS T), \ - CAST(3 * 281474976710656 + 2 * 4294967296 + rowNumberInAllBlocks() AS T), \ - CAST(CAST('L1' AS String) AS T), \ - CAST(parseDateTime64BestEffort('2026-11-18T00:00:00Z', 6, 'UTC') AS T) \ - FROM s3('https://unload.s3.eu-west-2.amazonaws.com/run/data_0_1_0.snappy.parquet', {sqlload_key_id:String}, {sqlload_secret:String}, 'Parquet') \ - SETTINGS max_threads = 1, max_insert_threads = 1, input_format_parquet_preserve_order = 1" - ); + for test_case in &test_cases { + let target = WriteTarget { + table: &table, + staging: None, + columns: &test_case.columns, + write_columns: &test_case.columns, + compat: Compat::Sqlload, + load_id: "L1", + batch: 3, + direct: false, + builds: false, + constants: &constants, + }; + let insert = unloaded_insert( + ( + "`raw`.`orders__staging`", + &test_case.columns, + &|_: &ColumnType| "T".to_owned(), + ), + &target, + (&read, "2026-11-18T00:00:00Z"), + (&file, Some(&credentials).filter(|_| test_case.signed)), + ); + assert_eq!( + insert, test_case.expected_insert, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs index 805a3e70..d7ffe8c7 100644 --- a/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs @@ -162,7 +162,7 @@ fn group_statement( } /// The rows a `COPY INTO`'s `result` reports it inserted. -fn inserted_rows(result: &SqlRows) -> Result { +pub(in crate::adapters) fn inserted_rows(result: &SqlRows) -> Result { let reported = result .columns .iter() diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs index 30bac952..6dba39a6 100644 --- a/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs @@ -2,3 +2,5 @@ pub(crate) mod copy_select; pub(crate) mod direct_load; pub(crate) mod s3_copy; pub(crate) mod table_writer; +pub(crate) mod unloaded_copy; +pub(crate) mod unloaded_load; diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs index ea13bf88..82d41097 100644 --- a/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs @@ -12,7 +12,7 @@ use crate::schema_plan::models::DesiredColumn; use arrow_schema::{Field, Schema}; /// The path `COPY INTO` reads each row from, as SQLLoad names it: `_metadata.file_path` is percent-encoded, except for `+`, which `url_decode` alone would read as a space. -const FILE_PATH: &str = "url_decode(replace(_metadata.file_path, '+', '%2B'))"; +pub(crate) const FILE_PATH: &str = "url_decode(replace(_metadata.file_path, '+', '%2B'))"; /// One batch read from S3: the folder `COPY INTO` reads (`s3://bucket/prefix/`), and for each file its key below it and the values SQLLoad gives every row of it (`None` for NULL), by column. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/unloaded_copy.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/unloaded_copy.rs new file mode 100644 index 00000000..ed4912d8 --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/unloaded_copy.rs @@ -0,0 +1,111 @@ +//! The `COPY INTO` that has Databricks read files a warehouse source unloaded to S3. + +use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS}; +use crate::adapters::contract::models::WriteTarget; +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::databricks::_helpers::write::copy_select::typed_value; +use crate::adapters::databricks::_helpers::write::s3_copy::FILE_PATH; +use crate::adapters::databricks::constants::S3_COPY_OPTIONS; +use crate::adapters::databricks::models::DatabricksSql; +use crate::column_types::models::ColumnType; +use crate::load_naming::main::metadata_role::metadata_role; +use crate::load_naming::models::MetadataRole; +use crate::postgres_source::models::ReadColumn; +use crate::schema_plan::constants::MERGE_ROW_COLUMN; +use crate::schema_plan::models::DesiredColumn; + +/// Some of one unload's files: the folder `COPY INTO` reads (`s3://bucket/prefix/`), their keys below it, and the place of the first among the unload's files, from 0, which a merge's row number counts by. +pub(crate) struct UnloadedGroup<'a> { + pub folder: &'a str, + pub keys: &'a [String], + pub first: usize, +} + +/// `COPY INTO into` every one of `columns`, typed as `target` lands them, from `group`'s files: a column of `read` from the files, turned back from whole microseconds where it is read as a timestamp (a time never loads this way); a column of `target`'s constants from its value; a merge's row number from the batch, the file (picked by its path, and a row from any other path fails the statement) and the row in it; the load id and load time are the batch's; any other column is NULL. +pub(crate) fn unloaded_copy_statement( + (into, columns): (&str, &[DesiredColumn]), + target: &WriteTarget<'_>, + (read, group): (&[ReadColumn], &UnloadedGroup<'_>), + loaded_at: &str, +) -> String { + let literal = |text: &str| DatabricksSql.render_loader_value_literal(text); + let values: Vec = columns + .iter() + .map(|column| { + let value = column_value(target, (read, group), column, loaded_at); + format!( + "{} AS {}", + typed_value(target, column, &value), + DatabricksSql.render_identifier(&column.name) + ) + }) + .collect(); + let keys: Vec = group.keys.iter().map(|key| literal(key)).collect(); + format!( + "COPY INTO {into} FROM (SELECT {} FROM {}) FILEFORMAT = PARQUET FILES = ({}) {S3_COPY_OPTIONS}", + values.join(", "), + literal(group.folder), + keys.join(", ") + ) +} + +/// Where `column`'s value comes from, before it is typed: the load id or load time, a merge's row number, a constant's value, the files' column of `read` it is, or NULL when the files lack it. +fn column_value( + target: &WriteTarget<'_>, + (read, group): (&[ReadColumn], &UnloadedGroup<'_>), + column: &DesiredColumn, + loaded_at: &str, +) -> String { + let literal = |text: &str| DatabricksSql.render_loader_value_literal(text); + let constant = target + .constants + .iter() + .find(|(name, _, _)| *name == column.name); + match (metadata_role(target.compat, &column.name), constant) { + (Some(MetadataRole::LoadId), _) => literal(target.load_id), + (Some(MetadataRole::LoadedAt), _) => literal(loaded_at), + _ if column.name == MERGE_ROW_COLUMN => { + let files: Vec = group + .keys + .iter() + .enumerate() + .map(|(index, key)| { + format!( + "WHEN {} THEN {}", + literal(&format!("{}{key}", group.folder)), + group.first + index + ) + }) + .collect(); + format!( + "{} * {UNLOADED_BATCH_ROWS} + (CASE {FILE_PATH} {} ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END) * {UNLOADED_FILE_ROWS} + _metadata.row_index", + target.batch, + files.join(" ") + ) + } + (_, Some((_, _, value))) => literal(value), + _ => read + .iter() + .find(|read| read.name == column.name) + .map_or_else( + || "NULL".to_owned(), + |read| { + unloaded_value( + &read.output, + &DatabricksSql.render_exact_identifier(&read.name), + ) + }, + ), + } +} + +/// File column `quoted` as the value it was unloaded from: a timestamp with time zone as the instant its whole microseconds count from the epoch, one without as the wall-clock time they count from midnight of 1970-01-01, neither depending on the session's time zone; anything else as it is. +fn unloaded_value(output: &ColumnType, quoted: &str) -> String { + match output { + ColumnType::TimestampTz => format!("timestamp_micros(CAST({quoted} AS BIGINT))"), + ColumnType::TimestampNtz => format!( + "timestampadd(MICROSECOND, CAST({quoted} AS BIGINT), TIMESTAMP_NTZ'1970-01-01 00:00:00')" + ), + _ => quoted.to_owned(), + } +} diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/unloaded_load.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/unloaded_load.rs new file mode 100644 index 00000000..0c94ef81 --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/unloaded_load.rs @@ -0,0 +1,64 @@ +//! A warehouse source's unloaded files loaded by Databricks `COPY`ing them from S3. + +use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch, UnloadedSource}; +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::databricks::_helpers::write::direct_load::inserted_rows; +use crate::adapters::databricks::_helpers::write::unloaded_copy::{ + UnloadedGroup, unloaded_copy_statement, +}; +use crate::adapters::databricks::constants::MAX_FILES_PER_COPY; +use crate::adapters::databricks::models::{DatabricksSql, DatabricksWriter}; +use crate::databricks_api::main::execute_statement::execute_statement; +use crate::loader::errors::LoaderError; + +impl DatabricksWriter { + /// Load `batch` by having Databricks `COPY` the unloaded files from S3, through the external location that covers them, into the staging or build table the load publishes from: one `COPY INTO` per 1000 files, the most `FILES` takes. Each commits on its own, so the batch always lands there, never straight in the table, and fails unless they load as many rows as the unload reports. + pub(in crate::adapters) async fn pull_unloaded( + &mut self, + batch: &UnloadedBatch<'_>, + ) -> Result { + let UnloadedSource::Files(files) = batch.source else { + return Err(LoaderError::Invalid( + "Databricks reads a warehouse source's rows only from its unloaded files" + .to_owned(), + )); + }; + let target = &batch.target; + let Some(staging) = target.staging else { + return Err(LoaderError::Invalid( + "Databricks loads a warehouse source's unloaded files only into a staging or build table".to_owned(), + )); + }; + let folder = files.folder.to_string(); + let columns = match target.builds { + true => batch.write_columns, + false => target.columns, + }; + let into = DatabricksSql.render_qualified_name(staging); + let mut rows = 0; + for (index, keys) in files.keys.chunks(MAX_FILES_PER_COPY).enumerate() { + let statement = unloaded_copy_statement( + (&into, columns), + target, + ( + batch.read, + &UnloadedGroup { + folder: &folder, + keys, + first: index * MAX_FILES_PER_COPY, + }, + ), + batch.loaded_at, + ); + let result = execute_statement(&self.destination.api, &statement).await?; + rows += inserted_rows(&result)?; + } + if rows != files.rows { + return Err(LoaderError::Invalid(format!( + "Databricks loaded {rows} rows from the unloaded files in {folder}, but the unload reports {}; the batch is not published", + files.rows + ))); + } + Ok(DirectLoaded::Staged(rows)) + } +} diff --git a/crates/sqlload/src/adapters/databricks/tests.rs b/crates/sqlload/src/adapters/databricks/tests.rs index 57161de5..f0e6de2f 100644 --- a/crates/sqlload/src/adapters/databricks/tests.rs +++ b/crates/sqlload/src/adapters/databricks/tests.rs @@ -8,3 +8,5 @@ mod s3_copy; mod stale_removals; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_unloaded_copy.rs"] +mod unloaded_copy; diff --git a/crates/sqlload/src/adapters/databricks/tests/test_types.rs b/crates/sqlload/src/adapters/databricks/tests/test_types.rs index 146d4e04..1f7d1cb4 100644 --- a/crates/sqlload/src/adapters/databricks/tests/test_types.rs +++ b/crates/sqlload/src/adapters/databricks/tests/test_types.rs @@ -34,3 +34,10 @@ pub(crate) struct LocationUrlTestCase { pub(crate) url: &'static str, pub(crate) expected_url: &'static str, } + +pub(crate) struct UnloadedCopyTestCase { + pub(crate) description: &'static str, + /// Whether the batch lands in a build table in its final types, rather than a staging table. + pub(crate) builds: bool, + pub(crate) expected_statement: &'static str, +} diff --git a/crates/sqlload/src/adapters/databricks/tests/test_unloaded_copy.rs b/crates/sqlload/src/adapters/databricks/tests/test_unloaded_copy.rs new file mode 100644 index 00000000..67c97e6d --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/tests/test_unloaded_copy.rs @@ -0,0 +1,122 @@ +use crate::adapters::contract::models::{TableName, WriteTarget}; +use crate::adapters::databricks::_helpers::write::unloaded_copy::{ + UnloadedGroup, unloaded_copy_statement, +}; +use crate::adapters::databricks::tests::test_types::UnloadedCopyTestCase; +use crate::column_types::models::ColumnType; +use crate::load_naming::models::Compat; +use crate::postgres_source::models::{ReadColumn, WireType}; +use crate::schema_plan::models::{ColumnOrigin, DesiredColumn}; + +#[test] +fn given_files_of_a_snowflake_unload_when_databricks_copies_them_then_timestamps_come_back_from_microseconds_and_merge_rows_count_by_file() + { + let test_cases = [ + UnloadedCopyTestCase { + description: "into a staging table: timestamps from microseconds whatever the session's time zone, JSON as text, the merge row by batch, file and row", + builds: false, + expected_statement: "COPY INTO `main`.`raw`.`orders_stage` FROM (SELECT \ + CAST(`ID` AS BIGINT) AS `ID`, \ + CAST(timestamp_micros(CAST(`at` AS BIGINT)) AS TIMESTAMP) AS `at`, \ + CAST(timestampadd(MICROSECOND, CAST(`wall` AS BIGINT), TIMESTAMP_NTZ'1970-01-01 00:00:00') AS TIMESTAMP_NTZ) AS `wall`, \ + CAST(`payload` AS STRING) AS `payload`, \ + CAST('eu' AS STRING) AS `region`, \ + CAST(NULL AS STRING) AS `missing`, \ + CAST(3 * 281474976710656 + (CASE url_decode(replace(_metadata.file_path, '+', '%2B')) WHEN 's3://lake/unload/run1/data_0_1_0.snappy.parquet' THEN 1000 WHEN 's3://lake/unload/run1/data_0_2_0.snappy.parquet' THEN 1001 ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END) * 4294967296 + _metadata.row_index AS BIGINT) AS `_sqlload_row`, \ + CAST('L1' AS STRING) AS `_sqlload_load_id`, \ + CAST('2026-11-18T00:00:00Z' AS TIMESTAMP) AS `_sqlload_loaded_at` \ + FROM 's3://lake/unload/run1/') FILEFORMAT = PARQUET FILES = ('data_0_1_0.snappy.parquet', 'data_0_2_0.snappy.parquet') FORMAT_OPTIONS ('mergeSchema' = 'true') COPY_OPTIONS ('force' = 'true')", + }, + UnloadedCopyTestCase { + description: "into a build table: final types, JSON parsed", + builds: true, + expected_statement: "COPY INTO `main`.`raw`.`orders_stage` FROM (SELECT \ + CAST(`ID` AS BIGINT) AS `ID`, \ + CAST(timestamp_micros(CAST(`at` AS BIGINT)) AS TIMESTAMP) AS `at`, \ + CAST(timestampadd(MICROSECOND, CAST(`wall` AS BIGINT), TIMESTAMP_NTZ'1970-01-01 00:00:00') AS TIMESTAMP_NTZ) AS `wall`, \ + parse_json(`payload`) AS `payload`, \ + CAST('eu' AS STRING) AS `region`, \ + CAST(NULL AS STRING) AS `missing`, \ + CAST(3 * 281474976710656 + (CASE url_decode(replace(_metadata.file_path, '+', '%2B')) WHEN 's3://lake/unload/run1/data_0_1_0.snappy.parquet' THEN 1000 WHEN 's3://lake/unload/run1/data_0_2_0.snappy.parquet' THEN 1001 ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END) * 4294967296 + _metadata.row_index AS BIGINT) AS `_sqlload_row`, \ + CAST('L1' AS STRING) AS `_sqlload_load_id`, \ + CAST('2026-11-18T00:00:00Z' AS TIMESTAMP) AS `_sqlload_loaded_at` \ + FROM 's3://lake/unload/run1/') FILEFORMAT = PARQUET FILES = ('data_0_1_0.snappy.parquet', 'data_0_2_0.snappy.parquet') FORMAT_OPTIONS ('mergeSchema' = 'true') COPY_OPTIONS ('force' = 'true')", + }, + ]; + let column = |name: &str, column_type: ColumnType| DesiredColumn { + name: name.to_owned(), + column_type, + origin: ColumnOrigin::Source, + }; + let read: Vec = [ + ("ID", ColumnType::Int), + ("at", ColumnType::TimestampTz), + ("wall", ColumnType::TimestampNtz), + ("payload", ColumnType::Json), + ] + .into_iter() + .map(|(name, output)| ReadColumn { + name: name.to_owned(), + expression: format!("\"{name}\""), + wire: WireType::Text, + output, + }) + .collect(); + let columns = [ + column("ID", ColumnType::Int), + column("at", ColumnType::TimestampTz), + column("wall", ColumnType::TimestampNtz), + column("payload", ColumnType::Json), + column("region", ColumnType::Text), + column("missing", ColumnType::Text), + column("_sqlload_row", ColumnType::Int), + column("_sqlload_load_id", ColumnType::Text), + column("_sqlload_loaded_at", ColumnType::TimestampTz), + ]; + let table = TableName { + database: Some("main".to_owned()), + schema: "raw".to_owned(), + table: "orders".to_owned(), + }; + let staging = TableName { + table: "orders_stage".to_owned(), + ..table.clone() + }; + let constants = [("region".to_owned(), ColumnType::Text, "eu".to_owned())]; + for test_case in &test_cases { + let target = WriteTarget { + table: &table, + staging: Some(&staging), + columns: &columns, + write_columns: &columns, + compat: Compat::Sqlload, + load_id: "L1", + batch: 3, + direct: false, + builds: test_case.builds, + constants: &constants, + }; + let keys = [ + "data_0_1_0.snappy.parquet".to_owned(), + "data_0_2_0.snappy.parquet".to_owned(), + ]; + let statement = unloaded_copy_statement( + ("`main`.`raw`.`orders_stage`", &columns), + &target, + ( + &read, + &UnloadedGroup { + folder: "s3://lake/unload/run1/", + keys: &keys, + first: 1000, + }, + ), + "2026-11-18T00:00:00Z", + ); + assert_eq!( + statement, test_case.expected_statement, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs b/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs index 0bfd8c48..3309950c 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs @@ -8,13 +8,25 @@ use crate::adapters::dispatch::models::Destination; use crate::adapters::duckdb::main::_check_files::check_files; use crate::files_source::models::FilesRead; use crate::loader::models::FilesSource; +use crate::routing::models::Store; +use crate::store_paths::models::StoreKind; -/// Check, at plan, that `destination` reads the files of `source` in S3 where they lie, for a table nothing sends through SQLLoad, with `read` its planned read: Databricks looks up its Unity Catalog external locations once per run, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers, and lists the folder when one covers it and it holds files to load; MotherDuck reads the first file to load on its servers. The error says why it can't, with the fix. +/// Check, at plan, that `destination` reads the files of `source` in S3 where they lie, or the S3 folder a warehouse source unloads to (`unload_to`), for a table nothing sends through SQLLoad, with `read` its planned read: Databricks looks up its Unity Catalog external locations once per run, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers, and lists the folder when one covers it and it holds files to load; MotherDuck reads the first file to load on its servers. An unload's folder is new each run, so for it Databricks only lists its locations, and the route then needs one to cover it. The error says why it can't, with the fix. pub async fn check_reach( destination: &Destination, - source: &FilesSource, + (source, unload_to): (Option<&FilesSource>, &[Store]), read: Option<&FilesRead>, ) -> Result<(), String> { + let source = source.filter(|source| matches!(source.location.store, StoreKind::S3 { .. })); + let unloads = unload_to + .iter() + .any(|store| matches!(store, Store::ObjectStore(url) if url.starts_with("s3://"))); + let Some(source) = source else { + return match (destination, unloads) { + (Destination::Databricks(databricks), true) => databricks_reach(databricks, None).await, + _ => Ok(()), + }; + }; let first = read.and_then(|read| read.files.first()); let folder = source.location.to_string(); match destination { diff --git a/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs b/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs index bb15d0d3..10d762c0 100644 --- a/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs +++ b/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs @@ -14,6 +14,7 @@ pub async fn load_unloaded( Writer::DuckDb(duckdb) => duckdb.pull_unloaded(batch).await.map(Some), Writer::Snowflake(snowflake) => snowflake.pull_unloaded(batch).await.map(Some), Writer::ClickHouse(clickhouse) => clickhouse.pull_unloaded(batch).await.map(Some), + Writer::Databricks(databricks) => databricks.pull_unloaded(batch).await.map(Some), _ => Ok(None), } } diff --git a/crates/sqlload/src/adapters/dispatch/main/write/mod.rs b/crates/sqlload/src/adapters/dispatch/main/write/mod.rs index ed36b379..75e01c52 100644 --- a/crates/sqlload/src/adapters/dispatch/main/write/mod.rs +++ b/crates/sqlload/src/adapters/dispatch/main/write/mod.rs @@ -2,3 +2,4 @@ pub mod file_keys; pub mod load_files; pub mod load_unloaded; pub mod open_table_writer; +pub mod unloads_directly; diff --git a/crates/sqlload/src/adapters/dispatch/main/write/unloads_directly.rs b/crates/sqlload/src/adapters/dispatch/main/write/unloads_directly.rs new file mode 100644 index 00000000..6e3f532f --- /dev/null +++ b/crates/sqlload/src/adapters/dispatch/main/write/unloads_directly.rs @@ -0,0 +1,8 @@ +//! Whether a destination loads a warehouse source's unload straight into the table. + +use crate::adapters::dispatch::models::Writer; + +/// Whether `writer` may load a warehouse source's unloaded files straight into the table, where it writes directly at all: not Databricks, whose `COPY`s commit one by one before their rows are checked against the unload's. +pub fn unloads_directly(writer: &Writer) -> bool { + !matches!(writer, Writer::Databricks(_)) +} diff --git a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs index 98967d73..016dc4b1 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs @@ -468,10 +468,7 @@ pub(crate) fn unload_readers_for( same_connection: false, }; use crate::snowflake_source::_helpers::catalog::source_column; - let column = match test_case.times { - true => source_column("at", "TIME(9)", true), - false => source_column("id", "NUMBER(38,0)", true), - }; + let column = source_column("c", test_case.column_type, true); unload_blocked( test_case.warehouse, (test_case.masks, &[column]), diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs index d36a8265..6f38a6ae 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs @@ -137,8 +137,8 @@ pub(crate) struct UnloadReadersTestCase { pub(crate) description: &'static str, pub(crate) warehouse: crate::column_types::models::Warehouse, pub(crate) masks: bool, - /// The table has a time column. - pub(crate) times: bool, + /// The Snowflake type of the table's one column. + pub(crate) column_type: &'static str, /// How the reason SQLLoad reads the files instead starts, or `None` when the target reads the S3 unload itself. pub(crate) expected_blocked: Option<&'static str>, } diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs b/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs index 7a40f193..861d98a7 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs @@ -92,49 +92,49 @@ fn given_a_snowflake_sources_s3_unload_when_routing_into_another_warehouse_then_ description: "Databricks reads the unload itself", warehouse: Warehouse::Databricks, masks: false, - times: false, + column_type: "NUMBER(38,0)", expected_blocked: None, }, UnloadReadersTestCase { description: "ClickHouse reads the unload itself", warehouse: Warehouse::ClickHouse, masks: false, - times: false, + column_type: "NUMBER(38,0)", expected_blocked: None, }, UnloadReadersTestCase { description: "a time column goes through SQLLoad into Databricks, which keeps times as text", warehouse: Warehouse::Databricks, masks: false, - times: true, + column_type: "TIME(9)", expected_blocked: Some("the table has a time column"), }, UnloadReadersTestCase { description: "a time column goes through SQLLoad into ClickHouse", warehouse: Warehouse::ClickHouse, masks: false, - times: true, + column_type: "TIME(9)", expected_blocked: Some("the table has a time column"), }, UnloadReadersTestCase { description: "masks need SQLLoad to see the rows", warehouse: Warehouse::ClickHouse, masks: true, - times: false, + column_type: "NUMBER(38,0)", expected_blocked: Some(MASKS), }, UnloadReadersTestCase { description: "DuckDB turns times back itself", warehouse: Warehouse::DuckDb, masks: false, - times: true, + column_type: "TIME(9)", expected_blocked: None, }, UnloadReadersTestCase { description: "Postgres never reads files itself", warehouse: Warehouse::Postgres, masks: false, - times: false, + column_type: "NUMBER(38,0)", expected_blocked: Some(UNLOADED_TIMES), }, ]; diff --git a/crates/sqlload/src/loader/_helpers/publish.rs b/crates/sqlload/src/loader/_helpers/publish.rs index b0290427..ed3630e3 100644 --- a/crates/sqlload/src/loader/_helpers/publish.rs +++ b/crates/sqlload/src/loader/_helpers/publish.rs @@ -23,6 +23,7 @@ use crate::adapters::dispatch::main::write::file_keys::file_keys; use crate::adapters::dispatch::main::write::load_files::load_files; use crate::adapters::dispatch::main::write::load_unloaded::load_unloaded; use crate::adapters::dispatch::main::write::open_table_writer::open_table_writer; +use crate::adapters::dispatch::main::write::unloads_directly::unloads_directly; use crate::adapters::dispatch::models::Destination; use crate::adapters::dispatch::models::Writer; use crate::adapters::files::models::ReplacedFiles; @@ -168,9 +169,12 @@ use crate::loader::_helpers::table_plan::{ added_constants, is_direct, pulls_unloaded, selects_rows, }; -/// Whether `append` can write the batches straight into the table: not a logged table (its rows wait for a changed `ctx.state`), and a writer that can (on Snowflake, a route loading files into the table itself). +/// Whether `append` can write the batches straight into the table: not a logged table (its rows wait for a changed `ctx.state`), and a writer that can (on Snowflake, a route loading files into the table itself), and, for an unload the destination reads itself, one that [loads it directly](unloads_directly). fn writes_directly(table: &TablePlan, writer: &Writer) -> bool { - table.mode == WriteMode::Append && table.logged.is_none() && writer.writes_directly() + table.mode == WriteMode::Append + && table.logged.is_none() + && writer.writes_directly() + && (!pulls_unloaded(table.route.as_ref()) || unloads_directly(writer)) } /// Where the batches go, from the publisher's fields: `staging` when there is one, and `columns` as the staged batches carry them. diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index 10500805..01d9013b 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -1206,7 +1206,7 @@ fn copy_warning(route: Option<&Route>) -> Option { )) } -/// The route a table's data takes into `destination`; `read` is a files table's planned read. For files in S3 that nothing sends through SQLLoad, it first checks that the destination reaches them (Databricks' external locations, MotherDuck's servers). +/// The route a table's data takes into `destination`; `read` is a files table's planned read. For files in S3, or an unload to S3, that nothing sends through SQLLoad, it first checks that the destination reaches them (Databricks' external locations, MotherDuck's servers). pub(crate) async fn table_route( spec: &TableSpec, read: Option<&FilesRead>, @@ -1217,10 +1217,8 @@ pub(crate) async fn table_route( let refused = unload_refused(destination, &reader.unload_to); reader.direct_blocked = reader.direct_blocked.or(refused); let in_place = reader.needs_loader.is_none() && reader.direct_blocked.is_none(); - if let Some(files) = spec.files.as_ref().filter(|_| in_place) - && let StoreKind::S3 { .. } = files.location.store - { - check_reach(destination, files, read).await?; + if in_place { + check_reach(destination, (spec.files.as_ref(), &reader.unload_to), read).await?; } plan_route( &reader, diff --git a/crates/sqlload/src/routing/_helpers/files_route.rs b/crates/sqlload/src/routing/_helpers/files_route.rs index f2cd52a3..c5291102 100644 --- a/crates/sqlload/src/routing/_helpers/files_route.rs +++ b/crates/sqlload/src/routing/_helpers/files_route.rs @@ -91,7 +91,7 @@ pub(crate) fn other_reads( } /// Why `writer` refuses `files` it cannot read where they lie, when it reads such files only itself. -fn must_pull(writer: &WriterAccepts, files: &FileSet) -> Option { +pub(crate) fn must_pull(writer: &WriterAccepts, files: &FileSet) -> Option { writer .must_pull .as_ref() diff --git a/crates/sqlload/src/routing/_helpers/warehouse_source.rs b/crates/sqlload/src/routing/_helpers/warehouse_source.rs index 58056c6c..de448edb 100644 --- a/crates/sqlload/src/routing/_helpers/warehouse_source.rs +++ b/crates/sqlload/src/routing/_helpers/warehouse_source.rs @@ -1,4 +1,5 @@ use crate::routing::_helpers::deliver::deliver_arrow; +use crate::routing::_helpers::files_route::must_pull; use crate::routing::_helpers::stores::{destination_pulls, ordered_stores}; use crate::routing::constants::{DOWNLOAD_FOLDER, UNLOAD_THRESHOLD_BYTES}; use crate::routing::errors::RouteError; @@ -22,7 +23,7 @@ pub(crate) fn should_unload(reader: &ReaderOffer, read_via: ReadVia) -> Result Date: Wed, 18 Nov 2026 21:18:42 +0000 Subject: [PATCH 5/7] wip: docs for Databricks and ClickHouse reading a Snowflake S3 unload --- website/public/index.html | 2 +- .../content/docs/docs/sources/snowflake.mdx | 22 +++++++++++++++++++ .../content/docs/docs/supported-systems.mdx | 2 +- .../content/docs/docs/targets/clickhouse.mdx | 5 +++++ .../content/docs/docs/targets/databricks.mdx | 6 +++++ .../src/content/docs/docs/targets/index.mdx | 2 ++ 6 files changed, 37 insertions(+), 2 deletions(-) diff --git a/website/public/index.html b/website/public/index.html index 5cca8e46..08417387 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -173,7 +173,7 @@

See every change first

02 · Load

Bulk copy, not row inserts

-

Files in S3 load into Snowflake straight from an external stage, and Parquet in S3 into Databricks through an external location. Parquet loads into DuckDB through DuckDB's own reader, and Parquet in S3 into MotherDuck on its servers. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Rows are staged as Parquet and copied in bulk.

+

Files in S3 load into Snowflake straight from an external stage, and Parquet in S3 into Databricks through an external location. Parquet loads into DuckDB through DuckDB's own reader, and Parquet in S3 into MotherDuck on its servers. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Snowflake tables unloaded to S3 load into DuckDB, Databricks and ClickHouse with no row passing through SQLLoad. Rows are staged as Parquet and copied in bulk.

Load methods and routes →

sqlload plan
diff --git a/website/src/content/docs/docs/sources/snowflake.mdx b/website/src/content/docs/docs/sources/snowflake.mdx
index 907030a2..1ad423a6 100644
--- a/website/src/content/docs/docs/sources/snowflake.mdx
+++ b/website/src/content/docs/docs/sources/snowflake.mdx
@@ -159,6 +159,28 @@ whatever its session time zone. A load fails before it commits when DuckDB loads
 of rows than `COPY INTO` reports, and the folder is removed either way. A table without a cursor
 commits once per run rather than about every 1 GB.
 
+### Databricks and ClickHouse read the files themselves
+
+With a stage given by `url`, Databricks and ClickHouse targets read the unloaded files from S3
+themselves too, so no rows pass through SQLLoad:
+
+```text
+route   unload Parquet to s3://example-bucket/unload/ -> Databricks COPY INTO (nothing through the loader)
+route   unload Parquet to s3://example-bucket/unload/ -> ClickHouse reads them (nothing through the loader)
+```
+
+- **Databricks** `COPY`s the files through the Unity Catalog external location that covers the
+  `url`, as for [Parquet files in S3](/docs/targets/databricks/#parquet-files-in-s3). A `url` no
+  external location covers fails `plan` with the fix; the files never go through SQLLoad instead.
+  Each batch lands in a staging or build table, an `append` too, and publishes only when the
+  `COPY`s load as many rows as `COPY INTO` reported unloading.
+- **ClickHouse** reads each file with `s3()` into the load's staging table, signed with the same
+  S3 credentials as SQLLoad, and publishes only when it loads as many rows as were unloaded.
+
+Timestamps come back exactly from their whole microseconds, whatever the session's time zone. A
+table with a `TIME` column, or a `mask`, goes through SQLLoad into either target, and `plan` says
+why: neither stores a time of day as a time.
+
 ### Snowflake loads the rows itself
 
 Into a Snowflake target, no rows pass through SQLLoad in two cases:
diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx
index 63e6910a..9fd512d3 100644
--- a/website/src/content/docs/docs/supported-systems.mdx
+++ b/website/src/content/docs/docs/supported-systems.mdx
@@ -9,7 +9,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes
 |--------|---------|---------|
 | Postgres | Tables, by `include` patterns | Binary `COPY` into Arrow, over TLS when the connection's `sslmode` asks |
 | Postgres changes (`postgres_cdc`) | Inserts, updates and deletes of tables, by `include` patterns | A snapshot, then a logical replication slot (`pgoutput`), in whole transactions. See [Postgres changes](/docs/sources/postgres-cdc/). |
-| Snowflake | Tables and views, by `include` patterns, and query results | Parquet unloaded with `COPY INTO` to the user stage or an S3 stage, then read by SQLLoad, or by DuckDB itself from an S3 stage. A Snowflake target selects or copies the rows inside Snowflake when it writes through the source's connection, or its external stage covers the unload; see [Snowflake loads the rows itself](/docs/sources/snowflake/#snowflake-loads-the-rows-itself). See [Snowflake](/docs/sources/snowflake/). |
+| Snowflake | Tables and views, by `include` patterns, and query results | Parquet unloaded with `COPY INTO` to the user stage or an S3 stage, then read by SQLLoad, or by DuckDB, Databricks or ClickHouse themselves from an S3 stage. A Snowflake target selects or copies the rows inside Snowflake when it writes through the source's connection, or its external stage covers the unload; see [Snowflake loads the rows itself](/docs/sources/snowflake/#snowflake-loads-the-rows-itself). See [Snowflake](/docs/sources/snowflake/). |
 | Databases with an ODBC driver: SQL Server, MySQL, MariaDB, OpenEdge and others | Tables and views, by `include` patterns, and query results | The driver, as Arrow. Tested with psqlODBC, Microsoft ODBC Driver 18 for SQL Server and MySQL Connector/ODBC; MariaDB Connector/ODBC and OpenEdge are checked by hand. See [ODBC](/docs/sources/odbc/#drivers). |
 | BigQuery (beta) | Tables and views, by `include` patterns, and query results | The Storage Read API, as Arrow in parallel streams; views and queries through a query job. See [BigQuery](/docs/sources/bigquery/). |
 | Databricks (beta) | Tables and views, by `include` patterns, and query results | One `SELECT` on the SQL warehouse, its result downloaded as Arrow files in parallel through the Statement Execution API's links. See [Databricks](/docs/sources/databricks/). |
diff --git a/website/src/content/docs/docs/targets/clickhouse.mdx b/website/src/content/docs/docs/targets/clickhouse.mdx
index 956bea85..ee0e224b 100644
--- a/website/src/content/docs/docs/targets/clickhouse.mdx
+++ b/website/src/content/docs/docs/targets/clickhouse.mdx
@@ -116,6 +116,11 @@ SQLLoad reads the files itself and sends Arrow instead when ClickHouse wouldn't
 - times of day, which ClickHouse reads from Parquet as timestamps;
 - a path that reads as a glob.
 
+A Snowflake source with a stage `url` in S3 skips SQLLoad the same way: ClickHouse reads the files
+Snowflake unloads, one `s3()` insert per file, and the batch publishes only when it loads as many
+rows as Snowflake reported unloading. See
+[Databricks and ClickHouse read the files themselves](/docs/sources/snowflake/#databricks-and-clickhouse-read-the-files-themselves).
+
 ## Write strategies
 
 | Strategy | Publishes |
diff --git a/website/src/content/docs/docs/targets/databricks.mdx b/website/src/content/docs/docs/targets/databricks.mdx
index efe8be36..942ced61 100644
--- a/website/src/content/docs/docs/targets/databricks.mdx
+++ b/website/src/content/docs/docs/targets/databricks.mdx
@@ -89,6 +89,12 @@ list, and files on an S3-compatible endpoint (set on the connection or in `AWS_E
 folder or over SFTP always go through SQLLoad. `copy_to_internal_stage` is for Snowflake and is
 refused here.
 
+A Snowflake source with a stage `url` in S3 that an external location covers loads the same way:
+Databricks `COPY`s the files Snowflake unloads, into a staging or build table even for `append`,
+and the batch publishes only when it loads as many rows as Snowflake reported unloading. A `url`
+no external location covers fails `plan`. See
+[Databricks and ClickHouse read the files themselves](/docs/sources/snowflake/#databricks-and-clickhouse-read-the-files-themselves).
+
 ## Write strategies
 
 Databricks' Statement Execution API runs every statement in its own transaction, so each write is
diff --git a/website/src/content/docs/docs/targets/index.mdx b/website/src/content/docs/docs/targets/index.mdx
index 35da076b..b28b9fd0 100644
--- a/website/src/content/docs/docs/targets/index.mdx
+++ b/website/src/content/docs/docs/targets/index.mdx
@@ -23,6 +23,8 @@ names the route for every table.
 A Snowflake source loading into Snowflake stays inside Snowflake: an `INSERT ... SELECT` on the
 source's own connection, or a `COPY` of its unloaded files through the target's external stage.
 See [Snowflake loads the rows itself](/docs/sources/snowflake/#snowflake-loads-the-rows-itself).
+Into DuckDB, Databricks and ClickHouse, a Snowflake source unloaded to an S3 `url` is read by the
+target itself; see [Databricks and ClickHouse read the files themselves](/docs/sources/snowflake/#databricks-and-clickhouse-read-the-files-themselves).
 
 There are no silent fallbacks. If the route a table needs can't be used, `plan` fails for that
 table and says how to fix it.

From 24734b6210bda9afb99fabd62d3a0517d932eafe Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Wed, 18 Nov 2026 21:55:00 +0000
Subject: [PATCH 6/7] wip: unload store url ends in a slash so an external
 location of that folder covers it; each unload is its own batch so merge
 ordinals stay unique across windows

---
 crates/sqlload/src/loader/_helpers/publish.rs |  3 +-
 crates/sqlload/src/loader/_helpers/specs.rs   |  2 +-
 crates/sqlload/src/loader/tests/helpers.rs    | 41 +++++++++++++++++++
 .../src/loader/tests/test_plan_lines.rs       | 29 ++++++++++++-
 crates/sqlload/src/loader/tests/test_types.rs |  7 ++++
 5 files changed, 78 insertions(+), 4 deletions(-)

diff --git a/crates/sqlload/src/loader/_helpers/publish.rs b/crates/sqlload/src/loader/_helpers/publish.rs
index ed3630e3..b93fb0b6 100644
--- a/crates/sqlload/src/loader/_helpers/publish.rs
+++ b/crates/sqlload/src/loader/_helpers/publish.rs
@@ -1077,7 +1077,7 @@ impl Publisher {
         }
     }
 
-    /// Load one unload's files, or the rows its query selects, by having the destination read them itself, where the plan chose that; the destination must load them so.
+    /// Load one unload's files, or the rows its query selects, by having the destination read them itself, where the plan chose that; the destination must load them so. Each unload is a batch of its own, so its rows' merge ordinals follow every earlier window's even when they wait in one staging table.
     pub(crate) async fn load_unloaded(
         &mut self,
         source: UnloadedSource<'_>,
@@ -1104,6 +1104,7 @@ impl Publisher {
                 ))
             })?;
         self.note_direct(&loaded);
+        self.batch += 1;
         Ok(loaded)
     }
 
diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs
index 1db158ec..3667c520 100644
--- a/crates/sqlload/src/loader/_helpers/specs.rs
+++ b/crates/sqlload/src/loader/_helpers/specs.rs
@@ -681,7 +681,7 @@ fn warehouse_source(
                 storage_integration: Some(integration),
                 ..
             }) => (
-                Store::ObjectStore(url.clone()),
+                Store::ObjectStore(format!("{}/", url.trim_end_matches('/'))),
                 format!(
                     "external: Snowflake writes to {url} through storage integration {integration}; the files are read there and removed after the read"
                 ),
diff --git a/crates/sqlload/src/loader/tests/helpers.rs b/crates/sqlload/src/loader/tests/helpers.rs
index 8943d491..dea2492e 100644
--- a/crates/sqlload/src/loader/tests/helpers.rs
+++ b/crates/sqlload/src/loader/tests/helpers.rs
@@ -637,3 +637,44 @@ pub(crate) fn merge_newest() -> WriteMode {
         not_null_keys: Vec::new(),
     }
 }
+
+/// The store a `snowflake` source with stage `url` unloads table `current` to, as Debug prints it (the whole route when it is no unload).
+pub(crate) fn unload_store(url: &str) -> Result {
+    let text = format!(
+        "[connections.wh]\nkind = \"duckdb\"\ndatabase = \":memory:\"\n[targets.dev]\nconnection = \"wh\"\nloader_schema = \"RAW\"\n[connections.sf]\nkind = \"snowflake\"\naccount = \"a\"\nuser = \"u\"\nauth = {{ method = \"pat\", token = \"t\" }}\ndatabase = \"D\"\n[sources.marts]\nkind = \"snowflake\"\nconnection = \"sf\"\nwrite_strategy = \"replace\"\nstage = {{ url = \"{url}\", storage_integration = \"I\" }}\n[sources.marts.tables.current]\ndescription = \"d\"\nsql = \"SELECT 1 AS id\"\n"
+    );
+    let project = crate::project::main::parse_project::parse_project(&text, &|_| None)
+        .map_err(|error| error.to_string())?;
+    let table = SourceTable {
+        schema: String::new(),
+        name: "current".to_owned(),
+        columns: vec![SourceColumn {
+            name: "id".to_owned(),
+            postgres_type: "NUMBER".to_owned(),
+            pg_type: crate::postgres_source::models::PgType::Other,
+            column_type: Ok(crate::column_types::models::ColumnType::Int),
+            nullable: true,
+            source_spelling: None,
+            repeat_of: None,
+        }],
+        primary_key: Vec::new(),
+        indexed_columns: Vec::new(),
+        estimated_rows: -1,
+        estimated_bytes: -1,
+        split_pages: None,
+        comment: None,
+        listed_keys: Vec::new(),
+    };
+    let resolved = resolve_tables(&project, "marts", &[table]);
+    let route = resolved
+        .specs
+        .first()
+        .and_then(|spec| spec.warehouse_source.as_ref())
+        .map(|source| format!("{:?}", source.route))
+        .ok_or_else(|| format!("no warehouse source: {:?}", resolved.problems))?;
+    Ok(route
+        .split(", stage: ")
+        .next()
+        .unwrap_or_default()
+        .replace("Unload { store: ", ""))
+}
diff --git a/crates/sqlload/src/loader/tests/test_plan_lines.rs b/crates/sqlload/src/loader/tests/test_plan_lines.rs
index d2395378..e9001918 100644
--- a/crates/sqlload/src/loader/tests/test_plan_lines.rs
+++ b/crates/sqlload/src/loader/tests/test_plan_lines.rs
@@ -2,9 +2,9 @@ use crate::loader::_helpers::table_plan::blocked_table;
 use crate::loader::main::render_plan::render_plan;
 use crate::loader::models::{ProjectPlan, PythonTable, SkippedTable};
 use crate::loader::tests::helpers::{
-    merge_failing_repeats, merge_newest, python_spec, replace_mode, streamed_route,
+    merge_failing_repeats, merge_newest, python_spec, replace_mode, streamed_route, unload_store,
 };
-use crate::loader::tests::test_types::{ModedTable, PlanLinesTestCase};
+use crate::loader::tests::test_types::{ModedTable, PlanLinesTestCase, UnloadStoreTestCase};
 
 #[test]
 fn given_tables_when_printing_the_plan_without_verbose_then_only_their_own_lines_show_grouped_by_source()
@@ -96,3 +96,28 @@ fn given_tables_when_printing_the_plan_without_verbose_then_only_their_own_lines
         );
     }
 }
+
+#[test]
+fn given_a_snowflake_stage_url_when_resolving_its_table_then_the_unload_store_is_a_folder_ending_in_a_slash()
+ {
+    let test_cases = [
+        UnloadStoreTestCase {
+            description: "a url without a trailing slash is the folder, so an external location of that folder covers it",
+            url: "s3://lake/unload",
+            expected_store: "Ok(\"ObjectStore(\\\"s3://lake/unload/\\\")\")",
+        },
+        UnloadStoreTestCase {
+            description: "a url with a trailing slash keeps it",
+            url: "s3://lake/unload/",
+            expected_store: "Ok(\"ObjectStore(\\\"s3://lake/unload/\\\")\")",
+        },
+    ];
+    for test_case in test_cases {
+        assert_eq!(
+            format!("{:?}", unload_store(test_case.url)),
+            test_case.expected_store,
+            "{}",
+            test_case.description
+        );
+    }
+}
diff --git a/crates/sqlload/src/loader/tests/test_types.rs b/crates/sqlload/src/loader/tests/test_types.rs
index 8158ba6c..368b4a93 100644
--- a/crates/sqlload/src/loader/tests/test_types.rs
+++ b/crates/sqlload/src/loader/tests/test_types.rs
@@ -250,3 +250,10 @@ pub(crate) struct PlanLinesTestCase {
     pub(crate) skipped: &'static [(&'static str, &'static str)],
     pub(crate) expected_output: &'static str,
 }
+
+/// A `snowflake` source whose stage `url` is written as `url`, and the store its unloads go to.
+pub(crate) struct UnloadStoreTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) url: &'static str,
+    pub(crate) expected_store: &'static str,
+}

From 863552842f6d83fb0f9193be00096c86cb9633e0 Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Wed, 18 Nov 2026 22:04:40 +0000
Subject: [PATCH 7/7] wip: an unloaded table's read splits into at most 32,767
 windows so its merge row numbers fit 64 bits

---
 .../src/batch_plan/main/split_windows.rs      |  9 ++--
 crates/sqlload/src/batch_plan/models.rs       |  2 +
 .../sqlload/src/batch_plan/tests/helpers.rs   |  1 +
 .../batch_plan/tests/test_split_windows.rs    | 41 +++++++++++++++++++
 .../src/batch_plan/tests/test_types.rs        |  2 +
 .../sqlload/src/loader/_helpers/table_plan.rs |  1 +
 crates/sqlload/src/loader/models.rs           | 10 +++++
 crates/sqlload/src/loader/tests/helpers.rs    | 21 +++++++---
 .../src/loader/tests/test_plan_lines.rs       | 34 ++++++++++++++-
 crates/sqlload/src/loader/tests/test_types.rs |  7 ++++
 .../content/docs/docs/concepts/cursors.mdx    |  2 +-
 11 files changed, 116 insertions(+), 14 deletions(-)

diff --git a/crates/sqlload/src/batch_plan/main/split_windows.rs b/crates/sqlload/src/batch_plan/main/split_windows.rs
index 65e8aebb..c8137f6d 100644
--- a/crates/sqlload/src/batch_plan/main/split_windows.rs
+++ b/crates/sqlload/src/batch_plan/main/split_windows.rs
@@ -1,12 +1,11 @@
 //! Split a cursor range into the windows of a table's `batch_size`.
 
 use crate::batch_plan::_helpers::bounds::{integer_plan, time_plan};
-use crate::batch_plan::constants::MAX_WINDOWS;
 use crate::batch_plan::models::{WindowPlan, WindowSplit};
 use crate::column_types::models::ColumnType;
 use crate::project::models::BatchStep;
 
-/// The windows `[from, from + step)`, ... up to the end (`--end-cursor-*`, else now; for a date cursor the end of today), counted but not listed; the first keeps the read's inclusiveness. A range without a start cannot be split, so `cursor_start` must give the first load one; a step giving more than 100,000 windows is refused.
+/// The windows `[from, from + step)`, ... up to the end (`--end-cursor-*`, else now; for a date cursor the end of today), counted but not listed; the first keeps the read's inclusiveness. A range without a start cannot be split, so `cursor_start` must give the first load one; a step giving more than `most` windows is refused.
 pub fn split_windows(split: &WindowSplit<'_>) -> Result {
     let Some(from) = split.from else {
         return Err(
@@ -28,10 +27,10 @@ pub fn split_windows(split: &WindowSplit<'_>) -> Result {
             return Err("batch_size = \"none\" reads in one piece; it has no windows".to_owned());
         }
     };
-    if plan.count > MAX_WINDOWS {
+    if plan.count > split.most {
         return Err(format!(
-            "batch_size splits the read into {} windows, more than {MAX_WINDOWS}; use a larger batch_size, or narrow the read with --start-cursor/--end-cursor",
-            plan.count
+            "batch_size splits the read into {} windows, more than {}; use a larger batch_size, or narrow the read with --start-cursor/--end-cursor",
+            plan.count, split.most
         ));
     }
     Ok(plan)
diff --git a/crates/sqlload/src/batch_plan/models.rs b/crates/sqlload/src/batch_plan/models.rs
index 2274f5ca..9217b8a5 100644
--- a/crates/sqlload/src/batch_plan/models.rs
+++ b/crates/sqlload/src/batch_plan/models.rs
@@ -25,6 +25,8 @@ pub struct WindowSplit<'a> {
     pub now: DateTime,
     /// The table's `timezone`, which a bound without an offset is read in while the cursor's type is unknown.
     pub zone: chrono_tz::Tz,
+    /// The most windows the read may be split into (see [`crate::loader::models::TableSpec::most_windows`]).
+    pub most: u64,
 }
 
 /// Files that commit together, in key order, and the cursor values their rows carry when they replace whole partitions.
diff --git a/crates/sqlload/src/batch_plan/tests/helpers.rs b/crates/sqlload/src/batch_plan/tests/helpers.rs
index 6a807be9..b23778b9 100644
--- a/crates/sqlload/src/batch_plan/tests/helpers.rs
+++ b/crates/sqlload/src/batch_plan/tests/helpers.rs
@@ -38,6 +38,7 @@ pub(crate) fn windows(test_case: &SplitWindowsTestCase) -> Result, S
         cursor_type: test_case.cursor_type.as_ref(),
         now: now(),
         zone: test_case.zone,
+        most: test_case.most,
     };
     let plan = split_windows(&split)?;
     (0..plan.count)
diff --git a/crates/sqlload/src/batch_plan/tests/test_split_windows.rs b/crates/sqlload/src/batch_plan/tests/test_split_windows.rs
index 7a7c8001..7f493118 100644
--- a/crates/sqlload/src/batch_plan/tests/test_split_windows.rs
+++ b/crates/sqlload/src/batch_plan/tests/test_split_windows.rs
@@ -1,3 +1,4 @@
+use crate::batch_plan::constants::MAX_WINDOWS;
 use crate::batch_plan::tests::helpers::{owned_windows, windows};
 use crate::batch_plan::tests::test_types::SplitWindowsTestCase;
 use crate::column_types::models::ColumnType;
@@ -16,12 +17,41 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order()
             step: BatchStep::Duration(DAY),
             cursor_type: Some(ColumnType::Date),
             zone: chrono_tz::Tz::UTC,
+            most: MAX_WINDOWS,
             expected_windows: Ok(&[
                 "2026-11-03..2026-11-04",
                 "2026-11-04..2026-11-05",
                 "2026-11-05..2026-11-06",
             ]),
         },
+        SplitWindowsTestCase {
+            description: "a read split into exactly the most windows allowed is kept",
+            from: Some("2026-11-03"),
+            to: None,
+            inclusive: true,
+            step: BatchStep::Duration(DAY),
+            cursor_type: Some(ColumnType::Date),
+            zone: chrono_tz::Tz::UTC,
+            most: 3,
+            expected_windows: Ok(&[
+                "2026-11-03..2026-11-04",
+                "2026-11-04..2026-11-05",
+                "2026-11-05..2026-11-06",
+            ]),
+        },
+        SplitWindowsTestCase {
+            description: "one window more than the most allowed is refused",
+            from: Some("2026-11-03"),
+            to: None,
+            inclusive: true,
+            step: BatchStep::Duration(DAY),
+            cursor_type: Some(ColumnType::Date),
+            zone: chrono_tz::Tz::UTC,
+            most: 2,
+            expected_windows: Err(
+                "batch_size splits the read into 3 windows, more than 2; use a larger batch_size, or narrow the read with --start-cursor/--end-cursor",
+            ),
+        },
         SplitWindowsTestCase {
             description: "timestamp windows end now, the last one shorter",
             from: Some("2026-11-04 12:00:00.000000+00"),
@@ -30,6 +60,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order()
             step: BatchStep::Duration(DAY),
             cursor_type: Some(ColumnType::TimestampTz),
             zone: chrono_tz::Tz::UTC,
+            most: MAX_WINDOWS,
             expected_windows: Ok(&["2026-11-04 12:00:00.000000+00..2026-11-05 10:30:00.000000+00"]),
         },
         SplitWindowsTestCase {
@@ -40,6 +71,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order()
             step: BatchStep::Duration(DAY),
             cursor_type: Some(ColumnType::TimestampNtz),
             zone: chrono_tz::Tz::UTC,
+            most: MAX_WINDOWS,
             expected_windows: Ok(&[
                 "(2026-11-01 00:00:00.000000..2026-11-02 00:00:00.000000",
                 "2026-11-02 00:00:00.000000..2026-11-02 12:00:00.000000",
@@ -53,6 +85,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order()
             step: BatchStep::Integer(100),
             cursor_type: Some(ColumnType::Int),
             zone: chrono_tz::Tz::UTC,
+            most: MAX_WINDOWS,
             expected_windows: Ok(&["0..100", "100..200", "200..250"]),
         },
         SplitWindowsTestCase {
@@ -63,6 +96,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order()
             step: BatchStep::Duration(DAY),
             cursor_type: None,
             zone: chrono_tz::Tz::UTC,
+            most: MAX_WINDOWS,
             expected_windows: Ok(&["2026-11-04..2026-11-05", "2026-11-05..2026-11-06"]),
         },
         SplitWindowsTestCase {
@@ -73,6 +107,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order()
             step: BatchStep::Duration(DAY),
             cursor_type: None,
             zone: chrono_tz::Tz::Europe__Paris,
+            most: MAX_WINDOWS,
             expected_windows: Ok(&[
                 "2026-02-28 23:00:00.000000+00..2026-03-01 23:00:00.000000+00",
                 "2026-03-01 23:00:00.000000+00..2026-03-02 23:00:00.000000+00",
@@ -86,6 +121,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order()
             step: BatchStep::Duration(DAY),
             cursor_type: Some(ColumnType::Date),
             zone: chrono_tz::Tz::UTC,
+            most: MAX_WINDOWS,
             expected_windows: Err(
                 "batch_size needs a start to split the cursor range: set cursor_start for the first load",
             ),
@@ -98,6 +134,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order()
             step: BatchStep::Integer(10),
             cursor_type: Some(ColumnType::Int),
             zone: chrono_tz::Tz::UTC,
+            most: MAX_WINDOWS,
             expected_windows: Err(
                 "an integer batch needs --end-cursor to know where the last window ends, with --reload too (a Postgres source ends at its current maximum)",
             ),
@@ -110,6 +147,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order()
             step: BatchStep::Duration(std::time::Duration::from_secs(3600)),
             cursor_type: Some(ColumnType::Date),
             zone: chrono_tz::Tz::UTC,
+            most: MAX_WINDOWS,
             expected_windows: Err("a date cursor's batch_size must be whole days, not 1h"),
         },
         SplitWindowsTestCase {
@@ -120,6 +158,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order()
             step: BatchStep::Duration(std::time::Duration::from_secs(60)),
             cursor_type: Some(ColumnType::TimestampTz),
             zone: chrono_tz::Tz::UTC,
+            most: MAX_WINDOWS,
             expected_windows: Err(
                 "batch_size splits the read into 13675680 windows, more than 100000; use a larger batch_size, or narrow the read with --start-cursor/--end-cursor",
             ),
@@ -132,6 +171,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order()
             step: BatchStep::Integer(5),
             cursor_type: Some(ColumnType::Int),
             zone: chrono_tz::Tz::UTC,
+            most: MAX_WINDOWS,
             expected_windows: Ok(&[]),
         },
         SplitWindowsTestCase {
@@ -142,6 +182,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order()
             step: BatchStep::Duration(DAY),
             cursor_type: Some(ColumnType::Int),
             zone: chrono_tz::Tz::UTC,
+            most: MAX_WINDOWS,
             expected_windows: Err(
                 "a duration batch needs a date or timestamp cursor, but the cursor is int; give an integer step",
             ),
diff --git a/crates/sqlload/src/batch_plan/tests/test_types.rs b/crates/sqlload/src/batch_plan/tests/test_types.rs
index 49d7f978..7e66da2a 100644
--- a/crates/sqlload/src/batch_plan/tests/test_types.rs
+++ b/crates/sqlload/src/batch_plan/tests/test_types.rs
@@ -10,6 +10,8 @@ pub(crate) struct SplitWindowsTestCase {
     pub(crate) cursor_type: Option,
     /// The table's `timezone`, which a bound without an offset is read in while the cursor's type is unknown.
     pub(crate) zone: chrono_tz::Tz,
+    /// The most windows allowed.
+    pub(crate) most: u64,
     /// Each window as `from..to` (`(` when the start is not read), or the error.
     pub(crate) expected_windows: Result<&'static [&'static str], &'static str>,
 }
diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs
index 01d9013b..d285ecad 100644
--- a/crates/sqlload/src/loader/_helpers/table_plan.rs
+++ b/crates/sqlload/src/loader/_helpers/table_plan.rs
@@ -1749,6 +1749,7 @@ pub(crate) fn commit_rule(request: &CommitRequest<'_>) -> Result u64 {
+        match self.warehouse_source.as_ref().map(|source| &source.route) {
+            Some(WarehouseRoute::Unload { .. }) => (i64::MAX / UNLOADED_BATCH_ROWS).unsigned_abs(),
+            _ => MAX_WINDOWS,
+        }
+    }
+
     /// The query whose rows are the table, for a warehouse source's query table.
     pub fn query_sql(&self) -> Option<&str> {
         self.warehouse_source
diff --git a/crates/sqlload/src/loader/tests/helpers.rs b/crates/sqlload/src/loader/tests/helpers.rs
index dea2492e..323ecc23 100644
--- a/crates/sqlload/src/loader/tests/helpers.rs
+++ b/crates/sqlload/src/loader/tests/helpers.rs
@@ -638,8 +638,8 @@ pub(crate) fn merge_newest() -> WriteMode {
     }
 }
 
-/// The store a `snowflake` source with stage `url` unloads table `current` to, as Debug prints it (the whole route when it is no unload).
-pub(crate) fn unload_store(url: &str) -> Result {
+/// The spec of table `current` of a `snowflake` source with stage `url`, loading into DuckDB.
+pub(crate) fn unload_spec(url: &str) -> Result {
     let text = format!(
         "[connections.wh]\nkind = \"duckdb\"\ndatabase = \":memory:\"\n[targets.dev]\nconnection = \"wh\"\nloader_schema = \"RAW\"\n[connections.sf]\nkind = \"snowflake\"\naccount = \"a\"\nuser = \"u\"\nauth = {{ method = \"pat\", token = \"t\" }}\ndatabase = \"D\"\n[sources.marts]\nkind = \"snowflake\"\nconnection = \"sf\"\nwrite_strategy = \"replace\"\nstage = {{ url = \"{url}\", storage_integration = \"I\" }}\n[sources.marts.tables.current]\ndescription = \"d\"\nsql = \"SELECT 1 AS id\"\n"
     );
@@ -666,12 +666,21 @@ pub(crate) fn unload_store(url: &str) -> Result {
         listed_keys: Vec::new(),
     };
     let resolved = resolve_tables(&project, "marts", &[table]);
-    let route = resolved
+    resolved
         .specs
-        .first()
-        .and_then(|spec| spec.warehouse_source.as_ref())
+        .into_iter()
+        .next()
+        .ok_or_else(|| format!("no spec: {:?}", resolved.problems))
+}
+
+/// The store a `snowflake` source with stage `url` unloads table `current` to, as Debug prints it (the whole route when it is no unload).
+pub(crate) fn unload_store(url: &str) -> Result {
+    let spec = unload_spec(url)?;
+    let route = spec
+        .warehouse_source
+        .as_ref()
         .map(|source| format!("{:?}", source.route))
-        .ok_or_else(|| format!("no warehouse source: {:?}", resolved.problems))?;
+        .ok_or("no warehouse source")?;
     Ok(route
         .split(", stage: ")
         .next()
diff --git a/crates/sqlload/src/loader/tests/test_plan_lines.rs b/crates/sqlload/src/loader/tests/test_plan_lines.rs
index e9001918..98bf4a37 100644
--- a/crates/sqlload/src/loader/tests/test_plan_lines.rs
+++ b/crates/sqlload/src/loader/tests/test_plan_lines.rs
@@ -2,9 +2,12 @@ use crate::loader::_helpers::table_plan::blocked_table;
 use crate::loader::main::render_plan::render_plan;
 use crate::loader::models::{ProjectPlan, PythonTable, SkippedTable};
 use crate::loader::tests::helpers::{
-    merge_failing_repeats, merge_newest, python_spec, replace_mode, streamed_route, unload_store,
+    merge_failing_repeats, merge_newest, python_spec, replace_mode, streamed_route, unload_spec,
+    unload_store,
+};
+use crate::loader::tests::test_types::{
+    ModedTable, MostWindowsTestCase, PlanLinesTestCase, UnloadStoreTestCase,
 };
-use crate::loader::tests::test_types::{ModedTable, PlanLinesTestCase, UnloadStoreTestCase};
 
 #[test]
 fn given_tables_when_printing_the_plan_without_verbose_then_only_their_own_lines_show_grouped_by_source()
@@ -121,3 +124,30 @@ fn given_a_snowflake_stage_url_when_resolving_its_table_then_the_unload_store_is
         );
     }
 }
+
+#[test]
+fn given_a_table_when_splitting_its_read_into_windows_then_an_unloaded_table_takes_fewer_so_its_merge_row_numbers_fit_64_bits()
+ {
+    let test_cases = [
+        MostWindowsTestCase {
+            description: "a table SQLLoad reads takes up to 100,000 windows",
+            spec: || python_spec("{}"),
+            expected_most: Ok(100_000),
+        },
+        MostWindowsTestCase {
+            description: "an unloaded table's rows number from the window times 2^48, so it takes up to 32,767",
+            spec: || unload_spec("s3://lake/unload/"),
+            expected_most: Ok(32_767),
+        },
+    ];
+    for test_case in test_cases {
+        assert_eq!(
+            (test_case.spec)()
+                .map(|spec| spec.most_windows())
+                .map_err(|_| "no spec"),
+            test_case.expected_most,
+            "{}",
+            test_case.description
+        );
+    }
+}
diff --git a/crates/sqlload/src/loader/tests/test_types.rs b/crates/sqlload/src/loader/tests/test_types.rs
index 368b4a93..4a9e43bb 100644
--- a/crates/sqlload/src/loader/tests/test_types.rs
+++ b/crates/sqlload/src/loader/tests/test_types.rs
@@ -257,3 +257,10 @@ pub(crate) struct UnloadStoreTestCase {
     pub(crate) url: &'static str,
     pub(crate) expected_store: &'static str,
 }
+
+/// A table's spec, and the most cursor windows its read may be split into.
+pub(crate) struct MostWindowsTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) spec: fn() -> Result,
+    pub(crate) expected_most: Result,
+}
diff --git a/website/src/content/docs/docs/concepts/cursors.mdx b/website/src/content/docs/docs/concepts/cursors.mdx
index 17c0a32a..e557ae6d 100644
--- a/website/src/content/docs/docs/concepts/cursors.mdx
+++ b/website/src/content/docs/docs/concepts/cursors.mdx
@@ -229,7 +229,7 @@ and `merge` publish once per run (last row below). `plan -v` of one table shows
 
 | Table | Commits |
 |-------|---------|
-| `batch_size = "1d"` (date or timestamp cursor) or `batch_size = 1000` (integer cursor) | One cursor window at a time, up to 100,000 windows. |
+| `batch_size = "1d"` (date or timestamp cursor) or `batch_size = 1000` (integer cursor) | One cursor window at a time, up to 100,000 windows (32,767 for a Snowflake table read through an unload). |
 | A Postgres table whose `cursor_column` leads an index, without `batch_size` | About every 1 GB, reading in cursor order (NULLs first), only between distinct cursor values. `batch_size = "none"` opts out. |
 | Other cursor tables, text cursors, and `append` without a cursor or logged state | Once per read, since committing part of an unordered read could skip or duplicate rows. |
 | Files sources | At 1,000 files or about 1 GB; under `delete_insert` a partition is never split. |