diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/mod.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/mod.rs index ac656d3c5..f98d52a0b 100644 --- a/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/mod.rs +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/mod.rs @@ -3,4 +3,5 @@ pub(crate) mod publish; pub(crate) mod pull; pub(crate) mod sql_adapter; pub(crate) mod type_names; +pub(crate) mod unloaded; pub(crate) mod write; diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/pull.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/pull.rs index b1adcf673..0c101a12a 100644 --- a/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/pull.rs +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/pull.rs @@ -81,7 +81,6 @@ pub(crate) fn pull_insert( (file, credentials): (&PulledFile, Option<&PullCredentials>), (compat, load_id, loaded_at): (Compat, &str, &str), ) -> String { - let literal = |text: &str| ClickHouseSql.render_loader_value_literal(text); let identifier = |name: &str| ClickHouseSql.render_identifier(name); let cast = |value: String, column: &DesiredColumn| { format!("CAST({value} AS {})", sql_type(&column.column_type)) @@ -116,8 +115,26 @@ pub(crate) fn pull_insert( .iter() .map(|column| identifier(&column.name)) .collect(); + let numbered = columns + .iter() + .any(|column| column.name == SOURCE_ROW_COLUMN); + let settings = match numbered { + true => format!(" SETTINGS {ORDERED_READ_SETTINGS}"), + false => String::new(), + }; + format!( + "INSERT INTO {into} ({}) SELECT {} FROM {}{settings}", + targets.join(", "), + values.join(", "), + s3_parquet(&file.url, credentials) + ) +} + +/// `s3()` reading the Parquet file at `url`, signed with `credentials` (named as [query parameters](credential_parameters)), or unsigned (`NOSIGN`) when there are none. +pub(crate) fn s3_parquet(url: &str, credentials: Option<&PullCredentials>) -> String { + let literal = |text: &str| ClickHouseSql.render_loader_value_literal(text); let parameter = |name: &str| format!("{{{name}:String}}"); - let mut signed = vec![literal(&file.url)]; + let mut signed = vec![literal(url)]; match credentials { Some(credentials) => { signed.extend([parameter(KEY_ID_PARAMETER), parameter(SECRET_PARAMETER)]); @@ -131,19 +148,7 @@ pub(crate) fn pull_insert( None => signed.push("NOSIGN".to_owned()), } signed.push(literal("Parquet")); - let numbered = columns - .iter() - .any(|column| column.name == SOURCE_ROW_COLUMN); - let settings = match numbered { - true => format!(" SETTINGS {ORDERED_READ_SETTINGS}"), - false => String::new(), - }; - format!( - "INSERT INTO {into} ({}) SELECT {} FROM s3({}){settings}", - targets.join(", "), - values.join(", "), - signed.join(", ") - ) + format!("s3({})", signed.join(", ")) } /// The HTTP URL ClickHouse reads `key` of `bucket` at: under `endpoint` by path when there is one (or with the bucket already in it, `in_endpoint`, for virtual-hosted requests to a custom endpoint), otherwise on AWS, in `region` when known, by virtual host unless the bucket's name holds a dot, which AWS's certificate does not cover. Each path segment is percent-encoded, which ClickHouse decodes before signing. diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/unloaded.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/unloaded.rs new file mode 100644 index 000000000..b09190fd3 --- /dev/null +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/unloaded.rs @@ -0,0 +1,94 @@ +//! The statement that has ClickHouse read one file of a warehouse source's unload from S3 itself. + +use crate::adapters::clickhouse::_helpers::dialect::pull::{ + PullCredentials, SqlTypeOf, s3_parquet, +}; +use crate::adapters::clickhouse::_helpers::dialect::write::typed_literal; +use crate::adapters::clickhouse::constants::ORDERED_READ_SETTINGS; +use crate::adapters::clickhouse::models::ClickHouseSql; +use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS}; +use crate::adapters::contract::models::WriteTarget; +use crate::adapters::contract::types::SqlAdapter; +use crate::column_types::models::ColumnType; +use crate::load_naming::main::metadata_role::metadata_role; +use crate::load_naming::models::MetadataRole; +use crate::postgres_source::models::ReadColumn; +use crate::schema_plan::constants::MERGE_ROW_COLUMN; +use crate::schema_plan::models::DesiredColumn; + +/// One file of an unload that ClickHouse reads: its URL, and its place among the unload's files, from 0, which a merge's row number counts by. +pub(crate) struct UnloadedFile<'a> { + pub url: &'a str, + pub index: usize, +} + +/// `INSERT INTO into` every one of `columns` (each typed by `sql_type`) from one of a warehouse's unloaded Parquet `file`s, read by [`s3_parquet`] with `credentials`. A column of `read` comes from the file, turned back from whole microseconds where it is read as a timestamp (a time never loads this way); a column of `target`'s constants from its value; a merge's row number from the batch, the file and the row in it, read in stored order; the load id and load time (`loaded_at`) are the batch's; any other column is NULL. +pub(crate) fn unloaded_insert( + (into, columns, sql_type): (&str, &[DesiredColumn], SqlTypeOf<'_>), + target: &WriteTarget<'_>, + (read, loaded_at): (&[ReadColumn], &str), + (file, credentials): (&UnloadedFile<'_>, Option<&PullCredentials>), +) -> String { + let values: Vec = columns + .iter() + .map(|column| { + format!( + "CAST({} AS {})", + column_value(column, target, (read, loaded_at), file), + sql_type(&column.column_type) + ) + }) + .collect(); + let targets: Vec = columns + .iter() + .map(|column| ClickHouseSql.render_identifier(&column.name)) + .collect(); + let settings = match columns.iter().any(|column| column.name == MERGE_ROW_COLUMN) { + true => format!(" SETTINGS {ORDERED_READ_SETTINGS}"), + false => String::new(), + }; + format!( + "INSERT INTO {into} ({}) SELECT {} FROM {}{settings}", + targets.join(", "), + values.join(", "), + s3_parquet(file.url, credentials) + ) +} + +/// Where `column`'s value comes from, before its cast: the load id or load time, a merge's row number, a constant's value, the file's column of `read` it is, or NULL when the file lacks it. +fn column_value( + column: &DesiredColumn, + target: &WriteTarget<'_>, + (read, loaded_at): (&[ReadColumn], &str), + file: &UnloadedFile<'_>, +) -> String { + let constant = target + .constants + .iter() + .find(|(name, _, _)| *name == column.name); + let held = read.iter().find(|read| read.name == column.name); + match (metadata_role(target.compat, &column.name), constant, held) { + (Some(MetadataRole::LoadId), _, _) => typed_literal(target.load_id, &column.column_type), + (Some(MetadataRole::LoadedAt), _, _) => typed_literal(loaded_at, &column.column_type), + _ if column.name == MERGE_ROW_COLUMN => format!( + "{} * {UNLOADED_BATCH_ROWS} + {} * {UNLOADED_FILE_ROWS} + rowNumberInAllBlocks()", + target.batch, file.index + ), + (_, Some((_, column_type, value)), _) => typed_literal(value, column_type), + (_, None, Some(read)) => { + unloaded_value(&read.output, &ClickHouseSql.render_identifier(&read.name)) + } + (_, None, None) => "NULL".to_owned(), + } +} + +/// File column `quoted` as the value it was unloaded from: a timestamp, with or without time zone, from its whole microseconds as the UTC instant SQLLoad's reader gives (ClickHouse stores both as an instant); a binary through `substring`, which keeps a fixed-size value's trailing zero bytes; anything else as it is. +fn unloaded_value(output: &ColumnType, quoted: &str) -> String { + match output { + ColumnType::TimestampTz | ColumnType::TimestampNtz => { + format!("fromUnixTimestamp64Micro(CAST({quoted} AS Int64), 'UTC')") + } + ColumnType::Binary => format!("substring({quoted}, 1)"), + _ => quoted.to_owned(), + } +} diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/write/direct_load.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/write/direct_load.rs index 38ec802a4..ca427ae9a 100644 --- a/crates/sqlload/src/adapters/clickhouse/_helpers/write/direct_load.rs +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/write/direct_load.rs @@ -48,47 +48,7 @@ impl ClickHouseWriter { ))); } self.settle_rows().await?; - let builder = s3_builder( - bucket, - region.as_deref(), - endpoint.as_deref(), - profile.as_deref(), - ) - .map_err(|error| { - LoaderError::Invalid(format!( - "cannot sign ClickHouse's reads of s3://{bucket}/: {error}" - )) - })?; - let setting = |key: AmazonS3ConfigKey| builder.get_config_value(&key); - let region = setting(AmazonS3ConfigKey::Region); - let endpoint = - setting(AmazonS3ConfigKey::S3Endpoint).or(setting(AmazonS3ConfigKey::Endpoint)); - let in_endpoint = endpoint.is_some() - && switched_on(setting(AmazonS3ConfigKey::VirtualHostedStyleRequest)); - let unsigned = switched_on(setting(AmazonS3ConfigKey::SkipSignature)); - let unreadable = |error: String| { - LoaderError::Invalid(format!( - "cannot sign ClickHouse's reads of s3://{bucket}/: {error}" - )) - }; - let credentials = match unsigned { - true => None, - false => { - let store = builder - .build() - .map_err(|error| unreadable(error.to_string()))?; - let credential = store - .credentials() - .get_credential() - .await - .map_err(|error| unreadable(error.to_string()))?; - Some(PullCredentials { - key_id: credential.key_id.clone(), - secret: credential.secret_key.clone(), - token: credential.token.clone(), - }) - } - }; + let (region, endpoint, credentials) = s3_reads(bucket, (region, endpoint, profile)).await?; let parameters = credential_parameters(credentials.as_ref()); let target = &batch.target; let (into, columns) = match (target.staging, target.builds) { @@ -107,7 +67,7 @@ impl ClickHouseWriter { let url = object_url( (bucket, &key), region.as_deref(), - (endpoint.as_deref(), in_endpoint), + (endpoint.0.as_deref(), endpoint.1), ) .map_err(LoaderError::Invalid)?; let file = pulled_file(source, data, url, (batch, columns))?; @@ -131,6 +91,54 @@ impl ClickHouseWriter { } } +/// How ClickHouse reads objects of `bucket`, as SQLLoad's own S3 reads would (`region`, `endpoint` and `profile` from the connection): the region, the endpoint with whether the bucket is already in it (virtual-hosted requests to a custom endpoint), and the credentials SQLLoad resolves to sign `s3()` with, `None` for unsigned reads. +pub(in crate::adapters) async fn s3_reads( + bucket: &str, + (region, endpoint, profile): (&Option, &Option, &Option), +) -> Result< + ( + Option, + (Option, bool), + Option, + ), + LoaderError, +> { + let unreadable = |error: String| { + LoaderError::Invalid(format!( + "cannot sign ClickHouse's reads of s3://{bucket}/: {error}" + )) + }; + let builder = s3_builder( + bucket, + region.as_deref(), + endpoint.as_deref(), + profile.as_deref(), + ) + .map_err(|error| unreadable(error.to_string()))?; + let setting = |key: AmazonS3ConfigKey| builder.get_config_value(&key); + let region = setting(AmazonS3ConfigKey::Region); + let endpoint = setting(AmazonS3ConfigKey::S3Endpoint).or(setting(AmazonS3ConfigKey::Endpoint)); + let in_endpoint = + endpoint.is_some() && switched_on(setting(AmazonS3ConfigKey::VirtualHostedStyleRequest)); + if switched_on(setting(AmazonS3ConfigKey::SkipSignature)) { + return Ok((region, (endpoint, in_endpoint), None)); + } + let store = builder + .build() + .map_err(|error| unreadable(error.to_string()))?; + let credential = store + .credentials() + .get_credential() + .await + .map_err(|error| unreadable(error.to_string()))?; + let credentials = PullCredentials { + key_id: credential.key_id.clone(), + secret: credential.secret_key.clone(), + token: credential.token.clone(), + }; + Ok((region, (endpoint, in_endpoint), Some(credentials))) +} + /// Whether an S3 switch is on, read as object_store reads it from the environment (`true`, `1`, `yes`, `on` or `y`, in any case). fn switched_on(value: Option) -> bool { value.is_some_and(|value| { diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/write/mod.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/write/mod.rs index d2184caf1..14ca8d42d 100644 --- a/crates/sqlload/src/adapters/clickhouse/_helpers/write/mod.rs +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/write/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod arrow_batch; pub(crate) mod direct_load; pub(crate) mod table_writer; +pub(crate) mod unloaded_load; diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/write/unloaded_load.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/write/unloaded_load.rs new file mode 100644 index 000000000..23d8eca23 --- /dev/null +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/write/unloaded_load.rs @@ -0,0 +1,82 @@ +//! A warehouse source's unloaded files loaded by ClickHouse reading them from S3 itself. + +use crate::adapters::clickhouse::_helpers::dialect::pull::{credential_parameters, object_url}; +use crate::adapters::clickhouse::_helpers::dialect::unloaded::{UnloadedFile, unloaded_insert}; +use crate::adapters::clickhouse::_helpers::session::http::written_rows; +use crate::adapters::clickhouse::_helpers::write::direct_load::s3_reads; +use crate::adapters::clickhouse::models::{ClickHouseSql, ClickHouseWriter}; +use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch, UnloadedSource}; +use crate::adapters::contract::types::{SqlAdapter, TableWriter}; +use crate::column_types::models::ColumnType; +use crate::loader::errors::LoaderError; +use crate::store_paths::models::StoreKind; + +impl ClickHouseWriter { + /// Load `batch` by having ClickHouse read the unloaded files from S3 itself, one `INSERT ... SELECT FROM s3()` per file, into the staging or build table the load publishes from, signed with the credentials SQLLoad reads S3 with. ClickHouse must load as many rows as the unload reports, or the batch fails before anything publishes. + pub(in crate::adapters) async fn pull_unloaded( + &mut self, + batch: &UnloadedBatch<'_>, + ) -> Result { + let UnloadedSource::Files(files) = batch.source else { + return Err(LoaderError::Invalid( + "ClickHouse reads a warehouse source's rows only from its unloaded files" + .to_owned(), + )); + }; + let StoreKind::S3 { + bucket, + region, + endpoint, + profile, + } = &files.folder.store + else { + return Err(LoaderError::Invalid(format!( + "ClickHouse reads unloaded files only from S3, not {}", + files.folder + ))); + }; + if files.keys.is_empty() { + return Ok(DirectLoaded::Staged(0)); + } + self.settle_rows().await?; + let (region, endpoint, credentials) = s3_reads(bucket, (region, endpoint, profile)).await?; + let parameters = credential_parameters(credentials.as_ref()); + let target = &batch.target; + let (into, columns) = match (target.staging, target.builds) { + (Some(staging), false) => (staging, target.columns), + (staging, _) => (staging.unwrap_or(target.table), batch.write_columns), + }; + let builds = target.builds; + let sql_type = |column_type: &ColumnType| match builds { + true => ClickHouseSql.render_loader_logical_type(column_type).sql, + false => ClickHouseSql.render_staging_type(column_type), + }; + let into = ClickHouseSql.render_qualified_name(into); + let mut rows = 0; + for (index, key) in files.keys.iter().enumerate() { + let object = format!("{}{key}", files.folder.prefix); + let url = object_url( + (bucket, &object), + region.as_deref(), + (endpoint.0.as_deref(), endpoint.1), + ) + .map_err(LoaderError::Invalid)?; + let insert = unloaded_insert( + (&into, columns, &sql_type), + target, + (batch.read, batch.loaded_at), + (&UnloadedFile { url: &url, index }, credentials.as_ref()), + ); + rows += written_rows(&self.destination, &insert, ¶meters) + .await + .map_err(|error| LoaderError::Invalid(format!("{}{key}: {error}", files.folder)))?; + } + if rows != files.rows { + return Err(LoaderError::Invalid(format!( + "ClickHouse loaded {rows} rows from the unloaded files in {}, but the unload reports {}; the batch is not committed", + files.folder, files.rows + ))); + } + Ok(DirectLoaded::Staged(rows)) + } +} diff --git a/crates/sqlload/src/adapters/clickhouse/tests.rs b/crates/sqlload/src/adapters/clickhouse/tests.rs index 6d121a234..89c63d5a8 100644 --- a/crates/sqlload/src/adapters/clickhouse/tests.rs +++ b/crates/sqlload/src/adapters/clickhouse/tests.rs @@ -12,3 +12,5 @@ mod pull; mod rows; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_unloaded.rs"] +mod unloaded; diff --git a/crates/sqlload/src/adapters/clickhouse/tests/test_types.rs b/crates/sqlload/src/adapters/clickhouse/tests/test_types.rs index 4a6f0b562..86526a4d4 100644 --- a/crates/sqlload/src/adapters/clickhouse/tests/test_types.rs +++ b/crates/sqlload/src/adapters/clickhouse/tests/test_types.rs @@ -78,3 +78,11 @@ pub(crate) struct SortedInsertTestCase { pub(crate) key_columns: &'static [&'static str], pub(crate) expected_insert: &'static str, } + +pub(crate) struct UnloadedInsertTestCase { + pub(crate) description: &'static str, + pub(crate) columns: Vec, + /// Whether the read is signed, or unsigned for a public bucket. + pub(crate) signed: bool, + pub(crate) expected_insert: &'static str, +} diff --git a/crates/sqlload/src/adapters/clickhouse/tests/test_unloaded.rs b/crates/sqlload/src/adapters/clickhouse/tests/test_unloaded.rs new file mode 100644 index 000000000..c58f8869b --- /dev/null +++ b/crates/sqlload/src/adapters/clickhouse/tests/test_unloaded.rs @@ -0,0 +1,107 @@ +use crate::adapters::clickhouse::_helpers::dialect::pull::PullCredentials; +use crate::adapters::clickhouse::_helpers::dialect::unloaded::{UnloadedFile, unloaded_insert}; +use crate::adapters::clickhouse::tests::helpers::desired; +use crate::adapters::clickhouse::tests::test_types::UnloadedInsertTestCase; +use crate::adapters::contract::models::{TableName, WriteTarget}; +use crate::column_types::models::ColumnType; +use crate::load_naming::models::Compat; +use crate::postgres_source::models::{ReadColumn, WireType}; + +#[test] +fn given_a_snowflake_unload_when_clickhouse_reads_one_of_its_files_then_timestamps_come_back_from_microseconds_and_the_rest_as_unloaded() + { + let test_cases = [ + UnloadedInsertTestCase { + description: "a merge's staging table: timestamps from microseconds, the constant, a column the files lack as NULL, the merge row by batch, file and row read in stored order", + columns: vec![ + desired("id", ColumnType::Int), + desired("at", ColumnType::TimestampTz), + desired("wall", ColumnType::TimestampNtz), + desired("blob", ColumnType::Binary), + desired("region", ColumnType::Text), + desired("missing", ColumnType::Text), + desired("_sqlload_row", ColumnType::Int), + desired("_sqlload_load_id", ColumnType::Text), + desired("_sqlload_loaded_at", ColumnType::TimestampTz), + ], + signed: true, + expected_insert: "INSERT INTO `raw`.`orders__staging` (`id`, `at`, `wall`, `blob`, `region`, `missing`, `_sqlload_row`, `_sqlload_load_id`, `_sqlload_loaded_at`) SELECT \ + CAST(`id` AS T), \ + CAST(fromUnixTimestamp64Micro(CAST(`at` AS Int64), 'UTC') AS T), \ + CAST(fromUnixTimestamp64Micro(CAST(`wall` AS Int64), 'UTC') AS T), \ + CAST(substring(`blob`, 1) AS T), \ + CAST(CAST('eu' AS String) AS T), \ + CAST(NULL AS T), \ + CAST(3 * 281474976710656 + 2 * 4294967296 + rowNumberInAllBlocks() AS T), \ + CAST(CAST('L1' AS String) AS T), \ + CAST(parseDateTime64BestEffort('2026-11-18T00:00:00Z', 6, 'UTC') AS T) \ + FROM s3('https://unload.s3.eu-west-2.amazonaws.com/run/data_0_1_0.snappy.parquet', {sqlload_key_id:String}, {sqlload_secret:String}, 'Parquet') \ + SETTINGS max_threads = 1, max_insert_threads = 1, input_format_parquet_preserve_order = 1", + }, + UnloadedInsertTestCase { + description: "without a merge row the file is read unsigned from a public bucket, in any order", + columns: vec![desired("id", ColumnType::Int)], + signed: false, + expected_insert: "INSERT INTO `raw`.`orders__staging` (`id`) SELECT CAST(`id` AS T) \ + FROM s3('https://unload.s3.eu-west-2.amazonaws.com/run/data_0_1_0.snappy.parquet', NOSIGN, 'Parquet')", + }, + ]; + let read: Vec = [ + ("id", ColumnType::Int), + ("at", ColumnType::TimestampTz), + ("wall", ColumnType::TimestampNtz), + ("blob", ColumnType::Binary), + ] + .into_iter() + .map(|(name, output)| ReadColumn { + name: name.to_owned(), + expression: format!("\"{name}\""), + wire: WireType::Text, + output, + }) + .collect(); + let table = TableName { + database: None, + schema: "raw".to_owned(), + table: "orders".to_owned(), + }; + let constants = [("region".to_owned(), ColumnType::Text, "eu".to_owned())]; + let credentials = PullCredentials { + key_id: "AKIDEXAMPLE".to_owned(), + secret: "secret".to_owned(), + token: None, + }; + let file = UnloadedFile { + url: "https://unload.s3.eu-west-2.amazonaws.com/run/data_0_1_0.snappy.parquet", + index: 2, + }; + for test_case in &test_cases { + let target = WriteTarget { + table: &table, + staging: None, + columns: &test_case.columns, + write_columns: &test_case.columns, + compat: Compat::Sqlload, + load_id: "L1", + batch: 3, + direct: false, + builds: false, + constants: &constants, + }; + let insert = unloaded_insert( + ( + "`raw`.`orders__staging`", + &test_case.columns, + &|_: &ColumnType| "T".to_owned(), + ), + &target, + (&read, "2026-11-18T00:00:00Z"), + (&file, Some(&credentials).filter(|_| test_case.signed)), + ); + assert_eq!( + insert, test_case.expected_insert, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs index 805a3e705..d7ffe8c73 100644 --- a/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs @@ -162,7 +162,7 @@ fn group_statement( } /// The rows a `COPY INTO`'s `result` reports it inserted. -fn inserted_rows(result: &SqlRows) -> Result { +pub(in crate::adapters) fn inserted_rows(result: &SqlRows) -> Result { let reported = result .columns .iter() diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs index 30bac9520..6dba39a67 100644 --- a/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs @@ -2,3 +2,5 @@ pub(crate) mod copy_select; pub(crate) mod direct_load; pub(crate) mod s3_copy; pub(crate) mod table_writer; +pub(crate) mod unloaded_copy; +pub(crate) mod unloaded_load; diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs index ea13bf884..82d410970 100644 --- a/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs @@ -12,7 +12,7 @@ use crate::schema_plan::models::DesiredColumn; use arrow_schema::{Field, Schema}; /// The path `COPY INTO` reads each row from, as SQLLoad names it: `_metadata.file_path` is percent-encoded, except for `+`, which `url_decode` alone would read as a space. -const FILE_PATH: &str = "url_decode(replace(_metadata.file_path, '+', '%2B'))"; +pub(crate) const FILE_PATH: &str = "url_decode(replace(_metadata.file_path, '+', '%2B'))"; /// One batch read from S3: the folder `COPY INTO` reads (`s3://bucket/prefix/`), and for each file its key below it and the values SQLLoad gives every row of it (`None` for NULL), by column. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/unloaded_copy.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/unloaded_copy.rs new file mode 100644 index 000000000..ed4912d84 --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/unloaded_copy.rs @@ -0,0 +1,111 @@ +//! The `COPY INTO` that has Databricks read files a warehouse source unloaded to S3. + +use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS}; +use crate::adapters::contract::models::WriteTarget; +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::databricks::_helpers::write::copy_select::typed_value; +use crate::adapters::databricks::_helpers::write::s3_copy::FILE_PATH; +use crate::adapters::databricks::constants::S3_COPY_OPTIONS; +use crate::adapters::databricks::models::DatabricksSql; +use crate::column_types::models::ColumnType; +use crate::load_naming::main::metadata_role::metadata_role; +use crate::load_naming::models::MetadataRole; +use crate::postgres_source::models::ReadColumn; +use crate::schema_plan::constants::MERGE_ROW_COLUMN; +use crate::schema_plan::models::DesiredColumn; + +/// Some of one unload's files: the folder `COPY INTO` reads (`s3://bucket/prefix/`), their keys below it, and the place of the first among the unload's files, from 0, which a merge's row number counts by. +pub(crate) struct UnloadedGroup<'a> { + pub folder: &'a str, + pub keys: &'a [String], + pub first: usize, +} + +/// `COPY INTO into` every one of `columns`, typed as `target` lands them, from `group`'s files: a column of `read` from the files, turned back from whole microseconds where it is read as a timestamp (a time never loads this way); a column of `target`'s constants from its value; a merge's row number from the batch, the file (picked by its path, and a row from any other path fails the statement) and the row in it; the load id and load time are the batch's; any other column is NULL. +pub(crate) fn unloaded_copy_statement( + (into, columns): (&str, &[DesiredColumn]), + target: &WriteTarget<'_>, + (read, group): (&[ReadColumn], &UnloadedGroup<'_>), + loaded_at: &str, +) -> String { + let literal = |text: &str| DatabricksSql.render_loader_value_literal(text); + let values: Vec = columns + .iter() + .map(|column| { + let value = column_value(target, (read, group), column, loaded_at); + format!( + "{} AS {}", + typed_value(target, column, &value), + DatabricksSql.render_identifier(&column.name) + ) + }) + .collect(); + let keys: Vec = group.keys.iter().map(|key| literal(key)).collect(); + format!( + "COPY INTO {into} FROM (SELECT {} FROM {}) FILEFORMAT = PARQUET FILES = ({}) {S3_COPY_OPTIONS}", + values.join(", "), + literal(group.folder), + keys.join(", ") + ) +} + +/// Where `column`'s value comes from, before it is typed: the load id or load time, a merge's row number, a constant's value, the files' column of `read` it is, or NULL when the files lack it. +fn column_value( + target: &WriteTarget<'_>, + (read, group): (&[ReadColumn], &UnloadedGroup<'_>), + column: &DesiredColumn, + loaded_at: &str, +) -> String { + let literal = |text: &str| DatabricksSql.render_loader_value_literal(text); + let constant = target + .constants + .iter() + .find(|(name, _, _)| *name == column.name); + match (metadata_role(target.compat, &column.name), constant) { + (Some(MetadataRole::LoadId), _) => literal(target.load_id), + (Some(MetadataRole::LoadedAt), _) => literal(loaded_at), + _ if column.name == MERGE_ROW_COLUMN => { + let files: Vec = group + .keys + .iter() + .enumerate() + .map(|(index, key)| { + format!( + "WHEN {} THEN {}", + literal(&format!("{}{key}", group.folder)), + group.first + index + ) + }) + .collect(); + format!( + "{} * {UNLOADED_BATCH_ROWS} + (CASE {FILE_PATH} {} ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END) * {UNLOADED_FILE_ROWS} + _metadata.row_index", + target.batch, + files.join(" ") + ) + } + (_, Some((_, _, value))) => literal(value), + _ => read + .iter() + .find(|read| read.name == column.name) + .map_or_else( + || "NULL".to_owned(), + |read| { + unloaded_value( + &read.output, + &DatabricksSql.render_exact_identifier(&read.name), + ) + }, + ), + } +} + +/// File column `quoted` as the value it was unloaded from: a timestamp with time zone as the instant its whole microseconds count from the epoch, one without as the wall-clock time they count from midnight of 1970-01-01, neither depending on the session's time zone; anything else as it is. +fn unloaded_value(output: &ColumnType, quoted: &str) -> String { + match output { + ColumnType::TimestampTz => format!("timestamp_micros(CAST({quoted} AS BIGINT))"), + ColumnType::TimestampNtz => format!( + "timestampadd(MICROSECOND, CAST({quoted} AS BIGINT), TIMESTAMP_NTZ'1970-01-01 00:00:00')" + ), + _ => quoted.to_owned(), + } +} diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/unloaded_load.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/unloaded_load.rs new file mode 100644 index 000000000..0c94ef81d --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/unloaded_load.rs @@ -0,0 +1,64 @@ +//! A warehouse source's unloaded files loaded by Databricks `COPY`ing them from S3. + +use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch, UnloadedSource}; +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::databricks::_helpers::write::direct_load::inserted_rows; +use crate::adapters::databricks::_helpers::write::unloaded_copy::{ + UnloadedGroup, unloaded_copy_statement, +}; +use crate::adapters::databricks::constants::MAX_FILES_PER_COPY; +use crate::adapters::databricks::models::{DatabricksSql, DatabricksWriter}; +use crate::databricks_api::main::execute_statement::execute_statement; +use crate::loader::errors::LoaderError; + +impl DatabricksWriter { + /// Load `batch` by having Databricks `COPY` the unloaded files from S3, through the external location that covers them, into the staging or build table the load publishes from: one `COPY INTO` per 1000 files, the most `FILES` takes. Each commits on its own, so the batch always lands there, never straight in the table, and fails unless they load as many rows as the unload reports. + pub(in crate::adapters) async fn pull_unloaded( + &mut self, + batch: &UnloadedBatch<'_>, + ) -> Result { + let UnloadedSource::Files(files) = batch.source else { + return Err(LoaderError::Invalid( + "Databricks reads a warehouse source's rows only from its unloaded files" + .to_owned(), + )); + }; + let target = &batch.target; + let Some(staging) = target.staging else { + return Err(LoaderError::Invalid( + "Databricks loads a warehouse source's unloaded files only into a staging or build table".to_owned(), + )); + }; + let folder = files.folder.to_string(); + let columns = match target.builds { + true => batch.write_columns, + false => target.columns, + }; + let into = DatabricksSql.render_qualified_name(staging); + let mut rows = 0; + for (index, keys) in files.keys.chunks(MAX_FILES_PER_COPY).enumerate() { + let statement = unloaded_copy_statement( + (&into, columns), + target, + ( + batch.read, + &UnloadedGroup { + folder: &folder, + keys, + first: index * MAX_FILES_PER_COPY, + }, + ), + batch.loaded_at, + ); + let result = execute_statement(&self.destination.api, &statement).await?; + rows += inserted_rows(&result)?; + } + if rows != files.rows { + return Err(LoaderError::Invalid(format!( + "Databricks loaded {rows} rows from the unloaded files in {folder}, but the unload reports {}; the batch is not published", + files.rows + ))); + } + Ok(DirectLoaded::Staged(rows)) + } +} diff --git a/crates/sqlload/src/adapters/databricks/tests.rs b/crates/sqlload/src/adapters/databricks/tests.rs index 57161de5d..f0e6de2fd 100644 --- a/crates/sqlload/src/adapters/databricks/tests.rs +++ b/crates/sqlload/src/adapters/databricks/tests.rs @@ -8,3 +8,5 @@ mod s3_copy; mod stale_removals; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_unloaded_copy.rs"] +mod unloaded_copy; diff --git a/crates/sqlload/src/adapters/databricks/tests/test_types.rs b/crates/sqlload/src/adapters/databricks/tests/test_types.rs index 146d4e041..1f7d1cb4d 100644 --- a/crates/sqlload/src/adapters/databricks/tests/test_types.rs +++ b/crates/sqlload/src/adapters/databricks/tests/test_types.rs @@ -34,3 +34,10 @@ pub(crate) struct LocationUrlTestCase { pub(crate) url: &'static str, pub(crate) expected_url: &'static str, } + +pub(crate) struct UnloadedCopyTestCase { + pub(crate) description: &'static str, + /// Whether the batch lands in a build table in its final types, rather than a staging table. + pub(crate) builds: bool, + pub(crate) expected_statement: &'static str, +} diff --git a/crates/sqlload/src/adapters/databricks/tests/test_unloaded_copy.rs b/crates/sqlload/src/adapters/databricks/tests/test_unloaded_copy.rs new file mode 100644 index 000000000..67c97e6db --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/tests/test_unloaded_copy.rs @@ -0,0 +1,122 @@ +use crate::adapters::contract::models::{TableName, WriteTarget}; +use crate::adapters::databricks::_helpers::write::unloaded_copy::{ + UnloadedGroup, unloaded_copy_statement, +}; +use crate::adapters::databricks::tests::test_types::UnloadedCopyTestCase; +use crate::column_types::models::ColumnType; +use crate::load_naming::models::Compat; +use crate::postgres_source::models::{ReadColumn, WireType}; +use crate::schema_plan::models::{ColumnOrigin, DesiredColumn}; + +#[test] +fn given_files_of_a_snowflake_unload_when_databricks_copies_them_then_timestamps_come_back_from_microseconds_and_merge_rows_count_by_file() + { + let test_cases = [ + UnloadedCopyTestCase { + description: "into a staging table: timestamps from microseconds whatever the session's time zone, JSON as text, the merge row by batch, file and row", + builds: false, + expected_statement: "COPY INTO `main`.`raw`.`orders_stage` FROM (SELECT \ + CAST(`ID` AS BIGINT) AS `ID`, \ + CAST(timestamp_micros(CAST(`at` AS BIGINT)) AS TIMESTAMP) AS `at`, \ + CAST(timestampadd(MICROSECOND, CAST(`wall` AS BIGINT), TIMESTAMP_NTZ'1970-01-01 00:00:00') AS TIMESTAMP_NTZ) AS `wall`, \ + CAST(`payload` AS STRING) AS `payload`, \ + CAST('eu' AS STRING) AS `region`, \ + CAST(NULL AS STRING) AS `missing`, \ + CAST(3 * 281474976710656 + (CASE url_decode(replace(_metadata.file_path, '+', '%2B')) WHEN 's3://lake/unload/run1/data_0_1_0.snappy.parquet' THEN 1000 WHEN 's3://lake/unload/run1/data_0_2_0.snappy.parquet' THEN 1001 ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END) * 4294967296 + _metadata.row_index AS BIGINT) AS `_sqlload_row`, \ + CAST('L1' AS STRING) AS `_sqlload_load_id`, \ + CAST('2026-11-18T00:00:00Z' AS TIMESTAMP) AS `_sqlload_loaded_at` \ + FROM 's3://lake/unload/run1/') FILEFORMAT = PARQUET FILES = ('data_0_1_0.snappy.parquet', 'data_0_2_0.snappy.parquet') FORMAT_OPTIONS ('mergeSchema' = 'true') COPY_OPTIONS ('force' = 'true')", + }, + UnloadedCopyTestCase { + description: "into a build table: final types, JSON parsed", + builds: true, + expected_statement: "COPY INTO `main`.`raw`.`orders_stage` FROM (SELECT \ + CAST(`ID` AS BIGINT) AS `ID`, \ + CAST(timestamp_micros(CAST(`at` AS BIGINT)) AS TIMESTAMP) AS `at`, \ + CAST(timestampadd(MICROSECOND, CAST(`wall` AS BIGINT), TIMESTAMP_NTZ'1970-01-01 00:00:00') AS TIMESTAMP_NTZ) AS `wall`, \ + parse_json(`payload`) AS `payload`, \ + CAST('eu' AS STRING) AS `region`, \ + CAST(NULL AS STRING) AS `missing`, \ + CAST(3 * 281474976710656 + (CASE url_decode(replace(_metadata.file_path, '+', '%2B')) WHEN 's3://lake/unload/run1/data_0_1_0.snappy.parquet' THEN 1000 WHEN 's3://lake/unload/run1/data_0_2_0.snappy.parquet' THEN 1001 ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END) * 4294967296 + _metadata.row_index AS BIGINT) AS `_sqlload_row`, \ + CAST('L1' AS STRING) AS `_sqlload_load_id`, \ + CAST('2026-11-18T00:00:00Z' AS TIMESTAMP) AS `_sqlload_loaded_at` \ + FROM 's3://lake/unload/run1/') FILEFORMAT = PARQUET FILES = ('data_0_1_0.snappy.parquet', 'data_0_2_0.snappy.parquet') FORMAT_OPTIONS ('mergeSchema' = 'true') COPY_OPTIONS ('force' = 'true')", + }, + ]; + let column = |name: &str, column_type: ColumnType| DesiredColumn { + name: name.to_owned(), + column_type, + origin: ColumnOrigin::Source, + }; + let read: Vec = [ + ("ID", ColumnType::Int), + ("at", ColumnType::TimestampTz), + ("wall", ColumnType::TimestampNtz), + ("payload", ColumnType::Json), + ] + .into_iter() + .map(|(name, output)| ReadColumn { + name: name.to_owned(), + expression: format!("\"{name}\""), + wire: WireType::Text, + output, + }) + .collect(); + let columns = [ + column("ID", ColumnType::Int), + column("at", ColumnType::TimestampTz), + column("wall", ColumnType::TimestampNtz), + column("payload", ColumnType::Json), + column("region", ColumnType::Text), + column("missing", ColumnType::Text), + column("_sqlload_row", ColumnType::Int), + column("_sqlload_load_id", ColumnType::Text), + column("_sqlload_loaded_at", ColumnType::TimestampTz), + ]; + let table = TableName { + database: Some("main".to_owned()), + schema: "raw".to_owned(), + table: "orders".to_owned(), + }; + let staging = TableName { + table: "orders_stage".to_owned(), + ..table.clone() + }; + let constants = [("region".to_owned(), ColumnType::Text, "eu".to_owned())]; + for test_case in &test_cases { + let target = WriteTarget { + table: &table, + staging: Some(&staging), + columns: &columns, + write_columns: &columns, + compat: Compat::Sqlload, + load_id: "L1", + batch: 3, + direct: false, + builds: test_case.builds, + constants: &constants, + }; + let keys = [ + "data_0_1_0.snappy.parquet".to_owned(), + "data_0_2_0.snappy.parquet".to_owned(), + ]; + let statement = unloaded_copy_statement( + ("`main`.`raw`.`orders_stage`", &columns), + &target, + ( + &read, + &UnloadedGroup { + folder: "s3://lake/unload/run1/", + keys: &keys, + first: 1000, + }, + ), + "2026-11-18T00:00:00Z", + ); + assert_eq!( + statement, test_case.expected_statement, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs b/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs index 0bfd8c48c..3309950cc 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs @@ -8,13 +8,25 @@ use crate::adapters::dispatch::models::Destination; use crate::adapters::duckdb::main::_check_files::check_files; use crate::files_source::models::FilesRead; use crate::loader::models::FilesSource; +use crate::routing::models::Store; +use crate::store_paths::models::StoreKind; -/// Check, at plan, that `destination` reads the files of `source` in S3 where they lie, for a table nothing sends through SQLLoad, with `read` its planned read: Databricks looks up its Unity Catalog external locations once per run, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers, and lists the folder when one covers it and it holds files to load; MotherDuck reads the first file to load on its servers. The error says why it can't, with the fix. +/// Check, at plan, that `destination` reads the files of `source` in S3 where they lie, or the S3 folder a warehouse source unloads to (`unload_to`), for a table nothing sends through SQLLoad, with `read` its planned read: Databricks looks up its Unity Catalog external locations once per run, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers, and lists the folder when one covers it and it holds files to load; MotherDuck reads the first file to load on its servers. An unload's folder is new each run, so for it Databricks only lists its locations, and the route then needs one to cover it. The error says why it can't, with the fix. pub async fn check_reach( destination: &Destination, - source: &FilesSource, + (source, unload_to): (Option<&FilesSource>, &[Store]), read: Option<&FilesRead>, ) -> Result<(), String> { + let source = source.filter(|source| matches!(source.location.store, StoreKind::S3 { .. })); + let unloads = unload_to + .iter() + .any(|store| matches!(store, Store::ObjectStore(url) if url.starts_with("s3://"))); + let Some(source) = source else { + return match (destination, unloads) { + (Destination::Databricks(databricks), true) => databricks_reach(databricks, None).await, + _ => Ok(()), + }; + }; let first = read.and_then(|read| read.files.first()); let folder = source.location.to_string(); match destination { diff --git a/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs b/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs index ac6ff774d..05d3003ce 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs @@ -2,16 +2,17 @@ use crate::adapters::snowflake::main::copy::_stage_covers::stage_covers; use crate::adapters::snowflake::models::StageRoute; -use crate::column_types::models::Warehouse; +use crate::column_types::models::{ColumnType, Warehouse}; use crate::loader::constants::{UNLOADED_OUTSIDE_STAGE, UNLOADED_TIMES}; use crate::loader::models::WarehouseRoute; +use crate::postgres_source::models::SourceColumn; use crate::routing::models::Store; use crate::store_paths::main::resolve_location::resolve_location; -/// Why `warehouse` cannot load a table's unloaded files exactly by reading them itself, so SQLLoad reads them instead; `None` when it can (where it also reaches their store). Only DuckDB and Snowflake turn the files' whole microseconds back into times and timestamps; a table that `masks` columns needs SQLLoad to see the rows; Snowflake reads another session's unload only through the target's external `stage`, which must cover the S3 folder `route` unloads to, as a user or named stage of the source's is not its to read. +/// Why `warehouse` cannot load a table's unloaded files exactly by reading them itself, so SQLLoad reads them instead; `None` when it can (where it also reaches their store). DuckDB and Snowflake turn the files' whole microseconds back into times and timestamps, and Databricks and ClickHouse into timestamps, so a table with a time column among `columns` goes through SQLLoad into those; a table that `masks` columns needs SQLLoad to see the rows; Snowflake reads another session's unload only through the target's external `stage`, which must cover the S3 folder `route` unloads to, as a user or named stage of the source's is not its to read. pub fn unload_blocked( warehouse: Warehouse, - masks: bool, + (masks, columns): (bool, &[SourceColumn]), route: Option<&WarehouseRoute>, stage: Option<&StageRoute>, ) -> Option { @@ -26,12 +27,20 @@ pub fn unload_blocked( .is_ok_and(|unloaded| stage_covers(&stage.location, &unloaded)), _ => false, }; + let times = columns + .iter() + .any(|column| column.column_type == Ok(ColumnType::Time)); + let reads = matches!( + warehouse, + Warehouse::DuckDb | Warehouse::Snowflake | Warehouse::Databricks | Warehouse::ClickHouse + ); match (warehouse, masks) { - (Warehouse::DuckDb | Warehouse::Snowflake, true) => { - Some("the table masks columns".to_owned()) - } + (_, true) if reads => Some("the table masks columns".to_owned()), (Warehouse::Snowflake, false) if !covered => Some(UNLOADED_OUTSIDE_STAGE.to_owned()), - (Warehouse::DuckDb | Warehouse::Snowflake, false) => None, + (Warehouse::Databricks | Warehouse::ClickHouse, false) if times => Some(format!( + "the table has a time column, which {warehouse} doesn't store as a time" + )), + _ if reads => None, _ => Some(UNLOADED_TIMES.to_owned()), } } diff --git a/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs b/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs index 6ac0235d0..10d762c0e 100644 --- a/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs +++ b/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs @@ -13,6 +13,8 @@ pub async fn load_unloaded( #[cfg(feature = "duckdb")] Writer::DuckDb(duckdb) => duckdb.pull_unloaded(batch).await.map(Some), Writer::Snowflake(snowflake) => snowflake.pull_unloaded(batch).await.map(Some), + Writer::ClickHouse(clickhouse) => clickhouse.pull_unloaded(batch).await.map(Some), + Writer::Databricks(databricks) => databricks.pull_unloaded(batch).await.map(Some), _ => Ok(None), } } diff --git a/crates/sqlload/src/adapters/dispatch/main/write/mod.rs b/crates/sqlload/src/adapters/dispatch/main/write/mod.rs index ed36b3798..75e01c525 100644 --- a/crates/sqlload/src/adapters/dispatch/main/write/mod.rs +++ b/crates/sqlload/src/adapters/dispatch/main/write/mod.rs @@ -2,3 +2,4 @@ pub mod file_keys; pub mod load_files; pub mod load_unloaded; pub mod open_table_writer; +pub mod unloads_directly; diff --git a/crates/sqlload/src/adapters/dispatch/main/write/unloads_directly.rs b/crates/sqlload/src/adapters/dispatch/main/write/unloads_directly.rs new file mode 100644 index 000000000..6e3f532fe --- /dev/null +++ b/crates/sqlload/src/adapters/dispatch/main/write/unloads_directly.rs @@ -0,0 +1,8 @@ +//! Whether a destination loads a warehouse source's unload straight into the table. + +use crate::adapters::dispatch::models::Writer; + +/// Whether `writer` may load a warehouse source's unloaded files straight into the table, where it writes directly at all: not Databricks, whose `COPY`s commit one by one before their rows are checked against the unload's. +pub fn unloads_directly(writer: &Writer) -> bool { + !matches!(writer, Writer::Databricks(_)) +} diff --git a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs index c0e1db5d3..016dc4b10 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs @@ -452,12 +452,31 @@ pub(crate) fn unload_blocked_for(test_case: &UnloadBlockedTestCase) -> Option Option { + let route = WarehouseRoute::Unload { + store: Store::ObjectStore("s3://lake/raw/unload/".to_owned()), + stage: String::new(), + same_connection: false, + }; + use crate::snowflake_source::_helpers::catalog::source_column; + let column = source_column("c", test_case.column_type, true); + unload_blocked( + test_case.warehouse, + (test_case.masks, &[column]), + Some(&route), + None, + ) +} + /// How many requests the case's statements are sent as, each checked to fit `MAX_COPY_SQL_BYTES`, and whether the first commits only when every unloaded row loads; or the error up to its first comma. pub(crate) fn unloaded_requests_for( test_case: &UnloadedRequestsTestCase, diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs index 1d254396d..6f38a6ae2 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs @@ -133,6 +133,16 @@ pub(crate) struct UnloadBlockedTestCase { pub(crate) expected_blocked: Option<&'static str>, } +pub(crate) struct UnloadReadersTestCase { + pub(crate) description: &'static str, + pub(crate) warehouse: crate::column_types::models::Warehouse, + pub(crate) masks: bool, + /// The Snowflake type of the table's one column. + pub(crate) column_type: &'static str, + /// How the reason SQLLoad reads the files instead starts, or `None` when the target reads the S3 unload itself. + pub(crate) expected_blocked: Option<&'static str>, +} + /// How the statements loading one unload inside Snowflake are sent. pub(crate) struct UnloadedRequestsTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs b/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs index 4c0eb4237..861d98a7d 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs @@ -1,6 +1,7 @@ -use crate::adapters::snowflake::tests::helpers::unload_blocked_for; -use crate::adapters::snowflake::tests::test_types::UnloadBlockedTestCase; -use crate::loader::constants::UNLOADED_OUTSIDE_STAGE; +use crate::adapters::snowflake::tests::helpers::{unload_blocked_for, unload_readers_for}; +use crate::adapters::snowflake::tests::test_types::{UnloadBlockedTestCase, UnloadReadersTestCase}; +use crate::column_types::models::Warehouse; +use crate::loader::constants::{UNLOADED_OUTSIDE_STAGE, UNLOADED_TIMES}; const MASKS: &str = "the table masks columns"; const OUTSIDE: &str = UNLOADED_OUTSIDE_STAGE; @@ -82,3 +83,73 @@ fn given_a_snowflake_sources_unload_when_routing_into_snowflake_then_snowflake_c ); } } + +#[test] +fn given_a_snowflake_sources_s3_unload_when_routing_into_another_warehouse_then_databricks_and_clickhouse_read_it_unless_times_or_masks_need_sqlload() + { + let test_cases = [ + UnloadReadersTestCase { + description: "Databricks reads the unload itself", + warehouse: Warehouse::Databricks, + masks: false, + column_type: "NUMBER(38,0)", + expected_blocked: None, + }, + UnloadReadersTestCase { + description: "ClickHouse reads the unload itself", + warehouse: Warehouse::ClickHouse, + masks: false, + column_type: "NUMBER(38,0)", + expected_blocked: None, + }, + UnloadReadersTestCase { + description: "a time column goes through SQLLoad into Databricks, which keeps times as text", + warehouse: Warehouse::Databricks, + masks: false, + column_type: "TIME(9)", + expected_blocked: Some("the table has a time column"), + }, + UnloadReadersTestCase { + description: "a time column goes through SQLLoad into ClickHouse", + warehouse: Warehouse::ClickHouse, + masks: false, + column_type: "TIME(9)", + expected_blocked: Some("the table has a time column"), + }, + UnloadReadersTestCase { + description: "masks need SQLLoad to see the rows", + warehouse: Warehouse::ClickHouse, + masks: true, + column_type: "NUMBER(38,0)", + expected_blocked: Some(MASKS), + }, + UnloadReadersTestCase { + description: "DuckDB turns times back itself", + warehouse: Warehouse::DuckDb, + masks: false, + column_type: "TIME(9)", + expected_blocked: None, + }, + UnloadReadersTestCase { + description: "Postgres never reads files itself", + warehouse: Warehouse::Postgres, + masks: false, + column_type: "NUMBER(38,0)", + expected_blocked: Some(UNLOADED_TIMES), + }, + ]; + for test_case in &test_cases { + let blocked = unload_readers_for(test_case); + assert_eq!( + blocked.as_deref().map(|reason| { + test_case + .expected_blocked + .filter(|expected| reason.starts_with(expected)) + .unwrap_or(reason) + }), + test_case.expected_blocked, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/batch_plan/main/split_windows.rs b/crates/sqlload/src/batch_plan/main/split_windows.rs index 65e8aebbd..c8137f6d4 100644 --- a/crates/sqlload/src/batch_plan/main/split_windows.rs +++ b/crates/sqlload/src/batch_plan/main/split_windows.rs @@ -1,12 +1,11 @@ //! Split a cursor range into the windows of a table's `batch_size`. use crate::batch_plan::_helpers::bounds::{integer_plan, time_plan}; -use crate::batch_plan::constants::MAX_WINDOWS; use crate::batch_plan::models::{WindowPlan, WindowSplit}; use crate::column_types::models::ColumnType; use crate::project::models::BatchStep; -/// The windows `[from, from + step)`, ... up to the end (`--end-cursor-*`, else now; for a date cursor the end of today), counted but not listed; the first keeps the read's inclusiveness. A range without a start cannot be split, so `cursor_start` must give the first load one; a step giving more than 100,000 windows is refused. +/// The windows `[from, from + step)`, ... up to the end (`--end-cursor-*`, else now; for a date cursor the end of today), counted but not listed; the first keeps the read's inclusiveness. A range without a start cannot be split, so `cursor_start` must give the first load one; a step giving more than `most` windows is refused. pub fn split_windows(split: &WindowSplit<'_>) -> Result { let Some(from) = split.from else { return Err( @@ -28,10 +27,10 @@ pub fn split_windows(split: &WindowSplit<'_>) -> Result { return Err("batch_size = \"none\" reads in one piece; it has no windows".to_owned()); } }; - if plan.count > MAX_WINDOWS { + if plan.count > split.most { return Err(format!( - "batch_size splits the read into {} windows, more than {MAX_WINDOWS}; use a larger batch_size, or narrow the read with --start-cursor/--end-cursor", - plan.count + "batch_size splits the read into {} windows, more than {}; use a larger batch_size, or narrow the read with --start-cursor/--end-cursor", + plan.count, split.most )); } Ok(plan) diff --git a/crates/sqlload/src/batch_plan/models.rs b/crates/sqlload/src/batch_plan/models.rs index 2274f5caf..9217b8a57 100644 --- a/crates/sqlload/src/batch_plan/models.rs +++ b/crates/sqlload/src/batch_plan/models.rs @@ -25,6 +25,8 @@ pub struct WindowSplit<'a> { pub now: DateTime, /// The table's `timezone`, which a bound without an offset is read in while the cursor's type is unknown. pub zone: chrono_tz::Tz, + /// The most windows the read may be split into (see [`crate::loader::models::TableSpec::most_windows`]). + pub most: u64, } /// Files that commit together, in key order, and the cursor values their rows carry when they replace whole partitions. diff --git a/crates/sqlload/src/batch_plan/tests/helpers.rs b/crates/sqlload/src/batch_plan/tests/helpers.rs index 6a807be96..b23778b94 100644 --- a/crates/sqlload/src/batch_plan/tests/helpers.rs +++ b/crates/sqlload/src/batch_plan/tests/helpers.rs @@ -38,6 +38,7 @@ pub(crate) fn windows(test_case: &SplitWindowsTestCase) -> Result, S cursor_type: test_case.cursor_type.as_ref(), now: now(), zone: test_case.zone, + most: test_case.most, }; let plan = split_windows(&split)?; (0..plan.count) diff --git a/crates/sqlload/src/batch_plan/tests/test_split_windows.rs b/crates/sqlload/src/batch_plan/tests/test_split_windows.rs index 7a7c8001e..7f4931185 100644 --- a/crates/sqlload/src/batch_plan/tests/test_split_windows.rs +++ b/crates/sqlload/src/batch_plan/tests/test_split_windows.rs @@ -1,3 +1,4 @@ +use crate::batch_plan::constants::MAX_WINDOWS; use crate::batch_plan::tests::helpers::{owned_windows, windows}; use crate::batch_plan::tests::test_types::SplitWindowsTestCase; use crate::column_types::models::ColumnType; @@ -16,12 +17,41 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order() step: BatchStep::Duration(DAY), cursor_type: Some(ColumnType::Date), zone: chrono_tz::Tz::UTC, + most: MAX_WINDOWS, expected_windows: Ok(&[ "2026-11-03..2026-11-04", "2026-11-04..2026-11-05", "2026-11-05..2026-11-06", ]), }, + SplitWindowsTestCase { + description: "a read split into exactly the most windows allowed is kept", + from: Some("2026-11-03"), + to: None, + inclusive: true, + step: BatchStep::Duration(DAY), + cursor_type: Some(ColumnType::Date), + zone: chrono_tz::Tz::UTC, + most: 3, + expected_windows: Ok(&[ + "2026-11-03..2026-11-04", + "2026-11-04..2026-11-05", + "2026-11-05..2026-11-06", + ]), + }, + SplitWindowsTestCase { + description: "one window more than the most allowed is refused", + from: Some("2026-11-03"), + to: None, + inclusive: true, + step: BatchStep::Duration(DAY), + cursor_type: Some(ColumnType::Date), + zone: chrono_tz::Tz::UTC, + most: 2, + expected_windows: Err( + "batch_size splits the read into 3 windows, more than 2; use a larger batch_size, or narrow the read with --start-cursor/--end-cursor", + ), + }, SplitWindowsTestCase { description: "timestamp windows end now, the last one shorter", from: Some("2026-11-04 12:00:00.000000+00"), @@ -30,6 +60,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order() step: BatchStep::Duration(DAY), cursor_type: Some(ColumnType::TimestampTz), zone: chrono_tz::Tz::UTC, + most: MAX_WINDOWS, expected_windows: Ok(&["2026-11-04 12:00:00.000000+00..2026-11-05 10:30:00.000000+00"]), }, SplitWindowsTestCase { @@ -40,6 +71,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order() step: BatchStep::Duration(DAY), cursor_type: Some(ColumnType::TimestampNtz), zone: chrono_tz::Tz::UTC, + most: MAX_WINDOWS, expected_windows: Ok(&[ "(2026-11-01 00:00:00.000000..2026-11-02 00:00:00.000000", "2026-11-02 00:00:00.000000..2026-11-02 12:00:00.000000", @@ -53,6 +85,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order() step: BatchStep::Integer(100), cursor_type: Some(ColumnType::Int), zone: chrono_tz::Tz::UTC, + most: MAX_WINDOWS, expected_windows: Ok(&["0..100", "100..200", "200..250"]), }, SplitWindowsTestCase { @@ -63,6 +96,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order() step: BatchStep::Duration(DAY), cursor_type: None, zone: chrono_tz::Tz::UTC, + most: MAX_WINDOWS, expected_windows: Ok(&["2026-11-04..2026-11-05", "2026-11-05..2026-11-06"]), }, SplitWindowsTestCase { @@ -73,6 +107,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order() step: BatchStep::Duration(DAY), cursor_type: None, zone: chrono_tz::Tz::Europe__Paris, + most: MAX_WINDOWS, expected_windows: Ok(&[ "2026-02-28 23:00:00.000000+00..2026-03-01 23:00:00.000000+00", "2026-03-01 23:00:00.000000+00..2026-03-02 23:00:00.000000+00", @@ -86,6 +121,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order() step: BatchStep::Duration(DAY), cursor_type: Some(ColumnType::Date), zone: chrono_tz::Tz::UTC, + most: MAX_WINDOWS, expected_windows: Err( "batch_size needs a start to split the cursor range: set cursor_start for the first load", ), @@ -98,6 +134,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order() step: BatchStep::Integer(10), cursor_type: Some(ColumnType::Int), zone: chrono_tz::Tz::UTC, + most: MAX_WINDOWS, expected_windows: Err( "an integer batch needs --end-cursor to know where the last window ends, with --reload too (a Postgres source ends at its current maximum)", ), @@ -110,6 +147,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order() step: BatchStep::Duration(std::time::Duration::from_secs(3600)), cursor_type: Some(ColumnType::Date), zone: chrono_tz::Tz::UTC, + most: MAX_WINDOWS, expected_windows: Err("a date cursor's batch_size must be whole days, not 1h"), }, SplitWindowsTestCase { @@ -120,6 +158,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order() step: BatchStep::Duration(std::time::Duration::from_secs(60)), cursor_type: Some(ColumnType::TimestampTz), zone: chrono_tz::Tz::UTC, + most: MAX_WINDOWS, expected_windows: Err( "batch_size splits the read into 13675680 windows, more than 100000; use a larger batch_size, or narrow the read with --start-cursor/--end-cursor", ), @@ -132,6 +171,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order() step: BatchStep::Integer(5), cursor_type: Some(ColumnType::Int), zone: chrono_tz::Tz::UTC, + most: MAX_WINDOWS, expected_windows: Ok(&[]), }, SplitWindowsTestCase { @@ -142,6 +182,7 @@ fn given_a_cursor_range_when_splitting_by_batch_then_windows_cover_it_in_order() step: BatchStep::Duration(DAY), cursor_type: Some(ColumnType::Int), zone: chrono_tz::Tz::UTC, + most: MAX_WINDOWS, expected_windows: Err( "a duration batch needs a date or timestamp cursor, but the cursor is int; give an integer step", ), diff --git a/crates/sqlload/src/batch_plan/tests/test_types.rs b/crates/sqlload/src/batch_plan/tests/test_types.rs index 49d7f978c..7e66da2a5 100644 --- a/crates/sqlload/src/batch_plan/tests/test_types.rs +++ b/crates/sqlload/src/batch_plan/tests/test_types.rs @@ -10,6 +10,8 @@ pub(crate) struct SplitWindowsTestCase { pub(crate) cursor_type: Option, /// The table's `timezone`, which a bound without an offset is read in while the cursor's type is unknown. pub(crate) zone: chrono_tz::Tz, + /// The most windows allowed. + pub(crate) most: u64, /// Each window as `from..to` (`(` when the start is not read), or the error. pub(crate) expected_windows: Result<&'static [&'static str], &'static str>, } diff --git a/crates/sqlload/src/loader/_helpers/publish.rs b/crates/sqlload/src/loader/_helpers/publish.rs index b02904274..b93fb0b65 100644 --- a/crates/sqlload/src/loader/_helpers/publish.rs +++ b/crates/sqlload/src/loader/_helpers/publish.rs @@ -23,6 +23,7 @@ use crate::adapters::dispatch::main::write::file_keys::file_keys; use crate::adapters::dispatch::main::write::load_files::load_files; use crate::adapters::dispatch::main::write::load_unloaded::load_unloaded; use crate::adapters::dispatch::main::write::open_table_writer::open_table_writer; +use crate::adapters::dispatch::main::write::unloads_directly::unloads_directly; use crate::adapters::dispatch::models::Destination; use crate::adapters::dispatch::models::Writer; use crate::adapters::files::models::ReplacedFiles; @@ -168,9 +169,12 @@ use crate::loader::_helpers::table_plan::{ added_constants, is_direct, pulls_unloaded, selects_rows, }; -/// Whether `append` can write the batches straight into the table: not a logged table (its rows wait for a changed `ctx.state`), and a writer that can (on Snowflake, a route loading files into the table itself). +/// Whether `append` can write the batches straight into the table: not a logged table (its rows wait for a changed `ctx.state`), and a writer that can (on Snowflake, a route loading files into the table itself), and, for an unload the destination reads itself, one that [loads it directly](unloads_directly). fn writes_directly(table: &TablePlan, writer: &Writer) -> bool { - table.mode == WriteMode::Append && table.logged.is_none() && writer.writes_directly() + table.mode == WriteMode::Append + && table.logged.is_none() + && writer.writes_directly() + && (!pulls_unloaded(table.route.as_ref()) || unloads_directly(writer)) } /// Where the batches go, from the publisher's fields: `staging` when there is one, and `columns` as the staged batches carry them. @@ -1073,7 +1077,7 @@ impl Publisher { } } - /// Load one unload's files, or the rows its query selects, by having the destination read them itself, where the plan chose that; the destination must load them so. + /// Load one unload's files, or the rows its query selects, by having the destination read them itself, where the plan chose that; the destination must load them so. Each unload is a batch of its own, so its rows' merge ordinals follow every earlier window's even when they wait in one staging table. pub(crate) async fn load_unloaded( &mut self, source: UnloadedSource<'_>, @@ -1100,6 +1104,7 @@ impl Publisher { )) })?; self.note_direct(&loaded); + self.batch += 1; Ok(loaded) } diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs index 1db158ec1..3667c5204 100644 --- a/crates/sqlload/src/loader/_helpers/specs.rs +++ b/crates/sqlload/src/loader/_helpers/specs.rs @@ -681,7 +681,7 @@ fn warehouse_source( storage_integration: Some(integration), .. }) => ( - Store::ObjectStore(url.clone()), + Store::ObjectStore(format!("{}/", url.trim_end_matches('/'))), format!( "external: Snowflake writes to {url} through storage integration {integration}; the files are read there and removed after the read" ), diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index 48f8fad60..d285ecade 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -1094,7 +1094,7 @@ pub(crate) fn reader_offer( true => direct_blocked, false => unload_blocked( spec.warehouse, - !spec.masked.columns.is_empty(), + (!spec.masked.columns.is_empty(), &spec.source_table.columns), route, stage, ), @@ -1206,7 +1206,7 @@ fn copy_warning(route: Option<&Route>) -> Option { )) } -/// The route a table's data takes into `destination`; `read` is a files table's planned read. For files in S3 that nothing sends through SQLLoad, it first checks that the destination reaches them (Databricks' external locations, MotherDuck's servers). +/// The route a table's data takes into `destination`; `read` is a files table's planned read. For files in S3, or an unload to S3, that nothing sends through SQLLoad, it first checks that the destination reaches them (Databricks' external locations, MotherDuck's servers). pub(crate) async fn table_route( spec: &TableSpec, read: Option<&FilesRead>, @@ -1217,10 +1217,8 @@ pub(crate) async fn table_route( let refused = unload_refused(destination, &reader.unload_to); reader.direct_blocked = reader.direct_blocked.or(refused); let in_place = reader.needs_loader.is_none() && reader.direct_blocked.is_none(); - if let Some(files) = spec.files.as_ref().filter(|_| in_place) - && let StoreKind::S3 { .. } = files.location.store - { - check_reach(destination, files, read).await?; + if in_place { + check_reach(destination, (spec.files.as_ref(), &reader.unload_to), read).await?; } plan_route( &reader, @@ -1751,6 +1749,7 @@ pub(crate) fn commit_rule(request: &CommitRequest<'_>) -> Result u64 { + match self.warehouse_source.as_ref().map(|source| &source.route) { + Some(WarehouseRoute::Unload { .. }) => (i64::MAX / UNLOADED_BATCH_ROWS).unsigned_abs(), + _ => MAX_WINDOWS, + } + } + /// The query whose rows are the table, for a warehouse source's query table. pub fn query_sql(&self) -> Option<&str> { self.warehouse_source diff --git a/crates/sqlload/src/loader/tests/helpers.rs b/crates/sqlload/src/loader/tests/helpers.rs index 8943d4918..323ecc23a 100644 --- a/crates/sqlload/src/loader/tests/helpers.rs +++ b/crates/sqlload/src/loader/tests/helpers.rs @@ -637,3 +637,53 @@ pub(crate) fn merge_newest() -> WriteMode { not_null_keys: Vec::new(), } } + +/// The spec of table `current` of a `snowflake` source with stage `url`, loading into DuckDB. +pub(crate) fn unload_spec(url: &str) -> Result { + let text = format!( + "[connections.wh]\nkind = \"duckdb\"\ndatabase = \":memory:\"\n[targets.dev]\nconnection = \"wh\"\nloader_schema = \"RAW\"\n[connections.sf]\nkind = \"snowflake\"\naccount = \"a\"\nuser = \"u\"\nauth = {{ method = \"pat\", token = \"t\" }}\ndatabase = \"D\"\n[sources.marts]\nkind = \"snowflake\"\nconnection = \"sf\"\nwrite_strategy = \"replace\"\nstage = {{ url = \"{url}\", storage_integration = \"I\" }}\n[sources.marts.tables.current]\ndescription = \"d\"\nsql = \"SELECT 1 AS id\"\n" + ); + let project = crate::project::main::parse_project::parse_project(&text, &|_| None) + .map_err(|error| error.to_string())?; + let table = SourceTable { + schema: String::new(), + name: "current".to_owned(), + columns: vec![SourceColumn { + name: "id".to_owned(), + postgres_type: "NUMBER".to_owned(), + pg_type: crate::postgres_source::models::PgType::Other, + column_type: Ok(crate::column_types::models::ColumnType::Int), + nullable: true, + source_spelling: None, + repeat_of: None, + }], + primary_key: Vec::new(), + indexed_columns: Vec::new(), + estimated_rows: -1, + estimated_bytes: -1, + split_pages: None, + comment: None, + listed_keys: Vec::new(), + }; + let resolved = resolve_tables(&project, "marts", &[table]); + resolved + .specs + .into_iter() + .next() + .ok_or_else(|| format!("no spec: {:?}", resolved.problems)) +} + +/// The store a `snowflake` source with stage `url` unloads table `current` to, as Debug prints it (the whole route when it is no unload). +pub(crate) fn unload_store(url: &str) -> Result { + let spec = unload_spec(url)?; + let route = spec + .warehouse_source + .as_ref() + .map(|source| format!("{:?}", source.route)) + .ok_or("no warehouse source")?; + Ok(route + .split(", stage: ") + .next() + .unwrap_or_default() + .replace("Unload { store: ", "")) +} diff --git a/crates/sqlload/src/loader/tests/test_plan_lines.rs b/crates/sqlload/src/loader/tests/test_plan_lines.rs index d23953789..98bf4a372 100644 --- a/crates/sqlload/src/loader/tests/test_plan_lines.rs +++ b/crates/sqlload/src/loader/tests/test_plan_lines.rs @@ -2,9 +2,12 @@ use crate::loader::_helpers::table_plan::blocked_table; use crate::loader::main::render_plan::render_plan; use crate::loader::models::{ProjectPlan, PythonTable, SkippedTable}; use crate::loader::tests::helpers::{ - merge_failing_repeats, merge_newest, python_spec, replace_mode, streamed_route, + merge_failing_repeats, merge_newest, python_spec, replace_mode, streamed_route, unload_spec, + unload_store, +}; +use crate::loader::tests::test_types::{ + ModedTable, MostWindowsTestCase, PlanLinesTestCase, UnloadStoreTestCase, }; -use crate::loader::tests::test_types::{ModedTable, PlanLinesTestCase}; #[test] fn given_tables_when_printing_the_plan_without_verbose_then_only_their_own_lines_show_grouped_by_source() @@ -96,3 +99,55 @@ fn given_tables_when_printing_the_plan_without_verbose_then_only_their_own_lines ); } } + +#[test] +fn given_a_snowflake_stage_url_when_resolving_its_table_then_the_unload_store_is_a_folder_ending_in_a_slash() + { + let test_cases = [ + UnloadStoreTestCase { + description: "a url without a trailing slash is the folder, so an external location of that folder covers it", + url: "s3://lake/unload", + expected_store: "Ok(\"ObjectStore(\\\"s3://lake/unload/\\\")\")", + }, + UnloadStoreTestCase { + description: "a url with a trailing slash keeps it", + url: "s3://lake/unload/", + expected_store: "Ok(\"ObjectStore(\\\"s3://lake/unload/\\\")\")", + }, + ]; + for test_case in test_cases { + assert_eq!( + format!("{:?}", unload_store(test_case.url)), + test_case.expected_store, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_table_when_splitting_its_read_into_windows_then_an_unloaded_table_takes_fewer_so_its_merge_row_numbers_fit_64_bits() + { + let test_cases = [ + MostWindowsTestCase { + description: "a table SQLLoad reads takes up to 100,000 windows", + spec: || python_spec("{}"), + expected_most: Ok(100_000), + }, + MostWindowsTestCase { + description: "an unloaded table's rows number from the window times 2^48, so it takes up to 32,767", + spec: || unload_spec("s3://lake/unload/"), + expected_most: Ok(32_767), + }, + ]; + for test_case in test_cases { + assert_eq!( + (test_case.spec)() + .map(|spec| spec.most_windows()) + .map_err(|_| "no spec"), + test_case.expected_most, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/loader/tests/test_types.rs b/crates/sqlload/src/loader/tests/test_types.rs index 8158ba6ce..4a9e43bb7 100644 --- a/crates/sqlload/src/loader/tests/test_types.rs +++ b/crates/sqlload/src/loader/tests/test_types.rs @@ -250,3 +250,17 @@ pub(crate) struct PlanLinesTestCase { pub(crate) skipped: &'static [(&'static str, &'static str)], pub(crate) expected_output: &'static str, } + +/// A `snowflake` source whose stage `url` is written as `url`, and the store its unloads go to. +pub(crate) struct UnloadStoreTestCase { + pub(crate) description: &'static str, + pub(crate) url: &'static str, + pub(crate) expected_store: &'static str, +} + +/// A table's spec, and the most cursor windows its read may be split into. +pub(crate) struct MostWindowsTestCase { + pub(crate) description: &'static str, + pub(crate) spec: fn() -> Result, + pub(crate) expected_most: Result, +} diff --git a/crates/sqlload/src/routing/_helpers/files_route.rs b/crates/sqlload/src/routing/_helpers/files_route.rs index f2cd52a33..c5291102b 100644 --- a/crates/sqlload/src/routing/_helpers/files_route.rs +++ b/crates/sqlload/src/routing/_helpers/files_route.rs @@ -91,7 +91,7 @@ pub(crate) fn other_reads( } /// Why `writer` refuses `files` it cannot read where they lie, when it reads such files only itself. -fn must_pull(writer: &WriterAccepts, files: &FileSet) -> Option { +pub(crate) fn must_pull(writer: &WriterAccepts, files: &FileSet) -> Option { writer .must_pull .as_ref() diff --git a/crates/sqlload/src/routing/_helpers/warehouse_source.rs b/crates/sqlload/src/routing/_helpers/warehouse_source.rs index 58056c6ce..de448edb8 100644 --- a/crates/sqlload/src/routing/_helpers/warehouse_source.rs +++ b/crates/sqlload/src/routing/_helpers/warehouse_source.rs @@ -1,4 +1,5 @@ use crate::routing::_helpers::deliver::deliver_arrow; +use crate::routing::_helpers::files_route::must_pull; use crate::routing::_helpers::stores::{destination_pulls, ordered_stores}; use crate::routing::constants::{DOWNLOAD_FOLDER, UNLOAD_THRESHOLD_BYTES}; use crate::routing::errors::RouteError; @@ -22,7 +23,7 @@ pub(crate) fn should_unload(reader: &ReaderOffer, read_via: ReadVia) -> ResultSee every change first
02 · Load

Bulk copy, not row inserts

-

Files in S3 load into Snowflake straight from an external stage, and Parquet in S3 into Databricks through an external location. Parquet loads into DuckDB through DuckDB's own reader, and Parquet in S3 into MotherDuck on its servers. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Rows are staged as Parquet and copied in bulk.

+

Files in S3 load into Snowflake straight from an external stage, and Parquet in S3 into Databricks through an external location. Parquet loads into DuckDB through DuckDB's own reader, and Parquet in S3 into MotherDuck on its servers. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Snowflake tables unloaded to S3 load into DuckDB, Databricks and ClickHouse with no row passing through SQLLoad. Rows are staged as Parquet and copied in bulk.

Load methods and routes →

sqlload plan
diff --git a/website/src/content/docs/docs/concepts/cursors.mdx b/website/src/content/docs/docs/concepts/cursors.mdx
index 17c0a32a8..e557ae6dc 100644
--- a/website/src/content/docs/docs/concepts/cursors.mdx
+++ b/website/src/content/docs/docs/concepts/cursors.mdx
@@ -229,7 +229,7 @@ and `merge` publish once per run (last row below). `plan -v` of one table shows
 
 | Table | Commits |
 |-------|---------|
-| `batch_size = "1d"` (date or timestamp cursor) or `batch_size = 1000` (integer cursor) | One cursor window at a time, up to 100,000 windows. |
+| `batch_size = "1d"` (date or timestamp cursor) or `batch_size = 1000` (integer cursor) | One cursor window at a time, up to 100,000 windows (32,767 for a Snowflake table read through an unload). |
 | A Postgres table whose `cursor_column` leads an index, without `batch_size` | About every 1 GB, reading in cursor order (NULLs first), only between distinct cursor values. `batch_size = "none"` opts out. |
 | Other cursor tables, text cursors, and `append` without a cursor or logged state | Once per read, since committing part of an unordered read could skip or duplicate rows. |
 | Files sources | At 1,000 files or about 1 GB; under `delete_insert` a partition is never split. |
diff --git a/website/src/content/docs/docs/sources/snowflake.mdx b/website/src/content/docs/docs/sources/snowflake.mdx
index 907030a2c..1ad423a6f 100644
--- a/website/src/content/docs/docs/sources/snowflake.mdx
+++ b/website/src/content/docs/docs/sources/snowflake.mdx
@@ -159,6 +159,28 @@ whatever its session time zone. A load fails before it commits when DuckDB loads
 of rows than `COPY INTO` reports, and the folder is removed either way. A table without a cursor
 commits once per run rather than about every 1 GB.
 
+### Databricks and ClickHouse read the files themselves
+
+With a stage given by `url`, Databricks and ClickHouse targets read the unloaded files from S3
+themselves too, so no rows pass through SQLLoad:
+
+```text
+route   unload Parquet to s3://example-bucket/unload/ -> Databricks COPY INTO (nothing through the loader)
+route   unload Parquet to s3://example-bucket/unload/ -> ClickHouse reads them (nothing through the loader)
+```
+
+- **Databricks** `COPY`s the files through the Unity Catalog external location that covers the
+  `url`, as for [Parquet files in S3](/docs/targets/databricks/#parquet-files-in-s3). A `url` no
+  external location covers fails `plan` with the fix; the files never go through SQLLoad instead.
+  Each batch lands in a staging or build table, an `append` too, and publishes only when the
+  `COPY`s load as many rows as `COPY INTO` reported unloading.
+- **ClickHouse** reads each file with `s3()` into the load's staging table, signed with the same
+  S3 credentials as SQLLoad, and publishes only when it loads as many rows as were unloaded.
+
+Timestamps come back exactly from their whole microseconds, whatever the session's time zone. A
+table with a `TIME` column, or a `mask`, goes through SQLLoad into either target, and `plan` says
+why: neither stores a time of day as a time.
+
 ### Snowflake loads the rows itself
 
 Into a Snowflake target, no rows pass through SQLLoad in two cases:
diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx
index 63e6910a1..9fd512d30 100644
--- a/website/src/content/docs/docs/supported-systems.mdx
+++ b/website/src/content/docs/docs/supported-systems.mdx
@@ -9,7 +9,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes
 |--------|---------|---------|
 | Postgres | Tables, by `include` patterns | Binary `COPY` into Arrow, over TLS when the connection's `sslmode` asks |
 | Postgres changes (`postgres_cdc`) | Inserts, updates and deletes of tables, by `include` patterns | A snapshot, then a logical replication slot (`pgoutput`), in whole transactions. See [Postgres changes](/docs/sources/postgres-cdc/). |
-| Snowflake | Tables and views, by `include` patterns, and query results | Parquet unloaded with `COPY INTO` to the user stage or an S3 stage, then read by SQLLoad, or by DuckDB itself from an S3 stage. A Snowflake target selects or copies the rows inside Snowflake when it writes through the source's connection, or its external stage covers the unload; see [Snowflake loads the rows itself](/docs/sources/snowflake/#snowflake-loads-the-rows-itself). See [Snowflake](/docs/sources/snowflake/). |
+| Snowflake | Tables and views, by `include` patterns, and query results | Parquet unloaded with `COPY INTO` to the user stage or an S3 stage, then read by SQLLoad, or by DuckDB, Databricks or ClickHouse themselves from an S3 stage. A Snowflake target selects or copies the rows inside Snowflake when it writes through the source's connection, or its external stage covers the unload; see [Snowflake loads the rows itself](/docs/sources/snowflake/#snowflake-loads-the-rows-itself). See [Snowflake](/docs/sources/snowflake/). |
 | Databases with an ODBC driver: SQL Server, MySQL, MariaDB, OpenEdge and others | Tables and views, by `include` patterns, and query results | The driver, as Arrow. Tested with psqlODBC, Microsoft ODBC Driver 18 for SQL Server and MySQL Connector/ODBC; MariaDB Connector/ODBC and OpenEdge are checked by hand. See [ODBC](/docs/sources/odbc/#drivers). |
 | BigQuery (beta) | Tables and views, by `include` patterns, and query results | The Storage Read API, as Arrow in parallel streams; views and queries through a query job. See [BigQuery](/docs/sources/bigquery/). |
 | Databricks (beta) | Tables and views, by `include` patterns, and query results | One `SELECT` on the SQL warehouse, its result downloaded as Arrow files in parallel through the Statement Execution API's links. See [Databricks](/docs/sources/databricks/). |
diff --git a/website/src/content/docs/docs/targets/clickhouse.mdx b/website/src/content/docs/docs/targets/clickhouse.mdx
index 956bea850..ee0e224be 100644
--- a/website/src/content/docs/docs/targets/clickhouse.mdx
+++ b/website/src/content/docs/docs/targets/clickhouse.mdx
@@ -116,6 +116,11 @@ SQLLoad reads the files itself and sends Arrow instead when ClickHouse wouldn't
 - times of day, which ClickHouse reads from Parquet as timestamps;
 - a path that reads as a glob.
 
+A Snowflake source with a stage `url` in S3 skips SQLLoad the same way: ClickHouse reads the files
+Snowflake unloads, one `s3()` insert per file, and the batch publishes only when it loads as many
+rows as Snowflake reported unloading. See
+[Databricks and ClickHouse read the files themselves](/docs/sources/snowflake/#databricks-and-clickhouse-read-the-files-themselves).
+
 ## Write strategies
 
 | Strategy | Publishes |
diff --git a/website/src/content/docs/docs/targets/databricks.mdx b/website/src/content/docs/docs/targets/databricks.mdx
index efe8be369..942ced613 100644
--- a/website/src/content/docs/docs/targets/databricks.mdx
+++ b/website/src/content/docs/docs/targets/databricks.mdx
@@ -89,6 +89,12 @@ list, and files on an S3-compatible endpoint (set on the connection or in `AWS_E
 folder or over SFTP always go through SQLLoad. `copy_to_internal_stage` is for Snowflake and is
 refused here.
 
+A Snowflake source with a stage `url` in S3 that an external location covers loads the same way:
+Databricks `COPY`s the files Snowflake unloads, into a staging or build table even for `append`,
+and the batch publishes only when it loads as many rows as Snowflake reported unloading. A `url`
+no external location covers fails `plan`. See
+[Databricks and ClickHouse read the files themselves](/docs/sources/snowflake/#databricks-and-clickhouse-read-the-files-themselves).
+
 ## Write strategies
 
 Databricks' Statement Execution API runs every statement in its own transaction, so each write is
diff --git a/website/src/content/docs/docs/targets/index.mdx b/website/src/content/docs/docs/targets/index.mdx
index 35da076b7..b28b9fd0a 100644
--- a/website/src/content/docs/docs/targets/index.mdx
+++ b/website/src/content/docs/docs/targets/index.mdx
@@ -23,6 +23,8 @@ names the route for every table.
 A Snowflake source loading into Snowflake stays inside Snowflake: an `INSERT ... SELECT` on the
 source's own connection, or a `COPY` of its unloaded files through the target's external stage.
 See [Snowflake loads the rows itself](/docs/sources/snowflake/#snowflake-loads-the-rows-itself).
+Into DuckDB, Databricks and ClickHouse, a Snowflake source unloaded to an S3 `url` is read by the
+target itself; see [Databricks and ClickHouse read the files themselves](/docs/sources/snowflake/#databricks-and-clickhouse-read-the-files-themselves).
 
 There are no silent fallbacks. If the route a table needs can't be used, `plan` fails for that
 table and says how to fix it.