From 5e0fa3d3de29ec8089293972942336e62fad0296 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:48:04 +0000 Subject: [PATCH 1/9] wip: Snowflake COPYs a Snowflake source's unloaded files from its external stage Incomplete: the cross-account COPY of unloaded files and its SQL test. Not yet run against Snowflake; the same-connection route is not started. --- .../src/adapters/contract/constants.rs | 5 + .../dispatch/main/route/unload_blocked.rs | 8 +- .../dispatch/main/write/load_unloaded.rs | 3 +- .../src/adapters/duckdb/_helpers/pull_sql.rs | 3 +- .../sqlload/src/adapters/duckdb/constants.rs | 4 - .../snowflake/_helpers/write/direct_load.rs | 2 +- .../adapters/snowflake/_helpers/write/mod.rs | 1 + .../snowflake/_helpers/write/unloaded_load.rs | 109 +++++++++++++++ .../snowflake/main/copy/_copy_unloaded.rs | 124 ++++++++++++++++++ .../src/adapters/snowflake/main/copy/mod.rs | 1 + .../sqlload/src/adapters/snowflake/models.rs | 25 ++++ .../sqlload/src/adapters/snowflake/tests.rs | 2 + .../src/adapters/snowflake/tests/helpers.rs | 53 ++++++++ .../snowflake/tests/test_copy_unloaded.rs | 91 +++++++++++++ .../adapters/snowflake/tests/test_types.rs | 14 ++ 15 files changed, 434 insertions(+), 11 deletions(-) create mode 100644 crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs create mode 100644 crates/sqlload/src/adapters/snowflake/main/copy/_copy_unloaded.rs create mode 100644 crates/sqlload/src/adapters/snowflake/tests/test_copy_unloaded.rs diff --git a/crates/sqlload/src/adapters/contract/constants.rs b/crates/sqlload/src/adapters/contract/constants.rs index bbd271978..83681d130 100644 --- a/crates/sqlload/src/adapters/contract/constants.rs +++ b/crates/sqlload/src/adapters/contract/constants.rs @@ -39,3 +39,8 @@ pub const COMMIT_STATEMENT: &str = "COMMIT"; /// Ends a transaction, undoing it. pub const ROLLBACK_STATEMENT: &str = "ROLLBACK"; + +/// The `_sqlload_row` span of one unloaded batch (2^48): its files' rows number from the batch times this. +pub const UNLOADED_BATCH_ROWS: i64 = 1 << 48; +/// The `_sqlload_row` span of one unloaded file in a batch (2^32). +pub const UNLOADED_FILE_ROWS: i64 = 1 << 32; diff --git a/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs b/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs index 13ab26c19..3a9be50b7 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs @@ -4,11 +4,13 @@ use crate::column_types::models::Warehouse; use crate::loader::constants::UNLOADED_TIMES; use crate::loader::models::TableSpec; -/// Why the destination cannot load the table's unloaded files exactly by reading them itself, so SQLLoad reads them instead; `None` when it can (where it also reaches their store). Only DuckDB turns the files' whole microseconds back into times and timestamps; masks need SQLLoad to see the rows. +/// Why the destination cannot load the table's unloaded files exactly by reading them itself, so SQLLoad reads them instead; `None` when it can (where it also reaches their store). Only DuckDB and Snowflake turn the files' whole microseconds back into times and timestamps; masks need SQLLoad to see the rows. pub fn unload_blocked(spec: &TableSpec) -> Option { match (spec.warehouse, spec.masked.columns.is_empty()) { - (Warehouse::DuckDb, true) => None, - (Warehouse::DuckDb, false) => Some("the table masks columns".to_owned()), + (Warehouse::DuckDb | Warehouse::Snowflake, true) => None, + (Warehouse::DuckDb | Warehouse::Snowflake, false) => { + Some("the table masks columns".to_owned()) + } _ => Some(UNLOADED_TIMES.to_owned()), } } diff --git a/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs b/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs index aa3f547b4..6ac0235d0 100644 --- a/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs +++ b/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs @@ -9,11 +9,10 @@ pub async fn load_unloaded( writer: &mut Writer, batch: &UnloadedBatch<'_>, ) -> Result, LoaderError> { - #[cfg(not(feature = "duckdb"))] - let _ = batch; match writer { #[cfg(feature = "duckdb")] Writer::DuckDb(duckdb) => duckdb.pull_unloaded(batch).await.map(Some), + Writer::Snowflake(snowflake) => snowflake.pull_unloaded(batch).await.map(Some), _ => Ok(None), } } diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs b/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs index 44a650294..0ef1f5f84 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs @@ -1,8 +1,9 @@ //! The statement that has DuckDB load Parquet files itself. +use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS}; use crate::adapters::contract::models::TableName; use crate::adapters::contract::types::SqlAdapter; -use crate::adapters::duckdb::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS, UTC_OFFSET}; +use crate::adapters::duckdb::constants::UTC_OFFSET; use crate::adapters::duckdb::models::DuckDbSql; use crate::column_types::main::is_variant::is_variant; use crate::column_types::models::ColumnType; diff --git a/crates/sqlload/src/adapters/duckdb/constants.rs b/crates/sqlload/src/adapters/duckdb/constants.rs index 16b8a15da..ce049936c 100644 --- a/crates/sqlload/src/adapters/duckdb/constants.rs +++ b/crates/sqlload/src/adapters/duckdb/constants.rs @@ -12,10 +12,6 @@ pub const MOTHERDUCK_TOKEN_VARIABLES: &[&str] = &["motherduck_token", "MOTHERDUC pub const MOTHERDUCK_UNSUPPORTED_DUCKDB: &str = "is not yet supported by MotherDuck"; /// What precedes the extension's own message in DuckDB's error. pub const EXTENSION_EXCEPTION: &str = "threw an exception: "; -/// The `_sqlload_row` span of one unloaded batch (2^48): its files' rows number from the batch times this. -pub const UNLOADED_BATCH_ROWS: i64 = 1 << 48; -/// The `_sqlload_row` span of one unloaded file in a batch (2^32). -pub const UNLOADED_FILE_ROWS: i64 = 1 << 32; /// MotherDuck's server version, such as `v1.5.6-2026-11-382`. pub const MOTHERDUCK_VERSION_QUERY: &str = "SELECT md_version()"; /// The first DuckDB major version that shreds a VARIANT column's common fields into columns, so queries on them are fast. diff --git a/crates/sqlload/src/adapters/snowflake/_helpers/write/direct_load.rs b/crates/sqlload/src/adapters/snowflake/_helpers/write/direct_load.rs index c33bbc020..c5301baf1 100644 --- a/crates/sqlload/src/adapters/snowflake/_helpers/write/direct_load.rs +++ b/crates/sqlload/src/adapters/snowflake/_helpers/write/direct_load.rs @@ -375,7 +375,7 @@ fn held_types(columns: &[DesiredColumn]) -> Vec { } /// The columns a staging table holds, as the staged rows carry them: JSON and UUIDs as text; also the columns that write a table of `held_types` on with no cast. -fn staged_as_text(columns: &[DesiredColumn]) -> Vec { +pub(in crate::adapters) fn staged_as_text(columns: &[DesiredColumn]) -> Vec { columns .iter() .cloned() diff --git a/crates/sqlload/src/adapters/snowflake/_helpers/write/mod.rs b/crates/sqlload/src/adapters/snowflake/_helpers/write/mod.rs index 97c6c530e..c27f7acfe 100644 --- a/crates/sqlload/src/adapters/snowflake/_helpers/write/mod.rs +++ b/crates/sqlload/src/adapters/snowflake/_helpers/write/mod.rs @@ -4,3 +4,4 @@ pub(crate) mod file_keys; pub(crate) mod row_stream; pub(crate) mod rows_loaded; pub(crate) mod table_writer; +pub(crate) mod unloaded_load; diff --git a/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs b/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs new file mode 100644 index 000000000..893b3e556 --- /dev/null +++ b/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs @@ -0,0 +1,109 @@ +//! A Snowflake source's unloaded files loaded by Snowflake `COPY`ing them itself. + +use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch}; +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::snowflake::_helpers::write::direct_load::staged_as_text; +use crate::adapters::snowflake::main::copy::_copy_unloaded::copy_unloaded; +use crate::adapters::snowflake::main::copy::_stage_covers::stage_covers; +use crate::adapters::snowflake::main::stage::_stage_reference::stage_reference; +use crate::adapters::snowflake::models::{SnowflakeSql, SnowflakeWriter, StageRoute, UnloadedCopy}; +use crate::loader::errors::LoaderError; +use crate::sql_api::main::execute_sql::execute_sql; + +impl SnowflakeWriter { + /// Load `batch` by having Snowflake `COPY` the unloaded files from the external stage it loads through, which covers them, where Arrow batches would go: the load's staging table, the build table, or the table itself. The `COPY`s run in one block, in one transaction, that commits only when they load as many rows as the unload reports; otherwise it rolls back and the batch fails. + pub(in crate::adapters) async fn pull_unloaded( + &mut self, + batch: &UnloadedBatch<'_>, + ) -> Result { + let target = &batch.target; + let loaded = |rows: u64| match target.direct { + true => DirectLoaded::Committed(rows), + false => DirectLoaded::Staged(rows), + }; + let files = batch.files; + if files.keys.is_empty() { + return Ok(loaded(0)); + } + let covering = match &self.route { + Some(StageRoute::External(stage)) if stage_covers(&stage.location, &files.folder) => { + files + .folder + .prefix + .strip_prefix(&stage.location.prefix) + .map(|folder| (stage_reference(stage), folder.to_owned())) + } + _ => None, + }; + let Some((stage, folder)) = covering else { + return Err(LoaderError::Invalid(format!( + "Snowflake cannot read the unloaded files in {}: the target's external stage does not cover them", + files.folder + ))); + }; + let staged; + let columns = match (target.staging, target.builds) { + (Some(_), false) => { + staged = staged_as_text(target.columns); + &staged + } + _ => batch.write_columns, + }; + let copies = copy_unloaded(&UnloadedCopy { + into: target.staging.unwrap_or(target.table), + columns, + read: batch.read, + constants: target.constants, + compat: target.compat, + batch: target.batch, + load_id: target.load_id, + loaded_at: batch.loaded_at, + stage: &stage, + folder: &folder, + keys: &files.keys, + filename_prefix: &files.folder.prefix, + }); + let block = checked_block(&copies, files.rows); + let answer = execute_sql(&self.destination.sql, &block).await?; + let returned = answer + .rows + .first() + .and_then(|row| row.first().cloned().flatten()) + .unwrap_or_default(); + let rows: u64 = returned.parse().map_err(|error| { + LoaderError::Invalid(format!( + "the unloaded files' COPY returned {returned:?}, not a row count: {error}" + )) + })?; + if rows != files.rows { + return Err(LoaderError::Invalid(format!( + "Snowflake loaded {rows} rows from the unloaded files in {}, but the unload reports {}; the batch is not committed", + files.folder, files.rows + ))); + } + Ok(loaded(rows)) + } +} + +/// One Snowflake Scripting block running `statements` (`COPY`s) in a transaction and returning the rows they loaded, committed only when that is `expected` and rolled back otherwise. +pub(in crate::adapters) fn checked_block(statements: &[String], expected: u64) -> String { + let mut body = vec![ + "DECLARE loaded INTEGER DEFAULT 0; copied INTEGER;".to_owned(), + "BEGIN".to_owned(), + "BEGIN TRANSACTION;".to_owned(), + ]; + for statement in statements { + body.push(format!("{statement};")); + body.push("SELECT COALESCE(SUM(\"rows_loaded\"), 0) INTO :copied FROM TABLE(RESULT_SCAN(LAST_QUERY_ID()));".to_owned()); + body.push("loaded := loaded + copied;".to_owned()); + } + body.push(format!( + "IF (loaded = {expected}) THEN COMMIT; ELSE ROLLBACK; END IF;" + )); + body.push("RETURN loaded;".to_owned()); + body.push("END;".to_owned()); + format!( + "EXECUTE IMMEDIATE {}", + SnowflakeSql.render_loader_value_literal(&body.join("\n")) + ) +} diff --git a/crates/sqlload/src/adapters/snowflake/main/copy/_copy_unloaded.rs b/crates/sqlload/src/adapters/snowflake/main/copy/_copy_unloaded.rs new file mode 100644 index 000000000..1e705d471 --- /dev/null +++ b/crates/sqlload/src/adapters/snowflake/main/copy/_copy_unloaded.rs @@ -0,0 +1,124 @@ +//! The statements loading a Snowflake source's unloaded Parquet files into a table, inside Snowflake. + +use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS}; +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::snowflake::constants::{COPY_FILE_FORMAT, MAX_COPY_FILES}; +use crate::adapters::snowflake::models::{SnowflakeSql, UnloadedCopy}; +use crate::column_types::models::ColumnType; +use crate::load_naming::main::metadata_role::metadata_role; +use crate::load_naming::models::MetadataRole; +use crate::schema_plan::constants::MERGE_ROW_COLUMN; +use crate::schema_plan::models::DesiredColumn; + +/// The `COPY`s loading the unload's files into `request.into`, every one of its columns filled as [`unloaded_value`] says; at most 1,000 files each. Each stops at the first bad value and never relies on Snowflake's load history. +pub(in crate::adapters) fn copy_unloaded(request: &UnloadedCopy<'_>) -> Vec { + let names: Vec = request + .columns + .iter() + .map(|column| SnowflakeSql.render_identifier(&column.name)) + .collect(); + request + .keys + .chunks(MAX_COPY_FILES) + .enumerate() + .map(|(chunk, keys)| { + let values: Vec = request + .columns + .iter() + .map(|column| column_value(request, column, (chunk * MAX_COPY_FILES, keys))) + .collect(); + let files: Vec = keys + .iter() + .map(|key| SnowflakeSql.render_loader_value_literal(key)) + .collect(); + format!( + "COPY INTO {} ({}) FROM (SELECT {} FROM {}/{}) FILES = ({}) {COPY_FILE_FORMAT} ON_ERROR = ABORT_STATEMENT FORCE = TRUE", + SnowflakeSql.render_qualified_name(request.into), + names.join(", "), + values.join(", "), + request.stage, + request.folder, + files.join(", ") + ) + }) + .collect() +} + +/// Where `column` comes from in the `COPY` of `keys`, the unload's files from the `first`th on: the batch's load id or load time, a merge's row number (the batch, then the file, then the row in it), the value of the constant it is, or the files' column it is read as; NULL when the files lack it. +fn column_value( + request: &UnloadedCopy<'_>, + column: &DesiredColumn, + (first, keys): (usize, &[String]), +) -> String { + let literal = |text: &str| SnowflakeSql.render_loader_value_literal(text); + let constant = request + .constants + .iter() + .find(|(name, _, _)| *name == column.name); + match (metadata_role(request.compat, &column.name), constant) { + (Some(MetadataRole::LoadId), _) => literal(request.load_id), + (Some(MetadataRole::LoadedAt), _) => { + format!("CAST({} AS TIMESTAMP_TZ(6))", literal(request.loaded_at)) + } + _ if column.name == MERGE_ROW_COLUMN => { + let files: Vec = keys + .iter() + .enumerate() + .map(|(index, key)| { + format!( + "WHEN {} THEN {}", + literal(&format!("{}{key}", request.filename_prefix)), + first + index + ) + }) + .collect(); + format!( + "{} * {UNLOADED_BATCH_ROWS} + (CASE METADATA$FILENAME {} END) * {UNLOADED_FILE_ROWS} + METADATA$FILE_ROW_NUMBER", + request.batch, + files.join(" ") + ) + } + (_, Some((_, _, value))) => cast(column, &literal(value)), + _ => request + .read + .iter() + .find(|read| read.name == column.name) + .map_or_else( + || "NULL".to_owned(), + |read| { + let field = format!("GET($1, {})", literal(&read.name)); + unloaded_value(column, &read.output, &field) + }, + ), + } +} + +/// An unload's `field`, read as `read`, as the value it was unloaded from, then as `column` holds it: a time or timestamp from its whole microseconds (a timestamp with time zone in UTC, as SQLLoad's reader gives it), JSON and arrays parsed from their text, anything else cast to its type. +pub(in crate::adapters) fn unloaded_value( + column: &DesiredColumn, + read: &ColumnType, + field: &str, +) -> String { + let micros = format!("CAST({field} AS NUMBER(38,0))"); + let value = match read { + ColumnType::TimestampNtz => format!("TO_TIMESTAMP_NTZ({micros}, 6)"), + ColumnType::TimestampTz => { + format!("CONVERT_TIMEZONE('UTC', TO_TIMESTAMP_TZ({micros}, 6))") + } + ColumnType::Time => format!("TIMEADD(MICROSECOND, {micros}, '00:00:00'::TIME)"), + _ => field.to_owned(), + }; + cast(column, &value) +} + +/// `value` as `column` holds it: JSON and arrays parsed from their text, anything else cast to its type. +fn cast(column: &DesiredColumn, value: &str) -> String { + match &column.column_type { + ColumnType::Json => format!("PARSE_JSON(CAST({value} AS VARCHAR))"), + ColumnType::Array(_) => format!("CAST(PARSE_JSON(CAST({value} AS VARCHAR)) AS ARRAY)"), + column_type => format!( + "CAST({value} AS {})", + SnowflakeSql.render_loader_logical_type(column_type).sql + ), + } +} diff --git a/crates/sqlload/src/adapters/snowflake/main/copy/mod.rs b/crates/sqlload/src/adapters/snowflake/main/copy/mod.rs index 1b2f7e790..ebf690dcb 100644 --- a/crates/sqlload/src/adapters/snowflake/main/copy/mod.rs +++ b/crates/sqlload/src/adapters/snowflake/main/copy/mod.rs @@ -1,5 +1,6 @@ pub(in crate::adapters) mod _copy_rows; pub(in crate::adapters) mod _copy_staged; +pub(in crate::adapters) mod _copy_unloaded; pub(in crate::adapters) mod _direct_check; pub(in crate::adapters) mod _direct_copies; pub(in crate::adapters) mod _direct_problem; diff --git a/crates/sqlload/src/adapters/snowflake/models.rs b/crates/sqlload/src/adapters/snowflake/models.rs index f0d462657..c885f9613 100644 --- a/crates/sqlload/src/adapters/snowflake/models.rs +++ b/crates/sqlload/src/adapters/snowflake/models.rs @@ -4,8 +4,10 @@ use crate::column_types::models::ColumnType; use crate::config::models::{AuthConfig, SnowflakeConfig}; use crate::file_store::models::OpenFolder; use crate::files_source::models::{DataFile, FilesEncoding, SourceFormat, StoredRead}; +use crate::load_naming::models::Compat; use crate::loader::errors::LoaderError; use crate::parquet_parts::models::PartWriter; +use crate::postgres_source::models::ReadColumn; use crate::project::models::MaskMethod; use crate::schema_plan::models::DesiredColumn; use crate::sql_api::models::SqlClient; @@ -111,6 +113,29 @@ pub struct CopyStaged<'a> { pub selected: Option<&'a [DesiredColumn]>, } +/// The `COPY`s of one batch of a Snowflake source's unloaded files: the table they fill and its columns, the files' read columns, the batch's own values, and where the files are. +pub struct UnloadedCopy<'a> { + pub into: &'a TableName, + pub columns: &'a [DesiredColumn], + pub read: &'a [ReadColumn], + /// The table's `constant_columns`: name, type and value as text. + pub constants: &'a [(String, ColumnType, String)], + pub compat: Compat, + /// The batch, from 1. + pub batch: u64, + pub load_id: &'a str, + /// The load time as an ISO 8601 UTC timestamp. + pub loaded_at: &'a str, + /// The stage as SQL, `@"DB"."SCHEMA"."STAGE"`. + pub stage: &'a str, + /// The unload's folder below the stage, ending in `/`. + pub folder: &'a str, + /// Each file's key below `folder`. + pub keys: &'a [String], + /// What `METADATA$FILENAME` starts each file's name with: the folder's bucket-relative key. + pub filename_prefix: &'a str, +} + /// Where one staged file goes on an internal stage. #[derive(Debug, Clone, Copy)] pub struct StagedFile<'a> { diff --git a/crates/sqlload/src/adapters/snowflake/tests.rs b/crates/sqlload/src/adapters/snowflake/tests.rs index 5b8f65c1e..6c1090bd5 100644 --- a/crates/sqlload/src/adapters/snowflake/tests.rs +++ b/crates/sqlload/src/adapters/snowflake/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/test_copy_staged.rs"] mod copy_staged; +#[path = "tests/test_copy_unloaded.rs"] +mod copy_unloaded; #[path = "tests/test_direct_copies.rs"] mod direct_copies; #[path = "tests/test_direct_problem.rs"] diff --git a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs index 6f4d668fc..005ed9dda 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs @@ -374,3 +374,56 @@ pub(crate) fn csv_in( ..FilesEncoding::default() } } + +/// The `COPY`s loading `files` unloaded files (`data_.parquet` under `lake/unload/run-1/`, on the stage's `unload/` folder) into `RAW.ORDERS`, batch 3 of load L1, with constant `region` = `'EU'` and the files' columns read as typed. +pub(crate) fn unloaded_copies( + columns: &[(&str, ColumnType, Option)], + files: usize, +) -> Vec { + use crate::adapters::snowflake::main::copy::_copy_unloaded::copy_unloaded; + use crate::adapters::snowflake::models::UnloadedCopy; + use crate::postgres_source::models::{ReadColumn, WireType}; + use crate::schema_plan::models::{ColumnOrigin, DesiredColumn}; + + let desired: Vec = columns + .iter() + .map(|(name, column_type, _)| DesiredColumn { + name: (*name).to_owned(), + column_type: column_type.clone(), + origin: ColumnOrigin::Source, + }) + .collect(); + let read: Vec = columns + .iter() + .filter_map(|(name, _, read)| { + Some(ReadColumn { + name: (*name).to_owned(), + expression: format!("\"{name}\""), + wire: WireType::Text, + output: read.clone()?, + }) + }) + .collect(); + let keys: Vec = (0..files) + .map(|file| format!("data_{file}.parquet")) + .collect(); + let table = TableName { + database: None, + schema: "raw".to_owned(), + table: "orders".to_owned(), + }; + copy_unloaded(&UnloadedCopy { + into: &table, + columns: &desired, + read: &read, + constants: &[("region".to_owned(), ColumnType::Text, "EU".to_owned())], + compat: crate::load_naming::models::Compat::Sqlload, + batch: 3, + load_id: "L1", + loaded_at: "2026-11-18T00:00:00Z", + stage: "@\"DB\".\"RAW\".\"LAKE\"", + folder: "unload/run-1/", + keys: &keys, + filename_prefix: "lake/unload/run-1/", + }) +} diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_copy_unloaded.rs b/crates/sqlload/src/adapters/snowflake/tests/test_copy_unloaded.rs new file mode 100644 index 000000000..2a91e791d --- /dev/null +++ b/crates/sqlload/src/adapters/snowflake/tests/test_copy_unloaded.rs @@ -0,0 +1,91 @@ +use crate::adapters::snowflake::tests::helpers::unloaded_copies; +use crate::adapters::snowflake::tests::test_types::CopyUnloadedTestCase; +use crate::column_types::models::ColumnType; + +const TYPED: &[(&str, ColumnType, Option)] = &[ + ("id", ColumnType::Int, Some(ColumnType::Int)), + ( + "seen_at", + ColumnType::TimestampNtz, + Some(ColumnType::TimestampNtz), + ), + ( + "placed_at", + ColumnType::TimestampTz, + Some(ColumnType::TimestampTz), + ), + ("opens", ColumnType::Time, Some(ColumnType::Time)), + ("meta", ColumnType::Json, Some(ColumnType::Json)), + ("region", ColumnType::Text, None), + ("_sqlload_load_id", ColumnType::Text, None), + ("_sqlload_loaded_at", ColumnType::TimestampTz, None), + ("gone", ColumnType::Text, None), +]; + +const NUMBERED: &[(&str, ColumnType, Option)] = &[ + ("id", ColumnType::Int, Some(ColumnType::Int)), + ("_sqlload_row", ColumnType::Int, None), +]; + +#[test] +fn given_a_batch_of_unloaded_files_when_copying_them_then_snowflake_turns_their_microseconds_back_and_fills_the_loads_columns() + { + let test_cases = [ + CopyUnloadedTestCase { + description: "times and timestamps from whole microseconds (a zoned one in UTC), JSON parsed, the constant, load id and load time as literals, a column the files lack NULL", + columns: TYPED, + files: 2, + expected_copies: 1, + expected_first: &[ + "COPY INTO \"RAW\".\"ORDERS\" (\"ID\", \"SEEN_AT\", \"PLACED_AT\", \"OPENS\", \"META\", \"REGION\", \"_SQLLOAD_LOAD_ID\", \"_SQLLOAD_LOADED_AT\", \"GONE\") FROM (SELECT ", + "CAST(GET($1, 'id') AS NUMBER(38,0)), ", + "CAST(TO_TIMESTAMP_NTZ(CAST(GET($1, 'seen_at') AS NUMBER(38,0)), 6) AS TIMESTAMP_NTZ(9)), ", + "CAST(CONVERT_TIMEZONE('UTC', TO_TIMESTAMP_TZ(CAST(GET($1, 'placed_at') AS NUMBER(38,0)), 6)) AS TIMESTAMP_TZ(9)), ", + "CAST(TIMEADD(MICROSECOND, CAST(GET($1, 'opens') AS NUMBER(38,0)), '00:00:00'::TIME) AS TIME(9)), ", + "PARSE_JSON(CAST(GET($1, 'meta') AS VARCHAR)), ", + "CAST('EU' AS VARCHAR), ", + "'L1', ", + "CAST('2026-11-18T00:00:00Z' AS TIMESTAMP_TZ(6)), ", + "NULL ", + "FROM @\"DB\".\"RAW\".\"LAKE\"/unload/run-1/) FILES = ('data_0.parquet', 'data_1.parquet') FILE_FORMAT = (TYPE = PARQUET USE_LOGICAL_TYPE = TRUE BINARY_AS_TEXT = FALSE) ON_ERROR = ABORT_STATEMENT FORCE = TRUE", + ], + expected_last: "FORCE = TRUE", + }, + CopyUnloadedTestCase { + description: "a merge numbers its rows by batch, then file, counting on across COPYs of 1,000 files, then row in the file", + columns: NUMBERED, + files: 1001, + expected_copies: 2, + expected_first: &[ + "3 * 281474976710656 + (CASE METADATA$FILENAME WHEN 'lake/unload/run-1/data_0.parquet' THEN 0 WHEN ", + "WHEN 'lake/unload/run-1/data_999.parquet' THEN 999 END) * 4294967296 + METADATA$FILE_ROW_NUMBER", + ], + expected_last: "(CASE METADATA$FILENAME WHEN 'lake/unload/run-1/data_1000.parquet' THEN 1000 END) * 4294967296 + METADATA$FILE_ROW_NUMBER FROM @\"DB\".\"RAW\".\"LAKE\"/unload/run-1/) FILES = ('data_1000.parquet')", + }, + ]; + for test_case in test_cases { + let copies = unloaded_copies(test_case.columns, test_case.files); + assert_eq!( + copies.len(), + test_case.expected_copies, + "{}", + test_case.description + ); + let (first, last) = (&copies[0], &copies[copies.len() - 1]); + let mut rest = first.as_str(); + for expected in test_case.expected_first { + let at = rest.find(expected); + assert!( + at.is_some(), + "{}: {expected:?} not in order in {first}", + test_case.description + ); + rest = at.map_or("", |at| &rest[at + expected.len()..]); + } + assert!( + last.contains(test_case.expected_last), + "{}: {last}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs index 5fb8f6ea2..7e9d3f5aa 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs @@ -1,4 +1,5 @@ use crate::adapters::snowflake::models::CopyColumn; +use crate::column_types::models::ColumnType; use crate::files_source::models::SourceFormat; use crate::project::models::MaskMethod; @@ -105,3 +106,16 @@ pub(crate) struct SnowflakeEncodingTestCase { /// Snowflake's `ENCODING` for it, or `None` when the direct route refuses it. pub(crate) expected_encoding: Option<&'static str>, } + +/// The `COPY`s of one batch of unloaded files. +pub(crate) struct CopyUnloadedTestCase { + pub(crate) description: &'static str, + /// The table's columns, each with the type the files' column is read as (`None` when the files lack it). + pub(crate) columns: &'static [(&'static str, ColumnType, Option)], + pub(crate) files: usize, + pub(crate) expected_copies: usize, + /// Text the first `COPY` holds, in order. + pub(crate) expected_first: &'static [&'static str], + /// Text the last `COPY` holds. + pub(crate) expected_last: &'static str, +} From 427782148b6fd20ccd58e4c345c20ad91d0bac3b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:06:16 +0000 Subject: [PATCH 2/9] wip: Snowflake selects a Snowflake source's rows itself on the same connection The destination loads a Snowflake source's rows inside Snowflake: an INSERT ... SELECT of the unload's query when the target writes through the source's own connection and database, otherwise COPYs of the unloaded files from the target's external stage, committed only when their rows match the unload's. Not yet run against Snowflake. --- .../sqlload/src/adapters/contract/models.rs | 15 +- .../adapters/duckdb/_helpers/unloaded_load.rs | 8 +- .../src/adapters/duckdb/tests/helpers.rs | 2 +- .../snowflake/_helpers/write/unloaded_load.rs | 99 +++++++----- .../snowflake/main/copy/_copy_unloaded.rs | 124 -------------- .../snowflake/main/copy/_unloaded_sql.rs | 151 ++++++++++++++++++ .../src/adapters/snowflake/main/copy/mod.rs | 2 +- .../sqlload/src/adapters/snowflake/models.rs | 30 ++-- .../sqlload/src/adapters/snowflake/tests.rs | 4 +- .../src/adapters/snowflake/tests/helpers.rs | 27 ++-- .../adapters/snowflake/tests/test_types.rs | 13 +- ..._copy_unloaded.rs => test_unloaded_sql.rs} | 47 ++++-- crates/sqlload/src/loader/_helpers/publish.rs | 36 +++-- crates/sqlload/src/loader/_helpers/render.rs | 3 + crates/sqlload/src/loader/_helpers/resolve.rs | 8 + crates/sqlload/src/loader/_helpers/sources.rs | 10 ++ crates/sqlload/src/loader/_helpers/specs.rs | 17 +- .../sqlload/src/loader/_helpers/table_plan.rs | 19 ++- crates/sqlload/src/loader/models.rs | 8 +- .../src/routing/_helpers/warehouse_source.rs | 10 +- crates/sqlload/src/routing/models.rs | 5 + .../src/routing/tests/test_plan_route.rs | 26 +++ .../sqlload/src/snowflake_source/main/mod.rs | 1 + .../src/snowflake_source/main/select_query.rs | 10 ++ 24 files changed, 438 insertions(+), 237 deletions(-) delete mode 100644 crates/sqlload/src/adapters/snowflake/main/copy/_copy_unloaded.rs create mode 100644 crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs rename crates/sqlload/src/adapters/snowflake/tests/{test_copy_unloaded.rs => test_unloaded_sql.rs} (67%) create mode 100644 crates/sqlload/src/snowflake_source/main/select_query.rs diff --git a/crates/sqlload/src/adapters/contract/models.rs b/crates/sqlload/src/adapters/contract/models.rs index 124e4af96..0ac0aa204 100644 --- a/crates/sqlload/src/adapters/contract/models.rs +++ b/crates/sqlload/src/adapters/contract/models.rs @@ -283,11 +283,11 @@ pub struct DirectBatch<'a> { pub source_folder: Option<&'a OpenFolder>, } -/// One unload's files the destination loads by reading them itself: times and timestamps held as whole microseconds, every other column as the type it is read as. +/// A warehouse source's rows the destination loads itself, without SQLLoad seeing them: times and timestamps held as whole microseconds, every other column as the type it is read as. #[derive(Debug, Clone, Copy)] pub struct UnloadedBatch<'a> { - pub files: &'a UnloadedFiles, - /// The columns the files hold, as they are read. + pub source: UnloadedSource<'a>, + /// The columns the rows hold, as they are read. pub read: &'a [ReadColumn], /// The table's columns as the load writes them. pub write_columns: &'a [DesiredColumn], @@ -296,6 +296,15 @@ pub struct UnloadedBatch<'a> { pub loaded_at: &'a str, } +/// Where the destination reads an unloaded batch's rows. +#[derive(Debug, Clone, Copy)] +pub enum UnloadedSource<'a> { + /// One unload's files, which it reads itself. + Files(&'a UnloadedFiles), + /// The unload's query, which it runs itself in the source's own session. + Query(&'a str), +} + /// A query a check before publish runs on a relation; each returns its values as text. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CheckQuery<'a> { diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/unloaded_load.rs b/crates/sqlload/src/adapters/duckdb/_helpers/unloaded_load.rs index 195905574..feabf3188 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/unloaded_load.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/unloaded_load.rs @@ -1,7 +1,7 @@ //! A warehouse source's unloaded files loaded by DuckDB reading them itself. use crate::adapters::contract::constants::{BEGIN_STATEMENT, COMMIT_STATEMENT}; -use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch}; +use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch, UnloadedSource}; use crate::adapters::duckdb::_helpers::pull_sql::{UnloadedColumns, unloaded_insert}; use crate::adapters::duckdb::main::_execute_statements::execute_statements; use crate::adapters::duckdb::main::_pull_files::pull_files; @@ -19,7 +19,11 @@ impl DuckDbWriter { true => DirectLoaded::Committed(rows), false => DirectLoaded::Staged(rows), }; - let files = batch.files; + let UnloadedSource::Files(files) = batch.source else { + return Err(LoaderError::Invalid( + "DuckDB reads a warehouse source's rows only from its unloaded files".to_owned(), + )); + }; if files.keys.is_empty() { return Ok(loaded(0)); } diff --git a/crates/sqlload/src/adapters/duckdb/tests/helpers.rs b/crates/sqlload/src/adapters/duckdb/tests/helpers.rs index 3a8120f5c..2515c4633 100644 --- a/crates/sqlload/src/adapters/duckdb/tests/helpers.rs +++ b/crates/sqlload/src/adapters/duckdb/tests/helpers.rs @@ -155,7 +155,7 @@ pub(crate) fn pulled_unloaded(reported: u64, with_files: bool) -> Result<(String table: "loaded".to_owned(), }; let batch = UnloadedBatch { - files: &files, + source: crate::adapters::contract::models::UnloadedSource::Files(&files), read: &read, write_columns: &columns, target: WriteTarget { diff --git a/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs b/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs index 893b3e556..53280c7bd 100644 --- a/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs +++ b/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs @@ -1,17 +1,19 @@ -//! A Snowflake source's unloaded files loaded by Snowflake `COPY`ing them itself. +//! A Snowflake source's rows loaded by Snowflake itself: `COPY`ing its unloaded files, or selecting them in the source's own session. -use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch}; +use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch, UnloadedSource}; use crate::adapters::contract::types::SqlAdapter; use crate::adapters::snowflake::_helpers::write::direct_load::staged_as_text; -use crate::adapters::snowflake::main::copy::_copy_unloaded::copy_unloaded; use crate::adapters::snowflake::main::copy::_stage_covers::stage_covers; +use crate::adapters::snowflake::main::copy::_unloaded_sql::unloaded_statements; use crate::adapters::snowflake::main::stage::_stage_reference::stage_reference; -use crate::adapters::snowflake::models::{SnowflakeSql, SnowflakeWriter, StageRoute, UnloadedCopy}; +use crate::adapters::snowflake::models::{ + SnowflakeSql, SnowflakeWriter, StageRoute, UnloadedFrom, UnloadedLoad, +}; use crate::loader::errors::LoaderError; use crate::sql_api::main::execute_sql::execute_sql; impl SnowflakeWriter { - /// Load `batch` by having Snowflake `COPY` the unloaded files from the external stage it loads through, which covers them, where Arrow batches would go: the load's staging table, the build table, or the table itself. The `COPY`s run in one block, in one transaction, that commits only when they load as many rows as the unload reports; otherwise it rolls back and the batch fails. + /// Load `batch` inside Snowflake, where Arrow batches would go: the load's staging table, the build table, or the table itself. Unloaded files are `COPY`d from the external stage the writer loads through, which must cover them, in one transaction that commits only when they load as many rows as the unload reports; a query's rows are selected by one `INSERT ... SELECT`. pub(in crate::adapters) async fn pull_unloaded( &mut self, batch: &UnloadedBatch<'_>, @@ -21,26 +23,6 @@ impl SnowflakeWriter { true => DirectLoaded::Committed(rows), false => DirectLoaded::Staged(rows), }; - let files = batch.files; - if files.keys.is_empty() { - return Ok(loaded(0)); - } - let covering = match &self.route { - Some(StageRoute::External(stage)) if stage_covers(&stage.location, &files.folder) => { - files - .folder - .prefix - .strip_prefix(&stage.location.prefix) - .map(|folder| (stage_reference(stage), folder.to_owned())) - } - _ => None, - }; - let Some((stage, folder)) = covering else { - return Err(LoaderError::Invalid(format!( - "Snowflake cannot read the unloaded files in {}: the target's external stage does not cover them", - files.folder - ))); - }; let staged; let columns = match (target.staging, target.builds) { (Some(_), false) => { @@ -49,7 +31,42 @@ impl SnowflakeWriter { } _ => batch.write_columns, }; - let copies = copy_unloaded(&UnloadedCopy { + let place: (String, String); + let (from, unloaded) = match batch.source { + UnloadedSource::Files(files) if files.keys.is_empty() => return Ok(loaded(0)), + UnloadedSource::Files(files) => { + let covering = match &self.route { + Some(StageRoute::External(stage)) + if stage_covers(&stage.location, &files.folder) => + { + files + .folder + .prefix + .strip_prefix(&stage.location.prefix) + .map(|folder| (stage_reference(stage), folder.to_owned())) + } + _ => None, + }; + let Some(covering) = covering else { + return Err(LoaderError::Invalid(format!( + "Snowflake cannot read the unloaded files in {}: the target's external stage does not cover them", + files.folder + ))); + }; + place = covering; + ( + UnloadedFrom::Files { + stage: &place.0, + folder: &place.1, + keys: &files.keys, + filename_prefix: &files.folder.prefix, + }, + Some(files), + ) + } + UnloadedSource::Query(query) => (UnloadedFrom::Query(query), None), + }; + let statements = unloaded_statements(&UnloadedLoad { into: target.staging.unwrap_or(target.table), columns, read: batch.read, @@ -58,13 +75,11 @@ impl SnowflakeWriter { batch: target.batch, load_id: target.load_id, loaded_at: batch.loaded_at, - stage: &stage, - folder: &folder, - keys: &files.keys, - filename_prefix: &files.folder.prefix, + from, }); - let block = checked_block(&copies, files.rows); - let answer = execute_sql(&self.destination.sql, &block).await?; + let expected = unloaded.map(|files| files.rows); + let answer = + execute_sql(&self.destination.sql, &counted_block(&statements, expected)).await?; let returned = answer .rows .first() @@ -72,10 +87,10 @@ impl SnowflakeWriter { .unwrap_or_default(); let rows: u64 = returned.parse().map_err(|error| { LoaderError::Invalid(format!( - "the unloaded files' COPY returned {returned:?}, not a row count: {error}" + "loading the source's rows inside Snowflake returned {returned:?}, not a row count: {error}" )) })?; - if rows != files.rows { + if let Some(files) = unloaded.filter(|files| files.rows != rows) { return Err(LoaderError::Invalid(format!( "Snowflake loaded {rows} rows from the unloaded files in {}, but the unload reports {}; the batch is not committed", files.folder, files.rows @@ -85,8 +100,8 @@ impl SnowflakeWriter { } } -/// One Snowflake Scripting block running `statements` (`COPY`s) in a transaction and returning the rows they loaded, committed only when that is `expected` and rolled back otherwise. -pub(in crate::adapters) fn checked_block(statements: &[String], expected: u64) -> String { +/// One Snowflake Scripting block running `statements` in a transaction and returning the rows they loaded: `COPY`s, which commit only when they load `expected` rows and roll back otherwise, or an `INSERT ... SELECT` (`expected` `None`), which commits. +pub(in crate::adapters) fn counted_block(statements: &[String], expected: Option) -> String { let mut body = vec![ "DECLARE loaded INTEGER DEFAULT 0; copied INTEGER;".to_owned(), "BEGIN".to_owned(), @@ -94,12 +109,16 @@ pub(in crate::adapters) fn checked_block(statements: &[String], expected: u64) - ]; for statement in statements { body.push(format!("{statement};")); - body.push("SELECT COALESCE(SUM(\"rows_loaded\"), 0) INTO :copied FROM TABLE(RESULT_SCAN(LAST_QUERY_ID()));".to_owned()); + body.push(match expected { + Some(_) => "SELECT COALESCE(SUM(\"rows_loaded\"), 0) INTO :copied FROM TABLE(RESULT_SCAN(LAST_QUERY_ID()));".to_owned(), + None => "copied := SQLROWCOUNT;".to_owned(), + }); body.push("loaded := loaded + copied;".to_owned()); } - body.push(format!( - "IF (loaded = {expected}) THEN COMMIT; ELSE ROLLBACK; END IF;" - )); + body.push(match expected { + Some(expected) => format!("IF (loaded = {expected}) THEN COMMIT; ELSE ROLLBACK; END IF;"), + None => "COMMIT;".to_owned(), + }); body.push("RETURN loaded;".to_owned()); body.push("END;".to_owned()); format!( diff --git a/crates/sqlload/src/adapters/snowflake/main/copy/_copy_unloaded.rs b/crates/sqlload/src/adapters/snowflake/main/copy/_copy_unloaded.rs deleted file mode 100644 index 1e705d471..000000000 --- a/crates/sqlload/src/adapters/snowflake/main/copy/_copy_unloaded.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! The statements loading a Snowflake source's unloaded Parquet files into a table, inside Snowflake. - -use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS}; -use crate::adapters::contract::types::SqlAdapter; -use crate::adapters::snowflake::constants::{COPY_FILE_FORMAT, MAX_COPY_FILES}; -use crate::adapters::snowflake::models::{SnowflakeSql, UnloadedCopy}; -use crate::column_types::models::ColumnType; -use crate::load_naming::main::metadata_role::metadata_role; -use crate::load_naming::models::MetadataRole; -use crate::schema_plan::constants::MERGE_ROW_COLUMN; -use crate::schema_plan::models::DesiredColumn; - -/// The `COPY`s loading the unload's files into `request.into`, every one of its columns filled as [`unloaded_value`] says; at most 1,000 files each. Each stops at the first bad value and never relies on Snowflake's load history. -pub(in crate::adapters) fn copy_unloaded(request: &UnloadedCopy<'_>) -> Vec { - let names: Vec = request - .columns - .iter() - .map(|column| SnowflakeSql.render_identifier(&column.name)) - .collect(); - request - .keys - .chunks(MAX_COPY_FILES) - .enumerate() - .map(|(chunk, keys)| { - let values: Vec = request - .columns - .iter() - .map(|column| column_value(request, column, (chunk * MAX_COPY_FILES, keys))) - .collect(); - let files: Vec = keys - .iter() - .map(|key| SnowflakeSql.render_loader_value_literal(key)) - .collect(); - format!( - "COPY INTO {} ({}) FROM (SELECT {} FROM {}/{}) FILES = ({}) {COPY_FILE_FORMAT} ON_ERROR = ABORT_STATEMENT FORCE = TRUE", - SnowflakeSql.render_qualified_name(request.into), - names.join(", "), - values.join(", "), - request.stage, - request.folder, - files.join(", ") - ) - }) - .collect() -} - -/// Where `column` comes from in the `COPY` of `keys`, the unload's files from the `first`th on: the batch's load id or load time, a merge's row number (the batch, then the file, then the row in it), the value of the constant it is, or the files' column it is read as; NULL when the files lack it. -fn column_value( - request: &UnloadedCopy<'_>, - column: &DesiredColumn, - (first, keys): (usize, &[String]), -) -> String { - let literal = |text: &str| SnowflakeSql.render_loader_value_literal(text); - let constant = request - .constants - .iter() - .find(|(name, _, _)| *name == column.name); - match (metadata_role(request.compat, &column.name), constant) { - (Some(MetadataRole::LoadId), _) => literal(request.load_id), - (Some(MetadataRole::LoadedAt), _) => { - format!("CAST({} AS TIMESTAMP_TZ(6))", literal(request.loaded_at)) - } - _ if column.name == MERGE_ROW_COLUMN => { - let files: Vec = keys - .iter() - .enumerate() - .map(|(index, key)| { - format!( - "WHEN {} THEN {}", - literal(&format!("{}{key}", request.filename_prefix)), - first + index - ) - }) - .collect(); - format!( - "{} * {UNLOADED_BATCH_ROWS} + (CASE METADATA$FILENAME {} END) * {UNLOADED_FILE_ROWS} + METADATA$FILE_ROW_NUMBER", - request.batch, - files.join(" ") - ) - } - (_, Some((_, _, value))) => cast(column, &literal(value)), - _ => request - .read - .iter() - .find(|read| read.name == column.name) - .map_or_else( - || "NULL".to_owned(), - |read| { - let field = format!("GET($1, {})", literal(&read.name)); - unloaded_value(column, &read.output, &field) - }, - ), - } -} - -/// An unload's `field`, read as `read`, as the value it was unloaded from, then as `column` holds it: a time or timestamp from its whole microseconds (a timestamp with time zone in UTC, as SQLLoad's reader gives it), JSON and arrays parsed from their text, anything else cast to its type. -pub(in crate::adapters) fn unloaded_value( - column: &DesiredColumn, - read: &ColumnType, - field: &str, -) -> String { - let micros = format!("CAST({field} AS NUMBER(38,0))"); - let value = match read { - ColumnType::TimestampNtz => format!("TO_TIMESTAMP_NTZ({micros}, 6)"), - ColumnType::TimestampTz => { - format!("CONVERT_TIMEZONE('UTC', TO_TIMESTAMP_TZ({micros}, 6))") - } - ColumnType::Time => format!("TIMEADD(MICROSECOND, {micros}, '00:00:00'::TIME)"), - _ => field.to_owned(), - }; - cast(column, &value) -} - -/// `value` as `column` holds it: JSON and arrays parsed from their text, anything else cast to its type. -fn cast(column: &DesiredColumn, value: &str) -> String { - match &column.column_type { - ColumnType::Json => format!("PARSE_JSON(CAST({value} AS VARCHAR))"), - ColumnType::Array(_) => format!("CAST(PARSE_JSON(CAST({value} AS VARCHAR)) AS ARRAY)"), - column_type => format!( - "CAST({value} AS {})", - SnowflakeSql.render_loader_logical_type(column_type).sql - ), - } -} diff --git a/crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs b/crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs new file mode 100644 index 000000000..05e93946a --- /dev/null +++ b/crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs @@ -0,0 +1,151 @@ +//! The statements loading a Snowflake source's rows into a table inside Snowflake: from its unloaded Parquet files, or selected from the source in the same session. + +use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS}; +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::snowflake::constants::{COPY_FILE_FORMAT, MAX_COPY_FILES}; +use crate::adapters::snowflake::models::{SnowflakeSql, UnloadedFrom, UnloadedLoad}; +use crate::column_types::models::ColumnType; +use crate::load_naming::main::metadata_role::metadata_role; +use crate::load_naming::models::MetadataRole; +use crate::schema_plan::constants::MERGE_ROW_COLUMN; +use crate::schema_plan::models::DesiredColumn; + +/// The statements loading `request`'s rows into `request.into`, every one of its columns filled as [`column_value`] says: from files, `COPY`s of at most 1,000 files each, stopping at the first bad value and never relying on Snowflake's load history; from a query, one `INSERT ... SELECT`. +pub(in crate::adapters) fn unloaded_statements(request: &UnloadedLoad<'_>) -> Vec { + let into = SnowflakeSql.render_qualified_name(request.into); + let names: Vec = request + .columns + .iter() + .map(|column| SnowflakeSql.render_identifier(&column.name)) + .collect(); + let values = |rows: &Rows<'_>| -> Vec { + request + .columns + .iter() + .map(|column| column_value(request, column, rows)) + .collect() + }; + match &request.from { + UnloadedFrom::Files { + stage, + folder, + keys, + .. + } => keys + .chunks(MAX_COPY_FILES) + .enumerate() + .map(|(chunk, chunk_keys)| { + let files: Vec = chunk_keys + .iter() + .map(|key| SnowflakeSql.render_loader_value_literal(key)) + .collect(); + format!( + "COPY INTO {into} ({}) FROM (SELECT {} FROM {stage}/{folder}) FILES = ({}) {COPY_FILE_FORMAT} ON_ERROR = ABORT_STATEMENT FORCE = TRUE", + names.join(", "), + values(&Rows::Files(chunk * MAX_COPY_FILES, chunk_keys)).join(", "), + files.join(", ") + ) + }) + .collect(), + UnloadedFrom::Query(query) => vec![format!( + "INSERT INTO {into} ({}) SELECT {} FROM ({query}) AS u", + names.join(", "), + values(&Rows::Query).join(", ") + )], + } +} + +/// Which rows one statement reads: the unload's files `keys`, the `first`th file on, or the query. +enum Rows<'a> { + Files(usize, &'a [String]), + Query, +} + +/// Where `column` comes from in a statement reading `rows`: the batch's load id or load time, a merge's row number (the batch, then the file, then the row in it; a query's rows in the order it gives them), the value of the constant it is, or the source's column it is read as; NULL when the source lacks it. +fn column_value(request: &UnloadedLoad<'_>, column: &DesiredColumn, rows: &Rows<'_>) -> String { + let literal = |text: &str| SnowflakeSql.render_loader_value_literal(text); + let constant = request + .constants + .iter() + .find(|(name, _, _)| *name == column.name); + match (metadata_role(request.compat, &column.name), constant) { + (Some(MetadataRole::LoadId), _) => literal(request.load_id), + (Some(MetadataRole::LoadedAt), _) => { + format!("CAST({} AS TIMESTAMP_TZ(6))", literal(request.loaded_at)) + } + _ if column.name == MERGE_ROW_COLUMN => { + let in_batch = match (rows, &request.from) { + ( + Rows::Files(first, keys), + UnloadedFrom::Files { + filename_prefix, .. + }, + ) => { + let files: Vec = keys + .iter() + .enumerate() + .map(|(index, key)| { + format!( + "WHEN {} THEN {}", + literal(&format!("{filename_prefix}{key}")), + first + index + ) + }) + .collect(); + format!( + "(CASE METADATA$FILENAME {} END) * {UNLOADED_FILE_ROWS} + METADATA$FILE_ROW_NUMBER", + files.join(" ") + ) + } + _ => "SEQ8()".to_owned(), + }; + format!("{} * {UNLOADED_BATCH_ROWS} + {in_batch}", request.batch) + } + (_, Some((_, _, value))) => cast(column, &literal(value)), + _ => request + .read + .iter() + .find(|read| read.name == column.name) + .map_or_else( + || "NULL".to_owned(), + |read| { + let field = match rows { + Rows::Files(..) => format!("GET($1, {})", literal(&read.name)), + Rows::Query => format!("u.{}", quoted(&read.name)), + }; + unloaded_value(column, &read.output, &field) + }, + ), + } +} + +/// `name` as the unload's query names its column: quoted exactly as written. +fn quoted(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +/// An unloaded `field`, read as `read`, as the value it was unloaded from, then as `column` holds it: a time or timestamp from its whole microseconds (a timestamp with time zone in UTC, as SQLLoad's reader gives it), anything else as it is. +fn unloaded_value(column: &DesiredColumn, read: &ColumnType, field: &str) -> String { + let micros = format!("CAST({field} AS NUMBER(38,0))"); + let value = match read { + ColumnType::TimestampNtz => format!("TO_TIMESTAMP_NTZ({micros}, 6)"), + ColumnType::TimestampTz => { + format!("CONVERT_TIMEZONE('UTC', TO_TIMESTAMP_TZ({micros}, 6))") + } + ColumnType::Time => format!("TIMEADD(MICROSECOND, {micros}, '00:00:00'::TIME)"), + _ => field.to_owned(), + }; + cast(column, &value) +} + +/// `value` as `column` holds it: JSON and arrays parsed from their text, anything else cast to its type. +fn cast(column: &DesiredColumn, value: &str) -> String { + match &column.column_type { + ColumnType::Json => format!("PARSE_JSON(CAST({value} AS VARCHAR))"), + ColumnType::Array(_) => format!("CAST(PARSE_JSON(CAST({value} AS VARCHAR)) AS ARRAY)"), + column_type => format!( + "CAST({value} AS {})", + SnowflakeSql.render_loader_logical_type(column_type).sql + ), + } +} diff --git a/crates/sqlload/src/adapters/snowflake/main/copy/mod.rs b/crates/sqlload/src/adapters/snowflake/main/copy/mod.rs index ebf690dcb..24cf6b432 100644 --- a/crates/sqlload/src/adapters/snowflake/main/copy/mod.rs +++ b/crates/sqlload/src/adapters/snowflake/main/copy/mod.rs @@ -1,8 +1,8 @@ pub(in crate::adapters) mod _copy_rows; pub(in crate::adapters) mod _copy_staged; -pub(in crate::adapters) mod _copy_unloaded; pub(in crate::adapters) mod _direct_check; pub(in crate::adapters) mod _direct_copies; pub(in crate::adapters) mod _direct_problem; pub(in crate::adapters) mod _script_copy_rows; pub(in crate::adapters) mod _stage_covers; +pub(in crate::adapters) mod _unloaded_sql; diff --git a/crates/sqlload/src/adapters/snowflake/models.rs b/crates/sqlload/src/adapters/snowflake/models.rs index c885f9613..afeec30ca 100644 --- a/crates/sqlload/src/adapters/snowflake/models.rs +++ b/crates/sqlload/src/adapters/snowflake/models.rs @@ -113,8 +113,8 @@ pub struct CopyStaged<'a> { pub selected: Option<&'a [DesiredColumn]>, } -/// The `COPY`s of one batch of a Snowflake source's unloaded files: the table they fill and its columns, the files' read columns, the batch's own values, and where the files are. -pub struct UnloadedCopy<'a> { +/// One batch of a Snowflake source's rows loaded inside Snowflake: the table they fill and its columns, the source's read columns, the batch's own values, and where the rows come from. +pub struct UnloadedLoad<'a> { pub into: &'a TableName, pub columns: &'a [DesiredColumn], pub read: &'a [ReadColumn], @@ -126,14 +126,24 @@ pub struct UnloadedCopy<'a> { pub load_id: &'a str, /// The load time as an ISO 8601 UTC timestamp. pub loaded_at: &'a str, - /// The stage as SQL, `@"DB"."SCHEMA"."STAGE"`. - pub stage: &'a str, - /// The unload's folder below the stage, ending in `/`. - pub folder: &'a str, - /// Each file's key below `folder`. - pub keys: &'a [String], - /// What `METADATA$FILENAME` starts each file's name with: the folder's bucket-relative key. - pub filename_prefix: &'a str, + pub from: UnloadedFrom<'a>, +} + +/// Where a batch loaded inside Snowflake reads the source's rows, each read column under its own name, times and timestamps as whole microseconds. +pub enum UnloadedFrom<'a> { + /// The unload's Parquet files. + Files { + /// The stage as SQL, `@"DB"."SCHEMA"."STAGE"`. + stage: &'a str, + /// The unload's folder below the stage, ending in `/`. + folder: &'a str, + /// Each file's key below `folder`. + keys: &'a [String], + /// What `METADATA$FILENAME` starts each file's name with: the folder's bucket-relative key. + filename_prefix: &'a str, + }, + /// The query the unload would run, selected in the same session. + Query(&'a str), } /// Where one staged file goes on an internal stage. diff --git a/crates/sqlload/src/adapters/snowflake/tests.rs b/crates/sqlload/src/adapters/snowflake/tests.rs index 6c1090bd5..83afe1e92 100644 --- a/crates/sqlload/src/adapters/snowflake/tests.rs +++ b/crates/sqlload/src/adapters/snowflake/tests.rs @@ -1,7 +1,5 @@ #[path = "tests/test_copy_staged.rs"] mod copy_staged; -#[path = "tests/test_copy_unloaded.rs"] -mod copy_unloaded; #[path = "tests/test_direct_copies.rs"] mod direct_copies; #[path = "tests/test_direct_problem.rs"] @@ -22,3 +20,5 @@ mod stale_folders; mod step_failure; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_unloaded_sql.rs"] +mod unloaded_sql; diff --git a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs index 005ed9dda..68b427608 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs @@ -375,13 +375,13 @@ pub(crate) fn csv_in( } } -/// The `COPY`s loading `files` unloaded files (`data_.parquet` under `lake/unload/run-1/`, on the stage's `unload/` folder) into `RAW.ORDERS`, batch 3 of load L1, with constant `region` = `'EU'` and the files' columns read as typed. -pub(crate) fn unloaded_copies( +/// The statements loading a Snowflake source's rows into `RAW.ORDERS`, batch 3 of load L1, with constant `region` = `'EU'` and the source's columns read as typed: from `files` unloaded files (`data_.parquet` under `lake/unload/run-1/`, on the stage's `unload/` folder), or with none, from the query `SELECT ...`. +pub(crate) fn unloaded_sql( columns: &[(&str, ColumnType, Option)], - files: usize, + files: Option, ) -> Vec { - use crate::adapters::snowflake::main::copy::_copy_unloaded::copy_unloaded; - use crate::adapters::snowflake::models::UnloadedCopy; + use crate::adapters::snowflake::main::copy::_unloaded_sql::unloaded_statements; + use crate::adapters::snowflake::models::{UnloadedFrom, UnloadedLoad}; use crate::postgres_source::models::{ReadColumn, WireType}; use crate::schema_plan::models::{ColumnOrigin, DesiredColumn}; @@ -404,7 +404,7 @@ pub(crate) fn unloaded_copies( }) }) .collect(); - let keys: Vec = (0..files) + let keys: Vec = (0..files.unwrap_or_default()) .map(|file| format!("data_{file}.parquet")) .collect(); let table = TableName { @@ -412,7 +412,7 @@ pub(crate) fn unloaded_copies( schema: "raw".to_owned(), table: "orders".to_owned(), }; - copy_unloaded(&UnloadedCopy { + unloaded_statements(&UnloadedLoad { into: &table, columns: &desired, read: &read, @@ -421,9 +421,14 @@ pub(crate) fn unloaded_copies( batch: 3, load_id: "L1", loaded_at: "2026-11-18T00:00:00Z", - stage: "@\"DB\".\"RAW\".\"LAKE\"", - folder: "unload/run-1/", - keys: &keys, - filename_prefix: "lake/unload/run-1/", + from: match files { + Some(_) => UnloadedFrom::Files { + stage: "@\"DB\".\"RAW\".\"LAKE\"", + folder: "unload/run-1/", + keys: &keys, + filename_prefix: "lake/unload/run-1/", + }, + None => UnloadedFrom::Query("SELECT ..."), + }, }) } diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs index 7e9d3f5aa..fb91982d0 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs @@ -107,15 +107,16 @@ pub(crate) struct SnowflakeEncodingTestCase { pub(crate) expected_encoding: Option<&'static str>, } -/// The `COPY`s of one batch of unloaded files. -pub(crate) struct CopyUnloadedTestCase { +/// The statements loading one batch of a Snowflake source's rows inside Snowflake. +pub(crate) struct UnloadedSqlTestCase { pub(crate) description: &'static str, /// The table's columns, each with the type the files' column is read as (`None` when the files lack it). pub(crate) columns: &'static [(&'static str, ColumnType, Option)], - pub(crate) files: usize, - pub(crate) expected_copies: usize, - /// Text the first `COPY` holds, in order. + /// How many unloaded files; `None` selects the rows with the unload's query. + pub(crate) files: Option, + pub(crate) expected_statements: usize, + /// Text the first statement holds, in order. pub(crate) expected_first: &'static [&'static str], - /// Text the last `COPY` holds. + /// Text the last statement holds. pub(crate) expected_last: &'static str, } diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_copy_unloaded.rs b/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_sql.rs similarity index 67% rename from crates/sqlload/src/adapters/snowflake/tests/test_copy_unloaded.rs rename to crates/sqlload/src/adapters/snowflake/tests/test_unloaded_sql.rs index 2a91e791d..6912dd874 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/test_copy_unloaded.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_sql.rs @@ -1,5 +1,5 @@ -use crate::adapters::snowflake::tests::helpers::unloaded_copies; -use crate::adapters::snowflake::tests::test_types::CopyUnloadedTestCase; +use crate::adapters::snowflake::tests::helpers::unloaded_sql; +use crate::adapters::snowflake::tests::test_types::UnloadedSqlTestCase; use crate::column_types::models::ColumnType; const TYPED: &[(&str, ColumnType, Option)] = &[ @@ -28,14 +28,14 @@ const NUMBERED: &[(&str, ColumnType, Option)] = &[ ]; #[test] -fn given_a_batch_of_unloaded_files_when_copying_them_then_snowflake_turns_their_microseconds_back_and_fills_the_loads_columns() +fn given_a_snowflake_sources_rows_when_loading_them_inside_snowflake_then_their_microseconds_turn_back_and_the_loads_columns_fill() { let test_cases = [ - CopyUnloadedTestCase { + UnloadedSqlTestCase { description: "times and timestamps from whole microseconds (a zoned one in UTC), JSON parsed, the constant, load id and load time as literals, a column the files lack NULL", columns: TYPED, - files: 2, - expected_copies: 1, + files: Some(2), + expected_statements: 1, expected_first: &[ "COPY INTO \"RAW\".\"ORDERS\" (\"ID\", \"SEEN_AT\", \"PLACED_AT\", \"OPENS\", \"META\", \"REGION\", \"_SQLLOAD_LOAD_ID\", \"_SQLLOAD_LOADED_AT\", \"GONE\") FROM (SELECT ", "CAST(GET($1, 'id') AS NUMBER(38,0)), ", @@ -51,27 +51,48 @@ fn given_a_batch_of_unloaded_files_when_copying_them_then_snowflake_turns_their_ ], expected_last: "FORCE = TRUE", }, - CopyUnloadedTestCase { + UnloadedSqlTestCase { description: "a merge numbers its rows by batch, then file, counting on across COPYs of 1,000 files, then row in the file", columns: NUMBERED, - files: 1001, - expected_copies: 2, + files: Some(1001), + expected_statements: 2, expected_first: &[ "3 * 281474976710656 + (CASE METADATA$FILENAME WHEN 'lake/unload/run-1/data_0.parquet' THEN 0 WHEN ", "WHEN 'lake/unload/run-1/data_999.parquet' THEN 999 END) * 4294967296 + METADATA$FILE_ROW_NUMBER", ], expected_last: "(CASE METADATA$FILENAME WHEN 'lake/unload/run-1/data_1000.parquet' THEN 1000 END) * 4294967296 + METADATA$FILE_ROW_NUMBER FROM @\"DB\".\"RAW\".\"LAKE\"/unload/run-1/) FILES = ('data_1000.parquet')", }, + UnloadedSqlTestCase { + description: "selected in the source's session: one INSERT reading the query's columns by name, turned back the same way; a merge numbers rows in the order the query gives them", + columns: &[ + ("id", ColumnType::Int, Some(ColumnType::Int)), + ( + "placed_at", + ColumnType::TimestampTz, + Some(ColumnType::TimestampTz), + ), + ("_sqlload_row", ColumnType::Int, None), + ], + files: None, + expected_statements: 1, + expected_first: &[ + "INSERT INTO \"RAW\".\"ORDERS\" (\"ID\", \"PLACED_AT\", \"_SQLLOAD_ROW\") SELECT ", + "CAST(u.\"id\" AS NUMBER(38,0)), ", + "CAST(CONVERT_TIMEZONE('UTC', TO_TIMESTAMP_TZ(CAST(u.\"placed_at\" AS NUMBER(38,0)), 6)) AS TIMESTAMP_TZ(9)), ", + "3 * 281474976710656 + SEQ8() FROM (SELECT ...) AS u", + ], + expected_last: "FROM (SELECT ...) AS u", + }, ]; for test_case in test_cases { - let copies = unloaded_copies(test_case.columns, test_case.files); + let statements = unloaded_sql(test_case.columns, test_case.files); assert_eq!( - copies.len(), - test_case.expected_copies, + statements.len(), + test_case.expected_statements, "{}", test_case.description ); - let (first, last) = (&copies[0], &copies[copies.len() - 1]); + let (first, last) = (&statements[0], &statements[statements.len() - 1]); let mut rest = first.as_str(); for expected in test_case.expected_first { let at = rest.find(expected); diff --git a/crates/sqlload/src/loader/_helpers/publish.rs b/crates/sqlload/src/loader/_helpers/publish.rs index cca2a421a..f6dd790a4 100644 --- a/crates/sqlload/src/loader/_helpers/publish.rs +++ b/crates/sqlload/src/loader/_helpers/publish.rs @@ -9,7 +9,7 @@ use crate::adapters::contract::models::TableShape; use crate::adapters::contract::models::WriteMode; use crate::adapters::contract::models::WriteStatementsRequest; use crate::adapters::contract::models::WriteTarget; -use crate::adapters::contract::models::{DirectBatch, DirectLoaded, UnloadedBatch}; +use crate::adapters::contract::models::{DirectBatch, DirectLoaded, UnloadedBatch, UnloadedSource}; use crate::adapters::contract::types::TableWriter; use crate::adapters::dispatch::main::dialect::source_dialect::source_dialect; use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter; @@ -82,7 +82,7 @@ use crate::loader::_helpers::sources::ReadPart; use crate::loader::_helpers::sources::files_source_parts; use crate::loader::_helpers::sources::read_source; use crate::loader::_helpers::sources::unarchived; -use crate::loader::_helpers::sources::unload_source; +use crate::loader::_helpers::sources::{selected_query, unload_source}; use crate::loader::_helpers::specs::checked_key; use crate::loader::_helpers::specs::{null_cursor_problem, null_cursor_rows}; use crate::loader::_helpers::table_plan::{ @@ -144,7 +144,6 @@ use crate::schema_plan::constants::{ use crate::schema_plan::models::ColumnOrigin; use crate::schema_plan::models::DesiredColumn; use crate::schema_plan::models::LiveColumn; -use crate::snowflake_source::models::UnloadedFiles; use crate::table_lease::main::lease_now::lease_now; use crate::table_lease::main::take_lease::take_lease; use crate::time_text::main::now_utc_text::now_utc_text; @@ -165,7 +164,7 @@ use std::collections::BTreeSet; use std::sync::Arc; use crate::loader::_helpers::table_plan::{ - added_constants, is_direct, publishes_once, pulls_unloaded, + added_constants, is_direct, publishes_once, pulls_unloaded, selects_rows, }; /// Whether `append` can write the batches straight into the table: not a logged table (its rows wait for a changed `ctx.state`), and a writer that can (on Snowflake, a route loading files into the table itself). @@ -1073,14 +1072,14 @@ impl Publisher { } } - /// Load one unload's `files` by having the destination read them itself, where the plan chose that; the destination must load them so. + /// Load one unload's files, or the rows its query selects, by having the destination read them itself, where the plan chose that; the destination must load them so. pub(crate) async fn load_unloaded( &mut self, - files: &UnloadedFiles, + source: UnloadedSource<'_>, ) -> Result { let columns = staged_columns(&self.table); let batch = UnloadedBatch { - files, + source, read: &self.table.read_columns, write_columns: &self.table.schema.write_columns, target: write_target( @@ -1091,26 +1090,33 @@ impl Publisher { ), loaded_at: &self.loaded_at, }; - let loaded = load_unloaded(&mut self.writer, &batch).await?.ok_or_else(|| { - LoaderError::Invalid(format!( - "{}: the plan has the destination load the unloaded files itself, but it cannot", - self.table.spec.destination.table - )) - })?; + let loaded = load_unloaded(&mut self.writer, &batch) + .await? + .ok_or_else(|| { + LoaderError::Invalid(format!( + "{}: the plan has the destination load the source's rows itself, but it cannot", + self.table.spec.destination.table + )) + })?; self.note_direct(&loaded); Ok(loaded) } - /// Pass `part` of the table's rows on: the destination loads the unloaded files itself where the plan chose that, otherwise SQLLoad stages the rows. + /// Pass `part` of the table's rows on: the destination selects them itself, or loads the unloaded files itself, where the plan chose that; otherwise SQLLoad stages the rows. async fn pass_part( &mut self, reader: &SourceReader, part: ReadPart, ) -> Result<(), LoaderError> { let table = self.table.clone(); + if selects_rows(table.route.as_ref()) { + let query = selected_query(&table, part)?; + self.load_unloaded(UnloadedSource::Query(&query)).await?; + return Ok(()); + } if pulls_unloaded(table.route.as_ref()) { unload_source(reader, &table, part, async |files| { - self.load_unloaded(files).await + self.load_unloaded(UnloadedSource::Files(files)).await }) .await?; return Ok(()); diff --git a/crates/sqlload/src/loader/_helpers/render.rs b/crates/sqlload/src/loader/_helpers/render.rs index dcda32667..e8e9107cf 100644 --- a/crates/sqlload/src/loader/_helpers/render.rs +++ b/crates/sqlload/src/loader/_helpers/render.rs @@ -1001,6 +1001,9 @@ fn route_text(table: &TablePlan, route: &Route) -> String { real.unwrap_or_else(|| files.format.to_string()), files.store ), + [Step::DestinationSelects] => format!( + "{warehouse} INSERT ... SELECT on the source's connection (nothing through the loader)" + ), [Step::Unload(store), Step::DestinationPulls(_)] => { format!("unload Parquet to {store} -> {warehouse} {load} (nothing through the loader)") } diff --git a/crates/sqlload/src/loader/_helpers/resolve.rs b/crates/sqlload/src/loader/_helpers/resolve.rs index bc6d45291..72109be2c 100644 --- a/crates/sqlload/src/loader/_helpers/resolve.rs +++ b/crates/sqlload/src/loader/_helpers/resolve.rs @@ -746,6 +746,8 @@ fn setting_name( #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct ResolvedTarget { pub(crate) name: String, + /// The connection it writes through, when it writes in that connection's own database (a target `database` naming another leaves it `None`): a source on that connection reads in the same database, so the target can select its rows itself. + pub(crate) session_connection: Option, pub(crate) warehouse: Warehouse, pub(crate) database: Option, pub(crate) schema: String, @@ -831,6 +833,7 @@ pub(crate) fn source_target( if !sql_adapter(warehouse).runs_sql() { return Ok(ResolvedTarget { name, + session_connection: None, warehouse, database: None, schema: String::new(), @@ -850,6 +853,11 @@ pub(crate) fn source_target( .ok_or_else(|| format!("target {name:?} has no loader_schema"))?; Ok(ResolvedTarget { name, + session_connection: target + .database + .as_ref() + .is_none_or(|own| Some(own) == database.as_ref()) + .then(|| target.connection.clone()), warehouse, database: match connection { ConnectionConfig::MotherDuck { database, .. } => target diff --git a/crates/sqlload/src/loader/_helpers/sources.rs b/crates/sqlload/src/loader/_helpers/sources.rs index 3d7d11de2..7c5f2b53c 100644 --- a/crates/sqlload/src/loader/_helpers/sources.rs +++ b/crates/sqlload/src/loader/_helpers/sources.rs @@ -126,6 +126,7 @@ use crate::sheets_source::models::{Grid, HeaderRule, KeySource, SheetsClient}; use crate::snowflake_source::constants::SESSION_SCHEMA; use crate::snowflake_source::main::connect_snowflake_source::connect_snowflake_source; use crate::snowflake_source::main::read_catalog::read_catalog as read_snowflake_catalog; +use crate::snowflake_source::main::select_query::select_query; use crate::snowflake_source::main::unload_files::unload_files; use crate::snowflake_source::main::unload_rows::unload_rows; use crate::snowflake_source::models::{UnloadRequest, UnloadedFiles}; @@ -1745,6 +1746,15 @@ pub(crate) async fn unload_source( unload_files(snowflake, &unload_request(table, filter.as_deref()), load).await } +/// The query an unload of `part` of `table` would run, which a destination writing through the source's own connection runs itself. +pub(crate) fn selected_query(table: &TablePlan, part: ReadPart) -> Result { + let filter = table + .spec + .read_filter(part.filter.or_else(|| table.source_filter.clone())); + select_query(&unload_request(table, filter.as_deref())) + .map_err(|error| LoaderError::Invalid(error.to_string())) +} + /// What one unload of `table` reads: its table or query, its read columns, and `filter`. fn unload_request<'a>(table: &'a TablePlan, filter: Option<&'a str>) -> UnloadRequest<'a> { UnloadRequest { diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs index e9e0d0ca8..758b37281 100644 --- a/crates/sqlload/src/loader/_helpers/specs.rs +++ b/crates/sqlload/src/loader/_helpers/specs.rs @@ -544,7 +544,7 @@ pub(crate) fn build_spec( sql: overrides.sql.clone(), route: WarehouseRoute::Query, }), - _ => warehouse_source(context.source, overrides), + _ => warehouse_source(context, overrides), }, row_filter: layered(&settings.row_filter, &defaults.row_filter) .filter(|_| overrides.sql.is_none() && overrides.tombstones_of.is_none()) @@ -656,8 +656,12 @@ fn constant_type_problem( } } -/// How a `snowflake` source's table is read: unloaded to the source's `stage`, or to the user stage when it sets none. -fn warehouse_source(source: &SourceConfig, overrides: &TableOverride) -> Option { +/// How a `snowflake` source's table is read: unloaded to the source's `stage`, or to the user stage when it sets none; a Snowflake target writing through the source's own connection may select the rows itself. +fn warehouse_source( + context: &SourceContext<'_>, + overrides: &TableOverride, +) -> Option { + let source = context.source; (source.kind == SourceKind::Snowflake).then(|| { let (store, stage) = match &source.stage { Some(UnloadStage { @@ -686,7 +690,12 @@ fn warehouse_source(source: &SourceConfig, overrides: &TableOverride) -> Option< WarehouseSource { system: SNOWFLAKE_SOURCE, sql: overrides.sql.clone(), - route: WarehouseRoute::Unload { store, stage }, + route: WarehouseRoute::Unload { + store, + stage, + same_connection: context.target.warehouse == Warehouse::Snowflake + && context.target.session_connection.as_ref() == Some(&source.connection), + }, } }) } diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index c8a357d47..f5a4f9765 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -1079,6 +1079,16 @@ pub(crate) fn reader_offer( }) | None => Vec::new(), }; + let selected_by_destination = matches!( + &spec.warehouse_source, + Some(WarehouseSource { + route: WarehouseRoute::Unload { + same_connection: true, + .. + }, + .. + }) + ); let direct_blocked = match unload_to.is_empty() { true => direct_blocked, false => unload_blocked(spec), @@ -1101,6 +1111,7 @@ pub(crate) fn reader_offer( .files .as_ref() .is_some_and(|files| stage_misses(stage, &files.location)), + selected_by_destination, } } @@ -1130,12 +1141,18 @@ fn one_file_each(count: usize) -> Vec { .collect() } -/// Whether `route` has the destination load a warehouse source's unloaded files itself. +/// Whether `route` has the destination select a warehouse source's rows itself, inside the warehouse. +pub(crate) fn selects_rows(route: Option<&Route>) -> bool { + route.is_some_and(|route| matches!(route.steps.as_slice(), [Step::DestinationSelects])) +} + +/// Whether `route` has the destination load a warehouse source's rows itself: its unloaded files, or the rows it selects inside the warehouse. pub(crate) fn pulls_unloaded(route: Option<&Route>) -> bool { route.is_some_and(|route| { matches!( route.steps.as_slice(), [Step::Unload(_), Step::DestinationPulls(_)] + | [Step::DestinationSelects] | [ Step::Unload(_), Step::CopyFiles(Store::Local(_)), diff --git a/crates/sqlload/src/loader/models.rs b/crates/sqlload/src/loader/models.rs index b24678bba..97745da9b 100644 --- a/crates/sqlload/src/loader/models.rs +++ b/crates/sqlload/src/loader/models.rs @@ -315,8 +315,12 @@ pub struct WarehouseSource { /// How a warehouse source's rows leave the warehouse; each warehouse chooses its own. #[derive(Debug, Clone, PartialEq, Eq)] pub enum WarehouseRoute { - /// The warehouse writes Parquet into `store`, which SQLLoad reads back and removes; `stage` says why there, for `plan`. - Unload { store: Store, stage: String }, + /// The warehouse writes Parquet into `store`, which SQLLoad reads back and removes; `stage` says why there, for `plan`. `same_connection`: the target writes through the source's own connection, so it can select the rows itself instead. + Unload { + store: Store, + stage: String, + same_connection: bool, + }, /// SQLLoad runs the query and reads its rows as Arrow. Query, } diff --git a/crates/sqlload/src/routing/_helpers/warehouse_source.rs b/crates/sqlload/src/routing/_helpers/warehouse_source.rs index 2f0ce5f7d..3da718fb9 100644 --- a/crates/sqlload/src/routing/_helpers/warehouse_source.rs +++ b/crates/sqlload/src/routing/_helpers/warehouse_source.rs @@ -22,14 +22,20 @@ pub(crate) fn should_unload(reader: &ReaderOffer, read_via: ReadVia) -> Result Result { - let stores = ordered_stores(&reader.unload_to, overrides.stage)?; let unblocked = reader.direct_blocked.is_none() && reader.needs_loader.is_none(); + if unblocked && reader.selected_by_destination { + return Ok(Route { + steps: vec![Step::DestinationSelects], + reason: "the destination writes through the source's own connection, so it selects the rows itself; no data leaves the warehouse".to_owned(), + }); + } + let stores = ordered_stores(&reader.unload_to, overrides.stage)?; if let Some(store) = stores .iter() .find(|store| unblocked && destination_pulls(writer, store, FileFormat::Parquet)) diff --git a/crates/sqlload/src/routing/models.rs b/crates/sqlload/src/routing/models.rs index de5b766f3..36b504233 100644 --- a/crates/sqlload/src/routing/models.rs +++ b/crates/sqlload/src/routing/models.rs @@ -83,6 +83,8 @@ pub struct ReaderOffer { pub copy_to_internal_stage: bool, /// The destination's store paths name `existing_files`, but it cannot read them (the same bucket on another endpoint). pub out_of_reach: bool, + /// The destination can select a warehouse source's rows itself, as it writes through the source's own connection. + pub selected_by_destination: bool, } /// What a destination can take. @@ -152,6 +154,8 @@ pub enum Step { DestinationTakesArrow, /// The rows go in without files, this way. RowsInto(String), + /// The destination selects the source's rows itself, inside the warehouse; no data leaves it. + DestinationSelects, } impl std::fmt::Display for Step { @@ -174,6 +178,7 @@ impl std::fmt::Display for Step { } Self::DestinationTakesArrow => formatter.write_str("destination takes Arrow"), Self::RowsInto(how) => formatter.write_str(how), + Self::DestinationSelects => formatter.write_str("destination selects the rows itself"), } } } diff --git a/crates/sqlload/src/routing/tests/test_plan_route.rs b/crates/sqlload/src/routing/tests/test_plan_route.rs index e477c749a..f9be85617 100644 --- a/crates/sqlload/src/routing/tests/test_plan_route.rs +++ b/crates/sqlload/src/routing/tests/test_plan_route.rs @@ -168,6 +168,32 @@ fn given_a_source_and_destination_when_planning_then_the_cheapest_route_is_chose "destination takes Arrow", ], }, + PlanRouteTestCase { + description: "a warehouse writing through the source's own connection selects the rows itself", + reader: ReaderOffer { + selected_by_destination: true, + ..warehouse_source(BIG) + }, + writer: warehouse_destination(vec![object_store(LAKE)]), + overrides: RouteOverrides::default(), + expected_steps: &["destination selects the rows itself"], + }, + PlanRouteTestCase { + description: "on the source's own connection, a table that needs the loader is still unloaded and read back", + reader: ReaderOffer { + selected_by_destination: true, + needs_loader: Some(HMAC.to_owned()), + ..warehouse_source(BIG) + }, + writer: warehouse_destination(vec![object_store(LAKE)]), + overrides: RouteOverrides::default(), + expected_steps: &[ + "unload Parquet to s3://lake/unload/", + "read parquet into Arrow", + "write Parquet to s3://lake/", + "destination loads parquet files from s3://lake/", + ], + }, PlanRouteTestCase { description: "a local directory downloads an unload from the lake", reader: warehouse_source(BIG), diff --git a/crates/sqlload/src/snowflake_source/main/mod.rs b/crates/sqlload/src/snowflake_source/main/mod.rs index 0215ae2d6..946975449 100644 --- a/crates/sqlload/src/snowflake_source/main/mod.rs +++ b/crates/sqlload/src/snowflake_source/main/mod.rs @@ -3,5 +3,6 @@ pub mod count_rows; pub mod describe_source; pub mod read_catalog; pub mod read_columns; +pub mod select_query; pub mod unload_files; pub mod unload_rows; diff --git a/crates/sqlload/src/snowflake_source/main/select_query.rs b/crates/sqlload/src/snowflake_source/main/select_query.rs new file mode 100644 index 000000000..b12ceba9a --- /dev/null +++ b/crates/sqlload/src/snowflake_source/main/select_query.rs @@ -0,0 +1,10 @@ +//! The query an unload runs, for a destination that selects the rows itself in the same session. + +use crate::snowflake_source::_helpers::unload_query::unload_query; +use crate::snowflake_source::errors::SnowflakeSourceError; +use crate::snowflake_source::models::UnloadRequest; + +/// The `SELECT` an unload of `request` writes to Parquet: each read column under its own name, times and timestamps as whole microseconds, filtered as `request` says. +pub fn select_query(request: &UnloadRequest<'_>) -> Result { + unload_query(request).map_err(SnowflakeSourceError::Invalid) +} From 2c5bdd11259d499653c1460688edc9c2491fc8c7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:12:08 +0000 Subject: [PATCH 3/9] wip: Snowflake reads another connection's unload only from S3; plan wording for selected rows Live check on Snowflake: same-connection INSERT ... SELECT and the through-SQLLoad route load identical values (decimals, zoned and plain timestamps to the microsecond, times, VARIANT), first run and an incremental merge after the cursor. --- .../dispatch/main/route/unload_blocked.rs | 20 +++++++++++++++---- crates/sqlload/src/loader/_helpers/render.rs | 9 ++++++++- .../sqlload/src/loader/_helpers/table_plan.rs | 19 +++++++++--------- crates/sqlload/src/loader/constants.rs | 2 ++ .../src/routing/_helpers/warehouse_source.rs | 4 ++-- 5 files changed, 38 insertions(+), 16 deletions(-) diff --git a/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs b/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs index 3a9be50b7..156eed15f 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs @@ -1,16 +1,28 @@ //! Why the destination cannot load a warehouse source's unloaded files exactly by reading them itself. use crate::column_types::models::Warehouse; -use crate::loader::constants::UNLOADED_TIMES; -use crate::loader::models::TableSpec; +use crate::loader::constants::{UNLOADED_OUTSIDE_S3, UNLOADED_TIMES}; +use crate::loader::models::{TableSpec, WarehouseRoute, WarehouseSource}; +use crate::routing::models::Store; -/// Why the destination cannot load the table's unloaded files exactly by reading them itself, so SQLLoad reads them instead; `None` when it can (where it also reaches their store). Only DuckDB and Snowflake turn the files' whole microseconds back into times and timestamps; masks need SQLLoad to see the rows. +/// Why the destination cannot load the table's unloaded files exactly by reading them itself, so SQLLoad reads them instead; `None` when it can (where it also reaches their store). Only DuckDB and Snowflake turn the files' whole microseconds back into times and timestamps; masks need SQLLoad to see the rows; Snowflake reads another session's unload only from S3, through an external stage of its own, as a user or named stage of the source's is not its to read. pub fn unload_blocked(spec: &TableSpec) -> Option { + let in_s3 = matches!( + &spec.warehouse_source, + Some(WarehouseSource { + route: WarehouseRoute::Unload { + store: Store::ObjectStore(url), + .. + }, + .. + }) if url.starts_with("s3://") + ); match (spec.warehouse, spec.masked.columns.is_empty()) { - (Warehouse::DuckDb | Warehouse::Snowflake, true) => None, (Warehouse::DuckDb | Warehouse::Snowflake, false) => { Some("the table masks columns".to_owned()) } + (Warehouse::Snowflake, true) if !in_s3 => Some(UNLOADED_OUTSIDE_S3.to_owned()), + (Warehouse::DuckDb | Warehouse::Snowflake, true) => None, _ => Some(UNLOADED_TIMES.to_owned()), } } diff --git a/crates/sqlload/src/loader/_helpers/render.rs b/crates/sqlload/src/loader/_helpers/render.rs index e8e9107cf..8222130f8 100644 --- a/crates/sqlload/src/loader/_helpers/render.rs +++ b/crates/sqlload/src/loader/_helpers/render.rs @@ -29,7 +29,7 @@ use crate::loader::_helpers::python::{list_choices, unnested_table}; use crate::loader::_helpers::resolve::display_name; use crate::loader::_helpers::sources::{readers_note, unarchived}; use crate::loader::_helpers::table_plan::{ - archived_reason, is_direct, publishes_once, pulls_unloaded, + archived_reason, is_direct, publishes_once, pulls_unloaded, selects_rows, }; use crate::loader::constants::{ CONSTANT_NOTE, DRAINED_REASON, LABEL_WIDTH, NULL_KEYS_NOTE, PLAN_DETAIL_COLUMN, @@ -693,6 +693,7 @@ pub(crate) fn load_item(table: &TablePlan) -> PlanItem { route: WarehouseRoute::Unload { stage, .. }, .. }) = &spec.warehouse_source + && !selects_rows(table.route.as_ref()) { details.push(detail("stage", stage.as_str())); } @@ -843,6 +844,9 @@ pub(crate) fn batch_line(table: &TablePlan) -> String { thousands(MAX_BATCH_FILES as u64) ) } + (_, true) if selects_rows(table.route.as_ref()) => { + "the rows selected at once into a new table, swapped in once at the end".to_owned() + } (_, true) if pulls_unloaded(table.route.as_ref()) => { "each unload loaded at once into a new table, swapped in once at the end".to_owned() } @@ -865,6 +869,9 @@ pub(crate) fn batch_line(table: &TablePlan) -> String { _ => "", } ), + (CommitRule::Size, false) if selects_rows(table.route.as_ref()) => { + "one commit: the destination selects every row at once".to_owned() + } (CommitRule::Size, false) if pulls_unloaded(table.route.as_ref()) => { "one commit: the destination loads the whole unload at once".to_owned() } diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index f5a4f9765..24a6b0686 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -1079,16 +1079,17 @@ pub(crate) fn reader_offer( }) | None => Vec::new(), }; - let selected_by_destination = matches!( - &spec.warehouse_source, - Some(WarehouseSource { - route: WarehouseRoute::Unload { - same_connection: true, + let selected_by_destination = spec.masked.columns.is_empty() + && matches!( + &spec.warehouse_source, + Some(WarehouseSource { + route: WarehouseRoute::Unload { + same_connection: true, + .. + }, .. - }, - .. - }) - ); + }) + ); let direct_blocked = match unload_to.is_empty() { true => direct_blocked, false => unload_blocked(spec), diff --git a/crates/sqlload/src/loader/constants.rs b/crates/sqlload/src/loader/constants.rs index f4ef5f8c6..c48e9db50 100644 --- a/crates/sqlload/src/loader/constants.rs +++ b/crates/sqlload/src/loader/constants.rs @@ -108,6 +108,8 @@ pub const RESET: &str = "\x1b[0m"; /// The host name that always means this machine: plain text to it never crosses a network. pub const LOCALHOST: &str = "localhost"; +/// Why Snowflake does not read a Snowflake source's unload itself when the source unloads to a stage of its own. +pub const UNLOADED_OUTSIDE_S3: &str = "Snowflake reads another connection's unload itself only from S3: set the source's stage to an external one (url and storage_integration) inside the target's external stage"; /// Why a warehouse source's unloaded files go through SQLLoad rather than straight into the destination. pub const UNLOADED_TIMES: &str = "the warehouse writes times and timestamps as whole microseconds (its Parquet writer keeps only milliseconds), which SQLLoad turns back into them"; diff --git a/crates/sqlload/src/routing/_helpers/warehouse_source.rs b/crates/sqlload/src/routing/_helpers/warehouse_source.rs index 3da718fb9..58056c6ce 100644 --- a/crates/sqlload/src/routing/_helpers/warehouse_source.rs +++ b/crates/sqlload/src/routing/_helpers/warehouse_source.rs @@ -28,14 +28,14 @@ pub(crate) fn unload_route( writer: &WriterAccepts, overrides: RouteOverrides, ) -> Result { - let unblocked = reader.direct_blocked.is_none() && reader.needs_loader.is_none(); - if unblocked && reader.selected_by_destination { + if reader.selected_by_destination && reader.needs_loader.is_none() { return Ok(Route { steps: vec![Step::DestinationSelects], reason: "the destination writes through the source's own connection, so it selects the rows itself; no data leaves the warehouse".to_owned(), }); } let stores = ordered_stores(&reader.unload_to, overrides.stage)?; + let unblocked = reader.direct_blocked.is_none() && reader.needs_loader.is_none(); if let Some(store) = stores .iter() .find(|store| unblocked && destination_pulls(writer, store, FileFormat::Parquet)) From d962b26640ed6c7fbd73576fe61eabf4ece04c47 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:59:46 +0000 Subject: [PATCH 4/9] wip: Snowflake copies another connection's unload only when the target's external stage covers it --- .../dispatch/main/route/unload_blocked.rs | 41 +++++---- .../sqlload/src/adapters/snowflake/tests.rs | 2 + .../src/adapters/snowflake/tests/helpers.rs | 29 +++++- .../adapters/snowflake/tests/test_types.rs | 12 +++ .../snowflake/tests/test_unload_blocked.rs | 88 +++++++++++++++++++ .../sqlload/src/loader/_helpers/table_plan.rs | 7 +- crates/sqlload/src/loader/constants.rs | 4 +- 7 files changed, 163 insertions(+), 20 deletions(-) create mode 100644 crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs diff --git a/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs b/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs index 156eed15f..ac6ff774d 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs @@ -1,28 +1,37 @@ //! Why the destination cannot load a warehouse source's unloaded files exactly by reading them itself. +use crate::adapters::snowflake::main::copy::_stage_covers::stage_covers; +use crate::adapters::snowflake::models::StageRoute; use crate::column_types::models::Warehouse; -use crate::loader::constants::{UNLOADED_OUTSIDE_S3, UNLOADED_TIMES}; -use crate::loader::models::{TableSpec, WarehouseRoute, WarehouseSource}; +use crate::loader::constants::{UNLOADED_OUTSIDE_STAGE, UNLOADED_TIMES}; +use crate::loader::models::WarehouseRoute; use crate::routing::models::Store; +use crate::store_paths::main::resolve_location::resolve_location; -/// Why the destination cannot load the table's unloaded files exactly by reading them itself, so SQLLoad reads them instead; `None` when it can (where it also reaches their store). Only DuckDB and Snowflake turn the files' whole microseconds back into times and timestamps; masks need SQLLoad to see the rows; Snowflake reads another session's unload only from S3, through an external stage of its own, as a user or named stage of the source's is not its to read. -pub fn unload_blocked(spec: &TableSpec) -> Option { - let in_s3 = matches!( - &spec.warehouse_source, - Some(WarehouseSource { - route: WarehouseRoute::Unload { +/// Why `warehouse` cannot load a table's unloaded files exactly by reading them itself, so SQLLoad reads them instead; `None` when it can (where it also reaches their store). Only DuckDB and Snowflake turn the files' whole microseconds back into times and timestamps; a table that `masks` columns needs SQLLoad to see the rows; Snowflake reads another session's unload only through the target's external `stage`, which must cover the S3 folder `route` unloads to, as a user or named stage of the source's is not its to read. +pub fn unload_blocked( + warehouse: Warehouse, + masks: bool, + route: Option<&WarehouseRoute>, + stage: Option<&StageRoute>, +) -> Option { + let covered = match (route, stage) { + ( + Some(WarehouseRoute::Unload { store: Store::ObjectStore(url), .. - }, - .. - }) if url.starts_with("s3://") - ); - match (spec.warehouse, spec.masked.columns.is_empty()) { - (Warehouse::DuckDb | Warehouse::Snowflake, false) => { + }), + Some(StageRoute::External(stage)), + ) => resolve_location(None, &format!("{}/", url.trim_end_matches('/'))) + .is_ok_and(|unloaded| stage_covers(&stage.location, &unloaded)), + _ => false, + }; + match (warehouse, masks) { + (Warehouse::DuckDb | Warehouse::Snowflake, true) => { Some("the table masks columns".to_owned()) } - (Warehouse::Snowflake, true) if !in_s3 => Some(UNLOADED_OUTSIDE_S3.to_owned()), - (Warehouse::DuckDb | Warehouse::Snowflake, true) => None, + (Warehouse::Snowflake, false) if !covered => Some(UNLOADED_OUTSIDE_STAGE.to_owned()), + (Warehouse::DuckDb | Warehouse::Snowflake, false) => None, _ => Some(UNLOADED_TIMES.to_owned()), } } diff --git a/crates/sqlload/src/adapters/snowflake/tests.rs b/crates/sqlload/src/adapters/snowflake/tests.rs index 83afe1e92..61595548a 100644 --- a/crates/sqlload/src/adapters/snowflake/tests.rs +++ b/crates/sqlload/src/adapters/snowflake/tests.rs @@ -20,5 +20,7 @@ mod stale_folders; mod step_failure; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_unload_blocked.rs"] +mod unload_blocked; #[path = "tests/test_unloaded_sql.rs"] mod unloaded_sql; diff --git a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs index 68b427608..e4dd8d6de 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs @@ -1,18 +1,23 @@ use crate::adapters::contract::models::TableName; +use crate::adapters::dispatch::main::route::unload_blocked::unload_blocked; use crate::adapters::snowflake::_helpers::stage::create_stage::create_stage; use crate::adapters::snowflake::_helpers::stage::stale_folders::stale_folders; use crate::adapters::snowflake::main::copy::_copy_staged::copy_staged; use crate::adapters::snowflake::main::stage::_stage_lookup::stage_lookup; use crate::adapters::snowflake::main::stage::_stage_reference::stage_reference; use crate::adapters::snowflake::models::CopyStaged; +use crate::adapters::snowflake::models::StageRoute; use crate::adapters::snowflake::models::{CopyColumn, CopyValue}; use crate::adapters::snowflake::models::{ExternalStage, StageObject}; -use crate::adapters::snowflake::tests::test_types::CopyFrom; +use crate::adapters::snowflake::tests::test_types::{CopyFrom, UnloadBlockedTestCase}; use crate::column_types::models::ColumnType; +use crate::column_types::models::Warehouse; use crate::file_store::models::StoredFile; use crate::files_source::models::DataFile; use crate::files_source::models::StoredRead; use crate::http::errors::ApiError; +use crate::loader::models::WarehouseRoute; +use crate::routing::models::Store; use crate::schema_plan::models::{ColumnOrigin, DesiredColumn}; use crate::store_paths::models::{Location, StoreKind}; use chrono::{DateTime, Utc}; @@ -432,3 +437,25 @@ pub(crate) fn unloaded_sql( }, }) } + +/// Why Snowflake does not load the case's unloaded files itself, or `None` when it does. +pub(crate) fn unload_blocked_for(test_case: &UnloadBlockedTestCase) -> Option { + let route = WarehouseRoute::Unload { + store: match test_case.unload_to { + Some(folder) => Store::ObjectStore(folder.to_owned()), + None => Store::InternalStage("sqlload_unload".to_owned()), + }, + stage: String::new(), + same_connection: false, + }; + let stage = match test_case.external { + true => StageRoute::External(Box::new(stage())), + false => StageRoute::User, + }; + unload_blocked( + Warehouse::Snowflake, + test_case.masks, + Some(&route), + Some(&stage), + ) +} diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs index fb91982d0..15b051e59 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs @@ -120,3 +120,15 @@ pub(crate) struct UnloadedSqlTestCase { /// Text the last statement holds. pub(crate) expected_last: &'static str, } + +/// Whether Snowflake loads a Snowflake source's unloaded files itself. +pub(crate) struct UnloadBlockedTestCase { + pub(crate) description: &'static str, + /// Where the source unloads: an `s3://` folder, `@stage`, or `None` for the user stage. + pub(crate) unload_to: Option<&'static str>, + /// Whether the target loads through the external stage on `s3://lake/raw/`; otherwise its user stage. + pub(crate) external: bool, + pub(crate) masks: bool, + /// The reason SQLLoad reads the files instead, or `None` when Snowflake loads them itself. + pub(crate) expected_blocked: Option<&'static str>, +} diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs b/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs new file mode 100644 index 000000000..fbcb6bfba --- /dev/null +++ b/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs @@ -0,0 +1,88 @@ +use crate::adapters::snowflake::tests::helpers::unload_blocked_for; +use crate::adapters::snowflake::tests::test_types::UnloadBlockedTestCase; + +const OUTSIDE: &str = + "Snowflake reads another connection's unload itself only through the target's external stage"; + +#[test] +fn given_a_snowflake_sources_unload_when_routing_into_snowflake_then_snowflake_copies_only_what_the_targets_external_stage_covers() + { + let test_cases = [ + UnloadBlockedTestCase { + description: "an S3 folder below the target's external stage is copied by Snowflake", + unload_to: Some("s3://lake/raw/unload"), + external: true, + masks: false, + expected_blocked: None, + }, + UnloadBlockedTestCase { + description: "the stage's own folder, written with a slash, is covered too", + unload_to: Some("s3://lake/raw/"), + external: true, + masks: false, + expected_blocked: None, + }, + UnloadBlockedTestCase { + description: "another folder of the same bucket goes through SQLLoad", + unload_to: Some("s3://lake/elsewhere/"), + external: true, + masks: false, + expected_blocked: Some(OUTSIDE), + }, + UnloadBlockedTestCase { + description: "a folder whose name only starts like the stage's goes through SQLLoad", + unload_to: Some("s3://lake/rawish/"), + external: true, + masks: false, + expected_blocked: Some(OUTSIDE), + }, + UnloadBlockedTestCase { + description: "the same folder in another bucket goes through SQLLoad", + unload_to: Some("s3://other/raw/unload/"), + external: true, + masks: false, + expected_blocked: Some(OUTSIDE), + }, + UnloadBlockedTestCase { + description: "an S3 unload into a target without an external stage goes through SQLLoad", + unload_to: Some("s3://lake/raw/unload/"), + external: false, + masks: false, + expected_blocked: Some(OUTSIDE), + }, + UnloadBlockedTestCase { + description: "the source's user stage is not the target's to read", + unload_to: None, + external: true, + masks: false, + expected_blocked: Some(OUTSIDE), + }, + UnloadBlockedTestCase { + description: "a named stage of the source's is not the target's to read", + unload_to: Some("@ops.loading.unload"), + external: true, + masks: false, + expected_blocked: Some(OUTSIDE), + }, + UnloadBlockedTestCase { + description: "masks need SQLLoad to see the rows, even under the stage", + unload_to: Some("s3://lake/raw/unload/"), + external: true, + masks: true, + expected_blocked: Some("the table masks columns"), + }, + ]; + for test_case in &test_cases { + let blocked = unload_blocked_for(test_case); + match test_case.expected_blocked { + None => assert_eq!(blocked, None, "{}", test_case.description), + Some(expected) => assert!( + blocked + .as_deref() + .is_some_and(|reason| reason.starts_with(expected)), + "{}: {blocked:?}", + test_case.description + ), + } + } +} diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index 24a6b0686..5de2f6d55 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -1092,7 +1092,12 @@ pub(crate) fn reader_offer( ); let direct_blocked = match unload_to.is_empty() { true => direct_blocked, - false => unload_blocked(spec), + false => unload_blocked( + spec.warehouse, + !spec.masked.columns.is_empty(), + spec.warehouse_source.as_ref().map(|source| &source.route), + stage, + ), }; ReaderOffer { arrow: unload_to.is_empty(), diff --git a/crates/sqlload/src/loader/constants.rs b/crates/sqlload/src/loader/constants.rs index c48e9db50..d394aed4d 100644 --- a/crates/sqlload/src/loader/constants.rs +++ b/crates/sqlload/src/loader/constants.rs @@ -108,8 +108,8 @@ pub const RESET: &str = "\x1b[0m"; /// The host name that always means this machine: plain text to it never crosses a network. pub const LOCALHOST: &str = "localhost"; -/// Why Snowflake does not read a Snowflake source's unload itself when the source unloads to a stage of its own. -pub const UNLOADED_OUTSIDE_S3: &str = "Snowflake reads another connection's unload itself only from S3: set the source's stage to an external one (url and storage_integration) inside the target's external stage"; +/// Why Snowflake does not read a Snowflake source's unload itself when no external stage of the target covers it. +pub const UNLOADED_OUTSIDE_STAGE: &str = "Snowflake reads another connection's unload itself only through the target's external stage: set the source's stage to an external one (url and storage_integration) whose url is inside the target's external stage url"; /// Why a warehouse source's unloaded files go through SQLLoad rather than straight into the destination. pub const UNLOADED_TIMES: &str = "the warehouse writes times and timestamps as whole microseconds (its Parquet writer keeps only milliseconds), which SQLLoad turns back into them"; From 6f20a514f7094c524d59c2596348bc24ec4713ef Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:01:52 +0000 Subject: [PATCH 5/9] wip: document Snowflake sources loading into Snowflake inside Snowflake --- website/public/index.html | 2 +- .../content/docs/docs/sources/snowflake.mdx | 25 ++++++++++++++++++- .../content/docs/docs/supported-systems.mdx | 2 +- .../src/content/docs/docs/targets/index.mdx | 4 +++ .../content/docs/docs/targets/snowflake.mdx | 7 ++++++ 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/website/public/index.html b/website/public/index.html index eded67b22..f3ce5d142 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -170,7 +170,7 @@

See every change first

02 · Load

Bulk copy, not row inserts

-

Files in S3 load into Snowflake straight from an external stage. Parquet loads into DuckDB through DuckDB's own reader. Rows are staged as Parquet and copied in bulk.

+

Files in S3 load into Snowflake straight from an external stage. Parquet loads into DuckDB through DuckDB's own reader. Snowflake tables load into Snowflake inside Snowflake. Rows are staged as Parquet and copied in bulk.

Load methods and routes →

sqlload plan
diff --git a/website/src/content/docs/docs/sources/snowflake.mdx b/website/src/content/docs/docs/sources/snowflake.mdx
index aa4bd2063..ccafde968 100644
--- a/website/src/content/docs/docs/sources/snowflake.mdx
+++ b/website/src/content/docs/docs/sources/snowflake.mdx
@@ -7,7 +7,8 @@ A Snowflake source loads tables and views of one
 database, and the results of queries you write.
 Snowflake unloads each read as Parquet with `COPY INTO`, and SQLLoad reads the files and loads
 them into the source's target. A DuckDB target reads them from S3
-itself.
+itself, and a Snowflake target loads the rows inside Snowflake; see
+[Snowflake loads the rows itself](#snowflake-loads-the-rows-itself).
 
 ```toml
 [connections.warehouse]
@@ -158,6 +159,28 @@ whatever its session time zone. A load fails before it commits when DuckDB loads
 of rows than `COPY INTO` reports, and the folder is removed either way. A table without a cursor
 commits once per run rather than about every 1 GB.
 
+### Snowflake loads the rows itself
+
+Into a Snowflake target, no rows pass through SQLLoad in two cases:
+
+- **The target writes through the source's connection**: the same `connection`, with no other
+  `database`. Snowflake selects the rows with one `INSERT ... SELECT`, and nothing is unloaded.
+- **The source's `stage` `url` is inside the target's external stage `url`**, on another connection
+  or account. Snowflake unloads the files there, and the target `COPY`s them through its own stage.
+  The load commits only when it loads as many rows as `COPY INTO` reported unloading.
+
+`plan` shows which:
+
+```text
+route   Snowflake INSERT ... SELECT on the source's connection (nothing through the loader)
+route   unload Parquet to s3://example-bucket/stage/unload/ -> Snowflake COPY (nothing through the loader)
+```
+
+Otherwise SQLLoad reads the files and stages them for the target, as for any other source: with the
+user stage, a named stage, or an S3 folder the target's stage doesn't cover. A table with a `mask`
+always goes through SQLLoad, which masks the values. Times and timestamps come back exactly, as
+with DuckDB.
+
 Without a stage SQLLoad downloads the files from the user stage
 unchanged, 16 at a time, into a temporary folder, and DuckDB reads them from there:
 
diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx
index 3fc7dee3c..64ade4134 100644
--- a/website/src/content/docs/docs/supported-systems.mdx
+++ b/website/src/content/docs/docs/supported-systems.mdx
@@ -9,7 +9,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes
 |--------|---------|---------|
 | Postgres | Tables, by `include` patterns | Binary `COPY` into Arrow, over TLS when the connection's `sslmode` asks |
 | Postgres changes (`postgres_cdc`) | Inserts, updates and deletes of tables, by `include` patterns | A snapshot, then a logical replication slot (`pgoutput`), in whole transactions. See [Postgres changes](/docs/sources/postgres-cdc/). |
-| Snowflake | Tables and views, by `include` patterns, and query results | Parquet unloaded with `COPY INTO` to the user stage or an S3 stage, then read by SQLLoad, or by DuckDB itself from an S3 stage. See [Snowflake](/docs/sources/snowflake/). |
+| Snowflake | Tables and views, by `include` patterns, and query results | Parquet unloaded with `COPY INTO` to the user stage or an S3 stage, then read by SQLLoad, or by DuckDB itself from an S3 stage. A Snowflake target selects or copies the rows inside Snowflake. See [Snowflake](/docs/sources/snowflake/). |
 | Databases with an ODBC driver: SQL Server, MySQL, MariaDB, OpenEdge and others | Tables and views, by `include` patterns, and query results | The driver, as Arrow. Tested with psqlODBC, Microsoft ODBC Driver 18 for SQL Server and MySQL Connector/ODBC; MariaDB Connector/ODBC and OpenEdge are checked by hand. See [ODBC](/docs/sources/odbc/#drivers). |
 | BigQuery (beta) | Tables and views, by `include` patterns, and query results | The Storage Read API, as Arrow in parallel streams; views and queries through a query job. See [BigQuery](/docs/sources/bigquery/). |
 | Databricks (beta) | Tables and views, by `include` patterns, and query results | One `SELECT` on the SQL warehouse, its result downloaded as Arrow files in parallel through the Statement Execution API's links. See [Databricks](/docs/sources/databricks/). |
diff --git a/website/src/content/docs/docs/targets/index.mdx b/website/src/content/docs/docs/targets/index.mdx
index c084f7251..cc500e889 100644
--- a/website/src/content/docs/docs/targets/index.mdx
+++ b/website/src/content/docs/docs/targets/index.mdx
@@ -20,6 +20,10 @@ names the route for every table.
 | ClickHouse | In S3: ClickHouse reads them with `s3()` into a staging table. In a folder: read by SQLLoad, sent as Arrow | Read by SQLLoad, sent as Arrow into a staging table | Sent as Arrow into a staging table, then swapped in by table or partition |
 | Postgres | Read by SQLLoad, then binary `COPY` into a temporary staging table | As for Parquet files | Binary `COPY` into a temporary staging table, then one transaction per commit |
 
+A Snowflake source loading into Snowflake stays inside Snowflake: an `INSERT ... SELECT` on the
+source's own connection, or a `COPY` of its unloaded files through the target's external stage.
+See [Snowflake loads the rows itself](/docs/sources/snowflake/#snowflake-loads-the-rows-itself).
+
 There are no silent fallbacks. If the route a table needs can't be used, `plan` fails for that
 table and says how to fix it.
 
diff --git a/website/src/content/docs/docs/targets/snowflake.mdx b/website/src/content/docs/docs/targets/snowflake.mdx
index 67e445d50..18f9d688b 100644
--- a/website/src/content/docs/docs/targets/snowflake.mdx
+++ b/website/src/content/docs/docs/targets/snowflake.mdx
@@ -105,3 +105,10 @@ this gives the same values as SQLLoad's own reader.
   [CSV encodings](/docs/sources/files/#csv-encodings).
 
 Files are never read and re-uploaded by SQLLoad otherwise.
+
+## Snowflake sources into Snowflake
+
+A [Snowflake source](/docs/sources/snowflake/) loads into this target without its rows passing
+through SQLLoad: with one `INSERT ... SELECT` when the target writes through the source's own
+connection, or a `COPY` from the target's external stage when the source unloads inside it. See
+[Snowflake loads the rows itself](/docs/sources/snowflake/#snowflake-loads-the-rows-itself).

From 46d49fa0f9630f67988312d357de316996902e20 Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Wed, 18 Nov 2026 17:31:47 +0000
Subject: [PATCH 6/9] wip: Snowflake-into-Snowflake review fixes: source
 warehouse, script limit, unexpected file, doc scope

---
 .../snowflake/_helpers/write/unloaded_load.rs | 80 ++++++++++++++-----
 .../snowflake/main/copy/_unloaded_sql.rs      |  2 +-
 .../sqlload/src/adapters/snowflake/tests.rs   |  2 +
 .../src/adapters/snowflake/tests/helpers.rs   | 41 +++++++++-
 .../adapters/snowflake/tests/test_types.rs    | 13 +++
 .../snowflake/tests/test_unloaded_requests.rs | 64 +++++++++++++++
 .../snowflake/tests/test_unloaded_sql.rs      |  4 +-
 crates/sqlload/src/loader/_helpers/specs.rs   |  5 +-
 website/public/index.html                     |  2 +-
 .../content/docs/docs/sources/snowflake.mdx   |  3 +-
 .../content/docs/docs/supported-systems.mdx   |  2 +-
 11 files changed, 191 insertions(+), 27 deletions(-)
 create mode 100644 crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs

diff --git a/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs b/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs
index 53280c7bd..c7deb4a1b 100644
--- a/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs
+++ b/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs
@@ -3,6 +3,7 @@
 use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch, UnloadedSource};
 use crate::adapters::contract::types::SqlAdapter;
 use crate::adapters::snowflake::_helpers::write::direct_load::staged_as_text;
+use crate::adapters::snowflake::constants::MAX_SCRIPT_BYTES;
 use crate::adapters::snowflake::main::copy::_stage_covers::stage_covers;
 use crate::adapters::snowflake::main::copy::_unloaded_sql::unloaded_statements;
 use crate::adapters::snowflake::main::stage::_stage_reference::stage_reference;
@@ -77,19 +78,26 @@ impl SnowflakeWriter {
             loaded_at: batch.loaded_at,
             from,
         });
-        let expected = unloaded.map(|files| files.rows);
-        let answer =
-            execute_sql(&self.destination.sql, &counted_block(&statements, expected)).await?;
-        let returned = answer
-            .rows
-            .first()
-            .and_then(|row| row.first().cloned().flatten())
-            .unwrap_or_default();
-        let rows: u64 = returned.parse().map_err(|error| {
-            LoaderError::Invalid(format!(
-                "loading the source's rows inside Snowflake returned {returned:?}, not a row count: {error}"
-            ))
-        })?;
+        let requests = unloaded_requests(
+            &statements,
+            unloaded.map(|files| (files.rows, files.keys.len())),
+            target.direct,
+        )
+        .map_err(LoaderError::Invalid)?;
+        let mut rows: u64 = 0;
+        for request in &requests {
+            let answer = execute_sql(&self.destination.sql, request).await?;
+            let returned = answer
+                .rows
+                .first()
+                .and_then(|row| row.first().cloned().flatten())
+                .unwrap_or_default();
+            rows += returned.parse::().map_err(|error| {
+                LoaderError::Invalid(format!(
+                    "loading the source's rows inside Snowflake returned {returned:?}, not a row count: {error}"
+                ))
+            })?;
+        }
         if let Some(files) = unloaded.filter(|files| files.rows != rows) {
             return Err(LoaderError::Invalid(format!(
                 "Snowflake loaded {rows} rows from the unloaded files in {}, but the unload reports {}; the batch is not committed",
@@ -100,8 +108,44 @@ impl SnowflakeWriter {
     }
 }
 
-/// One Snowflake Scripting block running `statements` in a transaction and returning the rows they loaded: `COPY`s, which commit only when they load `expected` rows and roll back otherwise, or an `INSERT ... SELECT` (`expected` `None`), which commits.
-pub(in crate::adapters) fn counted_block(statements: &[String], expected: Option) -> String {
+/// The requests loading `statements`, each a [`counted_block`]: one, whose `COPY`s commit only when they load all of the unload's rows (`unloaded`: its rows and files), when it fits `MAX_SCRIPT_BYTES`, as does an `INSERT ... SELECT` (`unloaded` `None`); otherwise several, each committing its `COPY`s into the staging or build table, which publishes only once their rows add up to the unload's, as the caller checks. A `direct` batch commits into the table itself, so it fails instead, before loading anything, as a part committed alone could not be taken back.
+pub(in crate::adapters) fn unloaded_requests(
+    statements: &[String],
+    unloaded: Option<(u64, usize)>,
+    direct: bool,
+) -> Result, String> {
+    let Some((rows, files)) = unloaded else {
+        return Ok(vec![counted_block(statements, false, None)]);
+    };
+    let whole = counted_block(statements, true, Some(rows));
+    if whole.len() <= MAX_SCRIPT_BYTES {
+        return Ok(vec![whole]);
+    }
+    if direct {
+        return Err(format!(
+            "the unload wrote {files} files, more than one transaction of Snowflake COPYs can name, and this append commits into the table itself. Fix: set cursor_column and batch_size, so each read unloads fewer files"
+        ));
+    }
+    let mut groups: Vec> = Vec::new();
+    for statement in statements {
+        let fits = groups.last().is_some_and(|group| {
+            let mut grown = group.clone();
+            grown.push(statement.clone());
+            counted_block(&grown, true, None).len() <= MAX_SCRIPT_BYTES
+        });
+        match (fits, groups.last_mut()) {
+            (true, Some(group)) => group.push(statement.clone()),
+            _ => groups.push(vec![statement.clone()]),
+        }
+    }
+    Ok(groups
+        .iter()
+        .map(|group| counted_block(group, true, None))
+        .collect())
+}
+
+/// One Snowflake Scripting block running `statements` in a transaction and returning the rows they loaded: `COPY`s (`copies`), counted from their results, which commit only when they load `expected` rows (when given) and roll back otherwise, or an `INSERT ... SELECT`, which commits.
+fn counted_block(statements: &[String], copies: bool, expected: Option) -> String {
     let mut body = vec![
         "DECLARE loaded INTEGER DEFAULT 0; copied INTEGER;".to_owned(),
         "BEGIN".to_owned(),
@@ -109,9 +153,9 @@ pub(in crate::adapters) fn counted_block(statements: &[String], expected: Option
     ];
     for statement in statements {
         body.push(format!("{statement};"));
-        body.push(match expected {
-            Some(_) => "SELECT COALESCE(SUM(\"rows_loaded\"), 0) INTO :copied FROM TABLE(RESULT_SCAN(LAST_QUERY_ID()));".to_owned(),
-            None => "copied := SQLROWCOUNT;".to_owned(),
+        body.push(match copies {
+            true => "SELECT COALESCE(SUM(\"rows_loaded\"), 0) INTO :copied FROM TABLE(RESULT_SCAN(LAST_QUERY_ID()));".to_owned(),
+            false => "copied := SQLROWCOUNT;".to_owned(),
         });
         body.push("loaded := loaded + copied;".to_owned());
     }
diff --git a/crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs b/crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs
index 05e93946a..31982b005 100644
--- a/crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs
+++ b/crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs
@@ -93,7 +93,7 @@ fn column_value(request: &UnloadedLoad<'_>, column: &DesiredColumn, rows: &Rows<
                         })
                         .collect();
                     format!(
-                        "(CASE METADATA$FILENAME {} END) * {UNLOADED_FILE_ROWS} + METADATA$FILE_ROW_NUMBER",
+                        "(CASE METADATA$FILENAME {} ELSE TO_NUMBER('sqlload: unexpected unloaded file ' || METADATA$FILENAME) END) * {UNLOADED_FILE_ROWS} + METADATA$FILE_ROW_NUMBER",
                         files.join(" ")
                     )
                 }
diff --git a/crates/sqlload/src/adapters/snowflake/tests.rs b/crates/sqlload/src/adapters/snowflake/tests.rs
index 61595548a..bd84e9bfb 100644
--- a/crates/sqlload/src/adapters/snowflake/tests.rs
+++ b/crates/sqlload/src/adapters/snowflake/tests.rs
@@ -22,5 +22,7 @@ mod step_failure;
 mod test_types;
 #[path = "tests/test_unload_blocked.rs"]
 mod unload_blocked;
+#[path = "tests/test_unloaded_requests.rs"]
+mod unloaded_requests;
 #[path = "tests/test_unloaded_sql.rs"]
 mod unloaded_sql;
diff --git a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs
index e4dd8d6de..0e8291c59 100644
--- a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs
+++ b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs
@@ -9,7 +9,9 @@ use crate::adapters::snowflake::models::CopyStaged;
 use crate::adapters::snowflake::models::StageRoute;
 use crate::adapters::snowflake::models::{CopyColumn, CopyValue};
 use crate::adapters::snowflake::models::{ExternalStage, StageObject};
-use crate::adapters::snowflake::tests::test_types::{CopyFrom, UnloadBlockedTestCase};
+use crate::adapters::snowflake::tests::test_types::{
+    CopyFrom, UnloadBlockedTestCase, UnloadedRequestsTestCase,
+};
 use crate::column_types::models::ColumnType;
 use crate::column_types::models::Warehouse;
 use crate::file_store::models::StoredFile;
@@ -459,3 +461,40 @@ pub(crate) fn unload_blocked_for(test_case: &UnloadBlockedTestCase) -> Option Result, String> {
+    use crate::adapters::snowflake::_helpers::write::unloaded_load::unloaded_requests;
+    use crate::adapters::snowflake::constants::MAX_SCRIPT_BYTES;
+
+    let (statements, unloaded) = match test_case.copies {
+        Some((count, length)) => (
+            (0..count)
+                .map(|index| format!("COPY INTO t /* {index} */ {}", "x".repeat(length)))
+                .collect::>(),
+            Some((10, count * 1_000)),
+        ),
+        None => (vec!["INSERT INTO t SELECT 1".to_owned()], None),
+    };
+    let requests = unloaded_requests(&statements, unloaded, test_case.direct)?;
+    assert!(
+        requests
+            .iter()
+            .all(|request| request.len() <= MAX_SCRIPT_BYTES),
+        "{}: a request is longer than the limit",
+        test_case.description
+    );
+    let sent: usize = requests
+        .iter()
+        .map(|request| request.matches("COPY INTO t /*").count())
+        .sum();
+    assert_eq!(
+        sent,
+        test_case.copies.map_or(0, |(count, _)| count),
+        "{}: every COPY is sent once",
+        test_case.description
+    );
+    Ok(requests)
+}
diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs
index 15b051e59..e56941911 100644
--- a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs
+++ b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs
@@ -132,3 +132,16 @@ pub(crate) struct UnloadBlockedTestCase {
     /// The reason SQLLoad reads the files instead, or `None` when Snowflake loads them itself.
     pub(crate) expected_blocked: Option<&'static str>,
 }
+
+/// How the statements loading one unload inside Snowflake are sent.
+pub(crate) struct UnloadedRequestsTestCase {
+    pub(crate) description: &'static str,
+    /// How many `COPY`s, and how long each is; `None` is one `INSERT ... SELECT`.
+    pub(crate) copies: Option<(usize, usize)>,
+    /// Whether the batch commits into the table itself.
+    pub(crate) direct: bool,
+    /// How many requests, or the start of the error.
+    pub(crate) expected_requests: Result,
+    /// Whether the (first) request commits only when every unloaded row loads.
+    pub(crate) expected_checks_count: bool,
+}
diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs b/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs
new file mode 100644
index 000000000..da1bf93ac
--- /dev/null
+++ b/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs
@@ -0,0 +1,64 @@
+use crate::adapters::snowflake::tests::helpers::unloaded_requests_for;
+use crate::adapters::snowflake::tests::test_types::UnloadedRequestsTestCase;
+
+#[test]
+fn given_an_unload_loaded_inside_snowflake_when_sending_its_statements_then_they_fit_the_script_limit_and_a_direct_append_never_commits_part()
+ {
+    let test_cases = [
+        UnloadedRequestsTestCase {
+            description: "a few COPYs go in one request that commits only when every unloaded row loads",
+            copies: Some((3, 100_000)),
+            direct: false,
+            expected_requests: Ok(1),
+            expected_checks_count: true,
+        },
+        UnloadedRequestsTestCase {
+            description: "COPYs past the script limit are split, each request committing its part into the staging table",
+            copies: Some((20, 120_000)),
+            direct: false,
+            expected_requests: Ok(2),
+            expected_checks_count: false,
+        },
+        UnloadedRequestsTestCase {
+            description: "a direct append past the limit fails before loading anything",
+            copies: Some((20, 120_000)),
+            direct: true,
+            expected_requests: Err("the unload wrote 20000 files"),
+            expected_checks_count: false,
+        },
+        UnloadedRequestsTestCase {
+            description: "a direct append within the limit goes in one counted request",
+            copies: Some((3, 100_000)),
+            direct: true,
+            expected_requests: Ok(1),
+            expected_checks_count: true,
+        },
+        UnloadedRequestsTestCase {
+            description: "an INSERT ... SELECT is one request",
+            copies: None,
+            direct: true,
+            expected_requests: Ok(1),
+            expected_checks_count: false,
+        },
+    ];
+    for test_case in &test_cases {
+        let requests = unloaded_requests_for(test_case);
+        match (&requests, test_case.expected_requests) {
+            (Ok(requests), Ok(expected)) => {
+                assert_eq!(requests.len(), expected, "{}", test_case.description);
+                assert_eq!(
+                    requests[0].contains("IF (loaded = "),
+                    test_case.expected_checks_count,
+                    "{}",
+                    test_case.description
+                );
+            }
+            (Err(error), Err(expected)) => assert!(
+                error.starts_with(expected),
+                "{}: {error}",
+                test_case.description
+            ),
+            _ => panic!("{}: {requests:?}", test_case.description),
+        }
+    }
+}
diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_sql.rs b/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_sql.rs
index 6912dd874..f7fd6bc99 100644
--- a/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_sql.rs
+++ b/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_sql.rs
@@ -58,9 +58,9 @@ fn given_a_snowflake_sources_rows_when_loading_them_inside_snowflake_then_their_
             expected_statements: 2,
             expected_first: &[
                 "3 * 281474976710656 + (CASE METADATA$FILENAME WHEN 'lake/unload/run-1/data_0.parquet' THEN 0 WHEN ",
-                "WHEN 'lake/unload/run-1/data_999.parquet' THEN 999 END) * 4294967296 + METADATA$FILE_ROW_NUMBER",
+                "WHEN 'lake/unload/run-1/data_999.parquet' THEN 999 ELSE TO_NUMBER('sqlload: unexpected unloaded file ' || METADATA$FILENAME) END) * 4294967296 + METADATA$FILE_ROW_NUMBER",
             ],
-            expected_last: "(CASE METADATA$FILENAME WHEN 'lake/unload/run-1/data_1000.parquet' THEN 1000 END) * 4294967296 + METADATA$FILE_ROW_NUMBER FROM @\"DB\".\"RAW\".\"LAKE\"/unload/run-1/) FILES = ('data_1000.parquet')",
+            expected_last: "(CASE METADATA$FILENAME WHEN 'lake/unload/run-1/data_1000.parquet' THEN 1000 ELSE TO_NUMBER('sqlload: unexpected unloaded file ' || METADATA$FILENAME) END) * 4294967296 + METADATA$FILE_ROW_NUMBER FROM @\"DB\".\"RAW\".\"LAKE\"/unload/run-1/) FILES = ('data_1000.parquet')",
         },
         UnloadedSqlTestCase {
             description: "selected in the source's session: one INSERT reading the query's columns by name, turned back the same way; a merge numbers rows in the order the query gives them",
diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs
index 758b37281..e06782506 100644
--- a/crates/sqlload/src/loader/_helpers/specs.rs
+++ b/crates/sqlload/src/loader/_helpers/specs.rs
@@ -656,7 +656,7 @@ fn constant_type_problem(
     }
 }
 
-/// How a `snowflake` source's table is read: unloaded to the source's `stage`, or to the user stage when it sets none; a Snowflake target writing through the source's own connection may select the rows itself.
+/// How a `snowflake` source's table is read: unloaded to the source's `stage`, or to the user stage when it sets none; a Snowflake target writing through the source's own connection, on its warehouse (the source sets no `warehouse` of its own), may select the rows itself.
 fn warehouse_source(
     context: &SourceContext<'_>,
     overrides: &TableOverride,
@@ -694,7 +694,8 @@ fn warehouse_source(
                 store,
                 stage,
                 same_connection: context.target.warehouse == Warehouse::Snowflake
-                    && context.target.session_connection.as_ref() == Some(&source.connection),
+                    && context.target.session_connection.as_ref() == Some(&source.connection)
+                    && source.warehouse.is_none(),
             },
         }
     })
diff --git a/website/public/index.html b/website/public/index.html
index f3ce5d142..f2078c4d4 100644
--- a/website/public/index.html
+++ b/website/public/index.html
@@ -170,7 +170,7 @@ 

See every change first

02 · Load

Bulk copy, not row inserts

-

Files in S3 load into Snowflake straight from an external stage. Parquet loads into DuckDB through DuckDB's own reader. Snowflake tables load into Snowflake inside Snowflake. Rows are staged as Parquet and copied in bulk.

+

Files in S3 load into Snowflake straight from an external stage. Parquet loads into DuckDB through DuckDB's own reader. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Rows are staged as Parquet and copied in bulk.

Load methods and routes →

sqlload plan
diff --git a/website/src/content/docs/docs/sources/snowflake.mdx b/website/src/content/docs/docs/sources/snowflake.mdx
index ccafde968..34304d670 100644
--- a/website/src/content/docs/docs/sources/snowflake.mdx
+++ b/website/src/content/docs/docs/sources/snowflake.mdx
@@ -164,7 +164,8 @@ commits once per run rather than about every 1 GB.
 Into a Snowflake target, no rows pass through SQLLoad in two cases:
 
 - **The target writes through the source's connection**: the same `connection`, with no other
-  `database`. Snowflake selects the rows with one `INSERT ... SELECT`, and nothing is unloaded.
+  `database`, and the source sets no `warehouse` of its own. Snowflake selects the rows with one
+  `INSERT ... SELECT` on the connection's warehouse, and nothing is unloaded.
 - **The source's `stage` `url` is inside the target's external stage `url`**, on another connection
   or account. Snowflake unloads the files there, and the target `COPY`s them through its own stage.
   The load commits only when it loads as many rows as `COPY INTO` reported unloading.
diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx
index 64ade4134..1c87b78f7 100644
--- a/website/src/content/docs/docs/supported-systems.mdx
+++ b/website/src/content/docs/docs/supported-systems.mdx
@@ -9,7 +9,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes
 |--------|---------|---------|
 | Postgres | Tables, by `include` patterns | Binary `COPY` into Arrow, over TLS when the connection's `sslmode` asks |
 | Postgres changes (`postgres_cdc`) | Inserts, updates and deletes of tables, by `include` patterns | A snapshot, then a logical replication slot (`pgoutput`), in whole transactions. See [Postgres changes](/docs/sources/postgres-cdc/). |
-| Snowflake | Tables and views, by `include` patterns, and query results | Parquet unloaded with `COPY INTO` to the user stage or an S3 stage, then read by SQLLoad, or by DuckDB itself from an S3 stage. A Snowflake target selects or copies the rows inside Snowflake. See [Snowflake](/docs/sources/snowflake/). |
+| Snowflake | Tables and views, by `include` patterns, and query results | Parquet unloaded with `COPY INTO` to the user stage or an S3 stage, then read by SQLLoad, or by DuckDB itself from an S3 stage. A Snowflake target selects or copies the rows inside Snowflake when it writes through the source's connection, or its external stage covers the unload; see [Snowflake loads the rows itself](/docs/sources/snowflake/#snowflake-loads-the-rows-itself). See [Snowflake](/docs/sources/snowflake/). |
 | Databases with an ODBC driver: SQL Server, MySQL, MariaDB, OpenEdge and others | Tables and views, by `include` patterns, and query results | The driver, as Arrow. Tested with psqlODBC, Microsoft ODBC Driver 18 for SQL Server and MySQL Connector/ODBC; MariaDB Connector/ODBC and OpenEdge are checked by hand. See [ODBC](/docs/sources/odbc/#drivers). |
 | BigQuery (beta) | Tables and views, by `include` patterns, and query results | The Storage Read API, as Arrow in parallel streams; views and queries through a query job. See [BigQuery](/docs/sources/bigquery/). |
 | Databricks (beta) | Tables and views, by `include` patterns, and query results | One `SELECT` on the SQL warehouse, its result downloaded as Arrow files in parallel through the Statement Execution API's links. See [Databricks](/docs/sources/databricks/). |

From 32eee0dbfc0c5bd7c92d5a491420c25c9ae2b1a7 Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Wed, 18 Nov 2026 17:38:01 +0000
Subject: [PATCH 7/9] wip: unloaded COPY blocks within one statement's limit;
 clearer refusal for a direct append

---
 .../snowflake/_helpers/write/unloaded_load.rs        | 12 ++++++------
 .../sqlload/src/adapters/snowflake/tests/helpers.rs  |  6 +++---
 .../snowflake/tests/test_unloaded_requests.rs        |  2 +-
 website/src/content/docs/docs/sources/snowflake.mdx  |  4 +++-
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs b/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs
index c7deb4a1b..3328c484e 100644
--- a/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs
+++ b/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs
@@ -3,7 +3,7 @@
 use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch, UnloadedSource};
 use crate::adapters::contract::types::SqlAdapter;
 use crate::adapters::snowflake::_helpers::write::direct_load::staged_as_text;
-use crate::adapters::snowflake::constants::MAX_SCRIPT_BYTES;
+use crate::adapters::snowflake::constants::MAX_COPY_SQL_BYTES;
 use crate::adapters::snowflake::main::copy::_stage_covers::stage_covers;
 use crate::adapters::snowflake::main::copy::_unloaded_sql::unloaded_statements;
 use crate::adapters::snowflake::main::stage::_stage_reference::stage_reference;
@@ -14,7 +14,7 @@ use crate::loader::errors::LoaderError;
 use crate::sql_api::main::execute_sql::execute_sql;
 
 impl SnowflakeWriter {
-    /// Load `batch` inside Snowflake, where Arrow batches would go: the load's staging table, the build table, or the table itself. Unloaded files are `COPY`d from the external stage the writer loads through, which must cover them, in one transaction that commits only when they load as many rows as the unload reports; a query's rows are selected by one `INSERT ... SELECT`.
+    /// Load `batch` inside Snowflake, where Arrow batches would go: the load's staging table, the build table, or the table itself. Unloaded files are `COPY`d from the external stage the writer loads through, which must cover them, as [`unloaded_requests`] sends them, and the batch fails unless they load as many rows as the unload reports; a query's rows are selected by one `INSERT ... SELECT`.
     pub(in crate::adapters) async fn pull_unloaded(
         &mut self,
         batch: &UnloadedBatch<'_>,
@@ -108,7 +108,7 @@ impl SnowflakeWriter {
     }
 }
 
-/// The requests loading `statements`, each a [`counted_block`]: one, whose `COPY`s commit only when they load all of the unload's rows (`unloaded`: its rows and files), when it fits `MAX_SCRIPT_BYTES`, as does an `INSERT ... SELECT` (`unloaded` `None`); otherwise several, each committing its `COPY`s into the staging or build table, which publishes only once their rows add up to the unload's, as the caller checks. A `direct` batch commits into the table itself, so it fails instead, before loading anything, as a part committed alone could not be taken back.
+/// The requests loading `statements`, each a [`counted_block`]: one, whose `COPY`s commit only when they load all of the unload's rows (`unloaded`: its rows and files), when it fits `MAX_COPY_SQL_BYTES`, as does an `INSERT ... SELECT` (`unloaded` `None`); otherwise several, each committing its `COPY`s into the staging or build table, which publishes only once their rows add up to the unload's, as the caller checks. A `direct` batch commits into the table itself, so it fails instead, before loading anything, as a part committed alone could not be taken back.
 pub(in crate::adapters) fn unloaded_requests(
     statements: &[String],
     unloaded: Option<(u64, usize)>,
@@ -118,12 +118,12 @@ pub(in crate::adapters) fn unloaded_requests(
         return Ok(vec![counted_block(statements, false, None)]);
     };
     let whole = counted_block(statements, true, Some(rows));
-    if whole.len() <= MAX_SCRIPT_BYTES {
+    if whole.len() <= MAX_COPY_SQL_BYTES {
         return Ok(vec![whole]);
     }
     if direct {
         return Err(format!(
-            "the unload wrote {files} files, more than one transaction of Snowflake COPYs can name, and this append commits into the table itself. Fix: set cursor_column and batch_size, so each read unloads fewer files"
+            "the unload wrote {files} files, whose COPYs are too long to run as one Snowflake statement, and this append commits into the table itself, so they can't be split. Fix: set cursor_column with a smaller batch_size, so each read unloads fewer files"
         ));
     }
     let mut groups: Vec> = Vec::new();
@@ -131,7 +131,7 @@ pub(in crate::adapters) fn unloaded_requests(
         let fits = groups.last().is_some_and(|group| {
             let mut grown = group.clone();
             grown.push(statement.clone());
-            counted_block(&grown, true, None).len() <= MAX_SCRIPT_BYTES
+            counted_block(&grown, true, None).len() <= MAX_COPY_SQL_BYTES
         });
         match (fits, groups.last_mut()) {
             (true, Some(group)) => group.push(statement.clone()),
diff --git a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs
index 0e8291c59..9aae0db8f 100644
--- a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs
+++ b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs
@@ -462,12 +462,12 @@ pub(crate) fn unload_blocked_for(test_case: &UnloadBlockedTestCase) -> Option Result, String> {
     use crate::adapters::snowflake::_helpers::write::unloaded_load::unloaded_requests;
-    use crate::adapters::snowflake::constants::MAX_SCRIPT_BYTES;
+    use crate::adapters::snowflake::constants::MAX_COPY_SQL_BYTES;
 
     let (statements, unloaded) = match test_case.copies {
         Some((count, length)) => (
@@ -482,7 +482,7 @@ pub(crate) fn unloaded_requests_for(
     assert!(
         requests
             .iter()
-            .all(|request| request.len() <= MAX_SCRIPT_BYTES),
+            .all(|request| request.len() <= MAX_COPY_SQL_BYTES),
         "{}: a request is longer than the limit",
         test_case.description
     );
diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs b/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs
index da1bf93ac..b3032e4b2 100644
--- a/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs
+++ b/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs
@@ -16,7 +16,7 @@ fn given_an_unload_loaded_inside_snowflake_when_sending_its_statements_then_they
             description: "COPYs past the script limit are split, each request committing its part into the staging table",
             copies: Some((20, 120_000)),
             direct: false,
-            expected_requests: Ok(2),
+            expected_requests: Ok(3),
             expected_checks_count: false,
         },
         UnloadedRequestsTestCase {
diff --git a/website/src/content/docs/docs/sources/snowflake.mdx b/website/src/content/docs/docs/sources/snowflake.mdx
index 34304d670..907030a2c 100644
--- a/website/src/content/docs/docs/sources/snowflake.mdx
+++ b/website/src/content/docs/docs/sources/snowflake.mdx
@@ -168,7 +168,9 @@ Into a Snowflake target, no rows pass through SQLLoad in two cases:
   `INSERT ... SELECT` on the connection's warehouse, and nothing is unloaded.
 - **The source's `stage` `url` is inside the target's external stage `url`**, on another connection
   or account. Snowflake unloads the files there, and the target `COPY`s them through its own stage.
-  The load commits only when it loads as many rows as `COPY INTO` reported unloading.
+  The load commits only when it loads as many rows as `COPY INTO` reported unloading. An `append`
+  that writes straight into its table fails, before loading anything, when one read unloads too
+  many files to `COPY` in one statement (thousands); a smaller `batch_size` splits the read.
 
 `plan` shows which:
 

From dd2c4ea66eef83bfd5a6270ede49c4eef403da65 Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Wed, 18 Nov 2026 18:07:12 +0000
Subject: [PATCH 8/9] wip: Snowflake-into-Snowflake within fensu: value helpers
 out of the entry module, branch-free tests, loader files under the line limit

---
 .../snowflake/_helpers/dialect/mod.rs         |   1 +
 .../_helpers/dialect/unloaded_values.rs       | 103 +++++++++++++
 .../snowflake/main/copy/_unloaded_sql.rs      | 137 +++---------------
 .../sqlload/src/adapters/snowflake/models.rs  |   9 ++
 .../src/adapters/snowflake/tests/helpers.rs   |  60 ++++----
 .../adapters/snowflake/tests/test_types.rs    |   6 +-
 .../snowflake/tests/test_unload_blocked.rs    |  24 ++-
 .../snowflake/tests/test_unloaded_requests.rs |  40 ++---
 crates/sqlload/src/loader/_helpers/publish.rs |  30 +++-
 crates/sqlload/src/loader/_helpers/render.rs  |  20 +--
 crates/sqlload/src/loader/_helpers/sources.rs |  15 +-
 crates/sqlload/src/loader/_helpers/specs.rs   |   4 +-
 .../sqlload/src/loader/_helpers/table_plan.rs |  28 +---
 crates/sqlload/src/loader/constants.rs        |   6 +-
 14 files changed, 240 insertions(+), 243 deletions(-)
 create mode 100644 crates/sqlload/src/adapters/snowflake/_helpers/dialect/unloaded_values.rs

diff --git a/crates/sqlload/src/adapters/snowflake/_helpers/dialect/mod.rs b/crates/sqlload/src/adapters/snowflake/_helpers/dialect/mod.rs
index 611e59dda..8f69dab79 100644
--- a/crates/sqlload/src/adapters/snowflake/_helpers/dialect/mod.rs
+++ b/crates/sqlload/src/adapters/snowflake/_helpers/dialect/mod.rs
@@ -2,4 +2,5 @@ pub(crate) mod copy_select;
 pub(crate) mod sql_adapter;
 pub(crate) mod text_values;
 pub(crate) mod type_names;
+pub(crate) mod unloaded_values;
 pub(crate) mod values;
diff --git a/crates/sqlload/src/adapters/snowflake/_helpers/dialect/unloaded_values.rs b/crates/sqlload/src/adapters/snowflake/_helpers/dialect/unloaded_values.rs
new file mode 100644
index 000000000..8a4368e38
--- /dev/null
+++ b/crates/sqlload/src/adapters/snowflake/_helpers/dialect/unloaded_values.rs
@@ -0,0 +1,103 @@
+//! The value each column of a Snowflake source's batch loaded inside Snowflake takes, from the source's unloaded files or its query.
+
+use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS};
+use crate::adapters::contract::types::SqlAdapter;
+use crate::adapters::snowflake::models::{SnowflakeSql, UnloadedFrom, UnloadedLoad, UnloadedRows};
+use crate::column_types::models::ColumnType;
+use crate::load_naming::main::metadata_role::metadata_role;
+use crate::load_naming::models::MetadataRole;
+use crate::schema_plan::constants::MERGE_ROW_COLUMN;
+use crate::schema_plan::models::DesiredColumn;
+
+/// Where `column` comes from in a statement reading `rows`: the batch's load id or load time, a merge's row number (the batch, then the file, then the row in it; a query's rows in the order it gives them), the value of the constant it is, or the source's column it is read as; NULL when the source lacks it.
+pub(crate) fn column_value(
+    request: &UnloadedLoad<'_>,
+    column: &DesiredColumn,
+    rows: &UnloadedRows<'_>,
+) -> String {
+    let literal = |text: &str| SnowflakeSql.render_loader_value_literal(text);
+    let constant = request
+        .constants
+        .iter()
+        .find(|(name, _, _)| *name == column.name);
+    match (metadata_role(request.compat, &column.name), constant) {
+        (Some(MetadataRole::LoadId), _) => literal(request.load_id),
+        (Some(MetadataRole::LoadedAt), _) => {
+            format!("CAST({} AS TIMESTAMP_TZ(6))", literal(request.loaded_at))
+        }
+        _ if column.name == MERGE_ROW_COLUMN => {
+            let in_batch = match (rows, &request.from) {
+                (
+                    UnloadedRows::Files(first, keys),
+                    UnloadedFrom::Files {
+                        filename_prefix, ..
+                    },
+                ) => {
+                    let files: Vec = keys
+                        .iter()
+                        .enumerate()
+                        .map(|(index, key)| {
+                            format!(
+                                "WHEN {} THEN {}",
+                                literal(&format!("{filename_prefix}{key}")),
+                                first + index
+                            )
+                        })
+                        .collect();
+                    format!(
+                        "(CASE METADATA$FILENAME {} ELSE TO_NUMBER('sqlload: unexpected unloaded file ' || METADATA$FILENAME) END) * {UNLOADED_FILE_ROWS} + METADATA$FILE_ROW_NUMBER",
+                        files.join(" ")
+                    )
+                }
+                _ => "SEQ8()".to_owned(),
+            };
+            format!("{} * {UNLOADED_BATCH_ROWS} + {in_batch}", request.batch)
+        }
+        (_, Some((_, _, value))) => cast(column, &literal(value)),
+        _ => request
+            .read
+            .iter()
+            .find(|read| read.name == column.name)
+            .map_or_else(
+                || "NULL".to_owned(),
+                |read| {
+                    let field = match rows {
+                        UnloadedRows::Files(..) => format!("GET($1, {})", literal(&read.name)),
+                        UnloadedRows::Query => format!("u.{}", quoted(&read.name)),
+                    };
+                    unloaded_value(column, &read.output, &field)
+                },
+            ),
+    }
+}
+
+/// `name` as the unload's query names its column: quoted exactly as written.
+fn quoted(name: &str) -> String {
+    format!("\"{}\"", name.replace('"', "\"\""))
+}
+
+/// An unloaded `field`, read as `read`, as the value it was unloaded from, then as `column` holds it: a time or timestamp from its whole microseconds (a timestamp with time zone in UTC, as SQLLoad's reader gives it), anything else as it is.
+fn unloaded_value(column: &DesiredColumn, read: &ColumnType, field: &str) -> String {
+    let micros = format!("CAST({field} AS NUMBER(38,0))");
+    let value = match read {
+        ColumnType::TimestampNtz => format!("TO_TIMESTAMP_NTZ({micros}, 6)"),
+        ColumnType::TimestampTz => {
+            format!("CONVERT_TIMEZONE('UTC', TO_TIMESTAMP_TZ({micros}, 6))")
+        }
+        ColumnType::Time => format!("TIMEADD(MICROSECOND, {micros}, '00:00:00'::TIME)"),
+        _ => field.to_owned(),
+    };
+    cast(column, &value)
+}
+
+/// `value` as `column` holds it: JSON and arrays parsed from their text, anything else cast to its type.
+fn cast(column: &DesiredColumn, value: &str) -> String {
+    match &column.column_type {
+        ColumnType::Json => format!("PARSE_JSON(CAST({value} AS VARCHAR))"),
+        ColumnType::Array(_) => format!("CAST(PARSE_JSON(CAST({value} AS VARCHAR)) AS ARRAY)"),
+        column_type => format!(
+            "CAST({value} AS {})",
+            SnowflakeSql.render_loader_logical_type(column_type).sql
+        ),
+    }
+}
diff --git a/crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs b/crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs
index 31982b005..9f1c8124e 100644
--- a/crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs
+++ b/crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs
@@ -1,14 +1,9 @@
 //! The statements loading a Snowflake source's rows into a table inside Snowflake: from its unloaded Parquet files, or selected from the source in the same session.
 
-use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS};
 use crate::adapters::contract::types::SqlAdapter;
+use crate::adapters::snowflake::_helpers::dialect::unloaded_values::column_value;
 use crate::adapters::snowflake::constants::{COPY_FILE_FORMAT, MAX_COPY_FILES};
-use crate::adapters::snowflake::models::{SnowflakeSql, UnloadedFrom, UnloadedLoad};
-use crate::column_types::models::ColumnType;
-use crate::load_naming::main::metadata_role::metadata_role;
-use crate::load_naming::models::MetadataRole;
-use crate::schema_plan::constants::MERGE_ROW_COLUMN;
-use crate::schema_plan::models::DesiredColumn;
+use crate::adapters::snowflake::models::{SnowflakeSql, UnloadedFrom, UnloadedLoad, UnloadedRows};
 
 /// The statements loading `request`'s rows into `request.into`, every one of its columns filled as [`column_value`] says: from files, `COPY`s of at most 1,000 files each, stopping at the first bad value and never relying on Snowflake's load history; from a query, one `INSERT ... SELECT`.
 pub(in crate::adapters) fn unloaded_statements(request: &UnloadedLoad<'_>) -> Vec {
@@ -18,13 +13,7 @@ pub(in crate::adapters) fn unloaded_statements(request: &UnloadedLoad<'_>) -> Ve
         .iter()
         .map(|column| SnowflakeSql.render_identifier(&column.name))
         .collect();
-    let values = |rows: &Rows<'_>| -> Vec {
-        request
-            .columns
-            .iter()
-            .map(|column| column_value(request, column, rows))
-            .collect()
-    };
+    let names = names.join(", ");
     match &request.from {
         UnloadedFrom::Files {
             stage,
@@ -35,117 +24,35 @@ pub(in crate::adapters) fn unloaded_statements(request: &UnloadedLoad<'_>) -> Ve
             .chunks(MAX_COPY_FILES)
             .enumerate()
             .map(|(chunk, chunk_keys)| {
-                let files: Vec = chunk_keys
-                    .iter()
-                    .map(|key| SnowflakeSql.render_loader_value_literal(key))
-                    .collect();
                 format!(
-                    "COPY INTO {into} ({}) FROM (SELECT {} FROM {stage}/{folder}) FILES = ({}) {COPY_FILE_FORMAT} ON_ERROR = ABORT_STATEMENT FORCE = TRUE",
-                    names.join(", "),
-                    values(&Rows::Files(chunk * MAX_COPY_FILES, chunk_keys)).join(", "),
-                    files.join(", ")
+                    "COPY INTO {into} ({names}) FROM (SELECT {} FROM {stage}/{folder}) FILES = ({}) {COPY_FILE_FORMAT} ON_ERROR = ABORT_STATEMENT FORCE = TRUE",
+                    values(request, UnloadedRows::Files(chunk * MAX_COPY_FILES, chunk_keys)),
+                    file_list(chunk_keys)
                 )
             })
             .collect(),
         UnloadedFrom::Query(query) => vec![format!(
-            "INSERT INTO {into} ({}) SELECT {} FROM ({query}) AS u",
-            names.join(", "),
-            values(&Rows::Query).join(", ")
+            "INSERT INTO {into} ({names}) SELECT {} FROM ({query}) AS u",
+            values(request, UnloadedRows::Query)
         )],
     }
 }
 
-/// Which rows one statement reads: the unload's files `keys`, the `first`th file on, or the query.
-enum Rows<'a> {
-    Files(usize, &'a [String]),
-    Query,
-}
-
-/// Where `column` comes from in a statement reading `rows`: the batch's load id or load time, a merge's row number (the batch, then the file, then the row in it; a query's rows in the order it gives them), the value of the constant it is, or the source's column it is read as; NULL when the source lacks it.
-fn column_value(request: &UnloadedLoad<'_>, column: &DesiredColumn, rows: &Rows<'_>) -> String {
-    let literal = |text: &str| SnowflakeSql.render_loader_value_literal(text);
-    let constant = request
-        .constants
+/// Each of `request`'s columns as a statement reading `rows` fills it.
+fn values(request: &UnloadedLoad<'_>, rows: UnloadedRows<'_>) -> String {
+    let values: Vec = request
+        .columns
         .iter()
-        .find(|(name, _, _)| *name == column.name);
-    match (metadata_role(request.compat, &column.name), constant) {
-        (Some(MetadataRole::LoadId), _) => literal(request.load_id),
-        (Some(MetadataRole::LoadedAt), _) => {
-            format!("CAST({} AS TIMESTAMP_TZ(6))", literal(request.loaded_at))
-        }
-        _ if column.name == MERGE_ROW_COLUMN => {
-            let in_batch = match (rows, &request.from) {
-                (
-                    Rows::Files(first, keys),
-                    UnloadedFrom::Files {
-                        filename_prefix, ..
-                    },
-                ) => {
-                    let files: Vec = keys
-                        .iter()
-                        .enumerate()
-                        .map(|(index, key)| {
-                            format!(
-                                "WHEN {} THEN {}",
-                                literal(&format!("{filename_prefix}{key}")),
-                                first + index
-                            )
-                        })
-                        .collect();
-                    format!(
-                        "(CASE METADATA$FILENAME {} ELSE TO_NUMBER('sqlload: unexpected unloaded file ' || METADATA$FILENAME) END) * {UNLOADED_FILE_ROWS} + METADATA$FILE_ROW_NUMBER",
-                        files.join(" ")
-                    )
-                }
-                _ => "SEQ8()".to_owned(),
-            };
-            format!("{} * {UNLOADED_BATCH_ROWS} + {in_batch}", request.batch)
-        }
-        (_, Some((_, _, value))) => cast(column, &literal(value)),
-        _ => request
-            .read
-            .iter()
-            .find(|read| read.name == column.name)
-            .map_or_else(
-                || "NULL".to_owned(),
-                |read| {
-                    let field = match rows {
-                        Rows::Files(..) => format!("GET($1, {})", literal(&read.name)),
-                        Rows::Query => format!("u.{}", quoted(&read.name)),
-                    };
-                    unloaded_value(column, &read.output, &field)
-                },
-            ),
-    }
-}
-
-/// `name` as the unload's query names its column: quoted exactly as written.
-fn quoted(name: &str) -> String {
-    format!("\"{}\"", name.replace('"', "\"\""))
-}
-
-/// An unloaded `field`, read as `read`, as the value it was unloaded from, then as `column` holds it: a time or timestamp from its whole microseconds (a timestamp with time zone in UTC, as SQLLoad's reader gives it), anything else as it is.
-fn unloaded_value(column: &DesiredColumn, read: &ColumnType, field: &str) -> String {
-    let micros = format!("CAST({field} AS NUMBER(38,0))");
-    let value = match read {
-        ColumnType::TimestampNtz => format!("TO_TIMESTAMP_NTZ({micros}, 6)"),
-        ColumnType::TimestampTz => {
-            format!("CONVERT_TIMEZONE('UTC', TO_TIMESTAMP_TZ({micros}, 6))")
-        }
-        ColumnType::Time => format!("TIMEADD(MICROSECOND, {micros}, '00:00:00'::TIME)"),
-        _ => field.to_owned(),
-    };
-    cast(column, &value)
+        .map(|column| column_value(request, column, &rows))
+        .collect();
+    values.join(", ")
 }
 
-/// `value` as `column` holds it: JSON and arrays parsed from their text, anything else cast to its type.
-fn cast(column: &DesiredColumn, value: &str) -> String {
-    match &column.column_type {
-        ColumnType::Json => format!("PARSE_JSON(CAST({value} AS VARCHAR))"),
-        ColumnType::Array(_) => format!("CAST(PARSE_JSON(CAST({value} AS VARCHAR)) AS ARRAY)"),
-        column_type => format!(
-            "CAST({value} AS {})",
-            SnowflakeSql.render_loader_logical_type(column_type).sql
-        ),
-    }
+/// `keys` as the `FILES` list of a `COPY`.
+fn file_list(keys: &[String]) -> String {
+    let files: Vec = keys
+        .iter()
+        .map(|key| SnowflakeSql.render_loader_value_literal(key))
+        .collect();
+    files.join(", ")
 }
diff --git a/crates/sqlload/src/adapters/snowflake/models.rs b/crates/sqlload/src/adapters/snowflake/models.rs
index afeec30ca..ec3cd00c4 100644
--- a/crates/sqlload/src/adapters/snowflake/models.rs
+++ b/crates/sqlload/src/adapters/snowflake/models.rs
@@ -114,6 +114,7 @@ pub struct CopyStaged<'a> {
 }
 
 /// One batch of a Snowflake source's rows loaded inside Snowflake: the table they fill and its columns, the source's read columns, the batch's own values, and where the rows come from.
+#[derive(Debug)]
 pub struct UnloadedLoad<'a> {
     pub into: &'a TableName,
     pub columns: &'a [DesiredColumn],
@@ -130,6 +131,7 @@ pub struct UnloadedLoad<'a> {
 }
 
 /// Where a batch loaded inside Snowflake reads the source's rows, each read column under its own name, times and timestamps as whole microseconds.
+#[derive(Debug)]
 pub enum UnloadedFrom<'a> {
     /// The unload's Parquet files.
     Files {
@@ -146,6 +148,13 @@ pub enum UnloadedFrom<'a> {
     Query(&'a str),
 }
 
+/// Which rows one statement of a batch loaded inside Snowflake reads: the unload's files `keys`, the `first`th file on, or the query.
+#[derive(Debug, Clone, Copy)]
+pub enum UnloadedRows<'a> {
+    Files(usize, &'a [String]),
+    Query,
+}
+
 /// Where one staged file goes on an internal stage.
 #[derive(Debug, Clone, Copy)]
 pub struct StagedFile<'a> {
diff --git a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs
index 9aae0db8f..814b854e1 100644
--- a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs
+++ b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs
@@ -428,32 +428,29 @@ pub(crate) fn unloaded_sql(
         batch: 3,
         load_id: "L1",
         loaded_at: "2026-11-18T00:00:00Z",
-        from: match files {
-            Some(_) => UnloadedFrom::Files {
-                stage: "@\"DB\".\"RAW\".\"LAKE\"",
-                folder: "unload/run-1/",
-                keys: &keys,
-                filename_prefix: "lake/unload/run-1/",
-            },
-            None => UnloadedFrom::Query("SELECT ..."),
-        },
+        from: files.map_or(UnloadedFrom::Query("SELECT ..."), |_| UnloadedFrom::Files {
+            stage: "@\"DB\".\"RAW\".\"LAKE\"",
+            folder: "unload/run-1/",
+            keys: &keys,
+            filename_prefix: "lake/unload/run-1/",
+        }),
     })
 }
 
 /// Why Snowflake does not load the case's unloaded files itself, or `None` when it does.
 pub(crate) fn unload_blocked_for(test_case: &UnloadBlockedTestCase) -> Option {
     let route = WarehouseRoute::Unload {
-        store: match test_case.unload_to {
-            Some(folder) => Store::ObjectStore(folder.to_owned()),
-            None => Store::InternalStage("sqlload_unload".to_owned()),
-        },
+        store: test_case.unload_to.map_or_else(
+            || Store::InternalStage("sqlload_unload".to_owned()),
+            |folder| Store::ObjectStore(folder.to_owned()),
+        ),
         stage: String::new(),
         same_connection: false,
     };
-    let stage = match test_case.external {
-        true => StageRoute::External(Box::new(stage())),
-        false => StageRoute::User,
-    };
+    let stage = test_case
+        .external
+        .then(|| StageRoute::External(Box::new(stage())))
+        .unwrap_or(StageRoute::User);
     unload_blocked(
         Warehouse::Snowflake,
         test_case.masks,
@@ -462,23 +459,26 @@ pub(crate) fn unload_blocked_for(test_case: &UnloadBlockedTestCase) -> Option Result, String> {
+) -> Result<(usize, bool), String> {
     use crate::adapters::snowflake::_helpers::write::unloaded_load::unloaded_requests;
     use crate::adapters::snowflake::constants::MAX_COPY_SQL_BYTES;
 
-    let (statements, unloaded) = match test_case.copies {
-        Some((count, length)) => (
-            (0..count)
-                .map(|index| format!("COPY INTO t /* {index} */ {}", "x".repeat(length)))
-                .collect::>(),
-            Some((10, count * 1_000)),
-        ),
-        None => (vec!["INSERT INTO t SELECT 1".to_owned()], None),
-    };
-    let requests = unloaded_requests(&statements, unloaded, test_case.direct)?;
+    let (statements, unloaded) = test_case.copies.map_or_else(
+        || (vec!["INSERT INTO t SELECT 1".to_owned()], None),
+        |(count, length)| {
+            (
+                (0..count)
+                    .map(|index| format!("COPY INTO t /* {index} */ {}", "x".repeat(length)))
+                    .collect::>(),
+                Some((10, count * 1_000)),
+            )
+        },
+    );
+    let requests = unloaded_requests(&statements, unloaded, test_case.direct)
+        .map_err(|error| error.split(',').next().unwrap_or_default().to_owned())?;
     assert!(
         requests
             .iter()
@@ -496,5 +496,5 @@ pub(crate) fn unloaded_requests_for(
         "{}: every COPY is sent once",
         test_case.description
     );
-    Ok(requests)
+    Ok((requests.len(), requests[0].contains("IF (loaded = ")))
 }
diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs
index e56941911..1d254396d 100644
--- a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs
+++ b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs
@@ -140,8 +140,6 @@ pub(crate) struct UnloadedRequestsTestCase {
     pub(crate) copies: Option<(usize, usize)>,
     /// Whether the batch commits into the table itself.
     pub(crate) direct: bool,
-    /// How many requests, or the start of the error.
-    pub(crate) expected_requests: Result,
-    /// Whether the (first) request commits only when every unloaded row loads.
-    pub(crate) expected_checks_count: bool,
+    /// How many requests, and whether the first commits only when every unloaded row loads; or the error up to its first comma.
+    pub(crate) expected_requests: Result<(usize, bool), &'static str>,
 }
diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs b/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs
index fbcb6bfba..4c0eb4237 100644
--- a/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs
+++ b/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs
@@ -1,8 +1,9 @@
 use crate::adapters::snowflake::tests::helpers::unload_blocked_for;
 use crate::adapters::snowflake::tests::test_types::UnloadBlockedTestCase;
+use crate::loader::constants::UNLOADED_OUTSIDE_STAGE;
 
-const OUTSIDE: &str =
-    "Snowflake reads another connection's unload itself only through the target's external stage";
+const MASKS: &str = "the table masks columns";
+const OUTSIDE: &str = UNLOADED_OUTSIDE_STAGE;
 
 #[test]
 fn given_a_snowflake_sources_unload_when_routing_into_snowflake_then_snowflake_copies_only_what_the_targets_external_stage_covers()
@@ -69,20 +70,15 @@ fn given_a_snowflake_sources_unload_when_routing_into_snowflake_then_snowflake_c
             unload_to: Some("s3://lake/raw/unload/"),
             external: true,
             masks: true,
-            expected_blocked: Some("the table masks columns"),
+            expected_blocked: Some(MASKS),
         },
     ];
     for test_case in &test_cases {
-        let blocked = unload_blocked_for(test_case);
-        match test_case.expected_blocked {
-            None => assert_eq!(blocked, None, "{}", test_case.description),
-            Some(expected) => assert!(
-                blocked
-                    .as_deref()
-                    .is_some_and(|reason| reason.starts_with(expected)),
-                "{}: {blocked:?}",
-                test_case.description
-            ),
-        }
+        assert_eq!(
+            unload_blocked_for(test_case).as_deref(),
+            test_case.expected_blocked,
+            "{}",
+            test_case.description
+        );
     }
 }
diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs b/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs
index b3032e4b2..f47be94b9 100644
--- a/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs
+++ b/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs
@@ -9,56 +9,42 @@ fn given_an_unload_loaded_inside_snowflake_when_sending_its_statements_then_they
             description: "a few COPYs go in one request that commits only when every unloaded row loads",
             copies: Some((3, 100_000)),
             direct: false,
-            expected_requests: Ok(1),
-            expected_checks_count: true,
+            expected_requests: Ok((1, true)),
         },
         UnloadedRequestsTestCase {
             description: "COPYs past the script limit are split, each request committing its part into the staging table",
             copies: Some((20, 120_000)),
             direct: false,
-            expected_requests: Ok(3),
-            expected_checks_count: false,
+            expected_requests: Ok((3, false)),
         },
         UnloadedRequestsTestCase {
             description: "a direct append past the limit fails before loading anything",
             copies: Some((20, 120_000)),
             direct: true,
             expected_requests: Err("the unload wrote 20000 files"),
-            expected_checks_count: false,
         },
         UnloadedRequestsTestCase {
             description: "a direct append within the limit goes in one counted request",
             copies: Some((3, 100_000)),
             direct: true,
-            expected_requests: Ok(1),
-            expected_checks_count: true,
+            expected_requests: Ok((1, true)),
         },
         UnloadedRequestsTestCase {
             description: "an INSERT ... SELECT is one request",
             copies: None,
             direct: true,
-            expected_requests: Ok(1),
-            expected_checks_count: false,
+            expected_requests: Ok((1, false)),
         },
     ];
     for test_case in &test_cases {
-        let requests = unloaded_requests_for(test_case);
-        match (&requests, test_case.expected_requests) {
-            (Ok(requests), Ok(expected)) => {
-                assert_eq!(requests.len(), expected, "{}", test_case.description);
-                assert_eq!(
-                    requests[0].contains("IF (loaded = "),
-                    test_case.expected_checks_count,
-                    "{}",
-                    test_case.description
-                );
-            }
-            (Err(error), Err(expected)) => assert!(
-                error.starts_with(expected),
-                "{}: {error}",
-                test_case.description
-            ),
-            _ => panic!("{}: {requests:?}", test_case.description),
-        }
+        assert_eq!(
+            unloaded_requests_for(test_case)
+                .as_ref()
+                .copied()
+                .map_err(String::as_str),
+            test_case.expected_requests,
+            "{}",
+            test_case.description
+        );
     }
 }
diff --git a/crates/sqlload/src/loader/_helpers/publish.rs b/crates/sqlload/src/loader/_helpers/publish.rs
index f6dd790a4..b02904274 100644
--- a/crates/sqlload/src/loader/_helpers/publish.rs
+++ b/crates/sqlload/src/loader/_helpers/publish.rs
@@ -82,7 +82,7 @@ use crate::loader::_helpers::sources::ReadPart;
 use crate::loader::_helpers::sources::files_source_parts;
 use crate::loader::_helpers::sources::read_source;
 use crate::loader::_helpers::sources::unarchived;
-use crate::loader::_helpers::sources::{selected_query, unload_source};
+use crate::loader::_helpers::sources::{unload_request, unload_source};
 use crate::loader::_helpers::specs::checked_key;
 use crate::loader::_helpers::specs::{null_cursor_problem, null_cursor_rows};
 use crate::loader::_helpers::table_plan::{
@@ -144,6 +144,7 @@ use crate::schema_plan::constants::{
 use crate::schema_plan::models::ColumnOrigin;
 use crate::schema_plan::models::DesiredColumn;
 use crate::schema_plan::models::LiveColumn;
+use crate::snowflake_source::main::select_query::select_query;
 use crate::table_lease::main::lease_now::lease_now;
 use crate::table_lease::main::take_lease::take_lease;
 use crate::time_text::main::now_utc_text::now_utc_text;
@@ -164,7 +165,7 @@ use std::collections::BTreeSet;
 use std::sync::Arc;
 
 use crate::loader::_helpers::table_plan::{
-    added_constants, is_direct, publishes_once, pulls_unloaded, selects_rows,
+    added_constants, is_direct, pulls_unloaded, selects_rows,
 };
 
 /// Whether `append` can write the batches straight into the table: not a logged table (its rows wait for a changed `ctx.state`), and a writer that can (on Snowflake, a route loading files into the table itself).
@@ -1932,3 +1933,28 @@ fn estimated_bytes(estimate: i64) -> Option {
         Ok(bytes) => Some(bytes),
     }
 }
+
+/// Whether `table`, a `delete_insert`, `append` or deduplicating `merge` (a key read in two batches keeps its last or newest row) read in windows or cursor-ordered batches, stages them all and publishes once at the end (see [`publishes_runs_once`](crate::adapters::contract::types::SqlAdapter::publishes_runs_once)); a files source's groups still commit one by one, as their values would not fit one statement, and a merge that fails on repeated keys commits each batch, as a key updated during the read is read twice.
+pub(crate) fn publishes_once(table: &TablePlan) -> bool {
+    matches!(
+        table.mode,
+        WriteMode::DeleteInsert { .. }
+            | WriteMode::Append
+            | WriteMode::Merge {
+                dedupe: Some(_),
+                ..
+            }
+    ) && matches!(
+        table.commits,
+        CommitRule::Windows(_) | CommitRule::Ordered { .. }
+    ) && sql_adapter(table.spec.warehouse).publishes_runs_once()
+}
+
+/// The query an unload of `part` of `table` would run, which a destination writing through the source's own connection runs itself.
+fn selected_query(table: &TablePlan, part: ReadPart) -> Result {
+    let filter = table
+        .spec
+        .read_filter(part.filter.or_else(|| table.source_filter.clone()));
+    select_query(&unload_request(table, filter.as_deref()))
+        .map_err(|error| LoaderError::Invalid(error.to_string()))
+}
diff --git a/crates/sqlload/src/loader/_helpers/render.rs b/crates/sqlload/src/loader/_helpers/render.rs
index 8222130f8..bf8a6824d 100644
--- a/crates/sqlload/src/loader/_helpers/render.rs
+++ b/crates/sqlload/src/loader/_helpers/render.rs
@@ -24,16 +24,16 @@ use crate::column_parse::main::parse_lines::parse_lines;
 use crate::column_types::models::ColumnType;
 use crate::files_source::models::{FilesOrder, SourceFormat};
 use crate::load_naming::main::is_load_column::is_load_column;
-use crate::loader::_helpers::publish::{published_once, records_files, send_lines};
+use crate::loader::_helpers::publish::{published_once, publishes_once, records_files, send_lines};
 use crate::loader::_helpers::python::{list_choices, unnested_table};
 use crate::loader::_helpers::resolve::display_name;
 use crate::loader::_helpers::sources::{readers_note, unarchived};
 use crate::loader::_helpers::table_plan::{
-    archived_reason, is_direct, publishes_once, pulls_unloaded, selects_rows,
+    archived_reason, is_direct, pulls_unloaded, selects_rows,
 };
 use crate::loader::constants::{
     CONSTANT_NOTE, DRAINED_REASON, LABEL_WIDTH, NULL_KEYS_NOTE, PLAN_DETAIL_COLUMN,
-    PRIMARY_KEY_PLACEHOLDER, RESULT_SCHEMA_VERSION,
+    PRIMARY_KEY_PLACEHOLDER, RESULT_SCHEMA_VERSION, SELECTED_ROUTE,
 };
 use crate::loader::models::{
     CommitRule, FilesOutput, FilesSource, ObjectChange, PlanDetail, PlanItem, PlanView,
@@ -844,11 +844,8 @@ pub(crate) fn batch_line(table: &TablePlan) -> String {
                 thousands(MAX_BATCH_FILES as u64)
             )
         }
-        (_, true) if selects_rows(table.route.as_ref()) => {
-            "the rows selected at once into a new table, swapped in once at the end".to_owned()
-        }
         (_, true) if pulls_unloaded(table.route.as_ref()) => {
-            "each unload loaded at once into a new table, swapped in once at the end".to_owned()
+            "every row loaded at once into a new table, swapped in once at the end".to_owned()
         }
         (_, true) if sql_adapter(spec.warehouse).keeps_tables() => {
             "about 1 GB at a time into a staging table, then the table emptied and refilled in one transaction".to_owned()
@@ -869,11 +866,8 @@ pub(crate) fn batch_line(table: &TablePlan) -> String {
                 _ => "",
             }
         ),
-        (CommitRule::Size, false) if selects_rows(table.route.as_ref()) => {
-            "one commit: the destination selects every row at once".to_owned()
-        }
         (CommitRule::Size, false) if pulls_unloaded(table.route.as_ref()) => {
-            "one commit: the destination loads the whole unload at once".to_owned()
+            "one commit: the destination loads every row at once".to_owned()
         }
         (CommitRule::Size, false) => "about every 1 GB".to_owned(),
         (CommitRule::Whole(reason), false) => {
@@ -1008,9 +1002,7 @@ fn route_text(table: &TablePlan, route: &Route) -> String {
             real.unwrap_or_else(|| files.format.to_string()),
             files.store
         ),
-        [Step::DestinationSelects] => format!(
-            "{warehouse} INSERT ... SELECT on the source's connection (nothing through the loader)"
-        ),
+        [Step::DestinationSelects] => format!("{warehouse} {SELECTED_ROUTE}"),
         [Step::Unload(store), Step::DestinationPulls(_)] => {
             format!("unload Parquet to {store} -> {warehouse} {load} (nothing through the loader)")
         }
diff --git a/crates/sqlload/src/loader/_helpers/sources.rs b/crates/sqlload/src/loader/_helpers/sources.rs
index 7c5f2b53c..da2a69cf9 100644
--- a/crates/sqlload/src/loader/_helpers/sources.rs
+++ b/crates/sqlload/src/loader/_helpers/sources.rs
@@ -126,7 +126,6 @@ use crate::sheets_source::models::{Grid, HeaderRule, KeySource, SheetsClient};
 use crate::snowflake_source::constants::SESSION_SCHEMA;
 use crate::snowflake_source::main::connect_snowflake_source::connect_snowflake_source;
 use crate::snowflake_source::main::read_catalog::read_catalog as read_snowflake_catalog;
-use crate::snowflake_source::main::select_query::select_query;
 use crate::snowflake_source::main::unload_files::unload_files;
 use crate::snowflake_source::main::unload_rows::unload_rows;
 use crate::snowflake_source::models::{UnloadRequest, UnloadedFiles};
@@ -1746,17 +1745,11 @@ pub(crate) async fn unload_source(
     unload_files(snowflake, &unload_request(table, filter.as_deref()), load).await
 }
 
-/// The query an unload of `part` of `table` would run, which a destination writing through the source's own connection runs itself.
-pub(crate) fn selected_query(table: &TablePlan, part: ReadPart) -> Result {
-    let filter = table
-        .spec
-        .read_filter(part.filter.or_else(|| table.source_filter.clone()));
-    select_query(&unload_request(table, filter.as_deref()))
-        .map_err(|error| LoaderError::Invalid(error.to_string()))
-}
-
 /// What one unload of `table` reads: its table or query, its read columns, and `filter`.
-fn unload_request<'a>(table: &'a TablePlan, filter: Option<&'a str>) -> UnloadRequest<'a> {
+pub(crate) fn unload_request<'a>(
+    table: &'a TablePlan,
+    filter: Option<&'a str>,
+) -> UnloadRequest<'a> {
     UnloadRequest {
         table: &table.spec.source_table,
         sql: table.spec.query_sql(),
diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs
index e06782506..da1fa8b19 100644
--- a/crates/sqlload/src/loader/_helpers/specs.rs
+++ b/crates/sqlload/src/loader/_helpers/specs.rs
@@ -693,8 +693,8 @@ fn warehouse_source(
             route: WarehouseRoute::Unload {
                 store,
                 stage,
-                same_connection: context.target.warehouse == Warehouse::Snowflake
-                    && context.target.session_connection.as_ref() == Some(&source.connection)
+                same_connection: context.target.session_connection.as_ref()
+                    == Some(&source.connection)
                     && source.warehouse.is_none(),
             },
         }
diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs
index 5de2f6d55..d51ae4786 100644
--- a/crates/sqlload/src/loader/_helpers/table_plan.rs
+++ b/crates/sqlload/src/loader/_helpers/table_plan.rs
@@ -1079,14 +1079,12 @@ pub(crate) fn reader_offer(
         })
         | None => Vec::new(),
     };
+    let route = spec.warehouse_source.as_ref().map(|source| &source.route);
     let selected_by_destination = spec.masked.columns.is_empty()
         && matches!(
-            &spec.warehouse_source,
-            Some(WarehouseSource {
-                route: WarehouseRoute::Unload {
-                    same_connection: true,
-                    ..
-                },
+            route,
+            Some(WarehouseRoute::Unload {
+                same_connection: true,
                 ..
             })
         );
@@ -1095,7 +1093,7 @@ pub(crate) fn reader_offer(
         false => unload_blocked(
             spec.warehouse,
             !spec.masked.columns.is_empty(),
-            spec.warehouse_source.as_ref().map(|source| &source.route),
+            route,
             stage,
         ),
     };
@@ -1121,22 +1119,6 @@ pub(crate) fn reader_offer(
     }
 }
 
-/// Whether `table`, a `delete_insert`, `append` or deduplicating `merge` (a key read in two batches keeps its last or newest row) read in windows or cursor-ordered batches, stages them all and publishes once at the end (see [`publishes_runs_once`](crate::adapters::contract::types::SqlAdapter::publishes_runs_once)); a files source's groups still commit one by one, as their values would not fit one statement, and a merge that fails on repeated keys commits each batch, as a key updated during the read is read twice.
-pub(crate) fn publishes_once(table: &TablePlan) -> bool {
-    matches!(
-        table.mode,
-        WriteMode::DeleteInsert { .. }
-            | WriteMode::Append
-            | WriteMode::Merge {
-                dedupe: Some(_),
-                ..
-            }
-    ) && matches!(
-        table.commits,
-        CommitRule::Windows(_) | CommitRule::Ordered { .. }
-    ) && sql_adapter(table.spec.warehouse).publishes_runs_once()
-}
-
 /// One commit per file, in the order planned: a files source's merge, or `delete_insert` by `replace_key`, applies each file on its own, so a key repeated across files ends with the newest file's value (or rows).
 fn one_file_each(count: usize) -> Vec {
     (0..count)
diff --git a/crates/sqlload/src/loader/constants.rs b/crates/sqlload/src/loader/constants.rs
index d394aed4d..09865f007 100644
--- a/crates/sqlload/src/loader/constants.rs
+++ b/crates/sqlload/src/loader/constants.rs
@@ -16,6 +16,10 @@ pub const PLAN_DETAIL_COLUMN: usize = 52;
 /// The width of a plan detail's label and the space after it, as `route   `.
 pub const LABEL_WIDTH: usize = 8;
 
+/// The route of a table a Snowflake target selects from a source on its own connection, after the warehouse's name.
+pub const SELECTED_ROUTE: &str =
+    "INSERT ... SELECT on the source's connection (nothing through the loader)";
+
 /// How a skipped files source's reason ends after `no files to load in `, when `after_load` emptied its folder.
 pub const DRAINED_REASON: &str = ": after_load deleted or moved the ones already loaded";
 
@@ -193,7 +197,7 @@ pub const TOMBSTONE_SUFFIX: &str = "__sqlload_deletes";
 /// Why `--sync-deletes` fails a plan none of whose selected tables sets `deletes`.
 pub const NO_DELETES_TABLES: &str = "--sync-deletes reads every key of each selected table with deletes and marks its rows of the keys the source no longer holds, but none of the selected tables sets deletes. Fix: set deletes = \"soft\" or \"hard\" on a merge table, or drop --sync-deletes";
 
-/// Rows a merge [published once per run](crate::loader::_helpers::table_plan::publishes_once) stages before it commits anyway, at its next cut: matching the staged keys holds them all in one set on the server (about 90 bytes a key on ClickHouse), which this keeps under about 1 GiB.
+/// Rows a merge [published once per run](crate::loader::_helpers::publish::publishes_once) stages before it commits anyway, at its next cut: matching the staged keys holds them all in one set on the server (about 90 bytes a key on ClickHouse), which this keeps under about 1 GiB.
 pub const MERGE_RUN_ROWS: u64 = 10_000_000;
 
 /// How `plan` words a run published in one commit.

From 5b447f0d2f300d3e6d249a3c076380b00c8008a7 Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Wed, 18 Nov 2026 18:11:37 +0000
Subject: [PATCH 9/9] wip: unload_blocked test helper picks its stage without
 if/else chains

---
 crates/sqlload/src/adapters/snowflake/tests/helpers.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs
index 814b854e1..c0e1db5d3 100644
--- a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs
+++ b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs
@@ -447,9 +447,8 @@ pub(crate) fn unload_blocked_for(test_case: &UnloadBlockedTestCase) -> Option