diff --git a/crates/sqlload/src/adapters/contract/constants.rs b/crates/sqlload/src/adapters/contract/constants.rs index bbd271978..83681d130 100644 --- a/crates/sqlload/src/adapters/contract/constants.rs +++ b/crates/sqlload/src/adapters/contract/constants.rs @@ -39,3 +39,8 @@ pub const COMMIT_STATEMENT: &str = "COMMIT"; /// Ends a transaction, undoing it. pub const ROLLBACK_STATEMENT: &str = "ROLLBACK"; + +/// The `_sqlload_row` span of one unloaded batch (2^48): its files' rows number from the batch times this. +pub const UNLOADED_BATCH_ROWS: i64 = 1 << 48; +/// The `_sqlload_row` span of one unloaded file in a batch (2^32). +pub const UNLOADED_FILE_ROWS: i64 = 1 << 32; diff --git a/crates/sqlload/src/adapters/contract/models.rs b/crates/sqlload/src/adapters/contract/models.rs index 124e4af96..0ac0aa204 100644 --- a/crates/sqlload/src/adapters/contract/models.rs +++ b/crates/sqlload/src/adapters/contract/models.rs @@ -283,11 +283,11 @@ pub struct DirectBatch<'a> { pub source_folder: Option<&'a OpenFolder>, } -/// One unload's files the destination loads by reading them itself: times and timestamps held as whole microseconds, every other column as the type it is read as. +/// A warehouse source's rows the destination loads itself, without SQLLoad seeing them: times and timestamps held as whole microseconds, every other column as the type it is read as. #[derive(Debug, Clone, Copy)] pub struct UnloadedBatch<'a> { - pub files: &'a UnloadedFiles, - /// The columns the files hold, as they are read. + pub source: UnloadedSource<'a>, + /// The columns the rows hold, as they are read. pub read: &'a [ReadColumn], /// The table's columns as the load writes them. pub write_columns: &'a [DesiredColumn], @@ -296,6 +296,15 @@ pub struct UnloadedBatch<'a> { pub loaded_at: &'a str, } +/// Where the destination reads an unloaded batch's rows. +#[derive(Debug, Clone, Copy)] +pub enum UnloadedSource<'a> { + /// One unload's files, which it reads itself. + Files(&'a UnloadedFiles), + /// The unload's query, which it runs itself in the source's own session. + Query(&'a str), +} + /// A query a check before publish runs on a relation; each returns its values as text. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CheckQuery<'a> { diff --git a/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs b/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs index 13ab26c19..ac6ff774d 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs @@ -1,14 +1,37 @@ //! Why the destination cannot load a warehouse source's unloaded files exactly by reading them itself. +use crate::adapters::snowflake::main::copy::_stage_covers::stage_covers; +use crate::adapters::snowflake::models::StageRoute; use crate::column_types::models::Warehouse; -use crate::loader::constants::UNLOADED_TIMES; -use crate::loader::models::TableSpec; +use crate::loader::constants::{UNLOADED_OUTSIDE_STAGE, UNLOADED_TIMES}; +use crate::loader::models::WarehouseRoute; +use crate::routing::models::Store; +use crate::store_paths::main::resolve_location::resolve_location; -/// Why the destination cannot load the table's unloaded files exactly by reading them itself, so SQLLoad reads them instead; `None` when it can (where it also reaches their store). Only DuckDB turns the files' whole microseconds back into times and timestamps; masks need SQLLoad to see the rows. -pub fn unload_blocked(spec: &TableSpec) -> Option { - match (spec.warehouse, spec.masked.columns.is_empty()) { - (Warehouse::DuckDb, true) => None, - (Warehouse::DuckDb, false) => Some("the table masks columns".to_owned()), +/// Why `warehouse` cannot load a table's unloaded files exactly by reading them itself, so SQLLoad reads them instead; `None` when it can (where it also reaches their store). Only DuckDB and Snowflake turn the files' whole microseconds back into times and timestamps; a table that `masks` columns needs SQLLoad to see the rows; Snowflake reads another session's unload only through the target's external `stage`, which must cover the S3 folder `route` unloads to, as a user or named stage of the source's is not its to read. +pub fn unload_blocked( + warehouse: Warehouse, + masks: bool, + route: Option<&WarehouseRoute>, + stage: Option<&StageRoute>, +) -> Option { + let covered = match (route, stage) { + ( + Some(WarehouseRoute::Unload { + store: Store::ObjectStore(url), + .. + }), + Some(StageRoute::External(stage)), + ) => resolve_location(None, &format!("{}/", url.trim_end_matches('/'))) + .is_ok_and(|unloaded| stage_covers(&stage.location, &unloaded)), + _ => false, + }; + match (warehouse, masks) { + (Warehouse::DuckDb | Warehouse::Snowflake, true) => { + Some("the table masks columns".to_owned()) + } + (Warehouse::Snowflake, false) if !covered => Some(UNLOADED_OUTSIDE_STAGE.to_owned()), + (Warehouse::DuckDb | Warehouse::Snowflake, false) => None, _ => Some(UNLOADED_TIMES.to_owned()), } } diff --git a/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs b/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs index aa3f547b4..6ac0235d0 100644 --- a/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs +++ b/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs @@ -9,11 +9,10 @@ pub async fn load_unloaded( writer: &mut Writer, batch: &UnloadedBatch<'_>, ) -> Result, LoaderError> { - #[cfg(not(feature = "duckdb"))] - let _ = batch; match writer { #[cfg(feature = "duckdb")] Writer::DuckDb(duckdb) => duckdb.pull_unloaded(batch).await.map(Some), + Writer::Snowflake(snowflake) => snowflake.pull_unloaded(batch).await.map(Some), _ => Ok(None), } } diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs b/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs index 44a650294..0ef1f5f84 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs @@ -1,8 +1,9 @@ //! The statement that has DuckDB load Parquet files itself. +use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS}; use crate::adapters::contract::models::TableName; use crate::adapters::contract::types::SqlAdapter; -use crate::adapters::duckdb::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS, UTC_OFFSET}; +use crate::adapters::duckdb::constants::UTC_OFFSET; use crate::adapters::duckdb::models::DuckDbSql; use crate::column_types::main::is_variant::is_variant; use crate::column_types::models::ColumnType; diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/unloaded_load.rs b/crates/sqlload/src/adapters/duckdb/_helpers/unloaded_load.rs index 195905574..feabf3188 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/unloaded_load.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/unloaded_load.rs @@ -1,7 +1,7 @@ //! A warehouse source's unloaded files loaded by DuckDB reading them itself. use crate::adapters::contract::constants::{BEGIN_STATEMENT, COMMIT_STATEMENT}; -use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch}; +use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch, UnloadedSource}; use crate::adapters::duckdb::_helpers::pull_sql::{UnloadedColumns, unloaded_insert}; use crate::adapters::duckdb::main::_execute_statements::execute_statements; use crate::adapters::duckdb::main::_pull_files::pull_files; @@ -19,7 +19,11 @@ impl DuckDbWriter { true => DirectLoaded::Committed(rows), false => DirectLoaded::Staged(rows), }; - let files = batch.files; + let UnloadedSource::Files(files) = batch.source else { + return Err(LoaderError::Invalid( + "DuckDB reads a warehouse source's rows only from its unloaded files".to_owned(), + )); + }; if files.keys.is_empty() { return Ok(loaded(0)); } diff --git a/crates/sqlload/src/adapters/duckdb/constants.rs b/crates/sqlload/src/adapters/duckdb/constants.rs index 16b8a15da..ce049936c 100644 --- a/crates/sqlload/src/adapters/duckdb/constants.rs +++ b/crates/sqlload/src/adapters/duckdb/constants.rs @@ -12,10 +12,6 @@ pub const MOTHERDUCK_TOKEN_VARIABLES: &[&str] = &["motherduck_token", "MOTHERDUC pub const MOTHERDUCK_UNSUPPORTED_DUCKDB: &str = "is not yet supported by MotherDuck"; /// What precedes the extension's own message in DuckDB's error. pub const EXTENSION_EXCEPTION: &str = "threw an exception: "; -/// The `_sqlload_row` span of one unloaded batch (2^48): its files' rows number from the batch times this. -pub const UNLOADED_BATCH_ROWS: i64 = 1 << 48; -/// The `_sqlload_row` span of one unloaded file in a batch (2^32). -pub const UNLOADED_FILE_ROWS: i64 = 1 << 32; /// MotherDuck's server version, such as `v1.5.6-2026-11-382`. pub const MOTHERDUCK_VERSION_QUERY: &str = "SELECT md_version()"; /// The first DuckDB major version that shreds a VARIANT column's common fields into columns, so queries on them are fast. diff --git a/crates/sqlload/src/adapters/duckdb/tests/helpers.rs b/crates/sqlload/src/adapters/duckdb/tests/helpers.rs index 3a8120f5c..2515c4633 100644 --- a/crates/sqlload/src/adapters/duckdb/tests/helpers.rs +++ b/crates/sqlload/src/adapters/duckdb/tests/helpers.rs @@ -155,7 +155,7 @@ pub(crate) fn pulled_unloaded(reported: u64, with_files: bool) -> Result<(String table: "loaded".to_owned(), }; let batch = UnloadedBatch { - files: &files, + source: crate::adapters::contract::models::UnloadedSource::Files(&files), read: &read, write_columns: &columns, target: WriteTarget { diff --git a/crates/sqlload/src/adapters/snowflake/_helpers/dialect/mod.rs b/crates/sqlload/src/adapters/snowflake/_helpers/dialect/mod.rs index 611e59dda..8f69dab79 100644 --- a/crates/sqlload/src/adapters/snowflake/_helpers/dialect/mod.rs +++ b/crates/sqlload/src/adapters/snowflake/_helpers/dialect/mod.rs @@ -2,4 +2,5 @@ pub(crate) mod copy_select; pub(crate) mod sql_adapter; pub(crate) mod text_values; pub(crate) mod type_names; +pub(crate) mod unloaded_values; pub(crate) mod values; diff --git a/crates/sqlload/src/adapters/snowflake/_helpers/dialect/unloaded_values.rs b/crates/sqlload/src/adapters/snowflake/_helpers/dialect/unloaded_values.rs new file mode 100644 index 000000000..8a4368e38 --- /dev/null +++ b/crates/sqlload/src/adapters/snowflake/_helpers/dialect/unloaded_values.rs @@ -0,0 +1,103 @@ +//! The value each column of a Snowflake source's batch loaded inside Snowflake takes, from the source's unloaded files or its query. + +use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS}; +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::snowflake::models::{SnowflakeSql, UnloadedFrom, UnloadedLoad, UnloadedRows}; +use crate::column_types::models::ColumnType; +use crate::load_naming::main::metadata_role::metadata_role; +use crate::load_naming::models::MetadataRole; +use crate::schema_plan::constants::MERGE_ROW_COLUMN; +use crate::schema_plan::models::DesiredColumn; + +/// Where `column` comes from in a statement reading `rows`: the batch's load id or load time, a merge's row number (the batch, then the file, then the row in it; a query's rows in the order it gives them), the value of the constant it is, or the source's column it is read as; NULL when the source lacks it. +pub(crate) fn column_value( + request: &UnloadedLoad<'_>, + column: &DesiredColumn, + rows: &UnloadedRows<'_>, +) -> String { + let literal = |text: &str| SnowflakeSql.render_loader_value_literal(text); + let constant = request + .constants + .iter() + .find(|(name, _, _)| *name == column.name); + match (metadata_role(request.compat, &column.name), constant) { + (Some(MetadataRole::LoadId), _) => literal(request.load_id), + (Some(MetadataRole::LoadedAt), _) => { + format!("CAST({} AS TIMESTAMP_TZ(6))", literal(request.loaded_at)) + } + _ if column.name == MERGE_ROW_COLUMN => { + let in_batch = match (rows, &request.from) { + ( + UnloadedRows::Files(first, keys), + UnloadedFrom::Files { + filename_prefix, .. + }, + ) => { + let files: Vec = keys + .iter() + .enumerate() + .map(|(index, key)| { + format!( + "WHEN {} THEN {}", + literal(&format!("{filename_prefix}{key}")), + first + index + ) + }) + .collect(); + format!( + "(CASE METADATA$FILENAME {} ELSE TO_NUMBER('sqlload: unexpected unloaded file ' || METADATA$FILENAME) END) * {UNLOADED_FILE_ROWS} + METADATA$FILE_ROW_NUMBER", + files.join(" ") + ) + } + _ => "SEQ8()".to_owned(), + }; + format!("{} * {UNLOADED_BATCH_ROWS} + {in_batch}", request.batch) + } + (_, Some((_, _, value))) => cast(column, &literal(value)), + _ => request + .read + .iter() + .find(|read| read.name == column.name) + .map_or_else( + || "NULL".to_owned(), + |read| { + let field = match rows { + UnloadedRows::Files(..) => format!("GET($1, {})", literal(&read.name)), + UnloadedRows::Query => format!("u.{}", quoted(&read.name)), + }; + unloaded_value(column, &read.output, &field) + }, + ), + } +} + +/// `name` as the unload's query names its column: quoted exactly as written. +fn quoted(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +/// An unloaded `field`, read as `read`, as the value it was unloaded from, then as `column` holds it: a time or timestamp from its whole microseconds (a timestamp with time zone in UTC, as SQLLoad's reader gives it), anything else as it is. +fn unloaded_value(column: &DesiredColumn, read: &ColumnType, field: &str) -> String { + let micros = format!("CAST({field} AS NUMBER(38,0))"); + let value = match read { + ColumnType::TimestampNtz => format!("TO_TIMESTAMP_NTZ({micros}, 6)"), + ColumnType::TimestampTz => { + format!("CONVERT_TIMEZONE('UTC', TO_TIMESTAMP_TZ({micros}, 6))") + } + ColumnType::Time => format!("TIMEADD(MICROSECOND, {micros}, '00:00:00'::TIME)"), + _ => field.to_owned(), + }; + cast(column, &value) +} + +/// `value` as `column` holds it: JSON and arrays parsed from their text, anything else cast to its type. +fn cast(column: &DesiredColumn, value: &str) -> String { + match &column.column_type { + ColumnType::Json => format!("PARSE_JSON(CAST({value} AS VARCHAR))"), + ColumnType::Array(_) => format!("CAST(PARSE_JSON(CAST({value} AS VARCHAR)) AS ARRAY)"), + column_type => format!( + "CAST({value} AS {})", + SnowflakeSql.render_loader_logical_type(column_type).sql + ), + } +} diff --git a/crates/sqlload/src/adapters/snowflake/_helpers/write/direct_load.rs b/crates/sqlload/src/adapters/snowflake/_helpers/write/direct_load.rs index c33bbc020..c5301baf1 100644 --- a/crates/sqlload/src/adapters/snowflake/_helpers/write/direct_load.rs +++ b/crates/sqlload/src/adapters/snowflake/_helpers/write/direct_load.rs @@ -375,7 +375,7 @@ fn held_types(columns: &[DesiredColumn]) -> Vec { } /// The columns a staging table holds, as the staged rows carry them: JSON and UUIDs as text; also the columns that write a table of `held_types` on with no cast. -fn staged_as_text(columns: &[DesiredColumn]) -> Vec { +pub(in crate::adapters) fn staged_as_text(columns: &[DesiredColumn]) -> Vec { columns .iter() .cloned() diff --git a/crates/sqlload/src/adapters/snowflake/_helpers/write/mod.rs b/crates/sqlload/src/adapters/snowflake/_helpers/write/mod.rs index 97c6c530e..c27f7acfe 100644 --- a/crates/sqlload/src/adapters/snowflake/_helpers/write/mod.rs +++ b/crates/sqlload/src/adapters/snowflake/_helpers/write/mod.rs @@ -4,3 +4,4 @@ pub(crate) mod file_keys; pub(crate) mod row_stream; pub(crate) mod rows_loaded; pub(crate) mod table_writer; +pub(crate) mod unloaded_load; diff --git a/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs b/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs new file mode 100644 index 000000000..3328c484e --- /dev/null +++ b/crates/sqlload/src/adapters/snowflake/_helpers/write/unloaded_load.rs @@ -0,0 +1,172 @@ +//! A Snowflake source's rows loaded by Snowflake itself: `COPY`ing its unloaded files, or selecting them in the source's own session. + +use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch, UnloadedSource}; +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::snowflake::_helpers::write::direct_load::staged_as_text; +use crate::adapters::snowflake::constants::MAX_COPY_SQL_BYTES; +use crate::adapters::snowflake::main::copy::_stage_covers::stage_covers; +use crate::adapters::snowflake::main::copy::_unloaded_sql::unloaded_statements; +use crate::adapters::snowflake::main::stage::_stage_reference::stage_reference; +use crate::adapters::snowflake::models::{ + SnowflakeSql, SnowflakeWriter, StageRoute, UnloadedFrom, UnloadedLoad, +}; +use crate::loader::errors::LoaderError; +use crate::sql_api::main::execute_sql::execute_sql; + +impl SnowflakeWriter { + /// Load `batch` inside Snowflake, where Arrow batches would go: the load's staging table, the build table, or the table itself. Unloaded files are `COPY`d from the external stage the writer loads through, which must cover them, as [`unloaded_requests`] sends them, and the batch fails unless they load as many rows as the unload reports; a query's rows are selected by one `INSERT ... SELECT`. + pub(in crate::adapters) async fn pull_unloaded( + &mut self, + batch: &UnloadedBatch<'_>, + ) -> Result { + let target = &batch.target; + let loaded = |rows: u64| match target.direct { + true => DirectLoaded::Committed(rows), + false => DirectLoaded::Staged(rows), + }; + let staged; + let columns = match (target.staging, target.builds) { + (Some(_), false) => { + staged = staged_as_text(target.columns); + &staged + } + _ => batch.write_columns, + }; + let place: (String, String); + let (from, unloaded) = match batch.source { + UnloadedSource::Files(files) if files.keys.is_empty() => return Ok(loaded(0)), + UnloadedSource::Files(files) => { + let covering = match &self.route { + Some(StageRoute::External(stage)) + if stage_covers(&stage.location, &files.folder) => + { + files + .folder + .prefix + .strip_prefix(&stage.location.prefix) + .map(|folder| (stage_reference(stage), folder.to_owned())) + } + _ => None, + }; + let Some(covering) = covering else { + return Err(LoaderError::Invalid(format!( + "Snowflake cannot read the unloaded files in {}: the target's external stage does not cover them", + files.folder + ))); + }; + place = covering; + ( + UnloadedFrom::Files { + stage: &place.0, + folder: &place.1, + keys: &files.keys, + filename_prefix: &files.folder.prefix, + }, + Some(files), + ) + } + UnloadedSource::Query(query) => (UnloadedFrom::Query(query), None), + }; + let statements = unloaded_statements(&UnloadedLoad { + into: target.staging.unwrap_or(target.table), + columns, + read: batch.read, + constants: target.constants, + compat: target.compat, + batch: target.batch, + load_id: target.load_id, + loaded_at: batch.loaded_at, + from, + }); + let requests = unloaded_requests( + &statements, + unloaded.map(|files| (files.rows, files.keys.len())), + target.direct, + ) + .map_err(LoaderError::Invalid)?; + let mut rows: u64 = 0; + for request in &requests { + let answer = execute_sql(&self.destination.sql, request).await?; + let returned = answer + .rows + .first() + .and_then(|row| row.first().cloned().flatten()) + .unwrap_or_default(); + rows += returned.parse::().map_err(|error| { + LoaderError::Invalid(format!( + "loading the source's rows inside Snowflake returned {returned:?}, not a row count: {error}" + )) + })?; + } + if let Some(files) = unloaded.filter(|files| files.rows != rows) { + return Err(LoaderError::Invalid(format!( + "Snowflake loaded {rows} rows from the unloaded files in {}, but the unload reports {}; the batch is not committed", + files.folder, files.rows + ))); + } + Ok(loaded(rows)) + } +} + +/// The requests loading `statements`, each a [`counted_block`]: one, whose `COPY`s commit only when they load all of the unload's rows (`unloaded`: its rows and files), when it fits `MAX_COPY_SQL_BYTES`, as does an `INSERT ... SELECT` (`unloaded` `None`); otherwise several, each committing its `COPY`s into the staging or build table, which publishes only once their rows add up to the unload's, as the caller checks. A `direct` batch commits into the table itself, so it fails instead, before loading anything, as a part committed alone could not be taken back. +pub(in crate::adapters) fn unloaded_requests( + statements: &[String], + unloaded: Option<(u64, usize)>, + direct: bool, +) -> Result, String> { + let Some((rows, files)) = unloaded else { + return Ok(vec![counted_block(statements, false, None)]); + }; + let whole = counted_block(statements, true, Some(rows)); + if whole.len() <= MAX_COPY_SQL_BYTES { + return Ok(vec![whole]); + } + if direct { + return Err(format!( + "the unload wrote {files} files, whose COPYs are too long to run as one Snowflake statement, and this append commits into the table itself, so they can't be split. Fix: set cursor_column with a smaller batch_size, so each read unloads fewer files" + )); + } + let mut groups: Vec> = Vec::new(); + for statement in statements { + let fits = groups.last().is_some_and(|group| { + let mut grown = group.clone(); + grown.push(statement.clone()); + counted_block(&grown, true, None).len() <= MAX_COPY_SQL_BYTES + }); + match (fits, groups.last_mut()) { + (true, Some(group)) => group.push(statement.clone()), + _ => groups.push(vec![statement.clone()]), + } + } + Ok(groups + .iter() + .map(|group| counted_block(group, true, None)) + .collect()) +} + +/// One Snowflake Scripting block running `statements` in a transaction and returning the rows they loaded: `COPY`s (`copies`), counted from their results, which commit only when they load `expected` rows (when given) and roll back otherwise, or an `INSERT ... SELECT`, which commits. +fn counted_block(statements: &[String], copies: bool, expected: Option) -> String { + let mut body = vec![ + "DECLARE loaded INTEGER DEFAULT 0; copied INTEGER;".to_owned(), + "BEGIN".to_owned(), + "BEGIN TRANSACTION;".to_owned(), + ]; + for statement in statements { + body.push(format!("{statement};")); + body.push(match copies { + true => "SELECT COALESCE(SUM(\"rows_loaded\"), 0) INTO :copied FROM TABLE(RESULT_SCAN(LAST_QUERY_ID()));".to_owned(), + false => "copied := SQLROWCOUNT;".to_owned(), + }); + body.push("loaded := loaded + copied;".to_owned()); + } + body.push(match expected { + Some(expected) => format!("IF (loaded = {expected}) THEN COMMIT; ELSE ROLLBACK; END IF;"), + None => "COMMIT;".to_owned(), + }); + body.push("RETURN loaded;".to_owned()); + body.push("END;".to_owned()); + format!( + "EXECUTE IMMEDIATE {}", + SnowflakeSql.render_loader_value_literal(&body.join("\n")) + ) +} diff --git a/crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs b/crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs new file mode 100644 index 000000000..9f1c8124e --- /dev/null +++ b/crates/sqlload/src/adapters/snowflake/main/copy/_unloaded_sql.rs @@ -0,0 +1,58 @@ +//! The statements loading a Snowflake source's rows into a table inside Snowflake: from its unloaded Parquet files, or selected from the source in the same session. + +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::snowflake::_helpers::dialect::unloaded_values::column_value; +use crate::adapters::snowflake::constants::{COPY_FILE_FORMAT, MAX_COPY_FILES}; +use crate::adapters::snowflake::models::{SnowflakeSql, UnloadedFrom, UnloadedLoad, UnloadedRows}; + +/// The statements loading `request`'s rows into `request.into`, every one of its columns filled as [`column_value`] says: from files, `COPY`s of at most 1,000 files each, stopping at the first bad value and never relying on Snowflake's load history; from a query, one `INSERT ... SELECT`. +pub(in crate::adapters) fn unloaded_statements(request: &UnloadedLoad<'_>) -> Vec { + let into = SnowflakeSql.render_qualified_name(request.into); + let names: Vec = request + .columns + .iter() + .map(|column| SnowflakeSql.render_identifier(&column.name)) + .collect(); + let names = names.join(", "); + match &request.from { + UnloadedFrom::Files { + stage, + folder, + keys, + .. + } => keys + .chunks(MAX_COPY_FILES) + .enumerate() + .map(|(chunk, chunk_keys)| { + format!( + "COPY INTO {into} ({names}) FROM (SELECT {} FROM {stage}/{folder}) FILES = ({}) {COPY_FILE_FORMAT} ON_ERROR = ABORT_STATEMENT FORCE = TRUE", + values(request, UnloadedRows::Files(chunk * MAX_COPY_FILES, chunk_keys)), + file_list(chunk_keys) + ) + }) + .collect(), + UnloadedFrom::Query(query) => vec![format!( + "INSERT INTO {into} ({names}) SELECT {} FROM ({query}) AS u", + values(request, UnloadedRows::Query) + )], + } +} + +/// Each of `request`'s columns as a statement reading `rows` fills it. +fn values(request: &UnloadedLoad<'_>, rows: UnloadedRows<'_>) -> String { + let values: Vec = request + .columns + .iter() + .map(|column| column_value(request, column, &rows)) + .collect(); + values.join(", ") +} + +/// `keys` as the `FILES` list of a `COPY`. +fn file_list(keys: &[String]) -> String { + let files: Vec = keys + .iter() + .map(|key| SnowflakeSql.render_loader_value_literal(key)) + .collect(); + files.join(", ") +} diff --git a/crates/sqlload/src/adapters/snowflake/main/copy/mod.rs b/crates/sqlload/src/adapters/snowflake/main/copy/mod.rs index 1b2f7e790..24cf6b432 100644 --- a/crates/sqlload/src/adapters/snowflake/main/copy/mod.rs +++ b/crates/sqlload/src/adapters/snowflake/main/copy/mod.rs @@ -5,3 +5,4 @@ pub(in crate::adapters) mod _direct_copies; pub(in crate::adapters) mod _direct_problem; pub(in crate::adapters) mod _script_copy_rows; pub(in crate::adapters) mod _stage_covers; +pub(in crate::adapters) mod _unloaded_sql; diff --git a/crates/sqlload/src/adapters/snowflake/models.rs b/crates/sqlload/src/adapters/snowflake/models.rs index f0d462657..ec3cd00c4 100644 --- a/crates/sqlload/src/adapters/snowflake/models.rs +++ b/crates/sqlload/src/adapters/snowflake/models.rs @@ -4,8 +4,10 @@ use crate::column_types::models::ColumnType; use crate::config::models::{AuthConfig, SnowflakeConfig}; use crate::file_store::models::OpenFolder; use crate::files_source::models::{DataFile, FilesEncoding, SourceFormat, StoredRead}; +use crate::load_naming::models::Compat; use crate::loader::errors::LoaderError; use crate::parquet_parts::models::PartWriter; +use crate::postgres_source::models::ReadColumn; use crate::project::models::MaskMethod; use crate::schema_plan::models::DesiredColumn; use crate::sql_api::models::SqlClient; @@ -111,6 +113,48 @@ pub struct CopyStaged<'a> { pub selected: Option<&'a [DesiredColumn]>, } +/// One batch of a Snowflake source's rows loaded inside Snowflake: the table they fill and its columns, the source's read columns, the batch's own values, and where the rows come from. +#[derive(Debug)] +pub struct UnloadedLoad<'a> { + pub into: &'a TableName, + pub columns: &'a [DesiredColumn], + pub read: &'a [ReadColumn], + /// The table's `constant_columns`: name, type and value as text. + pub constants: &'a [(String, ColumnType, String)], + pub compat: Compat, + /// The batch, from 1. + pub batch: u64, + pub load_id: &'a str, + /// The load time as an ISO 8601 UTC timestamp. + pub loaded_at: &'a str, + pub from: UnloadedFrom<'a>, +} + +/// Where a batch loaded inside Snowflake reads the source's rows, each read column under its own name, times and timestamps as whole microseconds. +#[derive(Debug)] +pub enum UnloadedFrom<'a> { + /// The unload's Parquet files. + Files { + /// The stage as SQL, `@"DB"."SCHEMA"."STAGE"`. + stage: &'a str, + /// The unload's folder below the stage, ending in `/`. + folder: &'a str, + /// Each file's key below `folder`. + keys: &'a [String], + /// What `METADATA$FILENAME` starts each file's name with: the folder's bucket-relative key. + filename_prefix: &'a str, + }, + /// The query the unload would run, selected in the same session. + Query(&'a str), +} + +/// Which rows one statement of a batch loaded inside Snowflake reads: the unload's files `keys`, the `first`th file on, or the query. +#[derive(Debug, Clone, Copy)] +pub enum UnloadedRows<'a> { + Files(usize, &'a [String]), + Query, +} + /// Where one staged file goes on an internal stage. #[derive(Debug, Clone, Copy)] pub struct StagedFile<'a> { diff --git a/crates/sqlload/src/adapters/snowflake/tests.rs b/crates/sqlload/src/adapters/snowflake/tests.rs index 5b8f65c1e..bd84e9bfb 100644 --- a/crates/sqlload/src/adapters/snowflake/tests.rs +++ b/crates/sqlload/src/adapters/snowflake/tests.rs @@ -20,3 +20,9 @@ mod stale_folders; mod step_failure; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_unload_blocked.rs"] +mod unload_blocked; +#[path = "tests/test_unloaded_requests.rs"] +mod unloaded_requests; +#[path = "tests/test_unloaded_sql.rs"] +mod unloaded_sql; diff --git a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs index 6f4d668fc..c0e1db5d3 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/helpers.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/helpers.rs @@ -1,18 +1,25 @@ use crate::adapters::contract::models::TableName; +use crate::adapters::dispatch::main::route::unload_blocked::unload_blocked; use crate::adapters::snowflake::_helpers::stage::create_stage::create_stage; use crate::adapters::snowflake::_helpers::stage::stale_folders::stale_folders; use crate::adapters::snowflake::main::copy::_copy_staged::copy_staged; use crate::adapters::snowflake::main::stage::_stage_lookup::stage_lookup; use crate::adapters::snowflake::main::stage::_stage_reference::stage_reference; use crate::adapters::snowflake::models::CopyStaged; +use crate::adapters::snowflake::models::StageRoute; use crate::adapters::snowflake::models::{CopyColumn, CopyValue}; use crate::adapters::snowflake::models::{ExternalStage, StageObject}; -use crate::adapters::snowflake::tests::test_types::CopyFrom; +use crate::adapters::snowflake::tests::test_types::{ + CopyFrom, UnloadBlockedTestCase, UnloadedRequestsTestCase, +}; use crate::column_types::models::ColumnType; +use crate::column_types::models::Warehouse; use crate::file_store::models::StoredFile; use crate::files_source::models::DataFile; use crate::files_source::models::StoredRead; use crate::http::errors::ApiError; +use crate::loader::models::WarehouseRoute; +use crate::routing::models::Store; use crate::schema_plan::models::{ColumnOrigin, DesiredColumn}; use crate::store_paths::models::{Location, StoreKind}; use chrono::{DateTime, Utc}; @@ -374,3 +381,119 @@ pub(crate) fn csv_in( ..FilesEncoding::default() } } + +/// The statements loading a Snowflake source's rows into `RAW.ORDERS`, batch 3 of load L1, with constant `region` = `'EU'` and the source's columns read as typed: from `files` unloaded files (`data_.parquet` under `lake/unload/run-1/`, on the stage's `unload/` folder), or with none, from the query `SELECT ...`. +pub(crate) fn unloaded_sql( + columns: &[(&str, ColumnType, Option)], + files: Option, +) -> Vec { + use crate::adapters::snowflake::main::copy::_unloaded_sql::unloaded_statements; + use crate::adapters::snowflake::models::{UnloadedFrom, UnloadedLoad}; + use crate::postgres_source::models::{ReadColumn, WireType}; + use crate::schema_plan::models::{ColumnOrigin, DesiredColumn}; + + let desired: Vec = columns + .iter() + .map(|(name, column_type, _)| DesiredColumn { + name: (*name).to_owned(), + column_type: column_type.clone(), + origin: ColumnOrigin::Source, + }) + .collect(); + let read: Vec = columns + .iter() + .filter_map(|(name, _, read)| { + Some(ReadColumn { + name: (*name).to_owned(), + expression: format!("\"{name}\""), + wire: WireType::Text, + output: read.clone()?, + }) + }) + .collect(); + let keys: Vec = (0..files.unwrap_or_default()) + .map(|file| format!("data_{file}.parquet")) + .collect(); + let table = TableName { + database: None, + schema: "raw".to_owned(), + table: "orders".to_owned(), + }; + unloaded_statements(&UnloadedLoad { + into: &table, + columns: &desired, + read: &read, + constants: &[("region".to_owned(), ColumnType::Text, "EU".to_owned())], + compat: crate::load_naming::models::Compat::Sqlload, + batch: 3, + load_id: "L1", + loaded_at: "2026-11-18T00:00:00Z", + from: files.map_or(UnloadedFrom::Query("SELECT ..."), |_| UnloadedFrom::Files { + stage: "@\"DB\".\"RAW\".\"LAKE\"", + folder: "unload/run-1/", + keys: &keys, + filename_prefix: "lake/unload/run-1/", + }), + }) +} + +/// Why Snowflake does not load the case's unloaded files itself, or `None` when it does. +pub(crate) fn unload_blocked_for(test_case: &UnloadBlockedTestCase) -> Option { + let route = WarehouseRoute::Unload { + store: test_case.unload_to.map_or_else( + || Store::InternalStage("sqlload_unload".to_owned()), + |folder| Store::ObjectStore(folder.to_owned()), + ), + stage: String::new(), + same_connection: false, + }; + let stage = Some(StageRoute::External(Box::new(stage()))) + .filter(|_| test_case.external) + .unwrap_or(StageRoute::User); + unload_blocked( + Warehouse::Snowflake, + test_case.masks, + Some(&route), + Some(&stage), + ) +} + +/// How many requests the case's statements are sent as, each checked to fit `MAX_COPY_SQL_BYTES`, and whether the first commits only when every unloaded row loads; or the error up to its first comma. +pub(crate) fn unloaded_requests_for( + test_case: &UnloadedRequestsTestCase, +) -> Result<(usize, bool), String> { + use crate::adapters::snowflake::_helpers::write::unloaded_load::unloaded_requests; + use crate::adapters::snowflake::constants::MAX_COPY_SQL_BYTES; + + let (statements, unloaded) = test_case.copies.map_or_else( + || (vec!["INSERT INTO t SELECT 1".to_owned()], None), + |(count, length)| { + ( + (0..count) + .map(|index| format!("COPY INTO t /* {index} */ {}", "x".repeat(length))) + .collect::>(), + Some((10, count * 1_000)), + ) + }, + ); + let requests = unloaded_requests(&statements, unloaded, test_case.direct) + .map_err(|error| error.split(',').next().unwrap_or_default().to_owned())?; + assert!( + requests + .iter() + .all(|request| request.len() <= MAX_COPY_SQL_BYTES), + "{}: a request is longer than the limit", + test_case.description + ); + let sent: usize = requests + .iter() + .map(|request| request.matches("COPY INTO t /*").count()) + .sum(); + assert_eq!( + sent, + test_case.copies.map_or(0, |(count, _)| count), + "{}: every COPY is sent once", + test_case.description + ); + Ok((requests.len(), requests[0].contains("IF (loaded = "))) +} diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs index 5fb8f6ea2..1d254396d 100644 --- a/crates/sqlload/src/adapters/snowflake/tests/test_types.rs +++ b/crates/sqlload/src/adapters/snowflake/tests/test_types.rs @@ -1,4 +1,5 @@ use crate::adapters::snowflake::models::CopyColumn; +use crate::column_types::models::ColumnType; use crate::files_source::models::SourceFormat; use crate::project::models::MaskMethod; @@ -105,3 +106,40 @@ pub(crate) struct SnowflakeEncodingTestCase { /// Snowflake's `ENCODING` for it, or `None` when the direct route refuses it. pub(crate) expected_encoding: Option<&'static str>, } + +/// The statements loading one batch of a Snowflake source's rows inside Snowflake. +pub(crate) struct UnloadedSqlTestCase { + pub(crate) description: &'static str, + /// The table's columns, each with the type the files' column is read as (`None` when the files lack it). + pub(crate) columns: &'static [(&'static str, ColumnType, Option)], + /// How many unloaded files; `None` selects the rows with the unload's query. + pub(crate) files: Option, + pub(crate) expected_statements: usize, + /// Text the first statement holds, in order. + pub(crate) expected_first: &'static [&'static str], + /// Text the last statement holds. + pub(crate) expected_last: &'static str, +} + +/// Whether Snowflake loads a Snowflake source's unloaded files itself. +pub(crate) struct UnloadBlockedTestCase { + pub(crate) description: &'static str, + /// Where the source unloads: an `s3://` folder, `@stage`, or `None` for the user stage. + pub(crate) unload_to: Option<&'static str>, + /// Whether the target loads through the external stage on `s3://lake/raw/`; otherwise its user stage. + pub(crate) external: bool, + pub(crate) masks: bool, + /// The reason SQLLoad reads the files instead, or `None` when Snowflake loads them itself. + pub(crate) expected_blocked: Option<&'static str>, +} + +/// How the statements loading one unload inside Snowflake are sent. +pub(crate) struct UnloadedRequestsTestCase { + pub(crate) description: &'static str, + /// How many `COPY`s, and how long each is; `None` is one `INSERT ... SELECT`. + pub(crate) copies: Option<(usize, usize)>, + /// Whether the batch commits into the table itself. + pub(crate) direct: bool, + /// How many requests, and whether the first commits only when every unloaded row loads; or the error up to its first comma. + pub(crate) expected_requests: Result<(usize, bool), &'static str>, +} diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs b/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs new file mode 100644 index 000000000..4c0eb4237 --- /dev/null +++ b/crates/sqlload/src/adapters/snowflake/tests/test_unload_blocked.rs @@ -0,0 +1,84 @@ +use crate::adapters::snowflake::tests::helpers::unload_blocked_for; +use crate::adapters::snowflake::tests::test_types::UnloadBlockedTestCase; +use crate::loader::constants::UNLOADED_OUTSIDE_STAGE; + +const MASKS: &str = "the table masks columns"; +const OUTSIDE: &str = UNLOADED_OUTSIDE_STAGE; + +#[test] +fn given_a_snowflake_sources_unload_when_routing_into_snowflake_then_snowflake_copies_only_what_the_targets_external_stage_covers() + { + let test_cases = [ + UnloadBlockedTestCase { + description: "an S3 folder below the target's external stage is copied by Snowflake", + unload_to: Some("s3://lake/raw/unload"), + external: true, + masks: false, + expected_blocked: None, + }, + UnloadBlockedTestCase { + description: "the stage's own folder, written with a slash, is covered too", + unload_to: Some("s3://lake/raw/"), + external: true, + masks: false, + expected_blocked: None, + }, + UnloadBlockedTestCase { + description: "another folder of the same bucket goes through SQLLoad", + unload_to: Some("s3://lake/elsewhere/"), + external: true, + masks: false, + expected_blocked: Some(OUTSIDE), + }, + UnloadBlockedTestCase { + description: "a folder whose name only starts like the stage's goes through SQLLoad", + unload_to: Some("s3://lake/rawish/"), + external: true, + masks: false, + expected_blocked: Some(OUTSIDE), + }, + UnloadBlockedTestCase { + description: "the same folder in another bucket goes through SQLLoad", + unload_to: Some("s3://other/raw/unload/"), + external: true, + masks: false, + expected_blocked: Some(OUTSIDE), + }, + UnloadBlockedTestCase { + description: "an S3 unload into a target without an external stage goes through SQLLoad", + unload_to: Some("s3://lake/raw/unload/"), + external: false, + masks: false, + expected_blocked: Some(OUTSIDE), + }, + UnloadBlockedTestCase { + description: "the source's user stage is not the target's to read", + unload_to: None, + external: true, + masks: false, + expected_blocked: Some(OUTSIDE), + }, + UnloadBlockedTestCase { + description: "a named stage of the source's is not the target's to read", + unload_to: Some("@ops.loading.unload"), + external: true, + masks: false, + expected_blocked: Some(OUTSIDE), + }, + UnloadBlockedTestCase { + description: "masks need SQLLoad to see the rows, even under the stage", + unload_to: Some("s3://lake/raw/unload/"), + external: true, + masks: true, + expected_blocked: Some(MASKS), + }, + ]; + for test_case in &test_cases { + assert_eq!( + unload_blocked_for(test_case).as_deref(), + test_case.expected_blocked, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs b/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs new file mode 100644 index 000000000..f47be94b9 --- /dev/null +++ b/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_requests.rs @@ -0,0 +1,50 @@ +use crate::adapters::snowflake::tests::helpers::unloaded_requests_for; +use crate::adapters::snowflake::tests::test_types::UnloadedRequestsTestCase; + +#[test] +fn given_an_unload_loaded_inside_snowflake_when_sending_its_statements_then_they_fit_the_script_limit_and_a_direct_append_never_commits_part() + { + let test_cases = [ + UnloadedRequestsTestCase { + description: "a few COPYs go in one request that commits only when every unloaded row loads", + copies: Some((3, 100_000)), + direct: false, + expected_requests: Ok((1, true)), + }, + UnloadedRequestsTestCase { + description: "COPYs past the script limit are split, each request committing its part into the staging table", + copies: Some((20, 120_000)), + direct: false, + expected_requests: Ok((3, false)), + }, + UnloadedRequestsTestCase { + description: "a direct append past the limit fails before loading anything", + copies: Some((20, 120_000)), + direct: true, + expected_requests: Err("the unload wrote 20000 files"), + }, + UnloadedRequestsTestCase { + description: "a direct append within the limit goes in one counted request", + copies: Some((3, 100_000)), + direct: true, + expected_requests: Ok((1, true)), + }, + UnloadedRequestsTestCase { + description: "an INSERT ... SELECT is one request", + copies: None, + direct: true, + expected_requests: Ok((1, false)), + }, + ]; + for test_case in &test_cases { + assert_eq!( + unloaded_requests_for(test_case) + .as_ref() + .copied() + .map_err(String::as_str), + test_case.expected_requests, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_sql.rs b/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_sql.rs new file mode 100644 index 000000000..f7fd6bc99 --- /dev/null +++ b/crates/sqlload/src/adapters/snowflake/tests/test_unloaded_sql.rs @@ -0,0 +1,112 @@ +use crate::adapters::snowflake::tests::helpers::unloaded_sql; +use crate::adapters::snowflake::tests::test_types::UnloadedSqlTestCase; +use crate::column_types::models::ColumnType; + +const TYPED: &[(&str, ColumnType, Option)] = &[ + ("id", ColumnType::Int, Some(ColumnType::Int)), + ( + "seen_at", + ColumnType::TimestampNtz, + Some(ColumnType::TimestampNtz), + ), + ( + "placed_at", + ColumnType::TimestampTz, + Some(ColumnType::TimestampTz), + ), + ("opens", ColumnType::Time, Some(ColumnType::Time)), + ("meta", ColumnType::Json, Some(ColumnType::Json)), + ("region", ColumnType::Text, None), + ("_sqlload_load_id", ColumnType::Text, None), + ("_sqlload_loaded_at", ColumnType::TimestampTz, None), + ("gone", ColumnType::Text, None), +]; + +const NUMBERED: &[(&str, ColumnType, Option)] = &[ + ("id", ColumnType::Int, Some(ColumnType::Int)), + ("_sqlload_row", ColumnType::Int, None), +]; + +#[test] +fn given_a_snowflake_sources_rows_when_loading_them_inside_snowflake_then_their_microseconds_turn_back_and_the_loads_columns_fill() + { + let test_cases = [ + UnloadedSqlTestCase { + description: "times and timestamps from whole microseconds (a zoned one in UTC), JSON parsed, the constant, load id and load time as literals, a column the files lack NULL", + columns: TYPED, + files: Some(2), + expected_statements: 1, + expected_first: &[ + "COPY INTO \"RAW\".\"ORDERS\" (\"ID\", \"SEEN_AT\", \"PLACED_AT\", \"OPENS\", \"META\", \"REGION\", \"_SQLLOAD_LOAD_ID\", \"_SQLLOAD_LOADED_AT\", \"GONE\") FROM (SELECT ", + "CAST(GET($1, 'id') AS NUMBER(38,0)), ", + "CAST(TO_TIMESTAMP_NTZ(CAST(GET($1, 'seen_at') AS NUMBER(38,0)), 6) AS TIMESTAMP_NTZ(9)), ", + "CAST(CONVERT_TIMEZONE('UTC', TO_TIMESTAMP_TZ(CAST(GET($1, 'placed_at') AS NUMBER(38,0)), 6)) AS TIMESTAMP_TZ(9)), ", + "CAST(TIMEADD(MICROSECOND, CAST(GET($1, 'opens') AS NUMBER(38,0)), '00:00:00'::TIME) AS TIME(9)), ", + "PARSE_JSON(CAST(GET($1, 'meta') AS VARCHAR)), ", + "CAST('EU' AS VARCHAR), ", + "'L1', ", + "CAST('2026-11-18T00:00:00Z' AS TIMESTAMP_TZ(6)), ", + "NULL ", + "FROM @\"DB\".\"RAW\".\"LAKE\"/unload/run-1/) FILES = ('data_0.parquet', 'data_1.parquet') FILE_FORMAT = (TYPE = PARQUET USE_LOGICAL_TYPE = TRUE BINARY_AS_TEXT = FALSE) ON_ERROR = ABORT_STATEMENT FORCE = TRUE", + ], + expected_last: "FORCE = TRUE", + }, + UnloadedSqlTestCase { + description: "a merge numbers its rows by batch, then file, counting on across COPYs of 1,000 files, then row in the file", + columns: NUMBERED, + files: Some(1001), + expected_statements: 2, + expected_first: &[ + "3 * 281474976710656 + (CASE METADATA$FILENAME WHEN 'lake/unload/run-1/data_0.parquet' THEN 0 WHEN ", + "WHEN 'lake/unload/run-1/data_999.parquet' THEN 999 ELSE TO_NUMBER('sqlload: unexpected unloaded file ' || METADATA$FILENAME) END) * 4294967296 + METADATA$FILE_ROW_NUMBER", + ], + expected_last: "(CASE METADATA$FILENAME WHEN 'lake/unload/run-1/data_1000.parquet' THEN 1000 ELSE TO_NUMBER('sqlload: unexpected unloaded file ' || METADATA$FILENAME) END) * 4294967296 + METADATA$FILE_ROW_NUMBER FROM @\"DB\".\"RAW\".\"LAKE\"/unload/run-1/) FILES = ('data_1000.parquet')", + }, + UnloadedSqlTestCase { + description: "selected in the source's session: one INSERT reading the query's columns by name, turned back the same way; a merge numbers rows in the order the query gives them", + columns: &[ + ("id", ColumnType::Int, Some(ColumnType::Int)), + ( + "placed_at", + ColumnType::TimestampTz, + Some(ColumnType::TimestampTz), + ), + ("_sqlload_row", ColumnType::Int, None), + ], + files: None, + expected_statements: 1, + expected_first: &[ + "INSERT INTO \"RAW\".\"ORDERS\" (\"ID\", \"PLACED_AT\", \"_SQLLOAD_ROW\") SELECT ", + "CAST(u.\"id\" AS NUMBER(38,0)), ", + "CAST(CONVERT_TIMEZONE('UTC', TO_TIMESTAMP_TZ(CAST(u.\"placed_at\" AS NUMBER(38,0)), 6)) AS TIMESTAMP_TZ(9)), ", + "3 * 281474976710656 + SEQ8() FROM (SELECT ...) AS u", + ], + expected_last: "FROM (SELECT ...) AS u", + }, + ]; + for test_case in test_cases { + let statements = unloaded_sql(test_case.columns, test_case.files); + assert_eq!( + statements.len(), + test_case.expected_statements, + "{}", + test_case.description + ); + let (first, last) = (&statements[0], &statements[statements.len() - 1]); + let mut rest = first.as_str(); + for expected in test_case.expected_first { + let at = rest.find(expected); + assert!( + at.is_some(), + "{}: {expected:?} not in order in {first}", + test_case.description + ); + rest = at.map_or("", |at| &rest[at + expected.len()..]); + } + assert!( + last.contains(test_case.expected_last), + "{}: {last}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/loader/_helpers/publish.rs b/crates/sqlload/src/loader/_helpers/publish.rs index cca2a421a..b02904274 100644 --- a/crates/sqlload/src/loader/_helpers/publish.rs +++ b/crates/sqlload/src/loader/_helpers/publish.rs @@ -9,7 +9,7 @@ use crate::adapters::contract::models::TableShape; use crate::adapters::contract::models::WriteMode; use crate::adapters::contract::models::WriteStatementsRequest; use crate::adapters::contract::models::WriteTarget; -use crate::adapters::contract::models::{DirectBatch, DirectLoaded, UnloadedBatch}; +use crate::adapters::contract::models::{DirectBatch, DirectLoaded, UnloadedBatch, UnloadedSource}; use crate::adapters::contract::types::TableWriter; use crate::adapters::dispatch::main::dialect::source_dialect::source_dialect; use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter; @@ -82,7 +82,7 @@ use crate::loader::_helpers::sources::ReadPart; use crate::loader::_helpers::sources::files_source_parts; use crate::loader::_helpers::sources::read_source; use crate::loader::_helpers::sources::unarchived; -use crate::loader::_helpers::sources::unload_source; +use crate::loader::_helpers::sources::{unload_request, unload_source}; use crate::loader::_helpers::specs::checked_key; use crate::loader::_helpers::specs::{null_cursor_problem, null_cursor_rows}; use crate::loader::_helpers::table_plan::{ @@ -144,7 +144,7 @@ use crate::schema_plan::constants::{ use crate::schema_plan::models::ColumnOrigin; use crate::schema_plan::models::DesiredColumn; use crate::schema_plan::models::LiveColumn; -use crate::snowflake_source::models::UnloadedFiles; +use crate::snowflake_source::main::select_query::select_query; use crate::table_lease::main::lease_now::lease_now; use crate::table_lease::main::take_lease::take_lease; use crate::time_text::main::now_utc_text::now_utc_text; @@ -165,7 +165,7 @@ use std::collections::BTreeSet; use std::sync::Arc; use crate::loader::_helpers::table_plan::{ - added_constants, is_direct, publishes_once, pulls_unloaded, + added_constants, is_direct, pulls_unloaded, selects_rows, }; /// Whether `append` can write the batches straight into the table: not a logged table (its rows wait for a changed `ctx.state`), and a writer that can (on Snowflake, a route loading files into the table itself). @@ -1073,14 +1073,14 @@ impl Publisher { } } - /// Load one unload's `files` by having the destination read them itself, where the plan chose that; the destination must load them so. + /// Load one unload's files, or the rows its query selects, by having the destination read them itself, where the plan chose that; the destination must load them so. pub(crate) async fn load_unloaded( &mut self, - files: &UnloadedFiles, + source: UnloadedSource<'_>, ) -> Result { let columns = staged_columns(&self.table); let batch = UnloadedBatch { - files, + source, read: &self.table.read_columns, write_columns: &self.table.schema.write_columns, target: write_target( @@ -1091,26 +1091,33 @@ impl Publisher { ), loaded_at: &self.loaded_at, }; - let loaded = load_unloaded(&mut self.writer, &batch).await?.ok_or_else(|| { - LoaderError::Invalid(format!( - "{}: the plan has the destination load the unloaded files itself, but it cannot", - self.table.spec.destination.table - )) - })?; + let loaded = load_unloaded(&mut self.writer, &batch) + .await? + .ok_or_else(|| { + LoaderError::Invalid(format!( + "{}: the plan has the destination load the source's rows itself, but it cannot", + self.table.spec.destination.table + )) + })?; self.note_direct(&loaded); Ok(loaded) } - /// Pass `part` of the table's rows on: the destination loads the unloaded files itself where the plan chose that, otherwise SQLLoad stages the rows. + /// Pass `part` of the table's rows on: the destination selects them itself, or loads the unloaded files itself, where the plan chose that; otherwise SQLLoad stages the rows. async fn pass_part( &mut self, reader: &SourceReader, part: ReadPart, ) -> Result<(), LoaderError> { let table = self.table.clone(); + if selects_rows(table.route.as_ref()) { + let query = selected_query(&table, part)?; + self.load_unloaded(UnloadedSource::Query(&query)).await?; + return Ok(()); + } if pulls_unloaded(table.route.as_ref()) { unload_source(reader, &table, part, async |files| { - self.load_unloaded(files).await + self.load_unloaded(UnloadedSource::Files(files)).await }) .await?; return Ok(()); @@ -1926,3 +1933,28 @@ fn estimated_bytes(estimate: i64) -> Option { Ok(bytes) => Some(bytes), } } + +/// Whether `table`, a `delete_insert`, `append` or deduplicating `merge` (a key read in two batches keeps its last or newest row) read in windows or cursor-ordered batches, stages them all and publishes once at the end (see [`publishes_runs_once`](crate::adapters::contract::types::SqlAdapter::publishes_runs_once)); a files source's groups still commit one by one, as their values would not fit one statement, and a merge that fails on repeated keys commits each batch, as a key updated during the read is read twice. +pub(crate) fn publishes_once(table: &TablePlan) -> bool { + matches!( + table.mode, + WriteMode::DeleteInsert { .. } + | WriteMode::Append + | WriteMode::Merge { + dedupe: Some(_), + .. + } + ) && matches!( + table.commits, + CommitRule::Windows(_) | CommitRule::Ordered { .. } + ) && sql_adapter(table.spec.warehouse).publishes_runs_once() +} + +/// The query an unload of `part` of `table` would run, which a destination writing through the source's own connection runs itself. +fn selected_query(table: &TablePlan, part: ReadPart) -> Result { + let filter = table + .spec + .read_filter(part.filter.or_else(|| table.source_filter.clone())); + select_query(&unload_request(table, filter.as_deref())) + .map_err(|error| LoaderError::Invalid(error.to_string())) +} diff --git a/crates/sqlload/src/loader/_helpers/render.rs b/crates/sqlload/src/loader/_helpers/render.rs index dcda32667..bf8a6824d 100644 --- a/crates/sqlload/src/loader/_helpers/render.rs +++ b/crates/sqlload/src/loader/_helpers/render.rs @@ -24,16 +24,16 @@ use crate::column_parse::main::parse_lines::parse_lines; use crate::column_types::models::ColumnType; use crate::files_source::models::{FilesOrder, SourceFormat}; use crate::load_naming::main::is_load_column::is_load_column; -use crate::loader::_helpers::publish::{published_once, records_files, send_lines}; +use crate::loader::_helpers::publish::{published_once, publishes_once, records_files, send_lines}; use crate::loader::_helpers::python::{list_choices, unnested_table}; use crate::loader::_helpers::resolve::display_name; use crate::loader::_helpers::sources::{readers_note, unarchived}; use crate::loader::_helpers::table_plan::{ - archived_reason, is_direct, publishes_once, pulls_unloaded, + archived_reason, is_direct, pulls_unloaded, selects_rows, }; use crate::loader::constants::{ CONSTANT_NOTE, DRAINED_REASON, LABEL_WIDTH, NULL_KEYS_NOTE, PLAN_DETAIL_COLUMN, - PRIMARY_KEY_PLACEHOLDER, RESULT_SCHEMA_VERSION, + PRIMARY_KEY_PLACEHOLDER, RESULT_SCHEMA_VERSION, SELECTED_ROUTE, }; use crate::loader::models::{ CommitRule, FilesOutput, FilesSource, ObjectChange, PlanDetail, PlanItem, PlanView, @@ -693,6 +693,7 @@ pub(crate) fn load_item(table: &TablePlan) -> PlanItem { route: WarehouseRoute::Unload { stage, .. }, .. }) = &spec.warehouse_source + && !selects_rows(table.route.as_ref()) { details.push(detail("stage", stage.as_str())); } @@ -844,7 +845,7 @@ pub(crate) fn batch_line(table: &TablePlan) -> String { ) } (_, true) if pulls_unloaded(table.route.as_ref()) => { - "each unload loaded at once into a new table, swapped in once at the end".to_owned() + "every row loaded at once into a new table, swapped in once at the end".to_owned() } (_, true) if sql_adapter(spec.warehouse).keeps_tables() => { "about 1 GB at a time into a staging table, then the table emptied and refilled in one transaction".to_owned() @@ -866,7 +867,7 @@ pub(crate) fn batch_line(table: &TablePlan) -> String { } ), (CommitRule::Size, false) if pulls_unloaded(table.route.as_ref()) => { - "one commit: the destination loads the whole unload at once".to_owned() + "one commit: the destination loads every row at once".to_owned() } (CommitRule::Size, false) => "about every 1 GB".to_owned(), (CommitRule::Whole(reason), false) => { @@ -1001,6 +1002,7 @@ fn route_text(table: &TablePlan, route: &Route) -> String { real.unwrap_or_else(|| files.format.to_string()), files.store ), + [Step::DestinationSelects] => format!("{warehouse} {SELECTED_ROUTE}"), [Step::Unload(store), Step::DestinationPulls(_)] => { format!("unload Parquet to {store} -> {warehouse} {load} (nothing through the loader)") } diff --git a/crates/sqlload/src/loader/_helpers/resolve.rs b/crates/sqlload/src/loader/_helpers/resolve.rs index bc6d45291..72109be2c 100644 --- a/crates/sqlload/src/loader/_helpers/resolve.rs +++ b/crates/sqlload/src/loader/_helpers/resolve.rs @@ -746,6 +746,8 @@ fn setting_name( #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct ResolvedTarget { pub(crate) name: String, + /// The connection it writes through, when it writes in that connection's own database (a target `database` naming another leaves it `None`): a source on that connection reads in the same database, so the target can select its rows itself. + pub(crate) session_connection: Option, pub(crate) warehouse: Warehouse, pub(crate) database: Option, pub(crate) schema: String, @@ -831,6 +833,7 @@ pub(crate) fn source_target( if !sql_adapter(warehouse).runs_sql() { return Ok(ResolvedTarget { name, + session_connection: None, warehouse, database: None, schema: String::new(), @@ -850,6 +853,11 @@ pub(crate) fn source_target( .ok_or_else(|| format!("target {name:?} has no loader_schema"))?; Ok(ResolvedTarget { name, + session_connection: target + .database + .as_ref() + .is_none_or(|own| Some(own) == database.as_ref()) + .then(|| target.connection.clone()), warehouse, database: match connection { ConnectionConfig::MotherDuck { database, .. } => target diff --git a/crates/sqlload/src/loader/_helpers/sources.rs b/crates/sqlload/src/loader/_helpers/sources.rs index 3d7d11de2..da2a69cf9 100644 --- a/crates/sqlload/src/loader/_helpers/sources.rs +++ b/crates/sqlload/src/loader/_helpers/sources.rs @@ -1746,7 +1746,10 @@ pub(crate) async fn unload_source( } /// What one unload of `table` reads: its table or query, its read columns, and `filter`. -fn unload_request<'a>(table: &'a TablePlan, filter: Option<&'a str>) -> UnloadRequest<'a> { +pub(crate) fn unload_request<'a>( + table: &'a TablePlan, + filter: Option<&'a str>, +) -> UnloadRequest<'a> { UnloadRequest { table: &table.spec.source_table, sql: table.spec.query_sql(), diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs index e9e0d0ca8..da1fa8b19 100644 --- a/crates/sqlload/src/loader/_helpers/specs.rs +++ b/crates/sqlload/src/loader/_helpers/specs.rs @@ -544,7 +544,7 @@ pub(crate) fn build_spec( sql: overrides.sql.clone(), route: WarehouseRoute::Query, }), - _ => warehouse_source(context.source, overrides), + _ => warehouse_source(context, overrides), }, row_filter: layered(&settings.row_filter, &defaults.row_filter) .filter(|_| overrides.sql.is_none() && overrides.tombstones_of.is_none()) @@ -656,8 +656,12 @@ fn constant_type_problem( } } -/// How a `snowflake` source's table is read: unloaded to the source's `stage`, or to the user stage when it sets none. -fn warehouse_source(source: &SourceConfig, overrides: &TableOverride) -> Option { +/// How a `snowflake` source's table is read: unloaded to the source's `stage`, or to the user stage when it sets none; a Snowflake target writing through the source's own connection, on its warehouse (the source sets no `warehouse` of its own), may select the rows itself. +fn warehouse_source( + context: &SourceContext<'_>, + overrides: &TableOverride, +) -> Option { + let source = context.source; (source.kind == SourceKind::Snowflake).then(|| { let (store, stage) = match &source.stage { Some(UnloadStage { @@ -686,7 +690,13 @@ fn warehouse_source(source: &SourceConfig, overrides: &TableOverride) -> Option< WarehouseSource { system: SNOWFLAKE_SOURCE, sql: overrides.sql.clone(), - route: WarehouseRoute::Unload { store, stage }, + route: WarehouseRoute::Unload { + store, + stage, + same_connection: context.target.session_connection.as_ref() + == Some(&source.connection) + && source.warehouse.is_none(), + }, } }) } diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index c8a357d47..d51ae4786 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -1079,9 +1079,23 @@ pub(crate) fn reader_offer( }) | None => Vec::new(), }; + let route = spec.warehouse_source.as_ref().map(|source| &source.route); + let selected_by_destination = spec.masked.columns.is_empty() + && matches!( + route, + Some(WarehouseRoute::Unload { + same_connection: true, + .. + }) + ); let direct_blocked = match unload_to.is_empty() { true => direct_blocked, - false => unload_blocked(spec), + false => unload_blocked( + spec.warehouse, + !spec.masked.columns.is_empty(), + route, + stage, + ), }; ReaderOffer { arrow: unload_to.is_empty(), @@ -1101,25 +1115,10 @@ pub(crate) fn reader_offer( .files .as_ref() .is_some_and(|files| stage_misses(stage, &files.location)), + selected_by_destination, } } -/// Whether `table`, a `delete_insert`, `append` or deduplicating `merge` (a key read in two batches keeps its last or newest row) read in windows or cursor-ordered batches, stages them all and publishes once at the end (see [`publishes_runs_once`](crate::adapters::contract::types::SqlAdapter::publishes_runs_once)); a files source's groups still commit one by one, as their values would not fit one statement, and a merge that fails on repeated keys commits each batch, as a key updated during the read is read twice. -pub(crate) fn publishes_once(table: &TablePlan) -> bool { - matches!( - table.mode, - WriteMode::DeleteInsert { .. } - | WriteMode::Append - | WriteMode::Merge { - dedupe: Some(_), - .. - } - ) && matches!( - table.commits, - CommitRule::Windows(_) | CommitRule::Ordered { .. } - ) && sql_adapter(table.spec.warehouse).publishes_runs_once() -} - /// One commit per file, in the order planned: a files source's merge, or `delete_insert` by `replace_key`, applies each file on its own, so a key repeated across files ends with the newest file's value (or rows). fn one_file_each(count: usize) -> Vec { (0..count) @@ -1130,12 +1129,18 @@ fn one_file_each(count: usize) -> Vec { .collect() } -/// Whether `route` has the destination load a warehouse source's unloaded files itself. +/// Whether `route` has the destination select a warehouse source's rows itself, inside the warehouse. +pub(crate) fn selects_rows(route: Option<&Route>) -> bool { + route.is_some_and(|route| matches!(route.steps.as_slice(), [Step::DestinationSelects])) +} + +/// Whether `route` has the destination load a warehouse source's rows itself: its unloaded files, or the rows it selects inside the warehouse. pub(crate) fn pulls_unloaded(route: Option<&Route>) -> bool { route.is_some_and(|route| { matches!( route.steps.as_slice(), [Step::Unload(_), Step::DestinationPulls(_)] + | [Step::DestinationSelects] | [ Step::Unload(_), Step::CopyFiles(Store::Local(_)), diff --git a/crates/sqlload/src/loader/constants.rs b/crates/sqlload/src/loader/constants.rs index f4ef5f8c6..09865f007 100644 --- a/crates/sqlload/src/loader/constants.rs +++ b/crates/sqlload/src/loader/constants.rs @@ -16,6 +16,10 @@ pub const PLAN_DETAIL_COLUMN: usize = 52; /// The width of a plan detail's label and the space after it, as `route `. pub const LABEL_WIDTH: usize = 8; +/// The route of a table a Snowflake target selects from a source on its own connection, after the warehouse's name. +pub const SELECTED_ROUTE: &str = + "INSERT ... SELECT on the source's connection (nothing through the loader)"; + /// How a skipped files source's reason ends after `no files to load in `, when `after_load` emptied its folder. pub const DRAINED_REASON: &str = ": after_load deleted or moved the ones already loaded"; @@ -108,6 +112,8 @@ pub const RESET: &str = "\x1b[0m"; /// The host name that always means this machine: plain text to it never crosses a network. pub const LOCALHOST: &str = "localhost"; +/// Why Snowflake does not read a Snowflake source's unload itself when no external stage of the target covers it. +pub const UNLOADED_OUTSIDE_STAGE: &str = "Snowflake reads another connection's unload itself only through the target's external stage: set the source's stage to an external one (url and storage_integration) whose url is inside the target's external stage url"; /// Why a warehouse source's unloaded files go through SQLLoad rather than straight into the destination. pub const UNLOADED_TIMES: &str = "the warehouse writes times and timestamps as whole microseconds (its Parquet writer keeps only milliseconds), which SQLLoad turns back into them"; @@ -191,7 +197,7 @@ pub const TOMBSTONE_SUFFIX: &str = "__sqlload_deletes"; /// Why `--sync-deletes` fails a plan none of whose selected tables sets `deletes`. pub const NO_DELETES_TABLES: &str = "--sync-deletes reads every key of each selected table with deletes and marks its rows of the keys the source no longer holds, but none of the selected tables sets deletes. Fix: set deletes = \"soft\" or \"hard\" on a merge table, or drop --sync-deletes"; -/// Rows a merge [published once per run](crate::loader::_helpers::table_plan::publishes_once) stages before it commits anyway, at its next cut: matching the staged keys holds them all in one set on the server (about 90 bytes a key on ClickHouse), which this keeps under about 1 GiB. +/// Rows a merge [published once per run](crate::loader::_helpers::publish::publishes_once) stages before it commits anyway, at its next cut: matching the staged keys holds them all in one set on the server (about 90 bytes a key on ClickHouse), which this keeps under about 1 GiB. pub const MERGE_RUN_ROWS: u64 = 10_000_000; /// How `plan` words a run published in one commit. diff --git a/crates/sqlload/src/loader/models.rs b/crates/sqlload/src/loader/models.rs index b24678bba..97745da9b 100644 --- a/crates/sqlload/src/loader/models.rs +++ b/crates/sqlload/src/loader/models.rs @@ -315,8 +315,12 @@ pub struct WarehouseSource { /// How a warehouse source's rows leave the warehouse; each warehouse chooses its own. #[derive(Debug, Clone, PartialEq, Eq)] pub enum WarehouseRoute { - /// The warehouse writes Parquet into `store`, which SQLLoad reads back and removes; `stage` says why there, for `plan`. - Unload { store: Store, stage: String }, + /// The warehouse writes Parquet into `store`, which SQLLoad reads back and removes; `stage` says why there, for `plan`. `same_connection`: the target writes through the source's own connection, so it can select the rows itself instead. + Unload { + store: Store, + stage: String, + same_connection: bool, + }, /// SQLLoad runs the query and reads its rows as Arrow. Query, } diff --git a/crates/sqlload/src/routing/_helpers/warehouse_source.rs b/crates/sqlload/src/routing/_helpers/warehouse_source.rs index 2f0ce5f7d..58056c6ce 100644 --- a/crates/sqlload/src/routing/_helpers/warehouse_source.rs +++ b/crates/sqlload/src/routing/_helpers/warehouse_source.rs @@ -22,12 +22,18 @@ pub(crate) fn should_unload(reader: &ReaderOffer, read_via: ReadVia) -> Result Result { + if reader.selected_by_destination && reader.needs_loader.is_none() { + return Ok(Route { + steps: vec![Step::DestinationSelects], + reason: "the destination writes through the source's own connection, so it selects the rows itself; no data leaves the warehouse".to_owned(), + }); + } let stores = ordered_stores(&reader.unload_to, overrides.stage)?; let unblocked = reader.direct_blocked.is_none() && reader.needs_loader.is_none(); if let Some(store) = stores diff --git a/crates/sqlload/src/routing/models.rs b/crates/sqlload/src/routing/models.rs index de5b766f3..36b504233 100644 --- a/crates/sqlload/src/routing/models.rs +++ b/crates/sqlload/src/routing/models.rs @@ -83,6 +83,8 @@ pub struct ReaderOffer { pub copy_to_internal_stage: bool, /// The destination's store paths name `existing_files`, but it cannot read them (the same bucket on another endpoint). pub out_of_reach: bool, + /// The destination can select a warehouse source's rows itself, as it writes through the source's own connection. + pub selected_by_destination: bool, } /// What a destination can take. @@ -152,6 +154,8 @@ pub enum Step { DestinationTakesArrow, /// The rows go in without files, this way. RowsInto(String), + /// The destination selects the source's rows itself, inside the warehouse; no data leaves it. + DestinationSelects, } impl std::fmt::Display for Step { @@ -174,6 +178,7 @@ impl std::fmt::Display for Step { } Self::DestinationTakesArrow => formatter.write_str("destination takes Arrow"), Self::RowsInto(how) => formatter.write_str(how), + Self::DestinationSelects => formatter.write_str("destination selects the rows itself"), } } } diff --git a/crates/sqlload/src/routing/tests/test_plan_route.rs b/crates/sqlload/src/routing/tests/test_plan_route.rs index e477c749a..f9be85617 100644 --- a/crates/sqlload/src/routing/tests/test_plan_route.rs +++ b/crates/sqlload/src/routing/tests/test_plan_route.rs @@ -168,6 +168,32 @@ fn given_a_source_and_destination_when_planning_then_the_cheapest_route_is_chose "destination takes Arrow", ], }, + PlanRouteTestCase { + description: "a warehouse writing through the source's own connection selects the rows itself", + reader: ReaderOffer { + selected_by_destination: true, + ..warehouse_source(BIG) + }, + writer: warehouse_destination(vec![object_store(LAKE)]), + overrides: RouteOverrides::default(), + expected_steps: &["destination selects the rows itself"], + }, + PlanRouteTestCase { + description: "on the source's own connection, a table that needs the loader is still unloaded and read back", + reader: ReaderOffer { + selected_by_destination: true, + needs_loader: Some(HMAC.to_owned()), + ..warehouse_source(BIG) + }, + writer: warehouse_destination(vec![object_store(LAKE)]), + overrides: RouteOverrides::default(), + expected_steps: &[ + "unload Parquet to s3://lake/unload/", + "read parquet into Arrow", + "write Parquet to s3://lake/", + "destination loads parquet files from s3://lake/", + ], + }, PlanRouteTestCase { description: "a local directory downloads an unload from the lake", reader: warehouse_source(BIG), diff --git a/crates/sqlload/src/snowflake_source/main/mod.rs b/crates/sqlload/src/snowflake_source/main/mod.rs index 0215ae2d6..946975449 100644 --- a/crates/sqlload/src/snowflake_source/main/mod.rs +++ b/crates/sqlload/src/snowflake_source/main/mod.rs @@ -3,5 +3,6 @@ pub mod count_rows; pub mod describe_source; pub mod read_catalog; pub mod read_columns; +pub mod select_query; pub mod unload_files; pub mod unload_rows; diff --git a/crates/sqlload/src/snowflake_source/main/select_query.rs b/crates/sqlload/src/snowflake_source/main/select_query.rs new file mode 100644 index 000000000..b12ceba9a --- /dev/null +++ b/crates/sqlload/src/snowflake_source/main/select_query.rs @@ -0,0 +1,10 @@ +//! The query an unload runs, for a destination that selects the rows itself in the same session. + +use crate::snowflake_source::_helpers::unload_query::unload_query; +use crate::snowflake_source::errors::SnowflakeSourceError; +use crate::snowflake_source::models::UnloadRequest; + +/// The `SELECT` an unload of `request` writes to Parquet: each read column under its own name, times and timestamps as whole microseconds, filtered as `request` says. +pub fn select_query(request: &UnloadRequest<'_>) -> Result { + unload_query(request).map_err(SnowflakeSourceError::Invalid) +} diff --git a/website/public/index.html b/website/public/index.html index eded67b22..f2078c4d4 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -170,7 +170,7 @@

See every change first

02 · Load

Bulk copy, not row inserts

-

Files in S3 load into Snowflake straight from an external stage. Parquet loads into DuckDB through DuckDB's own reader. Rows are staged as Parquet and copied in bulk.

+

Files in S3 load into Snowflake straight from an external stage. Parquet loads into DuckDB through DuckDB's own reader. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Rows are staged as Parquet and copied in bulk.

Load methods and routes →

sqlload plan
diff --git a/website/src/content/docs/docs/sources/snowflake.mdx b/website/src/content/docs/docs/sources/snowflake.mdx
index aa4bd2063..907030a2c 100644
--- a/website/src/content/docs/docs/sources/snowflake.mdx
+++ b/website/src/content/docs/docs/sources/snowflake.mdx
@@ -7,7 +7,8 @@ A Snowflake source loads tables and views of one
 database, and the results of queries you write.
 Snowflake unloads each read as Parquet with `COPY INTO`, and SQLLoad reads the files and loads
 them into the source's target. A DuckDB target reads them from S3
-itself.
+itself, and a Snowflake target loads the rows inside Snowflake; see
+[Snowflake loads the rows itself](#snowflake-loads-the-rows-itself).
 
 ```toml
 [connections.warehouse]
@@ -158,6 +159,31 @@ whatever its session time zone. A load fails before it commits when DuckDB loads
 of rows than `COPY INTO` reports, and the folder is removed either way. A table without a cursor
 commits once per run rather than about every 1 GB.
 
+### Snowflake loads the rows itself
+
+Into a Snowflake target, no rows pass through SQLLoad in two cases:
+
+- **The target writes through the source's connection**: the same `connection`, with no other
+  `database`, and the source sets no `warehouse` of its own. Snowflake selects the rows with one
+  `INSERT ... SELECT` on the connection's warehouse, and nothing is unloaded.
+- **The source's `stage` `url` is inside the target's external stage `url`**, on another connection
+  or account. Snowflake unloads the files there, and the target `COPY`s them through its own stage.
+  The load commits only when it loads as many rows as `COPY INTO` reported unloading. An `append`
+  that writes straight into its table fails, before loading anything, when one read unloads too
+  many files to `COPY` in one statement (thousands); a smaller `batch_size` splits the read.
+
+`plan` shows which:
+
+```text
+route   Snowflake INSERT ... SELECT on the source's connection (nothing through the loader)
+route   unload Parquet to s3://example-bucket/stage/unload/ -> Snowflake COPY (nothing through the loader)
+```
+
+Otherwise SQLLoad reads the files and stages them for the target, as for any other source: with the
+user stage, a named stage, or an S3 folder the target's stage doesn't cover. A table with a `mask`
+always goes through SQLLoad, which masks the values. Times and timestamps come back exactly, as
+with DuckDB.
+
 Without a stage SQLLoad downloads the files from the user stage
 unchanged, 16 at a time, into a temporary folder, and DuckDB reads them from there:
 
diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx
index 3fc7dee3c..1c87b78f7 100644
--- a/website/src/content/docs/docs/supported-systems.mdx
+++ b/website/src/content/docs/docs/supported-systems.mdx
@@ -9,7 +9,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes
 |--------|---------|---------|
 | Postgres | Tables, by `include` patterns | Binary `COPY` into Arrow, over TLS when the connection's `sslmode` asks |
 | Postgres changes (`postgres_cdc`) | Inserts, updates and deletes of tables, by `include` patterns | A snapshot, then a logical replication slot (`pgoutput`), in whole transactions. See [Postgres changes](/docs/sources/postgres-cdc/). |
-| Snowflake | Tables and views, by `include` patterns, and query results | Parquet unloaded with `COPY INTO` to the user stage or an S3 stage, then read by SQLLoad, or by DuckDB itself from an S3 stage. See [Snowflake](/docs/sources/snowflake/). |
+| Snowflake | Tables and views, by `include` patterns, and query results | Parquet unloaded with `COPY INTO` to the user stage or an S3 stage, then read by SQLLoad, or by DuckDB itself from an S3 stage. A Snowflake target selects or copies the rows inside Snowflake when it writes through the source's connection, or its external stage covers the unload; see [Snowflake loads the rows itself](/docs/sources/snowflake/#snowflake-loads-the-rows-itself). See [Snowflake](/docs/sources/snowflake/). |
 | Databases with an ODBC driver: SQL Server, MySQL, MariaDB, OpenEdge and others | Tables and views, by `include` patterns, and query results | The driver, as Arrow. Tested with psqlODBC, Microsoft ODBC Driver 18 for SQL Server and MySQL Connector/ODBC; MariaDB Connector/ODBC and OpenEdge are checked by hand. See [ODBC](/docs/sources/odbc/#drivers). |
 | BigQuery (beta) | Tables and views, by `include` patterns, and query results | The Storage Read API, as Arrow in parallel streams; views and queries through a query job. See [BigQuery](/docs/sources/bigquery/). |
 | Databricks (beta) | Tables and views, by `include` patterns, and query results | One `SELECT` on the SQL warehouse, its result downloaded as Arrow files in parallel through the Statement Execution API's links. See [Databricks](/docs/sources/databricks/). |
diff --git a/website/src/content/docs/docs/targets/index.mdx b/website/src/content/docs/docs/targets/index.mdx
index c084f7251..cc500e889 100644
--- a/website/src/content/docs/docs/targets/index.mdx
+++ b/website/src/content/docs/docs/targets/index.mdx
@@ -20,6 +20,10 @@ names the route for every table.
 | ClickHouse | In S3: ClickHouse reads them with `s3()` into a staging table. In a folder: read by SQLLoad, sent as Arrow | Read by SQLLoad, sent as Arrow into a staging table | Sent as Arrow into a staging table, then swapped in by table or partition |
 | Postgres | Read by SQLLoad, then binary `COPY` into a temporary staging table | As for Parquet files | Binary `COPY` into a temporary staging table, then one transaction per commit |
 
+A Snowflake source loading into Snowflake stays inside Snowflake: an `INSERT ... SELECT` on the
+source's own connection, or a `COPY` of its unloaded files through the target's external stage.
+See [Snowflake loads the rows itself](/docs/sources/snowflake/#snowflake-loads-the-rows-itself).
+
 There are no silent fallbacks. If the route a table needs can't be used, `plan` fails for that
 table and says how to fix it.
 
diff --git a/website/src/content/docs/docs/targets/snowflake.mdx b/website/src/content/docs/docs/targets/snowflake.mdx
index 67e445d50..18f9d688b 100644
--- a/website/src/content/docs/docs/targets/snowflake.mdx
+++ b/website/src/content/docs/docs/targets/snowflake.mdx
@@ -105,3 +105,10 @@ this gives the same values as SQLLoad's own reader.
   [CSV encodings](/docs/sources/files/#csv-encodings).
 
 Files are never read and re-uploaded by SQLLoad otherwise.
+
+## Snowflake sources into Snowflake
+
+A [Snowflake source](/docs/sources/snowflake/) loads into this target without its rows passing
+through SQLLoad: with one `INSERT ... SELECT` when the target writes through the source's own
+connection, or a `COPY` from the target's external stage when the source unloads inside it. See
+[Snowflake loads the rows itself](/docs/sources/snowflake/#snowflake-loads-the-rows-itself).