diff --git a/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs b/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs new file mode 100644 index 00000000..0bfd8c48 --- /dev/null +++ b/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs @@ -0,0 +1,75 @@ +//! Whether a destination reaches the files it would read itself. + +use crate::adapters::databricks::main::stage::_check_folder::check_folder; +use crate::adapters::databricks::main::stage::_external_locations::external_locations; +use crate::adapters::databricks::models::DatabricksDestination; +use crate::adapters::dispatch::models::Destination; +#[cfg(feature = "duckdb")] +use crate::adapters::duckdb::main::_check_files::check_files; +use crate::files_source::models::FilesRead; +use crate::loader::models::FilesSource; + +/// Check, at plan, that `destination` reads the files of `source` in S3 where they lie, for a table nothing sends through SQLLoad, with `read` its planned read: Databricks looks up its Unity Catalog external locations once per run, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers, and lists the folder when one covers it and it holds files to load; MotherDuck reads the first file to load on its servers. The error says why it can't, with the fix. +pub async fn check_reach( + destination: &Destination, + source: &FilesSource, + read: Option<&FilesRead>, +) -> Result<(), String> { + let first = read.and_then(|read| read.files.first()); + let folder = source.location.to_string(); + match destination { + Destination::Databricks(databricks) => { + databricks_reach(databricks, first.map(|_| folder.as_str())).await + } + #[cfg(feature = "duckdb")] + Destination::DuckDb(duckdb) => { + let Some(data) = first else { + return Ok(()); + }; + let url = format!("{folder}{}", data.file.relative_key); + check_files(duckdb, &source.location.store, &url) + .await + .map_err(|error| { + format!( + "{url} cannot be read by MotherDuck on its servers: {}. Fix: MotherDuck's servers read the bucket from their own network, with the keys SQLLoad's S3 reads use (the AWS_* variables, else the connection's profile or AWS_PROFILE, else the AWS credential chain on this machine): give those keys s3:GetObject and s3:ListBucket on it, and kms:Decrypt on its KMS key if it has one, and let the bucket policy allow reads from outside your network or VPC endpoint; a bucket only your network may reach can't load into MotherDuck", + first_line(&error.to_string()) + ) + }) + } + _ => Ok(()), + } +} + +/// Databricks' external locations, and whether it can list `folder`, which holds files to load, when one of them covers it (a folder holding no files yet does not list). +async fn databricks_reach( + databricks: &DatabricksDestination, + folder: Option<&str>, +) -> Result<(), String> { + let locations = external_locations(databricks) + .await + .map_err(|error| format!("cannot list Databricks' external locations: {error}"))?; + let Some(folder) = folder else { + return Ok(()); + }; + let Some(url) = locations + .iter() + .find(|url| folder.starts_with(url.as_str())) + else { + return Ok(()); + }; + check_folder(databricks, folder).await.map_err(|error| { + format!( + "{folder} cannot be read by Databricks through the external location at {url}: {}. Fix: when Databricks refuses access, grant the service principal READ FILES on that external location", + first_line(&error.to_string()) + ) + }) +} + +/// The first line of `error`, without a trailing colon: the services follow it with pages of detail. +fn first_line(error: &str) -> &str { + error + .lines() + .next() + .unwrap_or_default() + .trim_end_matches([':', ' ']) +} diff --git a/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs b/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs deleted file mode 100644 index 2b741f08..00000000 --- a/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! Where a destination reads files itself, beyond its own stores. - -use crate::adapters::databricks::main::stage::_check_folder::check_folder; -use crate::adapters::databricks::main::stage::_external_locations::external_locations; -use crate::adapters::dispatch::models::Destination; - -/// Look up, once per run, where `destination` reads files itself beyond its own stores, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers: Databricks' Unity Catalog external locations; and check that it can read `folder`, which holds files to load, when one of them covers it (a folder holding no files yet does not list). Called at plan for a table whose files in S3 the destination would read where they lie; the error says why it can't, with the fix. -pub async fn list_reach(destination: &Destination, folder: Option<&str>) -> Result<(), String> { - let Destination::Databricks(databricks) = destination else { - return Ok(()); - }; - let locations = external_locations(databricks) - .await - .map_err(|error| format!("cannot list Databricks' external locations: {error}"))?; - let Some(folder) = folder else { - return Ok(()); - }; - let Some(url) = locations - .iter() - .find(|url| folder.starts_with(url.as_str())) - else { - return Ok(()); - }; - check_folder(databricks, folder).await.map_err(|error| { - let error = error.to_string(); - let first_line = error.lines().next().unwrap_or_default(); - format!( - "{folder} cannot be read by Databricks through the external location at {url}: {}. Fix: when Databricks refuses access, grant the service principal READ FILES on that external location", - first_line.trim_end_matches([':', ' ']) - ) - }) -} diff --git a/crates/sqlload/src/adapters/dispatch/main/route/mod.rs b/crates/sqlload/src/adapters/dispatch/main/route/mod.rs index 562ba168..ffe8c7bf 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/mod.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/mod.rs @@ -1,8 +1,9 @@ +pub mod check_reach; pub mod choose_stage; pub mod direct_blocked; -pub mod list_reach; pub mod route_warning; pub mod sends_rows; pub mod stage_misses; pub mod unload_blocked; +pub mod unload_refused; pub mod writer_accepts; diff --git a/crates/sqlload/src/adapters/dispatch/main/route/unload_refused.rs b/crates/sqlload/src/adapters/dispatch/main/route/unload_refused.rs new file mode 100644 index 00000000..9b70fb21 --- /dev/null +++ b/crates/sqlload/src/adapters/dispatch/main/route/unload_refused.rs @@ -0,0 +1,19 @@ +//! Whether a destination reads a warehouse's unloaded files itself. + +use crate::adapters::dispatch::models::Destination; +use crate::routing::models::Store; + +/// Why `destination` doesn't read the files a warehouse source unloads to `unload_to` itself, so SQLLoad reads them: MotherDuck reads only a files source's Parquet on its servers so far; `None` otherwise. +pub fn unload_refused(destination: &Destination, unload_to: &[Store]) -> Option { + if unload_to.is_empty() { + return None; + } + match destination { + #[cfg(feature = "duckdb")] + Destination::DuckDb(duckdb) if duckdb.motherduck => Some( + "MotherDuck reads only a files source's Parquet in S3 itself, not a warehouse's unload" + .to_owned(), + ), + _ => None, + } +} diff --git a/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs b/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs index c5fddcc8..09e0c928 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs @@ -7,12 +7,14 @@ use crate::adapters::snowflake::models::StageRoute; use crate::routing::models::WriterAccepts; use crate::routing::models::{FileFormat, Store}; -/// What `destination` takes over the chosen `stage`: Arrow batches (DuckDB also reads Parquet in S3 or on this machine itself, but MotherDuck does not; ClickHouse reads Parquet in S3), or what its stage route takes. +/// What `destination` takes over the chosen `stage`: Arrow batches (DuckDB also reads Parquet in S3 or on this machine itself, MotherDuck Parquet in S3 on its servers, and ClickHouse Parquet in S3), or what its stage route takes. pub fn writer_accepts(destination: &Destination, stage: Option<&StageRoute>) -> WriterAccepts { match destination { #[cfg(feature = "duckdb")] Destination::DuckDb(duckdb) if duckdb.motherduck => WriterAccepts { arrow: true, + pulls_from: vec![Store::ObjectStore("s3://".to_owned())], + pull_formats: vec![FileFormat::Parquet], ..WriterAccepts::default() }, #[cfg(feature = "duckdb")] diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/direct_load.rs b/crates/sqlload/src/adapters/duckdb/_helpers/direct_load.rs index 7194f6fe..851ecd8d 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/direct_load.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/direct_load.rs @@ -7,6 +7,7 @@ use crate::adapters::contract::models::{DirectBatch, DirectLoaded}; use crate::adapters::duckdb::_helpers::pull_sql::{PulledFile, pull_insert}; use crate::adapters::duckdb::main::_direct_problem::direct_problem; use crate::adapters::duckdb::main::_execute_statements::execute_statements; +use crate::adapters::duckdb::main::_open_bucket::open_bucket; use crate::adapters::duckdb::main::_pull_files::pull_files; use crate::adapters::duckdb::models::DuckDbWriter; use crate::files_source::constants::{SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN}; @@ -54,15 +55,16 @@ impl DuckDbWriter { let insert = pull_insert( target.staging.unwrap_or(target.table), (carried, &in_files), - &files, + (&files, self.destination.motherduck), (target.compat, target.load_id, batch.loaded_at), ); + open_bucket(&self.destination, &source.location.store).await?; let transaction = target.direct || target.builds; if transaction && !self.open_transaction { execute_statements(&self.destination, vec![BEGIN_STATEMENT.to_owned()]).await?; self.open_transaction = true; } - let rows = pull_files(&self.destination, &source.location.store, insert).await?; + let rows = pull_files(&self.destination, insert).await?; let expected = footer_rows(batch.footers); if rows != expected { return Err(LoaderError::Invalid(format!( diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs b/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs index 0ef1f5f8..164bc959 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs @@ -1,18 +1,26 @@ //! The statement that has DuckDB load Parquet files itself. +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::time::Instant; + use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS}; use crate::adapters::contract::models::TableName; use crate::adapters::contract::types::SqlAdapter; -use crate::adapters::duckdb::constants::UTC_OFFSET; -use crate::adapters::duckdb::models::DuckDbSql; +use crate::adapters::duckdb::_helpers::motherduck_connection::environment_variable; +use crate::adapters::duckdb::constants::{CHAIN_SECRET_REFRESH, UTC_OFFSET}; +use crate::adapters::duckdb::errors::DuckDbError; +use crate::adapters::duckdb::models::{DuckDbSql, SetSecret}; use crate::column_types::main::is_variant::is_variant; use crate::column_types::models::ColumnType; +use crate::file_store::main::s3_keys::s3_keys; use crate::files_source::constants::SOURCE_ROW_COLUMN; use crate::load_naming::main::metadata_role::metadata_role; use crate::load_naming::models::{Compat, MetadataRole}; use crate::postgres_source::models::ReadColumn; use crate::schema_plan::constants::MERGE_ROW_COLUMN; use crate::schema_plan::models::DesiredColumn; +use crate::stage_api::models::AwsCredentials; +use crate::store_paths::models::StoreKind; /// One file DuckDB reads: where it reads it, and the values SQLLoad gives every row of it (its name and version as the files source records them, and its partition values), by column (`None` for a partition folder the file lacks). #[derive(Debug, Clone, PartialEq, Eq)] @@ -21,11 +29,11 @@ pub(crate) struct PulledFile { pub constants: Vec<(String, Option)>, } -/// `INSERT INTO into` every one of `columns` from the Parquet `files`, matched by name so a file that lacks a column gives NULL. A column comes from the file's constants when it has one of that name (a list of every file's value, indexed by the row's `file_index`: no join, so DuckDB streams the rows into the table instead of holding them), otherwise from the files when at least one of them holds it (`in_files`, each column with its name in the files, which `naming`, `rename` or numbering may have changed: `f."Order ID"` loads into `order_id`), and is cast to its type; `_sqlload_source_row` counts each file's rows from 1; the load id and load time are the batch's; any other column is NULL. Every file must carry the same constant columns. +/// `INSERT INTO into` every one of `columns` from the Parquet `files`, matched by name so a file that lacks a column gives NULL. A column comes from the file's constants when it has one of that name (a list of every file's value, indexed by the row's `file_index`: no join, so DuckDB streams the rows into the table instead of holding them), otherwise from the files when at least one of them holds it (`in_files`, each column with its name in the files, which `naming`, `rename` or numbering may have changed: `f."Order ID"` loads into `order_id`), and is cast to its type; `_sqlload_source_row` counts each file's rows from 1; the load id and load time are the batch's; any other column is NULL. Every file must carry the same constant columns. Into MotherDuck (`remote`) the files are read on its servers. pub(crate) fn pull_insert( into: &TableName, (columns, in_files): (&[DesiredColumn], &[(String, String)]), - files: &[PulledFile], + (files, remote): (&[PulledFile], bool), (compat, load_id, loaded_at): (Compat, &str, &str), ) -> String { let literal = |text: &str| DuckDbSql.render_loader_value_literal(text); @@ -75,11 +83,12 @@ pub(crate) fn pull_insert( .map(|column| identifier(&column.name)) .collect(); format!( - "INSERT INTO {} ({}) SELECT {} FROM read_parquet([{}], union_by_name = true, file_row_number = true, hive_partitioning = false) AS f", + "INSERT INTO {} ({}) SELECT {} FROM read_parquet([{}], union_by_name = true, file_row_number = true, hive_partitioning = false{}) AS f", DuckDbSql.render_qualified_name(into), targets.join(", "), values.join(", "), - urls.join(", ") + urls.join(", "), + on_servers(remote) ) } @@ -215,6 +224,14 @@ fn unloaded_value(output: &ColumnType, quoted: &str) -> String { } } +/// The `read_parquet` option that has MotherDuck read the files on its servers (`remote`), so no row passes through this machine; none for SQLLoad's own DuckDB. +fn on_servers(remote: bool) -> &'static str { + match remote { + true => ", MD_RUN = REMOTE", + false => "", + } +} + /// The statement, with its parameters, that gives DuckDB a temporary secret for `bucket`: scoped to it and named after it (hex, so two buckets never share a name), in `region`, with `keys` (key id, secret and session token, empty when there is none) passed as parameters and never in SQL, or else the AWS credential chain. pub(crate) fn s3_secret( bucket: &str, @@ -246,3 +263,70 @@ pub(crate) fn default_region(variable: &dyn Fn(&str) -> Option) -> Strin .or_else(|| set("AWS_DEFAULT_REGION")) .unwrap_or_else(|| "us-east-1".to_owned()) } + +/// What DuckDB needs to read files in an S3 bucket itself: the extensions to load (installing them on first use) and the [temporary secret](s3_secret), with its parameters, whose keys come from the AWS credential chain when `chain`. +pub(crate) struct BucketAccess { + pub bucket: String, + pub extensions: &'static str, + pub statement: String, + pub parameters: Vec, + pub chain: bool, +} + +impl BucketAccess { + /// The secret as it is set `at`, with a hash of its statement and parameters in place of the keys. + pub(crate) fn set_at(&self, at: Instant) -> SetSecret { + let mut hasher = DefaultHasher::new(); + (&self.statement, &self.parameters).hash(&mut hasher); + SetSecret { + hash: hasher.finish(), + chain: self.chain, + at, + } + } +} + +/// Whether the secret `set` on the database for a bucket still serves `wanted`, the one a table would set `now`: the same statement and parameters, and, for keys DuckDB resolved through the AWS credential chain when it was set, set less than [`CHAIN_SECRET_REFRESH`] ago, as they may expire. +pub(crate) fn secret_current(set: Option<&SetSecret>, wanted: &SetSecret, now: Instant) -> bool { + set.is_some_and(|set| { + set.hash == wanted.hash + && (!set.chain || now.saturating_duration_since(set.at) < CHAIN_SECRET_REFRESH) + }) +} + +/// For files in S3, [`BucketAccess`] for their bucket: the secret in the connection's region, else `AWS_REGION` / `AWS_DEFAULT_REGION` (unset when empty), else us-east-1 as SQLLoad's reader does; with the keys SQLLoad's own S3 reads use (the `AWS_*` variables, else the shared credentials file under the connection's `profile` or `AWS_PROFILE`), passed as parameters and never in SQL, otherwise the AWS credential chain, resolved where DuckDB runs. `None` for files elsewhere. +pub(crate) fn bucket_access(store: &StoreKind) -> Result, DuckDbError> { + let StoreKind::S3 { + bucket, + region, + profile, + .. + } = store + else { + return Ok(None); + }; + let keys = s3_keys(profile.as_deref()) + .map_err(DuckDbError::Task)? + .map(secret_parameters); + let extensions = match keys { + Some(_) => "INSTALL httpfs; LOAD httpfs;", + None => "INSTALL httpfs; LOAD httpfs; INSTALL aws; LOAD aws;", + }; + let region = region + .clone() + .unwrap_or_else(|| default_region(&environment_variable)); + let (statement, parameters) = s3_secret(bucket, region, keys.as_ref()); + Ok(Some(BucketAccess { + bucket: bucket.clone(), + extensions, + statement, + parameters, + chain: keys.is_none(), + })) +} + +/// `keys` as the `KEY_ID`, `SECRET` and `SESSION_TOKEN` parameters of a DuckDB secret. +fn secret_parameters(keys: AwsCredentials) -> [String; 3] { + let token = keys.session_token.map(|token| token.0).unwrap_or_default(); + [keys.key_id, keys.secret.0, token] +} diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/shared_destination.rs b/crates/sqlload/src/adapters/duckdb/_helpers/shared_destination.rs index 844266e7..7f441d8d 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/shared_destination.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/shared_destination.rs @@ -18,5 +18,6 @@ pub(in crate::adapters::duckdb) fn shared_destination( memory: None, own_limit: None, unflushed: Arc::default(), + secrets: Arc::default(), }) } diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/unloaded_load.rs b/crates/sqlload/src/adapters/duckdb/_helpers/unloaded_load.rs index feabf318..98d6f16c 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/unloaded_load.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/unloaded_load.rs @@ -4,6 +4,7 @@ use crate::adapters::contract::constants::{BEGIN_STATEMENT, COMMIT_STATEMENT}; use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch, UnloadedSource}; use crate::adapters::duckdb::_helpers::pull_sql::{UnloadedColumns, unloaded_insert}; use crate::adapters::duckdb::main::_execute_statements::execute_statements; +use crate::adapters::duckdb::main::_open_bucket::open_bucket; use crate::adapters::duckdb::main::_pull_files::pull_files; use crate::adapters::duckdb::models::DuckDbWriter; use crate::loader::errors::LoaderError; @@ -48,12 +49,13 @@ impl DuckDbWriter { &urls, (target.load_id, batch.loaded_at), ); + open_bucket(&self.destination, &files.folder.store).await?; let transaction = target.direct || target.builds; if transaction && !self.open_transaction { execute_statements(&self.destination, vec![BEGIN_STATEMENT.to_owned()]).await?; self.open_transaction = true; } - let rows = pull_files(&self.destination, &files.folder.store, insert).await?; + let rows = pull_files(&self.destination, insert).await?; if rows != files.rows { return Err(LoaderError::Invalid(format!( "DuckDB loaded {rows} rows from the unloaded files in {}, but the unload reports {}; the batch is not committed", diff --git a/crates/sqlload/src/adapters/duckdb/constants.rs b/crates/sqlload/src/adapters/duckdb/constants.rs index ce049936..ab3ce865 100644 --- a/crates/sqlload/src/adapters/duckdb/constants.rs +++ b/crates/sqlload/src/adapters/duckdb/constants.rs @@ -28,3 +28,5 @@ pub const CHECKPOINT_WAIT: std::time::Duration = std::time::Duration::from_secs( pub const NO_AUTOCHECKPOINT: &str = "SET checkpoint_threshold = '1TB'"; /// The suffix of a DuckDB file's write-ahead log. pub const WAL_SUFFIX: &str = ".wal"; +/// How long a bucket secret on the AWS credential chain is used before it is set again: DuckDB resolves the chain's keys once, when the secret is set, and temporary keys from a role may have only five minutes left when fetched. +pub const CHAIN_SECRET_REFRESH: std::time::Duration = std::time::Duration::from_secs(240); diff --git a/crates/sqlload/src/adapters/duckdb/main/_check_files.rs b/crates/sqlload/src/adapters/duckdb/main/_check_files.rs new file mode 100644 index 00000000..6d3d29a2 --- /dev/null +++ b/crates/sqlload/src/adapters/duckdb/main/_check_files.rs @@ -0,0 +1,25 @@ +//! Check that MotherDuck can read a files source's files on its servers. + +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::duckdb::errors::DuckDbError; +use crate::adapters::duckdb::main::_open_bucket::open_bucket; +use crate::adapters::duckdb::main::_pull_files::pull_files; +use crate::adapters::duckdb::models::{DuckDbDestination, DuckDbSql}; +use crate::store_paths::models::StoreKind; + +/// Check that MotherDuck reads `url`, a Parquet file in `store`, on its servers, with the session's secret for the bucket as a load sets it; nothing for SQLLoad's own DuckDB, which reads with the same keys as SQLLoad's reader. +pub(in crate::adapters) async fn check_files( + destination: &DuckDbDestination, + store: &StoreKind, + url: &str, +) -> Result<(), DuckDbError> { + if !destination.motherduck { + return Ok(()); + } + let probe = format!( + "SELECT count(*) FROM read_parquet({}, hive_partitioning = false, MD_RUN = REMOTE)", + DuckDbSql.render_loader_value_literal(url) + ); + open_bucket(destination, store).await?; + pull_files(destination, probe).await.map(|_| ()) +} diff --git a/crates/sqlload/src/adapters/duckdb/main/_open_bucket.rs b/crates/sqlload/src/adapters/duckdb/main/_open_bucket.rs new file mode 100644 index 00000000..6bad05a5 --- /dev/null +++ b/crates/sqlload/src/adapters/duckdb/main/_open_bucket.rs @@ -0,0 +1,45 @@ +//! Give DuckDB, or MotherDuck reading on its servers, access to a bucket of files. + +use std::time::Instant; + +use crate::adapters::duckdb::_helpers::blocking::with_connection; +use crate::adapters::duckdb::_helpers::pull_sql::{bucket_access, secret_current}; +use crate::adapters::duckdb::errors::DuckDbError; +use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination}; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; +use crate::store_paths::models::StoreKind; + +/// Set [`bucket_access`] for files in `store` on `destination`'s database, unless the bucket's secret [is current](secret_current): for every table's connection at once, as tables setting one at the same time conflict. Call it before the table opens its transaction, never inside one, so the secret commits for every connection and a rollback can't undo it. On MotherDuck the secret is the session's own, which its servers read the files with, and nothing is stored in the account. +pub(in crate::adapters) async fn open_bucket( + destination: &DuckDbDestination, + store: &StoreKind, +) -> Result<(), DuckDbError> { + let Some(access) = bucket_access(store)? else { + return Ok(()); + }; + let now = Instant::now(); + let wanted = access.set_at(now); + let mut set = destination.secrets.lock().await; + if secret_current(set.get(&access.bucket), &wanted, now) { + return Ok(()); + } + set.remove(&access.bucket); + match &destination.connection { + DuckDbConnection::Bridge(bridge) => { + run_statement(bridge, access.extensions.to_owned(), Vec::new()).await?; + run_statement(bridge, access.statement, access.parameters).await?; + } + DuckDbConnection::Local(_) => { + let (extensions, statement, values) = + (access.extensions, access.statement, access.parameters); + with_connection(destination, move |connection| { + connection.execute_batch(extensions)?; + connection.execute(&statement, duckdb::params_from_iter(values))?; + Ok(()) + }) + .await?; + } + } + set.insert(access.bucket, wanted); + Ok(()) +} diff --git a/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs b/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs index 456faa70..9ecaa2d3 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs @@ -51,5 +51,6 @@ pub(crate) fn open_motherduck( }), own_limit: None, unflushed: Arc::default(), + secrets: Arc::default(), }) } diff --git a/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs b/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs index 60224db4..c9b988f7 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs @@ -59,5 +59,6 @@ pub(in crate::adapters) fn own_connection( memory: destination.memory, own_limit: destination.own_limit, unflushed: destination.unflushed.clone(), + secrets: destination.secrets.clone(), }) } diff --git a/crates/sqlload/src/adapters/duckdb/main/_pull_files.rs b/crates/sqlload/src/adapters/duckdb/main/_pull_files.rs index 15f24427..9a53b798 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_pull_files.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_pull_files.rs @@ -1,52 +1,28 @@ -//! Run a statement that has DuckDB read files itself, giving it access to the files' bucket first. +//! Run a statement that has DuckDB, or MotherDuck on its servers, read files itself. use crate::adapters::duckdb::_helpers::blocking::with_connection; -use crate::adapters::duckdb::_helpers::motherduck_connection::environment_variable; -use crate::adapters::duckdb::_helpers::pull_sql::{default_region, s3_secret}; use crate::adapters::duckdb::errors::DuckDbError; -use crate::adapters::duckdb::models::DuckDbDestination; -use crate::file_store::main::s3_keys::s3_keys; -use crate::stage_api::models::AwsCredentials; -use crate::store_paths::models::StoreKind; +use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination}; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; -/// Run `insert` and return the rows it loaded. For S3 files it first loads `httpfs` (installing it on first use) and sets a [temporary secret](s3_secret) for the bucket, in the connection's region, else `AWS_REGION` / `AWS_DEFAULT_REGION` (unset when empty), else us-east-1 as SQLLoad's reader does: the keys SQLLoad's own S3 reads use (the `AWS_*` variables, else the shared credentials file under the connection's `profile` or `AWS_PROFILE`), passed as parameters and never in SQL, otherwise the AWS credential chain. +/// Run `insert`, which reads files from a bucket [opened](crate::adapters::duckdb::main::_open_bucket::open_bucket) before the table's transaction, and return the rows it loaded. pub(in crate::adapters) async fn pull_files( destination: &DuckDbDestination, - store: &StoreKind, insert: String, ) -> Result { - let (bucket, keys) = match store { - StoreKind::S3 { - bucket, - region, - profile, - .. - } => ( - Some((bucket.clone(), region.clone())), - s3_keys(profile.as_deref()) - .map_err(DuckDbError::Task)? - .map(secret_parameters), - ), - StoreKind::Filesystem { .. } | StoreKind::Sftp { .. } => (None, None), + let DuckDbConnection::Bridge(bridge) = &destination.connection else { + return with_connection(destination, move |connection| { + Ok(connection.execute(&insert, [])? as u64) + }) + .await; }; - let region = default_region(&environment_variable); - with_connection(destination, move |connection| { - if let Some((bucket, bucket_region)) = bucket { - connection.execute_batch("INSTALL httpfs; LOAD httpfs;")?; - if keys.is_none() { - connection.execute_batch("INSTALL aws; LOAD aws;")?; - } - let (sql, parameters) = - s3_secret(&bucket, bucket_region.unwrap_or(region), keys.as_ref()); - connection.execute(&sql, duckdb::params_from_iter(parameters))?; - } - Ok(connection.execute(&insert, [])? as u64) + let rows = run_statement(bridge, insert, Vec::new()).await?; + let Some(count) = rows.first().and_then(|row| row.first()).cloned().flatten() else { + return Err(DuckDbError::MotherDuck( + "the statement reported no row count".to_owned(), + )); + }; + count.parse::().map_err(|error| { + DuckDbError::MotherDuck(format!("the statement reported {count:?} rows: {error}")) }) - .await -} - -/// `keys` as the `KEY_ID`, `SECRET` and `SESSION_TOKEN` parameters of a DuckDB secret. -fn secret_parameters(keys: AwsCredentials) -> [String; 3] { - let token = keys.session_token.map(|token| token.0).unwrap_or_default(); - [keys.key_id, keys.secret.0, token] } diff --git a/crates/sqlload/src/adapters/duckdb/main/mod.rs b/crates/sqlload/src/adapters/duckdb/main/mod.rs index dcc4374c..e09a7baf 100644 --- a/crates/sqlload/src/adapters/duckdb/main/mod.rs +++ b/crates/sqlload/src/adapters/duckdb/main/mod.rs @@ -1,7 +1,9 @@ pub(in crate::adapters) mod _append_batch; +pub(in crate::adapters) mod _check_files; pub(in crate::adapters) mod _describe_table; pub(in crate::adapters) mod _direct_problem; pub(in crate::adapters) mod _execute_statements; +pub(in crate::adapters) mod _open_bucket; pub(in crate::adapters) mod _open_destination; pub(in crate::adapters) mod _open_duckdb; pub(in crate::adapters) mod _open_motherduck; diff --git a/crates/sqlload/src/adapters/duckdb/models.rs b/crates/sqlload/src/adapters/duckdb/models.rs index ff79bc57..31077f85 100644 --- a/crates/sqlload/src/adapters/duckdb/models.rs +++ b/crates/sqlload/src/adapters/duckdb/models.rs @@ -11,7 +11,7 @@ pub struct DuckDbDestination { pub connection: DuckDbConnection, /// Held while a load's tables change the schema, one at a time: DuckDB fails concurrent catalog changes with write-write conflicts. pub ddl: Arc>, - /// A MotherDuck database (`md:`), which only takes Arrow batches: it never reads files itself. + /// A MotherDuck database (`md:`), which reads files itself only from S3, on its servers. pub motherduck: bool, /// The version of Python's duckdb package the MotherDuck bridge runs, such as `1.5.6`; empty for the bundled DuckDB. pub bridge_duckdb: String, @@ -21,6 +21,17 @@ pub struct DuckDbDestination { pub own_limit: Option, /// The bytes of large values committed to the database since its last checkpoint, which DuckDB holds in memory until then. pub unflushed: Arc, + /// The secret set on the database for each bucket, by bucket, shared by every table's connection: tables setting one at the same time conflict, so each is set before a table opens its transaction, and again only when it [is no longer current](crate::adapters::duckdb::_helpers::pull_sql::secret_current). + pub secrets: Arc>>, +} + +/// A bucket's secret as it was set: a hash of its statement and parameters, never the keys themselves; whether DuckDB resolved its keys through the AWS credential chain, once, when it was set; and when. +#[cfg(feature = "duckdb")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SetSecret { + pub hash: u64, + pub chain: bool, + pub at: std::time::Instant, } /// Where a destination's SQL runs. diff --git a/crates/sqlload/src/adapters/duckdb/tests/helpers.rs b/crates/sqlload/src/adapters/duckdb/tests/helpers.rs index 2515c463..844426e6 100644 --- a/crates/sqlload/src/adapters/duckdb/tests/helpers.rs +++ b/crates/sqlload/src/adapters/duckdb/tests/helpers.rs @@ -128,6 +128,7 @@ pub(crate) fn pulled_unloaded(reported: u64, with_files: bool) -> Result<(String memory: None, own_limit: None, unflushed: Arc::default(), + secrets: Arc::default(), }, open_transaction: false, }; diff --git a/crates/sqlload/src/adapters/duckdb/tests/test_motherduck.rs b/crates/sqlload/src/adapters/duckdb/tests/test_motherduck.rs index 8f7d9679..68b201aa 100644 --- a/crates/sqlload/src/adapters/duckdb/tests/test_motherduck.rs +++ b/crates/sqlload/src/adapters/duckdb/tests/test_motherduck.rs @@ -1,5 +1,6 @@ use crate::adapters::contract::models::TableName; use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::dispatch::main::route::unload_refused::unload_refused; use crate::adapters::dispatch::main::route::writer_accepts::writer_accepts; use crate::adapters::dispatch::main::session::drop_stale_staging::drop_stale_staging; use crate::adapters::dispatch::main::session::text_values::text_values; @@ -23,6 +24,8 @@ use crate::column_types::models::Warehouse; use crate::loader::_helpers::resolve::source_target; use crate::project::main::parse_project::parse_project; use crate::project::models::ConnectionConfig; +use crate::routing::main::plan_route::plan_route; +use crate::routing::models::{FileFormat, ReadVia, ReaderOffer, RouteOverrides, Step, Store}; /// What DuckDB 2.0.0 reports when MotherDuck's extension refuses to load on it, as Python's duckdb package 2.0.0 would in the bridge. const UNSUPPORTED_VERSION: &str = "Invalid Input Error: Initialization function \"motherduck_duckdb_cpp_init\" from file \"/home/u/.duckdb/extensions/v2.0.0/linux_amd64/motherduck.duckdb_extension\" threw an exception: \"\nYour DuckDB version (v2.0.0) is not yet supported by MotherDuck. The latest supported version is v1.5.6. Please downgrade to use MotherDuck.\n\nSee https://motherduck.com/docs/getting-started/connect-query-from-duckdb-cli for more information.\""; @@ -346,18 +349,21 @@ fn given_a_duckdb_connection_when_its_database_is_a_motherduck_path_then_it_is_r } #[test] -fn given_a_motherduck_destination_when_routing_then_it_takes_only_arrow_batches() +fn given_a_motherduck_destination_when_routing_then_it_reads_parquet_only_from_s3_itself() -> Result<(), String> { let test_cases = [ MotherDuckRouteTestCase { - description: "local DuckDB reads Parquet itself", + description: "local DuckDB reads Parquet in S3 and on this machine itself", motherduck: false, - expected_reads_files: true, + expected_pulls_from: vec![ + Store::ObjectStore("s3://".to_owned()), + Store::Local(String::new()), + ], }, MotherDuckRouteTestCase { - description: "MotherDuck never reads files itself", + description: "MotherDuck reads Parquet in S3 on its servers, and never files on this machine", motherduck: true, - expected_reads_files: false, + expected_pulls_from: vec![Store::ObjectStore("s3://".to_owned())], }, ]; for test_case in test_cases { @@ -366,8 +372,62 @@ fn given_a_motherduck_destination_when_routing_then_it_takes_only_arrow_batches( let accepts = writer_accepts(&Destination::DuckDb(destination), None); assert!(accepts.arrow, "{}", test_case.description); assert_eq!( - !accepts.pulls_from.is_empty() || accepts.reads_files_itself.is_some(), - test_case.expected_reads_files, + (accepts.pulls_from, accepts.pull_formats), + (test_case.expected_pulls_from, vec![FileFormat::Parquet]), + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_a_snowflake_source_unloading_to_s3_when_routing_into_motherduck_then_sqlload_reads_the_unloaded_files() +-> Result<(), String> { + let test_cases = [ + MotherDuckRouteTestCase { + description: "local DuckDB reads the unloaded files itself", + motherduck: false, + expected_pulls_from: vec![Store::ObjectStore("s3://lake/unload/".to_owned())], + }, + MotherDuckRouteTestCase { + description: "MotherDuck's files from an unload go through SQLLoad, as before", + motherduck: true, + expected_pulls_from: vec![], + }, + ]; + for test_case in test_cases { + let mut destination = open_duckdb(":memory:", None).map_err(|error| error.to_string())?; + destination.motherduck = test_case.motherduck; + let destination = Destination::DuckDb(destination); + let unload_to = vec![Store::ObjectStore("s3://lake/unload/".to_owned())]; + let reader = ReaderOffer { + arrow: true, + direct_blocked: unload_refused(&destination, &unload_to), + unload_to, + ..ReaderOffer::default() + }; + let route = plan_route( + &reader, + &writer_accepts(&destination, None), + RouteOverrides { + read_via: ReadVia::Unload, + ..RouteOverrides::default() + }, + ) + .map_err(|error| error.to_string())?; + let pulled: Vec = route + .steps + .iter() + .filter_map(|step| { + let Step::DestinationPulls(files) = step else { + return None; + }; + Some(files.store.clone()) + }) + .collect(); + assert_eq!( + pulled, test_case.expected_pulls_from, "{}", test_case.description ); diff --git a/crates/sqlload/src/adapters/duckdb/tests/test_pull_insert.rs b/crates/sqlload/src/adapters/duckdb/tests/test_pull_insert.rs index a1c6be2d..580aa21b 100644 --- a/crates/sqlload/src/adapters/duckdb/tests/test_pull_insert.rs +++ b/crates/sqlload/src/adapters/duckdb/tests/test_pull_insert.rs @@ -6,31 +6,43 @@ use crate::column_types::models::ColumnType; #[test] fn given_parquet_files_when_duckdb_reads_them_itself_then_each_column_comes_from_the_right_place() { - let test_cases = [PullInsertTestCase { - description: "file columns are cast, a partition comes from each file's own entry in a list indexed by the row's file (NULL where its folder lacks it), rows count from 1, the load id is the batch's, and a column no file holds is NULL", - columns: vec![ - column("id", ColumnType::Int), - column("day", ColumnType::Date), - column("note", ColumnType::Text), - column("_sqlload_source_row", ColumnType::Int), - column("_sqlload_load_id", ColumnType::Text), - ], - in_files: vec![ - ("id".to_owned(), "Order ID".to_owned()), - ("day".to_owned(), "day".to_owned()), - ], - files: vec![ - PulledFile { - url: "s3://b/x/a.parquet".to_owned(), - constants: vec![("day".to_owned(), Some("2026-01-01".to_owned()))], - }, - PulledFile { - url: "s3://b/x/it's.parquet".to_owned(), - constants: vec![("day".to_owned(), None)], - }, - ], - expected_sql: "INSERT INTO \"RAW\".\"orders\" (\"id\", \"day\", \"note\", \"_sqlload_source_row\", \"_sqlload_load_id\") SELECT CAST(f.\"Order ID\" AS BIGINT), CAST(['2026-01-01', NULL][CAST(f.file_index AS BIGINT) + 1] AS DATE), NULL, CAST(f.file_row_number + 1 AS BIGINT), CAST('L1' AS VARCHAR) FROM read_parquet(['s3://b/x/a.parquet', 's3://b/x/it''s.parquet'], union_by_name = true, file_row_number = true, hive_partitioning = false) AS f", - }]; + let files = vec![ + PulledFile { + url: "s3://b/x/a.parquet".to_owned(), + constants: vec![("day".to_owned(), Some("2026-01-01".to_owned()))], + }, + PulledFile { + url: "s3://b/x/it's.parquet".to_owned(), + constants: vec![("day".to_owned(), None)], + }, + ]; + let test_cases = [ + PullInsertTestCase { + description: "file columns are cast, a partition comes from each file's own entry in a list indexed by the row's file (NULL where its folder lacks it), rows count from 1, the load id is the batch's, and a column no file holds is NULL", + columns: vec![ + column("id", ColumnType::Int), + column("day", ColumnType::Date), + column("note", ColumnType::Text), + column("_sqlload_source_row", ColumnType::Int), + column("_sqlload_load_id", ColumnType::Text), + ], + in_files: vec![ + ("id".to_owned(), "Order ID".to_owned()), + ("day".to_owned(), "day".to_owned()), + ], + files: files.clone(), + remote: false, + expected_sql: "INSERT INTO \"RAW\".\"orders\" (\"id\", \"day\", \"note\", \"_sqlload_source_row\", \"_sqlload_load_id\") SELECT CAST(f.\"Order ID\" AS BIGINT), CAST(['2026-01-01', NULL][CAST(f.file_index AS BIGINT) + 1] AS DATE), NULL, CAST(f.file_row_number + 1 AS BIGINT), CAST('L1' AS VARCHAR) FROM read_parquet(['s3://b/x/a.parquet', 's3://b/x/it''s.parquet'], union_by_name = true, file_row_number = true, hive_partitioning = false) AS f", + }, + PullInsertTestCase { + description: "into MotherDuck the files are read on its servers", + columns: vec![column("id", ColumnType::Int)], + in_files: vec![("id".to_owned(), "id".to_owned())], + files: files.clone(), + remote: true, + expected_sql: "INSERT INTO \"RAW\".\"orders\" (\"id\") SELECT CAST(f.\"id\" AS BIGINT) FROM read_parquet(['s3://b/x/a.parquet', 's3://b/x/it''s.parquet'], union_by_name = true, file_row_number = true, hive_partitioning = false, MD_RUN = REMOTE) AS f", + }, + ]; let table = TableName { database: None, schema: "RAW".to_owned(), @@ -41,7 +53,7 @@ fn given_parquet_files_when_duckdb_reads_them_itself_then_each_column_comes_from pull_insert( &table, (&test_case.columns, &test_case.in_files), - &test_case.files, + (&test_case.files, test_case.remote), ( crate::load_naming::models::Compat::Sqlload, "L1", diff --git a/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs b/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs index 23f9bcc6..fa57c39c 100644 --- a/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs +++ b/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs @@ -1,6 +1,12 @@ use crate::adapters::contract::main::_custom_endpoint::custom_endpoint; -use crate::adapters::duckdb::_helpers::pull_sql::{default_region, s3_secret}; -use crate::adapters::duckdb::tests::test_types::{AwsVariablesTestCase, S3SecretTestCase}; +use std::time::{Duration, Instant}; + +use crate::adapters::duckdb::_helpers::pull_sql::{ + BucketAccess, default_region, s3_secret, secret_current, +}; +use crate::adapters::duckdb::tests::test_types::{ + AwsVariablesTestCase, S3SecretTestCase, SecretCurrentTestCase, +}; use crate::store_paths::models::StoreKind; #[test] @@ -103,3 +109,70 @@ fn given_aws_variables_when_duckdb_reads_s3_itself_then_an_empty_one_counts_as_u ); } } + +#[test] +fn given_a_buckets_secret_already_set_when_a_table_opens_the_bucket_then_it_is_set_again_only_when_its_keys_change_or_the_chain_may_have_expired() + { + let keys: &[&str] = &["AKIAEXAMPLE", "s3cret", "", "eu-west-2", "s3://lake/"]; + let rotated: &[&str] = &["AKIAEXAMPLE", "s3cret2", "", "eu-west-2", "s3://lake/"]; + let chain: &[&str] = &["eu-west-2", "s3://lake/"]; + let test_cases = [ + SecretCurrentTestCase { + description: "nothing set yet", + set: None, + wanted: (keys, false), + expected_current: false, + }, + SecretCurrentTestCase { + description: "the same static keys stay current however long ago they were set", + set: Some((keys, false, 3_600)), + wanted: (keys, false), + expected_current: true, + }, + SecretCurrentTestCase { + description: "changed keys are set again", + set: Some((keys, false, 1)), + wanted: (rotated, false), + expected_current: false, + }, + SecretCurrentTestCase { + description: "a chain secret set just under four minutes ago is current", + set: Some((chain, true, 239)), + wanted: (chain, true), + expected_current: true, + }, + SecretCurrentTestCase { + description: "a chain secret set four minutes ago is set again, as its keys may expire", + set: Some((chain, true, 240)), + wanted: (chain, true), + expected_current: false, + }, + SecretCurrentTestCase { + description: "keys appearing in the environment replace the chain secret", + set: Some((chain, true, 1)), + wanted: (keys, false), + expected_current: false, + }, + ]; + let now = Instant::now() + Duration::from_secs(7_200); + let access = |parameters: &[&str], chain: bool| BucketAccess { + bucket: "lake".to_owned(), + extensions: "", + statement: "CREATE OR REPLACE TEMPORARY SECRET sqlload_s3_6c616b65".to_owned(), + parameters: parameters.iter().map(|value| (*value).to_owned()).collect(), + chain, + }; + for test_case in test_cases { + let set = test_case.set.map(|(parameters, chain, age)| { + access(parameters, chain).set_at(now - Duration::from_secs(age)) + }); + let (parameters, chain) = test_case.wanted; + let wanted = access(parameters, chain).set_at(now); + assert_eq!( + secret_current(set.as_ref(), &wanted, now), + test_case.expected_current, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/adapters/duckdb/tests/test_types.rs b/crates/sqlload/src/adapters/duckdb/tests/test_types.rs index 20cd5f5d..ba852483 100644 --- a/crates/sqlload/src/adapters/duckdb/tests/test_types.rs +++ b/crates/sqlload/src/adapters/duckdb/tests/test_types.rs @@ -8,6 +8,8 @@ pub(crate) struct PullInsertTestCase { /// Each column the files hold, with its name in them. pub(crate) in_files: Vec<(String, String)>, pub(crate) files: Vec, + /// The target is MotherDuck, which reads the files on its servers. + pub(crate) remote: bool, pub(crate) expected_sql: &'static str, } @@ -110,7 +112,8 @@ pub(crate) struct DuckDbPathTestCase { pub(crate) struct MotherDuckRouteTestCase { pub(crate) description: &'static str, pub(crate) motherduck: bool, - pub(crate) expected_reads_files: bool, + /// The stores whose Parquet files the destination reads itself. + pub(crate) expected_pulls_from: Vec, } pub(crate) struct AttachedDatabaseTestCase { @@ -149,3 +152,12 @@ pub(crate) struct MergeDedupeTestCase { /// The table's rows after the merge, as `id:name`, sorted. pub(crate) expected_rows: &'static [&'static str], } + +pub(crate) struct SecretCurrentTestCase { + pub(crate) description: &'static str, + /// The secret set on the database for the bucket: its parameters, whether its keys come from the credential chain, and how many seconds ago it was set. + pub(crate) set: Option<(&'static [&'static str], bool, u64)>, + /// The secret a table would set now: its parameters, and whether its keys come from the credential chain. + pub(crate) wanted: (&'static [&'static str], bool), + pub(crate) expected_current: bool, +} diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index d5b1ccfd..48f8fad6 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -17,12 +17,13 @@ use crate::adapters::dispatch::main::dialect::files_destination::files_destinati use crate::adapters::dispatch::main::dialect::source_binds_text::source_binds_text; use crate::adapters::dispatch::main::dialect::source_dialect::source_dialect; use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter; +use crate::adapters::dispatch::main::route::check_reach::check_reach; use crate::adapters::dispatch::main::route::choose_stage::choose_stage; use crate::adapters::dispatch::main::route::direct_blocked::direct_blocked; -use crate::adapters::dispatch::main::route::list_reach::list_reach; use crate::adapters::dispatch::main::route::route_warning::route_warning; use crate::adapters::dispatch::main::route::stage_misses::stage_misses; use crate::adapters::dispatch::main::route::unload_blocked::unload_blocked; +use crate::adapters::dispatch::main::route::unload_refused::unload_refused; use crate::adapters::dispatch::main::route::writer_accepts::writer_accepts; use crate::adapters::dispatch::main::session::live_columns::live_columns; use crate::adapters::dispatch::main::session::single_text::single_text; @@ -1205,21 +1206,21 @@ fn copy_warning(route: Option<&Route>) -> Option { )) } -/// The route a table's data takes into `destination`; `read` is a files table's planned read. For files in S3 that nothing sends through SQLLoad, it first looks up where the destination reads files itself (Databricks' external locations). +/// The route a table's data takes into `destination`; `read` is a files table's planned read. For files in S3 that nothing sends through SQLLoad, it first checks that the destination reaches them (Databricks' external locations, MotherDuck's servers). pub(crate) async fn table_route( spec: &TableSpec, read: Option<&FilesRead>, stage: Option<&StageRoute>, destination: &Destination, ) -> Result { - let reader = reader_offer(spec, read, stage); + let mut reader = reader_offer(spec, read, stage); + let refused = unload_refused(destination, &reader.unload_to); + reader.direct_blocked = reader.direct_blocked.or(refused); let in_place = reader.needs_loader.is_none() && reader.direct_blocked.is_none(); if let Some(files) = spec.files.as_ref().filter(|_| in_place) && let StoreKind::S3 { .. } = files.location.store { - let folder = files.location.to_string(); - let holds_files = read.is_some_and(|read| !read.files.is_empty()); - list_reach(destination, holds_files.then_some(folder.as_str())).await?; + check_reach(destination, files, read).await?; } plan_route( &reader, diff --git a/website/public/index.html b/website/public/index.html index 5040bf01..5cca8e46 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -173,7 +173,7 @@

See every change first

02 · Load

Bulk copy, not row inserts

-

Files in S3 load into Snowflake straight from an external stage, and Parquet in S3 into Databricks through an external location. Parquet loads into DuckDB through DuckDB's own reader. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Rows are staged as Parquet and copied in bulk.

+

Files in S3 load into Snowflake straight from an external stage, and Parquet in S3 into Databricks through an external location. Parquet loads into DuckDB through DuckDB's own reader, and Parquet in S3 into MotherDuck on its servers. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Rows are staged as Parquet and copied in bulk.

Load methods and routes →

sqlload plan
diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx
index a785bdd2..63e6910a 100644
--- a/website/src/content/docs/docs/supported-systems.mdx
+++ b/website/src/content/docs/docs/supported-systems.mdx
@@ -33,7 +33,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes
 | ClickHouse, one self-hosted server | `append`, `merge`, `delete_insert`, `replace` | Arrow over HTTP into a staging table, published by swapping in a whole table or whole partitions; Parquet in S3 read by ClickHouse with `s3()`. No clusters, replicated or distributed tables; ClickHouse Cloud is untested. See [ClickHouse](/docs/targets/clickhouse/). |
 | Postgres | `append`, `merge`, `delete_insert`, `replace` | Binary `COPY` into a temporary staging table, then one transaction per commit. See [Postgres](/docs/targets/postgres/). |
 | DuckDB | `append`, `merge`, `delete_insert`, `replace` | Arrow appends; Parquet files read by DuckDB with `read_parquet`. |
-| MotherDuck (beta) | `append`, `merge`, `delete_insert`, `replace` | Batches inserted by a separate DuckDB 1.5 process (`sqlload[motherduck]`), as MotherDuck doesn't accept DuckDB 2.0 clients yet; files are read by SQLLoad. See [MotherDuck](/docs/targets/motherduck/). |
+| MotherDuck (beta) | `append`, `merge`, `delete_insert`, `replace` | Batches inserted by a separate DuckDB 1.5 process (`sqlload[motherduck]`), as MotherDuck doesn't accept DuckDB 2.0 clients yet; Parquet in S3 is read on MotherDuck's servers, other files by SQLLoad. See [MotherDuck](/docs/targets/motherduck/). |
 | Parquet or CSV files in S3 | `append`, `delete_insert`, `replace` | Parquet written by SQLLoad, zstd-compressed; or CSV: plain, gzip or zstd. |
 | Parquet or CSV files in a local folder | As for S3; `replace` swaps the folder in one rename | As for S3 |
 
@@ -55,7 +55,8 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes
 | Python | Yes | Yes | Yes | Yes | Yes | Yes | Yes |
 
 A MotherDuck target (beta) takes every source a DuckDB target
-takes, untested, but always through SQLLoad: files are read by SQLLoad, never by MotherDuck.
+takes, untested, through SQLLoad, except a files source's Parquet in S3, which MotherDuck's servers
+read themselves.
 
 A files source loads into a files target, read and rewritten
 by SQLLoad. `merge` isn't available for files sources or files
diff --git a/website/src/content/docs/docs/targets/duckdb.mdx b/website/src/content/docs/docs/targets/duckdb.mdx
index 2f517068..01120a09 100644
--- a/website/src/content/docs/docs/targets/duckdb.mdx
+++ b/website/src/content/docs/docs/targets/duckdb.mdx
@@ -29,7 +29,8 @@ A later batch whose files hold a column differently fails the load, naming the c
 
 DuckDB reads S3 with the keys SQLLoad finds for the [`s3` connection](/docs/configuration/#s3):
 from the environment or the shared credentials file. Without
-either, it uses the AWS credential chain. The first load from S3 installs DuckDB's `httpfs`
+either, it uses the AWS credential chain, which it resolves again every four minutes as its keys may
+expire. The first load from S3 installs DuckDB's `httpfs`
 extension into `~/.duckdb`, and `aws` too when it uses the chain. That needs the network once.
 Local folders need nothing extra.
 
diff --git a/website/src/content/docs/docs/targets/index.mdx b/website/src/content/docs/docs/targets/index.mdx
index a1319a67..35da076b 100644
--- a/website/src/content/docs/docs/targets/index.mdx
+++ b/website/src/content/docs/docs/targets/index.mdx
@@ -13,7 +13,7 @@ names the route for every table.
 | Into | Parquet files in S3 or a folder | CSV, JSON, NDJSON files | Postgres or Python rows |
 |------|---------------------------------|-------------------------|-------------------------|
 | DuckDB | DuckDB reads them with `read_parquet` | Read by SQLLoad, appended as Arrow | Appended as Arrow |
-| MotherDuck (beta) | Read by SQLLoad, appended as Arrow | Read by SQLLoad, appended as Arrow | Appended as Arrow |
+| MotherDuck (beta) | In S3: MotherDuck's servers read them with `read_parquet`. In a folder: read by SQLLoad, appended as Arrow | Read by SQLLoad, appended as Arrow | Appended as Arrow |
 | Snowflake | `COPY` from an external stage over the bucket | `COPY` from an external stage over the bucket | Written as Parquet to the `load_method`'s stage, then `COPY` (or streamed or inserted, if the `load_method` says so) |
 | Databricks (beta) | In S3: `COPY INTO` where they lie, through an external location. In a folder: staged by SQLLoad in the volume, then `COPY INTO` | Read by SQLLoad, staged as Parquet in the volume, then `COPY INTO` | Written as Parquet to the volume, then `COPY INTO` |
 | BigQuery (beta) | Read by SQLLoad, streamed as Arrow into a staging table, then one query job | As for Parquet files | Streamed as Arrow into a staging table, then one query job |
diff --git a/website/src/content/docs/docs/targets/motherduck.mdx b/website/src/content/docs/docs/targets/motherduck.mdx
index bfedab1f..9181f6af 100644
--- a/website/src/content/docs/docs/targets/motherduck.mdx
+++ b/website/src/content/docs/docs/targets/motherduck.mdx
@@ -70,9 +70,34 @@ connection installs MotherDuck's DuckDB extension into `~/.duckdb`, which needs
 ## How data arrives
 
 SQLLoad writes each batch to a temporary Parquet file on this machine, and the separate process
-inserts it with `read_parquet` inside the batch's transaction, then the file is removed. MotherDuck
-never reads your source files itself: files sources, including Parquet in S3 or a local folder, and a
-Snowflake source's unloaded files are read by SQLLoad. `plan` names that route for every table.
+inserts it with `read_parquet` inside the batch's transaction, then the file is removed. `plan` names
+the route for every table.
+
+### Parquet files in S3
+
+Parquet files of a files source in S3 load straight from the bucket: MotherDuck's servers read them
+with `read_parquet` (`MD_RUN = REMOTE`), and no row passes through this machine. MotherDuck reads
+the bucket with a secret that lasts only for the load's session, made from the keys SQLLoad finds
+for the [`s3` connection](/docs/configuration/#s3) (the environment, else the shared credentials
+file, else the AWS credential chain resolved on this machine, set again every four minutes as its
+keys may expire); nothing is stored in your MotherDuck account. A secret of your own in MotherDuck
+scoped to a narrower path of the bucket, such as `s3://example-bucket/events/`, wins over it.
+`plan` reads the first file to load on MotherDuck's servers, and fails the table with the fix when
+MotherDuck can't read it. MotherDuck's servers read from their own network, so a bucket whose
+policy only allows your network or VPC endpoint can't load into MotherDuck, and its keys need
+`kms:Decrypt` on a bucket's KMS key:
+
+```console
+$ sqlload plan
+…
+        └── route  DuckDB reads directly from s3://example-bucket/events/ (nothing through the loader)
+```
+
+The route loads the same values as SQLLoad's reader, and goes through SQLLoad's reader for the
+same reasons as on [DuckDB](/docs/targets/duckdb/), such as CSV or JSON files, masked columns or a
+custom S3 endpoint, which MotherDuck's servers don't reach. Each batch fails unless MotherDuck
+reports inserting as many rows as the files' footers hold. Files in a local folder or over SFTP,
+and a Snowflake source's unloaded files, are read by SQLLoad.
 
 `append`, `merge`, `delete_insert` and `replace` run the same statements as on DuckDB, and so do
 [`deletes` and `deletes_from`](/docs/concepts/write-strategies/#rows-deleted-at-the-source)
@@ -104,6 +129,10 @@ version, it goes by the separate process's `duckdb` and says so.
 - Live, against a MotherDuck account on DuckDB 1.5.6: create, load, every write strategy, an
   incremental load and a rerun that adds nothing, a new column, every portable type, `json_type = "variant"`
   with its warning, and dropping the test databases afterwards.
+- Live, Parquet in S3 read on MotherDuck's servers: `append`, `replace`, `delete_insert` on a
+  partition cursor and by `replace_key`, loading at the same time, each compared row for row,
+  lineage columns included, with the same files read by SQLLoad, and a second run. A bucket only
+  this machine may read failed `plan` with the fix.
 - Offline, on every change: every write strategy and portable type loaded through the separate
   process into a stand-in for MotherDuck and compared with the same load into DuckDB, an incremental
   rerun, the process ending mid-load, a value MotherDuck rejects, and the setup errors.