diff --git a/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs b/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs index aec662e55..35d5820f0 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs @@ -4,15 +4,29 @@ use crate::adapters::databricks::main::stage::_writer_accepts::writer_accepts as use crate::adapters::dispatch::models::Destination; use crate::adapters::snowflake::main::stage::_writer_accepts::writer_accepts as snowflake_accepts; use crate::adapters::snowflake::models::StageRoute; +#[cfg(feature = "duckdb")] +use crate::memory_budget::constants::DUCKDB_ROW_GROUP_ROWS; +#[cfg(feature = "duckdb")] +use crate::number_text::main::thousands::thousands; use crate::routing::models::WriterAccepts; use crate::routing::models::{FileFormat, Store}; -/// What `destination` takes over the chosen `stage`: Arrow batches (DuckDB also reads Parquet in S3, GCS or on this machine itself, MotherDuck Parquet in S3 or GCS on its servers, and ClickHouse Parquet in S3, BigQuery Parquet in GCS), or what its stage route takes. +/// What `destination` takes over the chosen `stage`: Arrow batches (DuckDB also reads Parquet in S3, GCS or on this machine itself; MotherDuck takes zstd Parquet chunks it `COPY`s, and reads Parquet in S3 or GCS on its servers; and ClickHouse Parquet in S3, BigQuery Parquet in GCS), or what its stage route takes. pub fn writer_accepts(destination: &Destination, stage: Option<&StageRoute>) -> WriterAccepts { match destination { #[cfg(feature = "duckdb")] Destination::DuckDb(duckdb) if duckdb.motherduck => WriterAccepts { - arrow: true, + rows_into: Some(match &duckdb.stage { + None => format!( + "zstd Parquet, about {} rows a file -> MotherDuck COPY", + thousands(DUCKDB_ROW_GROUP_ROWS) + ), + Some(stage) => format!( + "zstd Parquet, about {} rows a file -> {} -> MotherDuck reads them on its servers", + thousands(DUCKDB_ROW_GROUP_ROWS), + stage.folder.location + ), + }), pulls_from: vec![ Store::ObjectStore("s3://".to_owned()), Store::ObjectStore("gs://".to_owned()), diff --git a/crates/sqlload/src/adapters/dispatch/main/session/connect_destination.rs b/crates/sqlload/src/adapters/dispatch/main/session/connect_destination.rs index 58d2afa1f..8635ff0a7 100644 --- a/crates/sqlload/src/adapters/dispatch/main/session/connect_destination.rs +++ b/crates/sqlload/src/adapters/dispatch/main/session/connect_destination.rs @@ -6,7 +6,7 @@ use crate::adapters::contract::models::TableName; use crate::adapters::databricks::main::session::_connect_databricks::connect_databricks; use crate::adapters::dispatch::models::Destination; #[cfg(feature = "duckdb")] -use crate::adapters::duckdb::main::_open_destination::open_destination; +use crate::adapters::duckdb::main::_open_target::open_target; use crate::adapters::files::models::FilesDestination; use crate::adapters::postgres::main::session::_open_postgres::open_postgres; use crate::adapters::snowflake::main::session::_connect_snowflake::connect_snowflake; @@ -27,13 +27,9 @@ pub async fn connect_destination( .ok_or_else(|| LoaderError::Invalid(format!("target {name:?} does not exist")))?; let destination = match project.connections.get(&target.connection) { #[cfg(feature = "duckdb")] - Some( - connection @ (ConnectionConfig::DuckDb { .. } | ConnectionConfig::MotherDuck { .. }), - ) => Destination::DuckDb(open_destination( - connection, - target.database.as_deref(), - project.memory.budget, - )?), + Some(ConnectionConfig::DuckDb { .. } | ConnectionConfig::MotherDuck { .. }) => { + Destination::DuckDb(open_target(project, target)?) + } #[cfg(not(feature = "duckdb"))] Some(ConnectionConfig::DuckDb { .. } | ConnectionConfig::MotherDuck { .. }) => { return Err(LoaderError::Invalid(format!( diff --git a/crates/sqlload/src/adapters/dispatch/main/session/write_problems.rs b/crates/sqlload/src/adapters/dispatch/main/session/write_problems.rs index c0087044d..dff24f07b 100644 --- a/crates/sqlload/src/adapters/dispatch/main/session/write_problems.rs +++ b/crates/sqlload/src/adapters/dispatch/main/session/write_problems.rs @@ -3,11 +3,13 @@ use crate::adapters::contract::models::{TableName, WriteMode}; use crate::adapters::dispatch::models::Destination; #[cfg(feature = "duckdb")] +use crate::adapters::duckdb::main::_stage_problems::stage_problems; +#[cfg(feature = "duckdb")] use crate::adapters::duckdb::main::_variant_problems::variant_problems; use crate::adapters::postgres::main::session::_write_problems::write_problems as postgres_write_problems; use crate::loader::errors::LoaderError; -/// Why `mode` cannot run against `table` (`exists` when it does) (problems), and what it may fail on (warnings); `variants` when the table has `VARIANT` columns, which MotherDuck needs a recent enough DuckDB for. None where the warehouse's writes need nothing of the table or its server. +/// Why `mode` cannot run against `table` (`exists` when it does) (problems), and what it may fail on (warnings); `variants` when the table has `VARIANT` columns, which MotherDuck needs a recent enough DuckDB for; and a MotherDuck `external_stage` SQLLoad cannot write or MotherDuck cannot read. None where the warehouse's writes need nothing of the table or its server. pub async fn write_problems( destination: &Destination, table: &TableName, @@ -20,7 +22,14 @@ pub async fn write_problems( postgres_write_problems(postgres, table, (mode, exists)).await } #[cfg(feature = "duckdb")] - Destination::DuckDb(duckdb) if variants => Ok(variant_problems(duckdb).await), + Destination::DuckDb(duckdb) => { + let (mut problems, warnings) = match variants { + true => variant_problems(duckdb).await, + false => (Vec::new(), Vec::new()), + }; + problems.extend(stage_problems(duckdb).await); + Ok((problems, warnings)) + } _ => Ok((Vec::new(), Vec::new())), } } diff --git a/crates/sqlload/src/adapters/dispatch/main/write/open_table_writer.rs b/crates/sqlload/src/adapters/dispatch/main/write/open_table_writer.rs index 3f23c9a5d..f03405f97 100644 --- a/crates/sqlload/src/adapters/dispatch/main/write/open_table_writer.rs +++ b/crates/sqlload/src/adapters/dispatch/main/write/open_table_writer.rs @@ -23,6 +23,7 @@ pub fn open_table_writer( Destination::DuckDb(duckdb) => Ok(Writer::DuckDb(DuckDbWriter { destination: duckdb.clone(), open_transaction: false, + staged: Default::default(), })), Destination::Snowflake(snowflake) => { let external_folder = match &route { diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/shared_destination.rs b/crates/sqlload/src/adapters/duckdb/_helpers/shared_destination.rs index 7f441d8d3..0e1e5b6ea 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/shared_destination.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/shared_destination.rs @@ -19,5 +19,6 @@ pub(in crate::adapters::duckdb) fn shared_destination( own_limit: None, unflushed: Arc::default(), secrets: Arc::default(), + stage: None, }) } diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/table_writer.rs b/crates/sqlload/src/adapters/duckdb/_helpers/table_writer.rs index 322b57764..52e7e0f90 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/table_writer.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/table_writer.rs @@ -1,20 +1,27 @@ -//! DuckDB's write session: Arrow appends, in one transaction per batch when the batch goes straight into the table or a build table, and checkpoints under the memory budget. +//! DuckDB's write session: Arrow appends (or a MotherDuck `external_stage`'s chunks), in one transaction per batch when the batch goes straight into the table or a build table, and checkpoints under the memory budget. use arrow_array::RecordBatch; use crate::adapters::contract::constants::{BEGIN_STATEMENT, COMMIT_STATEMENT}; use crate::adapters::contract::models::WriteTarget; +use crate::adapters::contract::types::SqlAdapter; use crate::adapters::contract::types::TableWriter; use crate::adapters::duckdb::constants::{ CHECKPOINT_BUSY, CHECKPOINT_RETRY, CHECKPOINT_STATEMENT, CHECKPOINT_WAIT, }; use crate::adapters::duckdb::main::_append_batch::append_batch; use crate::adapters::duckdb::main::_execute_statements::execute_statements; -use crate::adapters::duckdb::models::DuckDbWriter; +use crate::adapters::duckdb::main::_open_bucket::open_bucket; +use crate::adapters::duckdb::main::_pull_files::pull_files; +use crate::adapters::duckdb::models::{DuckDbSql, DuckDbWriter, ExternalStage}; +use crate::adapters::motherduck_bridge::main::_chunk_parquet::chunk_parquet; use crate::column_types::main::is_variant::is_variant; +use crate::file_store::main::delete_files::delete_files; +use crate::file_store::main::write_file::write_file; use crate::loader::errors::LoaderError; use crate::memory_budget::main::large_values::large_values; use crate::memory_budget::models::DuckDbShare; +use std::sync::Arc; use std::sync::atomic::Ordering; impl TableWriter for DuckDbWriter { @@ -27,6 +34,9 @@ impl TableWriter for DuckDbWriter { target: &WriteTarget<'_>, batch: RecordBatch, ) -> Result<(), LoaderError> { + if let Some(stage) = self.destination.stage.clone() { + return self.write_chunk(&stage, target, batch).await; + } let into = target.staging.unwrap_or(target.table); if self.destination.memory.is_some() && let Some(rows) = large_values(&batch) @@ -62,9 +72,11 @@ impl TableWriter for DuckDbWriter { } async fn load_batch(&mut self, _target: &WriteTarget<'_>) -> Result<(), LoaderError> { + self.load_chunks().await?; if self.open_transaction { execute_statements(&self.destination, vec![COMMIT_STATEMENT.to_owned()]).await?; self.open_transaction = false; + self.remove_chunks(true).await; } Ok(()) } @@ -72,7 +84,7 @@ impl TableWriter for DuckDbWriter { fn next_batch(&mut self) {} async fn settle_rows(&mut self) -> Result<(), LoaderError> { - Ok(()) + self.load_chunks().await } fn reopen_rows(&mut self) {} @@ -84,6 +96,7 @@ impl TableWriter for DuckDbWriter { { tracing::warn!(%error, "cannot roll back the open batch"); } + self.remove_chunks(false).await; } async fn remove_batch_files(&mut self, _target: &WriteTarget<'_>) {} @@ -126,3 +139,122 @@ impl TableWriter for DuckDbWriter { Ok(()) } } + +impl DuckDbWriter { + /// Write `batch` to `stage` as one zstd Parquet chunk (`VARIANT` columns typed `JSON`), which waits there for [`Self::load_chunks`]; the batch's transaction, when it goes straight into the table or a build table, opens only when MotherDuck reads them, right after the bucket's secret is set, so it never reads with keys set before a long upload. + pub(in crate::adapters) async fn write_chunk( + &mut self, + stage: &Arc, + target: &WriteTarget<'_>, + batch: RecordBatch, + ) -> Result<(), LoaderError> { + let into = target.staging.unwrap_or(target.table); + let names: Vec = (batch.schema().fields().iter()) + .map(|field| DuckDbSql.render_identifier(field.name())) + .collect(); + let insert = format!( + "INSERT INTO {} ({list}) SELECT {list} FROM read_parquet(", + DuckDbSql.render_qualified_name(into), + list = names.join(", ") + ); + if insert != self.staged.insert { + self.load_chunks().await?; + self.staged.insert = insert; + } + self.staged.transaction = target.direct || target.builds; + let json: Vec = (target.write_columns.iter()) + .filter(|column| is_variant(&column.column_type)) + .map(|column| column.name.clone()) + .collect(); + let rows = batch.num_rows() as u64; + let body = tokio::task::spawn_blocking(move || { + let mut body: Vec = Vec::new(); + chunk_parquet(&mut body, &batch, &json).map(|()| body) + }) + .await + .map_err(|error| LoaderError::Invalid(format!("the Parquet thread failed: {error}")))? + .map_err(LoaderError::Invalid)?; + self.staged.written += 1; + let key = format!( + "{}/{}/{}-{}.parquet", + target.load_id, + key_safe(&format!("{}.{}", into.schema, into.table)), + target.batch, + self.staged.written + ); + write_file(&stage.folder, &key, body.into()) + .await + .map_err(|error| LoaderError::Invalid(error.to_string()))?; + self.staged.waiting.push((key, rows)); + Ok(()) + } + + /// Have MotherDuck's servers read the waiting chunks into their table (`MD_RUN = REMOTE`), failing unless it reports as many rows as their footers hold. Outside a transaction the rows are in, and the files go now; inside one, once it commits ([`Self::remove_chunks`]). + pub(in crate::adapters) async fn load_chunks(&mut self) -> Result<(), LoaderError> { + let Some(stage) = self.destination.stage.clone() else { + return Ok(()); + }; + if self.staged.waiting.is_empty() { + return Ok(()); + } + if !self.open_transaction { + open_bucket(&self.destination, &stage.folder.location.store).await?; + } + if self.staged.transaction && !self.open_transaction { + execute_statements(&self.destination, vec![BEGIN_STATEMENT.to_owned()]).await?; + self.open_transaction = true; + } + let waiting = std::mem::take(&mut self.staged.waiting); + let urls: Vec = (waiting.iter()) + .map(|(key, _)| { + DuckDbSql.render_loader_value_literal(&format!("{}{key}", stage.folder.location)) + }) + .collect(); + let insert = format!( + "{}[{}], hive_partitioning = false, MD_RUN = REMOTE)", + self.staged.insert, + urls.join(", ") + ); + let keys: Vec = waiting.iter().map(|(key, _)| key.clone()).collect(); + self.staged.loaded.extend(keys); + let rows = pull_files(&self.destination, insert).await?; + let expected: u64 = waiting.iter().map(|(_, rows)| rows).sum(); + if rows != expected { + return Err(LoaderError::Invalid(format!( + "MotherDuck loaded {rows} rows from the chunks staged in {}, but their footers hold {expected}; the batch is not committed", + stage.folder.location + ))); + } + if !self.open_transaction { + self.remove_chunks(true).await; + } + Ok(()) + } + + /// Remove the chunks MotherDuck read once they `committed`, unless the stage keeps them, and always those of a batch that failed (read or still waiting); a file that cannot be removed is only logged. + pub(in crate::adapters) async fn remove_chunks(&mut self, committed: bool) { + let Some(stage) = self.destination.stage.clone() else { + return; + }; + let mut keys = std::mem::take(&mut self.staged.loaded); + keys.extend(self.staged.waiting.drain(..).map(|(key, _)| key)); + if (stage.keep && committed) || keys.is_empty() { + return; + } + if let Err(error) = delete_files(&stage.folder, &keys).await { + tracing::warn!(stage = %stage.folder.location, %error, "staged chunks were left in the external_stage"); + } + } +} + +/// `name` as a part of an object key that `read_parquet` never takes for a glob: letters, digits, `_`, `-` and `.` as they are, any other byte as `~` and its hex. +fn key_safe(name: &str) -> String { + name.bytes() + .map( + |byte| match byte.is_ascii_alphanumeric() || b"_-.".contains(&byte) { + true => char::from(byte).to_string(), + false => format!("~{byte:02x}"), + }, + ) + .collect() +} diff --git a/crates/sqlload/src/adapters/duckdb/main/_append_batch.rs b/crates/sqlload/src/adapters/duckdb/main/_append_batch.rs index ad9d7d851..346d98d1e 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_append_batch.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_append_batch.rs @@ -11,7 +11,7 @@ use crate::adapters::duckdb::errors::DuckDbError; use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination}; use crate::adapters::motherduck_bridge::main::_append_batch::append_batch as bridge_append; -/// Insert `batch` into `table` by column name, inside the connection's open transaction if it has one: columns the batch lacks get their `DEFAULT`, and values are converted to the column's type, so bad JSON or UUIDs fail here; a column among `variants` (a `VARIANT` column) takes each value as JSON, so objects, arrays, numbers and booleans keep their types. MotherDuck's bridge reads the batch from a Parquet file instead. +/// Insert `batch` into `table` by column name, inside the connection's open transaction if it has one: columns the batch lacks get their `DEFAULT`, and values are converted to the column's type, so bad JSON or UUIDs fail here; a column among `variants` (a `VARIANT` column) takes each value as JSON, so objects, arrays, numbers and booleans keep their types. MotherDuck's bridge `COPY`s the batch from a zstd Parquet file instead, its `VARIANT` columns typed `JSON` there. pub(in crate::adapters) async fn append_batch( destination: &DuckDbDestination, (table, variants): (&TableName, &[&str]), @@ -37,8 +37,9 @@ pub(in crate::adapters) async fn append_batch( .collect(); let (list, values) = (names.join(", "), values.join(", ")); if let DuckDbConnection::Bridge(bridge) = &destination.connection { - let insert = format!("INSERT INTO {target} ({list}) SELECT {values} FROM read_parquet(?)"); - return Ok(bridge_append(bridge, insert, batch).await?); + let copy = format!("COPY {target} ({list}) FROM ? (FORMAT parquet)"); + let json: Vec = variants.iter().map(|name| (*name).to_owned()).collect(); + return Ok(bridge_append(bridge, copy, (batch, json)).await?); } with_connection(destination, move |connection| { connection diff --git a/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs b/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs index 9ecaa2d3b..6814702eb 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs @@ -17,7 +17,10 @@ pub(crate) fn open_motherduck( target_database: Option<&str>, (python, memory): (Option<&str>, Option), ) -> Result { - let ConnectionConfig::MotherDuck { database, token } = connection else { + let ConnectionConfig::MotherDuck { + database, token, .. + } = connection + else { return Err(DuckDbError::MotherDuck(format!( "a {} connection is not a MotherDuck one", connection.kind() @@ -52,5 +55,6 @@ pub(crate) fn open_motherduck( own_limit: None, unflushed: Arc::default(), secrets: Arc::default(), + stage: None, }) } diff --git a/crates/sqlload/src/adapters/duckdb/main/_open_target.rs b/crates/sqlload/src/adapters/duckdb/main/_open_target.rs new file mode 100644 index 000000000..c3032d779 --- /dev/null +++ b/crates/sqlload/src/adapters/duckdb/main/_open_target.rs @@ -0,0 +1,38 @@ +//! Open the DuckDB or MotherDuck database a target loads into. + +use std::sync::Arc; + +use crate::adapters::duckdb::main::_open_destination::open_destination; +use crate::adapters::duckdb::models::{DuckDbDestination, ExternalStage}; +use crate::file_store::main::open_folder::open_folder; +use crate::loader::errors::LoaderError; +use crate::project::main::motherduck_stage::motherduck_stage; +use crate::project::models::{ProjectConfig, TargetConfig}; + +/// Open the database of `target`, whose connection is a `duckdb` or `motherduck` one, with the `external_stage` of a MotherDuck connection and its bucket's client. +pub(in crate::adapters) fn open_target( + project: &ProjectConfig, + target: &TargetConfig, +) -> Result { + let connection = (project.connections.get(&target.connection)).ok_or_else(|| { + LoaderError::Invalid(format!("connection {:?} does not exist", target.connection)) + })?; + let destination = open_destination( + connection, + target.database.as_deref(), + project.memory.budget, + )?; + let Some((location, keep)) = + motherduck_stage(project, &target.connection).map_err(LoaderError::Invalid)? + else { + return Ok(destination); + }; + Ok(DuckDbDestination { + stage: Some(Arc::new(ExternalStage { + folder: open_folder(&location)?, + keep, + checked: tokio::sync::OnceCell::new(), + })), + ..destination + }) +} diff --git a/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs b/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs index c9b988f77..9ae09f8f6 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs @@ -60,5 +60,6 @@ pub(in crate::adapters) fn own_connection( own_limit: destination.own_limit, unflushed: destination.unflushed.clone(), secrets: destination.secrets.clone(), + stage: destination.stage.clone(), }) } diff --git a/crates/sqlload/src/adapters/duckdb/main/_stage_problems.rs b/crates/sqlload/src/adapters/duckdb/main/_stage_problems.rs new file mode 100644 index 000000000..1b4983f1c --- /dev/null +++ b/crates/sqlload/src/adapters/duckdb/main/_stage_problems.rs @@ -0,0 +1,50 @@ +//! Whether a MotherDuck target's `external_stage` works. + +use arrow_array::{Int64Array, RecordBatch}; + +use crate::adapters::duckdb::main::_check_files::check_files; +use crate::adapters::duckdb::models::{DuckDbDestination, ExternalStage}; +use crate::adapters::motherduck_bridge::main::_chunk_parquet::chunk_parquet; +use crate::file_store::main::delete_files::delete_files; +use crate::file_store::main::write_file::write_file; + +/// Why `destination`'s `external_stage` cannot load: SQLLoad cannot write a file under its prefix, or MotherDuck's servers cannot read it there. Checked once for the destination's tables, with a one-row file it removes; none without a stage. +pub(in crate::adapters) async fn stage_problems(destination: &DuckDbDestination) -> Vec { + let Some(stage) = &destination.stage else { + return Vec::new(); + }; + let checked = stage + .checked + .get_or_init(|| check_stage(destination, stage)) + .await; + checked.clone().err().into_iter().collect() +} + +/// Write a one-row file under the stage, have MotherDuck's servers count it, and remove it. +async fn check_stage(destination: &DuckDbDestination, stage: &ExternalStage) -> Result<(), String> { + let location = &stage.folder.location; + let key = format!("_sqlload_check/{}.parquet", uuid::Uuid::new_v4()); + let batch = RecordBatch::try_from_iter([( + "checked", + std::sync::Arc::new(Int64Array::from(vec![1])) as _, + )]) + .map_err(|error| error.to_string())?; + let mut body: Vec = Vec::new(); + chunk_parquet(&mut body, &batch, &[])?; + write_file(&stage.folder, &key, body.into()) + .await + .map_err(|error| { + format!( + "load_method external_stage: SQLLoad cannot write under {location}: {error}; its keys for the bucket need to put and delete objects there" + ) + })?; + let read = check_files(destination, &location.store, &format!("{location}{key}")).await; + if let Err(error) = delete_files(&stage.folder, &[key]).await { + tracing::warn!(stage = %location, %error, "the external_stage check left its file"); + } + read.map_err(|error| { + format!( + "load_method external_stage: MotherDuck's servers cannot read {location}: {error}. They read it from their own network with the keys SQLLoad finds for the bucket: give those keys read access there and let MotherDuck through the bucket's policy, or remove load_method to send the rows from this machine" + ) + }) +} diff --git a/crates/sqlload/src/adapters/duckdb/main/mod.rs b/crates/sqlload/src/adapters/duckdb/main/mod.rs index e09a7bafd..75c3c50e9 100644 --- a/crates/sqlload/src/adapters/duckdb/main/mod.rs +++ b/crates/sqlload/src/adapters/duckdb/main/mod.rs @@ -7,9 +7,11 @@ pub(in crate::adapters) mod _open_bucket; pub(in crate::adapters) mod _open_destination; pub(in crate::adapters) mod _open_duckdb; pub(in crate::adapters) mod _open_motherduck; +pub(in crate::adapters) mod _open_target; pub(in crate::adapters) mod _own_connection; pub(in crate::adapters) mod _pull_files; pub(in crate::adapters) mod _query_text; pub(in crate::adapters) mod _query_texts; +pub(in crate::adapters) mod _stage_problems; pub(in crate::adapters) mod _table_shape; pub(in crate::adapters) mod _variant_problems; diff --git a/crates/sqlload/src/adapters/duckdb/models.rs b/crates/sqlload/src/adapters/duckdb/models.rs index 31077f85c..f41a1343b 100644 --- a/crates/sqlload/src/adapters/duckdb/models.rs +++ b/crates/sqlload/src/adapters/duckdb/models.rs @@ -23,6 +23,35 @@ pub struct DuckDbDestination { pub unflushed: Arc, /// The secret set on the database for each bucket, by bucket, shared by every table's connection: tables setting one at the same time conflict, so each is set before a table opens its transaction, and again only when it [is no longer current](crate::adapters::duckdb::_helpers::pull_sql::secret_current). pub secrets: Arc>>, + /// A MotherDuck target's `external_stage`; `None` sends each chunk from this machine. + pub stage: Option>, +} + +/// Where a MotherDuck target's `external_stage` writes each chunk as zstd Parquet, which MotherDuck's servers read in the commit. +#[cfg(feature = "duckdb")] +#[derive(Debug)] +pub struct ExternalStage { + pub folder: crate::file_store::models::OpenFolder, + /// Keep each chunk's file after its commit. + pub keep: bool, + /// Whether SQLLoad writes the prefix and MotherDuck reads it, checked once for the destination's tables. + pub checked: tokio::sync::OnceCell>, +} + +/// The chunks a table wrote to its [`ExternalStage`] that MotherDuck hasn't read yet, and those read inside the open transaction, whose files go after it commits. +#[cfg(feature = "duckdb")] +#[derive(Debug, Default)] +pub struct StagedChunks { + /// The `INSERT INTO … SELECT … FROM` the waiting chunks load with, up to their file list. + pub insert: String, + /// Each waiting chunk's key under the stage, with its rows. + pub waiting: Vec<(String, u64)>, + /// Chunks loaded in the open transaction. + pub loaded: Vec, + /// Chunks written by this table so far, naming the next. + pub written: u64, + /// The waiting chunks go straight into the table or a build table, in a transaction [`DuckDbWriter::load_chunks`] opens. + pub transaction: bool, } /// A bucket's secret as it was set: a hash of its statement and parameters, never the keys themselves; whether DuckDB resolved its keys through the AWS credential chain, once, when it was set; and when. @@ -55,4 +84,6 @@ pub struct DuckDbWriter { pub destination: DuckDbDestination, /// A transaction is open on the table's connection. pub open_transaction: bool, + /// Under an [`ExternalStage`], its chunks. + pub staged: Box, } diff --git a/crates/sqlload/src/adapters/duckdb/tests/helpers.rs b/crates/sqlload/src/adapters/duckdb/tests/helpers.rs index 20b4f5fb8..16f7e40b5 100644 --- a/crates/sqlload/src/adapters/duckdb/tests/helpers.rs +++ b/crates/sqlload/src/adapters/duckdb/tests/helpers.rs @@ -129,8 +129,10 @@ pub(crate) fn pulled_unloaded(reported: u64, with_files: bool) -> Result<(String own_limit: None, unflushed: Arc::default(), secrets: Arc::default(), + stage: None, }, open_transaction: false, + staged: Default::default(), }; let files = UnloadedFiles { folder: Location { diff --git a/crates/sqlload/src/adapters/duckdb/tests/test_motherduck.rs b/crates/sqlload/src/adapters/duckdb/tests/test_motherduck.rs index 3111e3d85..faec1b3db 100644 --- a/crates/sqlload/src/adapters/duckdb/tests/test_motherduck.rs +++ b/crates/sqlload/src/adapters/duckdb/tests/test_motherduck.rs @@ -235,6 +235,7 @@ fn given_a_motherduck_connection_when_opening_then_the_targets_database_wins_and let connection = ConnectionConfig::MotherDuck { database: test_case.connection_database.map(str::to_owned), token: Some(Secret(String::new())), + load_method: None, }; let error = open_motherduck(&connection, test_case.target_database, (None, None)) .err() @@ -374,7 +375,12 @@ fn given_a_motherduck_destination_when_routing_then_it_reads_parquet_only_from_o let mut destination = open_duckdb(":memory:", None).map_err(|error| error.to_string())?; destination.motherduck = test_case.motherduck; let accepts = writer_accepts(&Destination::DuckDb(destination), None); - assert!(accepts.arrow, "{}", test_case.description); + assert_eq!( + (accepts.arrow, accepts.rows_into.is_some()), + (!test_case.motherduck, test_case.motherduck), + "{}: DuckDB takes Arrow, MotherDuck zstd Parquet chunks", + test_case.description + ); assert_eq!( (accepts.pulls_from, accepts.pull_formats), (test_case.expected_pulls_from, vec![FileFormat::Parquet]), diff --git a/crates/sqlload/src/adapters/motherduck_bridge/_helpers/json_parquet.rs b/crates/sqlload/src/adapters/motherduck_bridge/_helpers/json_parquet.rs new file mode 100644 index 000000000..caeff2b60 --- /dev/null +++ b/crates/sqlload/src/adapters/motherduck_bridge/_helpers/json_parquet.rs @@ -0,0 +1,40 @@ +//! Parquet schemas whose text columns hold JSON. + +use std::sync::Arc; + +use arrow_array::RecordBatch; +use parquet::arrow::ArrowSchemaConverter; +use parquet::basic::{ConvertedType, LogicalType, Type as PhysicalType}; +use parquet::schema::types::{SchemaDescriptor, Type, TypePtr}; + +/// The Parquet schema of `batch`, with its text columns named in `json` typed `JSON`. +pub(crate) fn json_schema( + batch: &RecordBatch, + json: &[String], +) -> parquet::errors::Result { + let converted = ArrowSchemaConverter::new().convert(&batch.schema())?; + let root = converted.root_schema(); + let fields = (root.get_fields().iter()) + .map(|field| json_field(field, json)) + .collect::>>()?; + Ok(SchemaDescriptor::new(Arc::new( + Type::group_type_builder(root.name()) + .with_fields(fields) + .build()?, + ))) +} + +/// `field` typed `JSON` when it is a text column named in `json`, else as it is. +fn json_field(field: &TypePtr, json: &[String]) -> parquet::errors::Result { + let text = field.is_primitive() && field.get_physical_type() == PhysicalType::BYTE_ARRAY; + if !(text && json.iter().any(|name| name == field.name())) { + return Ok(field.clone()); + } + Ok(Arc::new( + Type::primitive_type_builder(field.name(), PhysicalType::BYTE_ARRAY) + .with_repetition(field.get_basic_info().repetition()) + .with_logical_type(Some(LogicalType::Json)) + .with_converted_type(ConvertedType::JSON) + .build()?, + )) +} diff --git a/crates/sqlload/src/adapters/motherduck_bridge/_helpers/mod.rs b/crates/sqlload/src/adapters/motherduck_bridge/_helpers/mod.rs index 3c433d355..4b24d0175 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/_helpers/mod.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/_helpers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod json_parquet; pub(crate) mod process; diff --git a/crates/sqlload/src/adapters/motherduck_bridge/main/_append_batch.rs b/crates/sqlload/src/adapters/motherduck_bridge/main/_append_batch.rs index 63bee0f53..4b53ba246 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/main/_append_batch.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/main/_append_batch.rs @@ -1,17 +1,17 @@ -//! Hand an Arrow batch to the bridge. +//! Send an Arrow batch through the MotherDuck bridge as a zstd Parquet file. use arrow_array::RecordBatch; -use parquet::arrow::ArrowWriter; use crate::adapters::motherduck_bridge::errors::BridgeError; +use crate::adapters::motherduck_bridge::main::_chunk_parquet::chunk_parquet; use crate::adapters::motherduck_bridge::main::run_statement::run_statement; use crate::adapters::motherduck_bridge::models::BridgeConnection; -/// Run `insert`, whose one `?` is the path of a Parquet file holding `batch` (such as `INSERT INTO t (a) SELECT a FROM read_parquet(?)`), on `connection`. The file is a temporary one on this machine, which DuckDB reads there and removed once the statement ends: the bridge's DuckDB is another version than SQLLoad's, so Arrow cannot cross between them in memory. +/// Run `statement`, whose one `?` is the path of a zstd Parquet file holding `batch` (see [`chunk_parquet`]; such as `COPY t (a) FROM ? (FORMAT parquet)`), on `connection`. The file is a temporary one on this machine, which DuckDB reads there and removed once the statement ends: the bridge's DuckDB is another version than SQLLoad's, so Arrow cannot cross between them in memory. pub(in crate::adapters) async fn append_batch( connection: &BridgeConnection, - insert: String, - batch: RecordBatch, + statement: String, + (batch, json): (RecordBatch, Vec), ) -> Result<(), BridgeError> { let file = tokio::task::spawn_blocking(move || -> Result<_, String> { let file = tempfile::Builder::new() @@ -19,14 +19,11 @@ pub(in crate::adapters) async fn append_batch( .suffix(".parquet") .tempfile() .map_err(|error| error.to_string())?; - let mut writer = ArrowWriter::try_new( + chunk_parquet( file.reopen().map_err(|error| error.to_string())?, - batch.schema(), - None, - ) - .map_err(|error| error.to_string())?; - writer.write(&batch).map_err(|error| error.to_string())?; - writer.close().map_err(|error| error.to_string())?; + &batch, + &json, + )?; Ok(file) }) .await @@ -37,7 +34,7 @@ pub(in crate::adapters) async fn append_batch( )) })?; let path = file.path().display().to_string(); - run_statement(connection, insert, vec![path]).await?; + run_statement(connection, statement, vec![path]).await?; drop(file); Ok(()) } diff --git a/crates/sqlload/src/adapters/motherduck_bridge/main/_chunk_parquet.rs b/crates/sqlload/src/adapters/motherduck_bridge/main/_chunk_parquet.rs new file mode 100644 index 000000000..2975e5e87 --- /dev/null +++ b/crates/sqlload/src/adapters/motherduck_bridge/main/_chunk_parquet.rs @@ -0,0 +1,35 @@ +//! Write a chunk of rows as zstd Parquet for MotherDuck. + +use std::io::Write; + +use arrow_array::RecordBatch; +use parquet::arrow::arrow_writer::{ArrowWriter, ArrowWriterOptions}; +use parquet::basic::{Compression, ZstdLevel}; +use parquet::file::properties::WriterProperties; + +use crate::adapters::motherduck_bridge::_helpers::json_parquet::json_schema; + +/// Write `batch` to `out` as one zstd Parquet file, its text columns named in `json` typed Parquet `JSON`, so DuckDB parses them into a `VARIANT` column as `CAST(… AS JSON)` would. +pub(in crate::adapters) fn chunk_parquet( + out: W, + batch: &RecordBatch, + json: &[String], +) -> Result<(), String> { + let mut writer = + ArrowWriter::try_new_with_options(out, batch.schema(), zstd_options(batch, json)?) + .map_err(|error| error.to_string())?; + writer.write(batch).map_err(|error| error.to_string())?; + writer.close().map_err(|error| error.to_string())?; + Ok(()) +} + +/// zstd compression and [`json_schema`]. +fn zstd_options(batch: &RecordBatch, json: &[String]) -> Result { + let properties = WriterProperties::builder() + .set_compression(Compression::ZSTD(ZstdLevel::default())) + .build(); + let schema = json_schema(batch, json).map_err(|error| error.to_string())?; + Ok(ArrowWriterOptions::new() + .with_properties(properties) + .with_parquet_schema(schema)) +} diff --git a/crates/sqlload/src/adapters/motherduck_bridge/main/mod.rs b/crates/sqlload/src/adapters/motherduck_bridge/main/mod.rs index 8f3a94a78..7e9265592 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/main/mod.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/main/mod.rs @@ -1,4 +1,5 @@ pub(in crate::adapters) mod _append_batch; +pub(in crate::adapters) mod _chunk_parquet; pub(in crate::adapters) mod _open_bridge; pub(crate) mod fork_connection; pub(crate) mod run_statement; diff --git a/crates/sqlload/src/loader/_helpers/destination.rs b/crates/sqlload/src/loader/_helpers/destination.rs index 0c77f2c3b..5deb3fa3f 100644 --- a/crates/sqlload/src/loader/_helpers/destination.rs +++ b/crates/sqlload/src/loader/_helpers/destination.rs @@ -54,18 +54,18 @@ use crate::loader::_helpers::publish::{staged_columns, write_target}; use crate::loader::_helpers::resolve::display_name; use crate::loader::_helpers::sources::{ReadPart, read_source}; use crate::loader::_helpers::table_plan::blocked_plan; -use crate::loader::constants::{CATALOG_CONFLICT, CATALOG_CONFLICT_TRIES}; +use crate::loader::constants::{CATALOG_CONFLICT, CATALOG_CONFLICT_TRIES, COALESCED_BYTES}; use crate::loader::errors::LoaderError; use crate::loader::models::SourceReader; use crate::loader::models::{CommitRange, Publisher}; -use crate::loader::models::{Cuts, Pieces, RowGroupSeen}; +use crate::loader::models::{Cuts, Pending, Pieces, RowGroupSeen}; use crate::loader::models::{ FilesSink, FilesSource, LiveFiles, LoadRun, ProjectPlan, TablePlan, TableSpec, UnreadSharing, }; use crate::masking::main::mask_batch::mask_batch; use crate::memory_budget::constants::{ DUCKDB_COMMIT_PERCENT, DUCKDB_ROW_GROUP_PERCENT, DUCKDB_ROW_GROUP_ROWS, HOLDING_DIVISOR, - LARGE_VALUE_BYTES, STAGED_COPIES, + LARGE_VALUE_BYTES, STAGED_COPIES, TABLE_SHARE_PARTS, }; use crate::memory_budget::main::large_values::large_values; use crate::memory_budget::main::needed_budget::needed_budget; @@ -91,6 +91,7 @@ use crate::table_lease::main::release_lease::release_lease; use crate::table_lease::models::Lease; use arrow_array::RecordBatch; use arrow_cast::display::{ArrayFormatter, FormatOptions}; +use arrow_select::concat::concat_batches; use std::path::Path; use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; @@ -1341,12 +1342,49 @@ impl Publisher { if let Some(spool) = self.spool.as_mut() { return spool.hold(self.batch, &batch).map_err(LoaderError::Invalid); } + self.write_merged(self.batch, batch).await + } + + /// Hold `batch`, rows of batch `number`, with the rows before it, writing those first when the batch number or columns change or `batch` would take them past [`DUCKDB_ROW_GROUP_ROWS`] rows or [`COALESCED_BYTES`] (or less: the batch's bytes, or under a memory budget the part of the table's share a source batch takes); and write them all once they reach either. + async fn write_merged(&mut self, number: u64, batch: RecordBatch) -> Result<(), LoaderError> { + let (rows, bytes) = (batch.num_rows() as u64, batch.get_array_memory_size()); + let part = table_memory().map(|watched| { + usize::try_from(watched.guard.budget.table_share / TABLE_SHARE_PARTS) + .unwrap_or(usize::MAX) + }); + let most = (COALESCED_BYTES.min(self.batch_bytes)).min(part.unwrap_or(usize::MAX)); + let joins = (self.pending.batches.first()) + .is_none_or(|first| first.schema() == batch.schema()) + && self.pending.batch == number + && self.pending.rows + rows <= DUCKDB_ROW_GROUP_ROWS + && self.pending.bytes + bytes <= most; + if !joins { + self.write_pending().await?; + } + self.pending.batch = number; + self.pending.rows += rows; + self.pending.bytes += bytes; + self.pending.batches.push(batch); + if self.pending.rows >= DUCKDB_ROW_GROUP_ROWS || self.pending.bytes >= most { + self.write_pending().await?; + } + Ok(()) + } + + /// Write the rows held back (see [`Self::write_merged`]) as one batch, before anything reads, commits or alters what they go into. + pub(in crate::loader) async fn write_pending(&mut self) -> Result<(), LoaderError> { + let pending = std::mem::take(&mut self.pending); + let Some(first) = pending.batches.first() else { + return Ok(()); + }; + let batch = concat_batches(&first.schema(), &pending.batches) + .map_err(|error| LoaderError::Invalid(error.to_string()))?; let columns = staged_columns(&self.table); let target = write_target( &self.table, self.staging.as_ref(), &columns, - ((&self.load_id, self.batch), (self.direct, self.builds)), + ((&self.load_id, pending.batch), (self.direct, self.builds)), ); self.writer.write_batch(&target, batch).await } @@ -1442,9 +1480,10 @@ impl Publisher { Ok(()) } - /// Run a marker statement in the transaction the batch's rows commit in: the open one, or under `send = "at_commit"`, whose rows open it only as they are sent, the one that sends them. + /// Run a marker statement in the transaction the batch's rows commit in: with rows, once the flush that loads them has them in (under `send = "at_commit"` they open it only as they are sent, and a MotherDuck `external_stage` only as MotherDuck reads them), just before it commits; without, now. async fn in_last_transaction(&mut self, statement: String) -> Result<(), LoaderError> { - match self.spool.is_some() && self.batch_rows > 0 { + self.write_pending().await?; + match self.batch_rows > 0 { true => self.pieces.statement = Some(statement), false => execute(&self.destination, vec![statement]).await?, } @@ -1456,6 +1495,7 @@ impl Publisher { if !self.pieces.unfinished { return; } + self.pending = Pending::default(); self.writer.rollback_batch().await; let spec = &self.table.spec; let statements = unfinished_recovery( @@ -1619,38 +1659,42 @@ impl Publisher { (batch, files): (u64, &[std::path::PathBuf]), loads: bool, ) -> Result<(), LoaderError> { - let columns = staged_columns(&self.table); - let target = write_target( - &self.table, - self.staging.as_ref(), - &columns, - ((&self.load_id, batch), (self.direct, self.builds)), - ); - let sent = 'send: { + let written = 'write: { for file in files { let reader = match held_rows(file) { Ok(reader) => reader, - Err(error) => break 'send Err(LoaderError::Invalid(error)), + Err(error) => break 'write Err(LoaderError::Invalid(error)), }; for rows in reader { let rows = match rows { Ok(rows) => rows, - Err(error) => break 'send Err(LoaderError::Invalid(error.to_string())), + Err(error) => break 'write Err(LoaderError::Invalid(error.to_string())), }; - if let Err(error) = self.writer.write_batch(&target, rows).await { - break 'send Err(error); + if let Err(error) = self.write_merged(batch, rows).await { + break 'write Err(error); } } let _ = std::fs::remove_file(file); } + self.write_pending().await + }; + if written.is_err() { + self.pending = Pending::default(); + } + let columns = staged_columns(&self.table); + let target = write_target( + &self.table, + self.staging.as_ref(), + &columns, + ((&self.load_id, batch), (self.direct, self.builds)), + ); + let sent = 'send: { + if let Err(error) = written { + break 'send Err(error); + } if !loads { break 'send Ok(()); } - if let Some(statement) = self.pieces.statement.take() - && let Err(error) = execute(&self.destination, vec![statement]).await - { - break 'send Err(error); - } if self.direct || self.builds { match self .check_row_group(target.staging.unwrap_or(target.table)) @@ -1666,6 +1710,11 @@ impl Publisher { if let Err(error) = self.writer.settle_rows().await { break 'send Err(error); } + if let Some(statement) = self.pieces.statement.take() + && let Err(error) = execute(&self.destination, vec![statement]).await + { + break 'send Err(error); + } self.writer.load_batch(&target).await }; match sent { diff --git a/crates/sqlload/src/loader/_helpers/publish.rs b/crates/sqlload/src/loader/_helpers/publish.rs index 4a6ef0965..36600478f 100644 --- a/crates/sqlload/src/loader/_helpers/publish.rs +++ b/crates/sqlload/src/loader/_helpers/publish.rs @@ -109,6 +109,7 @@ use crate::loader::models::CommitRule; use crate::loader::models::FilesSink; use crate::loader::models::FilesSource; use crate::loader::models::FolderSwap; +use crate::loader::models::Pending; use crate::loader::models::Pieces; use crate::loader::models::Publisher; use crate::loader::models::ReplacedKeys; @@ -658,6 +659,7 @@ impl Publisher { transform: None, spool, pieces: Pieces::default(), + pending: Pending::default(), }; publisher.create_marker_table().await?; Ok(publisher) @@ -910,6 +912,7 @@ impl Publisher { } give_back(leased_folder(sink), &sink.lease).await; } + self.pending = Pending::default(); self.writer.rollback_batch().await; let warehouse = self.table.spec.warehouse; if let Some(staging) = &self.staging { @@ -952,6 +955,7 @@ impl Publisher { /// Wait until every row sent without files is in the staging table; under `send = "at_commit"`, the held rows are sent first, before a schema change makes them unlike the rows after it: the batches that ended are loaded, and the open batch's rows written, leaving it open for its end to load or commit as it would have. pub(crate) async fn settle_rows(&mut self) -> Result<(), LoaderError> { + self.write_pending().await?; if let Some(spool) = self.spool.as_mut() { let ended = spool.take_ended(self.batch); let open = spool.take_open_files(self.batch); @@ -970,6 +974,7 @@ impl Publisher { /// Load the batch where it goes: `COPY` its staged files (straight into the table, committing, or into the staging table) and remove them; DuckDB commits a straight batch's transaction. Then start the next batch. pub(crate) async fn flush_batch(&mut self) -> Result<(), LoaderError> { + self.write_pending().await?; match self.spool.is_some() { true if self.batch_rows == 0 => return Ok(()), true if self.direct => Box::pin(self.send_held()).await?, @@ -993,6 +998,9 @@ impl Publisher { return Ok(()); } self.writer.settle_rows().await?; + if let Some(statement) = self.pieces.statement.take() { + execute(&self.destination, vec![statement]).await?; + } self.writer.load_batch(&target).await?; self.writer.next_batch(); if self.direct || self.builds { diff --git a/crates/sqlload/src/loader/constants.rs b/crates/sqlload/src/loader/constants.rs index 68b0b7cad..640fb70ad 100644 --- a/crates/sqlload/src/loader/constants.rs +++ b/crates/sqlload/src/loader/constants.rs @@ -7,6 +7,9 @@ pub const RELOAD_FIX: &str = "run it once with --reload"; /// The same on a Postgres target, which never drops and recreates a table. pub const KEPT_TABLES_COLUMN_FIX: &str = "SQLLoad never drops and recreates a Postgres table (--reload), so add the column with ALTER TABLE, then empty the table or fill the column, and load again"; +/// The most Arrow memory of rows a table holds back to write together (see [`Pending`](crate::loader::models::Pending)), or the batch's own bytes when smaller. +pub const COALESCED_BYTES: usize = 64 * 1024 * 1024; + /// Rows per batch read from a database source. pub const SOURCE_BATCH_ROWS: usize = 65_536; diff --git a/crates/sqlload/src/loader/models.rs b/crates/sqlload/src/loader/models.rs index cc07586ce..ab7f0c6a8 100644 --- a/crates/sqlload/src/loader/models.rs +++ b/crates/sqlload/src/loader/models.rs @@ -1193,6 +1193,8 @@ pub struct Publisher { pub transform: Option<(TransformRun, usize)>, /// Under `send = "at_commit"`, the batches held locally until their commit sends them; `None` when each batch is sent as it is read. pub spool: Option, + /// Rows staged but not yet written, merged into one batch first (see [`Pending`]). + pub pending: Pending, /// The commit being cut into pieces under DuckDB's memory share. pub pieces: Pieces, } @@ -1208,6 +1210,17 @@ pub enum Cuts { Never(&'static str), } +/// Rows a table staged that its writer hasn't taken yet: a source that yields a page at a time, such as `sl.http`, has its pages merged into one batch of up to about 122,880 rows ([`DUCKDB_ROW_GROUP_ROWS`](crate::memory_budget::constants::DUCKDB_ROW_GROUP_ROWS)) or [`COALESCED_BYTES`](crate::loader::constants::COALESCED_BYTES), so a target gets one write for them, not one per page. +#[derive(Debug, Default)] +pub struct Pending { + /// The batch number they belong to. + pub batch: u64, + pub batches: Vec, + pub rows: u64, + /// Their Arrow memory. + pub bytes: usize, +} + /// A table's commits cut into pieces, each holding no more large values (those DuckDB keeps in memory until a checkpoint) than DuckDB's share commits at once. #[derive(Debug, Default)] pub struct Pieces { diff --git a/crates/sqlload/src/project/_helpers/references.rs b/crates/sqlload/src/project/_helpers/references.rs index 6eba0c70a..b2633a7e6 100644 --- a/crates/sqlload/src/project/_helpers/references.rs +++ b/crates/sqlload/src/project/_helpers/references.rs @@ -16,6 +16,7 @@ use crate::project::constants::{ MAX_BATCH_CONCURRENCY, MAX_POSTGRES_NAME_BYTES, MIN_BATCH_INTERVAL, SHAREABLE_CONNECTION_KINDS, STAGE_NAME_PARTS, TABLE_PLACEHOLDER, }; +use crate::project::main::motherduck_stage::motherduck_stage; use crate::project::models::{ CdcMode, ConnectionConfig, ConstantColumns, Deletes, ProjectConfig, RenameSetting, SourceConfig, SourceKind, TableSettings, WriteStrategy, @@ -43,6 +44,7 @@ pub(crate) fn reference_problems(project: &ProjectConfig) -> Vec { .values() .any(|target| target.connection == *name); problems.extend(stage_problems(name, connection, targeted)); + problems.extend(motherduck_stage(project, name).err()); problems.extend(odbc_connection_problems(name, connection)); if let ConnectionConfig::S3 { profile: Some(profile), diff --git a/crates/sqlload/src/project/main/mod.rs b/crates/sqlload/src/project/main/mod.rs index 6bf5e2fb5..3f69123b7 100644 --- a/crates/sqlload/src/project/main/mod.rs +++ b/crates/sqlload/src/project/main/mod.rs @@ -2,6 +2,7 @@ pub mod config_at; pub mod config_lines; pub mod is_project; pub mod motherduck_path; +pub mod motherduck_stage; pub mod parse_project; pub mod parse_python_project; pub mod postgres_address; diff --git a/crates/sqlload/src/project/main/motherduck_stage.rs b/crates/sqlload/src/project/main/motherduck_stage.rs new file mode 100644 index 000000000..e41e5c16c --- /dev/null +++ b/crates/sqlload/src/project/main/motherduck_stage.rs @@ -0,0 +1,58 @@ +//! Where a MotherDuck connection's `external_stage` writes its chunks. + +use crate::project::models::{ConnectionConfig, MotherDuckLoadMethod, ProjectConfig}; +use crate::store_paths::main::connection_store::connection_store; +use crate::store_paths::main::resolve_location::resolve_location; +use crate::store_paths::models::{Location, StoreKind}; + +/// The prefix MotherDuck connection `name`'s `external_stage` writes its chunks under, with the credentials of the `s3` or `gcs` connection it names, and whether it keeps them after their commit; `None` for a `local` load or another kind of connection. An error names the setting and the fix: the URL must be `s3://` or `gs://` in a bucket MotherDuck's servers reach (no custom endpoint), and `gs://` needs a `gcs` connection with an HMAC key, the only key MotherDuck reads GCS with. +pub fn motherduck_stage( + project: &ProjectConfig, + name: &str, +) -> Result, String> { + let Some(ConnectionConfig::MotherDuck { + load_method: + Some(MotherDuckLoadMethod::ExternalStage { + url, + connection, + keep_staged_files, + }), + .. + }) = project.connections.get(name) + else { + return Ok(None); + }; + let place = format!("[connections.{name}] load_method"); + let store = match connection { + None => None, + Some(bucket) => match project.connections.get(bucket).and_then(connection_store) { + Some(store @ (StoreKind::S3 { .. } | StoreKind::Gcs { .. })) => Some(store), + _ => { + return Err(format!( + "{place} connection {bucket:?} must name an s3 or gcs connection to the bucket of {url:?}" + )); + } + }, + }; + let location = resolve_location(connection.as_deref().zip(store.as_ref()), url) + .map_err(|error| format!("{place} url {url:?}: {error}"))?; + match &location.store { + StoreKind::S3 { + endpoint: Some(_), .. + } + | StoreKind::Gcs { + endpoint: Some(_), .. + } => Err(format!( + "{place} stages in a bucket with a custom endpoint, which MotherDuck's servers don't reach: stage in AWS S3 or GCS" + )), + StoreKind::S3 { .. } | StoreKind::Gcs { hmac: Some(_), .. } => { + Ok(Some((location, *keep_staged_files))) + } + StoreKind::Gcs { .. } => Err(format!( + "{place} url {url:?}: MotherDuck reads GCS only with an HMAC key; set connection to a gcs connection with hmac_key_id and hmac_secret" + )), + StoreKind::Filesystem { .. } | StoreKind::Sftp { .. } => Err(format!( + "{place} url {url:?} must be an s3:// or gs:// URL, which MotherDuck's servers read" + )), + } +} diff --git a/crates/sqlload/src/project/models.rs b/crates/sqlload/src/project/models.rs index 1c33e40eb..e52d8b9c8 100644 --- a/crates/sqlload/src/project/models.rs +++ b/crates/sqlload/src/project/models.rs @@ -122,6 +122,25 @@ pub enum LoadMethod { Insert {}, } +/// How a MotherDuck connection's rows get into MotherDuck; a stage it cannot use fails at plan, never falling back. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum MotherDuckLoadMethod { + /// zstd Parquet chunks sent from this machine, each loaded with `COPY`. + Local {}, + /// zstd Parquet chunks written under an S3 or GCS prefix, which MotherDuck's servers read in the commit (`MD_RUN = REMOTE`). + ExternalStage { + /// The prefix, `s3://bucket/prefix/` or `gs://bucket/prefix/`. + url: String, + /// An `s3` or `gcs` connection to the bucket, whose credentials SQLLoad writes with and MotherDuck reads with; required for `gs://`, whose HMAC key MotherDuck reads with. Without it, S3 keys come from the environment, the shared credentials file or the AWS credential chain. + #[serde(default)] + connection: Option, + /// Keep each chunk's file after its commit. + #[serde(default)] + keep_staged_files: bool, + }, +} + /// How to reach one system. #[derive(Debug, Clone, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] @@ -273,6 +292,9 @@ pub enum ConnectionConfig { /// The access token; the `motherduck_token` or `MOTHERDUCK_TOKEN` environment variable when unset. #[serde(default)] token: Option, + /// How rows get into MotherDuck; `local` when unset. + #[serde(default)] + load_method: Option, }, /// A database reached through an ODBC driver: a connection string, or a DSN with an optional user and password. Odbc { diff --git a/crates/sqlload/src/project/tests/helpers.rs b/crates/sqlload/src/project/tests/helpers.rs index a48eabcf0..bb1d8415a 100644 --- a/crates/sqlload/src/project/tests/helpers.rs +++ b/crates/sqlload/src/project/tests/helpers.rs @@ -129,3 +129,14 @@ pub(crate) fn parse_with_python( .map(|project| read(&project)) .map_err(|error| error.to_string()) } + +/// An `s3` connection to the bucket `lake`, in `eu-west-1`. +pub(crate) const LAKE: &str = + "[connections.lake]\nkind = \"s3\"\nbucket = \"lake\"\nregion = \"eu-west-1\"\n"; + +/// A project loading into MotherDuck connection `md` with `load_method`, and the connections in `more`. +pub(crate) fn md_stage(load_method: &str, more: &str) -> String { + format!( + "[connections.md]\nkind = \"motherduck\"\ndatabase = \"shop\"\nload_method = {load_method}\n{more}[targets.dev]\nconnection = \"md\"\nloader_schema = \"raw\"\n" + ) +} diff --git a/crates/sqlload/src/project/tests/test_parse_project.rs b/crates/sqlload/src/project/tests/test_parse_project.rs index f94b2d0d3..8dc395c32 100644 --- a/crates/sqlload/src/project/tests/test_parse_project.rs +++ b/crates/sqlload/src/project/tests/test_parse_project.rs @@ -1,8 +1,13 @@ +use crate::auth::models::Secret; +use crate::project::main::motherduck_stage::motherduck_stage; use crate::project::main::parse_project::parse_project; use crate::project::tests::helpers::{ CONNECTIONS, FILES_PROJECT, PROJECT, SNOWFLAKE_SOURCE, environment, }; +use crate::project::tests::helpers::{LAKE, md_stage}; +use crate::project::tests::test_types::MotherDuckStageTestCase; use crate::project::tests::test_types::{ParseProjectErrorTestCase, ParseProjectTestCase}; +use crate::store_paths::models::{HmacKey, Location, StoreKind}; const CONSTANT_SOURCES: &str = r#" [sources.app] @@ -871,3 +876,127 @@ fn given_column_names_on_a_sheets_source_when_every_table_using_them_is_headerle ); } } + +#[test] +fn given_a_motherduck_external_stage_when_parsing_then_it_stages_in_a_bucket_motherduck_reads() { + let s3 = |bucket: &str, region: Option<&str>| StoreKind::S3 { + bucket: bucket.to_owned(), + region: region.map(str::to_owned), + endpoint: None, + profile: None, + }; + let at = |store: StoreKind| Location { + store, + prefix: "md/".to_owned(), + }; + let test_cases = [ + MotherDuckStageTestCase { + description: "an s3 prefix, keys from the environment", + text: md_stage("{ type = \"external_stage\", url = \"s3://lake/md/\" }", ""), + expected_stage: Some(Some((at(s3("lake", None)), false))), + expected_error: "", + }, + MotherDuckStageTestCase { + description: "an s3 prefix with an s3 connection's settings, kept", + text: md_stage( + "{ type = \"external_stage\", url = \"s3://lake/md/\", connection = \"lake\", keep_staged_files = true }", + LAKE, + ), + expected_stage: Some(Some((at(s3("lake", Some("eu-west-1"))), true))), + expected_error: "", + }, + MotherDuckStageTestCase { + description: "a gs prefix with a gcs connection's HMAC key", + text: md_stage( + "{ type = \"external_stage\", url = \"gs://lake/md/\", connection = \"gcs\" }", + "[connections.gcs]\nkind = \"gcs\"\nbucket = \"lake\"\nhmac_key_id = \"K\"\nhmac_secret = \"S\"\n", + ), + expected_stage: Some(Some(( + at(StoreKind::Gcs { + bucket: "lake".to_owned(), + service_account_key: None, + endpoint: None, + hmac: Some(HmacKey { + key_id: "K".to_owned(), + secret: Secret("S".to_owned()), + }), + }), + false, + ))), + expected_error: "", + }, + MotherDuckStageTestCase { + description: "local sends from this machine", + text: md_stage("{ type = \"local\" }", ""), + expected_stage: Some(None), + expected_error: "", + }, + MotherDuckStageTestCase { + description: "gs needs a gcs connection's HMAC key", + text: md_stage("{ type = \"external_stage\", url = \"gs://lake/md/\" }", ""), + expected_stage: None, + expected_error: "[connections.md] load_method url \"gs://lake/md/\": MotherDuck reads GCS only with an HMAC key; set connection to a gcs connection with hmac_key_id and hmac_secret", + }, + MotherDuckStageTestCase { + description: "a custom endpoint is out of MotherDuck's reach", + text: md_stage( + "{ type = \"external_stage\", url = \"s3://lake/md/\", connection = \"lake\" }", + "[connections.lake]\nkind = \"s3\"\nbucket = \"lake\"\nendpoint = \"http://localhost:9000\"\n", + ), + expected_stage: None, + expected_error: "[connections.md] load_method stages in a bucket with a custom endpoint, which MotherDuck's servers don't reach: stage in AWS S3 or GCS", + }, + MotherDuckStageTestCase { + description: "a local folder is no stage", + text: md_stage("{ type = \"external_stage\", url = \"/tmp/md/\" }", ""), + expected_stage: None, + expected_error: "[connections.md] load_method url \"/tmp/md/\" must be an s3:// or gs:// URL, which MotherDuck's servers read", + }, + MotherDuckStageTestCase { + description: "the connection must be a bucket's", + text: md_stage( + "{ type = \"external_stage\", url = \"s3://lake/md/\", connection = \"files\" }", + "[connections.files]\nkind = \"filesystem\"\nroot = \"/data\"\n", + ), + expected_stage: None, + expected_error: "[connections.md] load_method connection \"files\" must name an s3 or gcs connection to the bucket of \"s3://lake/md/\"", + }, + MotherDuckStageTestCase { + description: "the connection's bucket must be the url's", + text: md_stage( + "{ type = \"external_stage\", url = \"s3://other/md/\", connection = \"lake\" }", + LAKE, + ), + expected_stage: None, + expected_error: "[connections.md] load_method url \"s3://other/md/\":", + }, + MotherDuckStageTestCase { + description: "an unknown setting is refused", + text: md_stage( + "{ type = \"external_stage\", url = \"s3://lake/md/\", name = \"N\" }", + "", + ), + expected_stage: None, + expected_error: "unknown field `name`", + }, + ]; + for test_case in test_cases { + let found = parse_project(&test_case.text, &|_| None) + .map_err(|error| error.to_string()) + .and_then(|project| motherduck_stage(&project, "md")); + assert_eq!( + found.as_ref().ok(), + test_case.expected_stage.as_ref(), + "{}: {found:?}", + test_case.description + ); + assert!( + found + .err() + .unwrap_or_default() + .contains(test_case.expected_error), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/project/tests/test_types.rs b/crates/sqlload/src/project/tests/test_types.rs index 2d3987fa5..e27f14564 100644 --- a/crates/sqlload/src/project/tests/test_types.rs +++ b/crates/sqlload/src/project/tests/test_types.rs @@ -45,3 +45,13 @@ pub(crate) struct ConnectionVariablesTestCase { pub(crate) connection: &'static str, pub(crate) expected_variables: &'static [&'static str], } + +/// A MotherDuck connection's `load_method` and where it stages. +pub(crate) struct MotherDuckStageTestCase { + pub(crate) description: &'static str, + pub(crate) text: String, + /// The stage and whether it keeps its files (`Some(None)` for a `local` load); `None` when the project is refused. + pub(crate) expected_stage: Option>, + /// Text the error message must contain; empty when there is none. + pub(crate) expected_error: &'static str, +} diff --git a/packages/sqlload/tests/test_motherduck.py b/packages/sqlload/tests/test_motherduck.py index 6d7e00388..91670cc62 100644 --- a/packages/sqlload/tests/test_motherduck.py +++ b/packages/sqlload/tests/test_motherduck.py @@ -49,6 +49,7 @@ #: database read-only, so a source's bridges (one per table) can hold it at once as MotherDuck allows. STAND_IN = r''' import os +import re import sys import urllib.parse @@ -71,6 +72,10 @@ def __init__(self, inner): def execute(self, sql, params=None): log(sql) + for path in [*(params or []), *re.findall(r"'([^']+[.]parquet)'", sql)]: + if str(path).endswith(".parquet") and os.path.exists(path): + found = _connect().execute("SELECT DISTINCT compression FROM parquet_metadata(?)", [path]).fetchall() + log("-- parquet " + ",".join(sorted(row[0] for row in found))) if os.environ.get("MD_DIE_ON") and os.environ["MD_DIE_ON"] in sql: sys.stderr.write(f"the stand-in for MotherDuck ended the bridge, signed in as {os.environ['MD_EXPECTED_TOKEN']}\n") sys.stderr.flush() @@ -302,7 +307,7 @@ def test_given_the_bridge_ends_mid_load_when_loading_then_the_table_fails_keeps_ give(project, "rows", [{"id": 1, "v": "a"}]) load(project) give(project, "rows", [{"id": 1, "v": "A"}, {"id": 2, "v": "b"}]) - done = sqlload(project, "load", MD_DIE_ON="read_parquet") + done = sqlload(project, "load", MD_DIE_ON="(FORMAT parquet)") assert done.returncode == 1, done.stdout + done.stderr assert "the MotherDuck bridge on" in done.stdout + done.stderr assert "the stand-in for MotherDuck ended the bridge, signed in as " in done.stdout + done.stderr @@ -401,13 +406,13 @@ def failing(ctx): """ -def inserts_between_reads(log: Path) -> list[int]: - """How many INSERTs reached MotherDuck before the first `-- read` note, then after each.""" +def chunks_between_reads(log: Path) -> list[int]: + """How many chunks of rows reached MotherDuck before the first `-- read` note, then after each.""" counts = [0] for line in log.read_text().splitlines(): if line.startswith("-- read"): counts.append(0) - elif line.startswith("INSERT"): + elif line.startswith("COPY"): counts[-1] += 1 return counts @@ -423,7 +428,7 @@ def test_given_send_when_a_table_loads_in_windows_then_at_commit_sends_each_wind project = make_project(tmp_path, MARKED.replace("{send}", send)) log = tmp_path / "statements.log" load(project, "windowed", "--start-cursor", "0", "--end-cursor", "3", MD_LOG=str(log)) - counts = inserts_between_reads(log) + counts = chunks_between_reads(log) if expected_before_first_read is None: assert counts[0] >= 1, log.read_text() else: @@ -433,6 +438,33 @@ def test_given_send_when_a_table_loads_in_windows_then_at_commit_sends_each_wind ] +PAGED = """ +import pyarrow as pa + + +def paged(ctx): + for page in range(400): + yield pa.table({"id": list(range(page * 50, page * 50 + 50)), "doc": ['{"page": %d}' % page] * 50}) + + +cloud.table(description="Rows.", name="paged", write_strategy="merge", unique_key="id", columns={"doc": "json"}{send})(paged) +""" + + +@pytest.mark.parametrize("send", ["", ", send='per_batch'"], ids=["at_commit", "per_batch"]) +def test_given_a_source_yielding_small_pages_when_loading_into_motherduck_then_the_rows_go_in_one_zstd_parquet_chunk( + tmp_path: Path, send: str +) -> None: + project = make_project(tmp_path, PAGED.replace("{send}", send)) + log = tmp_path / "statements.log" + load(project, "paged", MD_LOG=str(log)) + lines = log.read_text().splitlines() + staged = [line for line in lines if "read_parquet" in line or line.startswith("COPY")] + assert (len(staged), sorted({line for line in lines if line.startswith("-- parquet")})) == (1, ["-- parquet ZSTD"]), log.read_text() + assert staged[0].startswith('COPY "shop"."raw".'), staged + assert cloud(project, "SELECT count(*), count(DISTINCT id), max(doc.page::INT) FROM raw.cloud__paged") == [(20000, 20000, 399)] + + def test_given_send_at_commit_when_a_read_fails_part_way_then_nothing_reaches_motherduck_and_no_held_file_stays(tmp_path: Path) -> None: project = make_project(tmp_path, MARKED.replace("{send}", "")) (project / "sqlload.toml").write_text( @@ -443,7 +475,7 @@ def test_given_send_at_commit_when_a_read_fails_part_way_then_nothing_reaches_mo done = sqlload(project, "load", "failing", MD_LOG=str(log)) assert done.returncode == 1, done.stdout + done.stderr assert "the API went away" in done.stdout - assert inserts_between_reads(log) == [0, 0], log.read_text() + assert chunks_between_reads(log) == [0, 0], log.read_text() assert list((project / "held").iterdir()) == [] @@ -459,6 +491,7 @@ def test_given_send_when_planning_then_each_table_says_how_its_rows_are_sent(tmp assert done.returncode == 0, done.stdout + done.stderr assert "send at commit: staged locally, then one burst" in done.stdout, done.stdout assert "send n/a: DuckDB reads the files itself" in done.stdout, done.stdout + assert "read as Arrow -> zstd Parquet, about 122,880 rows a file -> MotherDuck COPY" in done.stdout, done.stdout SLOW_VARIANT = "MotherDuck runs DuckDB 1.5, which stores VARIANT unshredded: queries on its fields can be much slower than on JSON until MotherDuck moves to DuckDB 2.0; json_type = \"json\" keeps JSON" diff --git a/website/src/content/docs/docs/configuration.mdx b/website/src/content/docs/docs/configuration.mdx index 9bfb4d0ef..617de88b2 100644 --- a/website/src/content/docs/docs/configuration.mdx +++ b/website/src/content/docs/docs/configuration.mdx @@ -117,6 +117,7 @@ DuckDB 1.5 process: install `sqlload[motherduck]`; see [MotherDuck](/docs/target |-----|---------| | `database` | The MotherDuck database, with or without `md:`. Required, here or in the target. It can't hold the token. | | `token` | A MotherDuck access token. Use `${ENV:NAME}`. Default: the `motherduck_token` or `MOTHERDUCK_TOKEN` environment variable. | +| `load_method` | `{ type = "local" }` (default): zstd Parquet chunks sent from this machine. `{ type = "external_stage", url = "s3://…" \| "gs://…", connection = "…", keep_staged_files = false }`: chunks written to a bucket, which MotherDuck's servers read. See [Staging in a bucket](/docs/targets/motherduck/#staging-in-a-bucket). | ### `s3` diff --git a/website/src/content/docs/docs/targets/motherduck.mdx b/website/src/content/docs/docs/targets/motherduck.mdx index e38766705..985742352 100644 --- a/website/src/content/docs/docs/targets/motherduck.mdx +++ b/website/src/content/docs/docs/targets/motherduck.mdx @@ -58,6 +58,7 @@ loader_schema = "raw" |-----|---------| | `database` | The MotherDuck database, with or without `md:`. Required, here or in the target. It can't hold the token. | | `token` | A MotherDuck access token. Use `${ENV:NAME}`. Default: the `motherduck_token` or `MOTHERDUCK_TOKEN` environment variable. | +| `load_method` | How rows reach MotherDuck: `{ type = "local" }` (default) or `{ type = "external_stage", … }`. See [Staging in a bucket](#staging-in-a-bucket). | A target's `database` overrides the connection's, so one connection can serve, say, a `shop_dev` and a `shop` database. The database must exist. MotherDuck attaches the account's other databases @@ -69,9 +70,51 @@ connection installs MotherDuck's DuckDB extension into `~/.duckdb`, which needs ## How data arrives -SQLLoad writes each batch to a temporary Parquet file on this machine, and the separate process -inserts it with `read_parquet` inside the batch's transaction, then the file is removed. `plan` names -the route for every table. +SQLLoad gathers the rows into chunks of about 122,880 rows, DuckDB's row group size, however small +the pages a source yields (an API's pages of 50 rows, say). It writes each chunk to a temporary zstd +Parquet file on this machine, and the separate process loads it with `COPY` inside the batch's +transaction, then the file is removed. `VARIANT` columns are typed `JSON` in the file, so they load +parsed. `plan` names the route for every table: + +```console +$ sqlload plan -v +… + ├── route read as Arrow -> zstd Parquet, about 122,880 rows a file -> MotherDuck COPY +``` + +### Staging in a bucket + +With `load_method = { type = "external_stage", … }` on the connection, SQLLoad writes each chunk +under an S3 or GCS prefix instead, and MotherDuck's servers read the chunks there in the commit +(`read_parquet` with `MD_RUN = REMOTE`), so this machine only uploads to the bucket. It suits a +flaky or slow uplink to MotherDuck, a machine beside the bucket, or keeping the loaded files as an +archive. The default, `{ type = "local" }`, sends the chunks from this machine as above: + +```toml +[connections.shop] +kind = "motherduck" +database = "shop" +load_method = { type = "external_stage", url = "s3://example-bucket/motherduck/" } +``` + +| Key | Meaning | +|-----|---------| +| `url` | The prefix, `s3://bucket/prefix/` or `gs://bucket/prefix/`. Chunks go in a folder named for the load id, then one for the table. | +| `connection` | An [`s3`](/docs/configuration/#s3) or [`gcs`](/docs/configuration/#gcs) connection to the bucket, for its region, profile and keys. Required for `gs://`: MotherDuck reads GCS only with the connection's HMAC key. Without it, S3 keys come from the environment, the shared credentials file or the AWS credential chain. | +| `keep_staged_files` | Keep each chunk once MotherDuck has read it, as an archive of what was sent. Default `false`: SQLLoad removes them. A chunk of a batch that fails before MotherDuck reads it into its table is always removed; on a table loaded through a staging table, such as a `merge`, the chunks are read into the staging table first, so a commit that fails after that keeps them. | + +SQLLoad writes with the bucket's keys, and MotherDuck's servers read with the same keys, as a +secret that lasts only for the load's session, as for [Parquet files in S3](#parquet-files-in-s3). +`plan` writes a one-row file under the prefix, has MotherDuck's servers read it, and removes it; it +fails each table with the fix when either step fails, and never falls back to sending from this +machine. A custom S3 endpoint is refused, since MotherDuck's servers can't reach it. Each batch +fails unless MotherDuck reports loading as many rows as the chunks' footers hold. A load that is killed leaves its chunks under its load id; a lifecycle rule on the prefix removes such leftovers: + +```console +$ sqlload plan -v +… + ├── route read as Arrow -> zstd Parquet, about 122,880 rows a file -> s3://example-bucket/motherduck/ -> MotherDuck reads them on its servers +``` ### Parquet files in S3