Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,29 @@ use crate::adapters::databricks::main::stage::_writer_accepts::writer_accepts as
use crate::adapters::dispatch::models::Destination;
use crate::adapters::snowflake::main::stage::_writer_accepts::writer_accepts as snowflake_accepts;
use crate::adapters::snowflake::models::StageRoute;
#[cfg(feature = "duckdb")]
use crate::memory_budget::constants::DUCKDB_ROW_GROUP_ROWS;
#[cfg(feature = "duckdb")]
use crate::number_text::main::thousands::thousands;
use crate::routing::models::WriterAccepts;
use crate::routing::models::{FileFormat, Store};

/// What `destination` takes over the chosen `stage`: Arrow batches (DuckDB also reads Parquet in S3, GCS or on this machine itself, MotherDuck Parquet in S3 or GCS on its servers, and ClickHouse Parquet in S3, BigQuery Parquet in GCS), or what its stage route takes.
/// What `destination` takes over the chosen `stage`: Arrow batches (DuckDB also reads Parquet in S3, GCS or on this machine itself; MotherDuck takes zstd Parquet chunks it `COPY`s, and reads Parquet in S3 or GCS on its servers; and ClickHouse Parquet in S3, BigQuery Parquet in GCS), or what its stage route takes.
pub fn writer_accepts(destination: &Destination, stage: Option<&StageRoute>) -> WriterAccepts {
match destination {
#[cfg(feature = "duckdb")]
Destination::DuckDb(duckdb) if duckdb.motherduck => WriterAccepts {
arrow: true,
rows_into: Some(match &duckdb.stage {
None => format!(
"zstd Parquet, about {} rows a file -> MotherDuck COPY",
thousands(DUCKDB_ROW_GROUP_ROWS)
),
Some(stage) => format!(
"zstd Parquet, about {} rows a file -> {} -> MotherDuck reads them on its servers",
thousands(DUCKDB_ROW_GROUP_ROWS),
stage.folder.location
),
}),
pulls_from: vec![
Store::ObjectStore("s3://".to_owned()),
Store::ObjectStore("gs://".to_owned()),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use crate::adapters::contract::models::TableName;
use crate::adapters::databricks::main::session::_connect_databricks::connect_databricks;
use crate::adapters::dispatch::models::Destination;
#[cfg(feature = "duckdb")]
use crate::adapters::duckdb::main::_open_destination::open_destination;
use crate::adapters::duckdb::main::_open_target::open_target;
use crate::adapters::files::models::FilesDestination;
use crate::adapters::postgres::main::session::_open_postgres::open_postgres;
use crate::adapters::snowflake::main::session::_connect_snowflake::connect_snowflake;
Expand All @@ -27,13 +27,9 @@ pub async fn connect_destination(
.ok_or_else(|| LoaderError::Invalid(format!("target {name:?} does not exist")))?;
let destination = match project.connections.get(&target.connection) {
#[cfg(feature = "duckdb")]
Some(
connection @ (ConnectionConfig::DuckDb { .. } | ConnectionConfig::MotherDuck { .. }),
) => Destination::DuckDb(open_destination(
connection,
target.database.as_deref(),
project.memory.budget,
)?),
Some(ConnectionConfig::DuckDb { .. } | ConnectionConfig::MotherDuck { .. }) => {
Destination::DuckDb(open_target(project, target)?)
}
#[cfg(not(feature = "duckdb"))]
Some(ConnectionConfig::DuckDb { .. } | ConnectionConfig::MotherDuck { .. }) => {
return Err(LoaderError::Invalid(format!(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,13 @@
use crate::adapters::contract::models::{TableName, WriteMode};
use crate::adapters::dispatch::models::Destination;
#[cfg(feature = "duckdb")]
use crate::adapters::duckdb::main::_stage_problems::stage_problems;
#[cfg(feature = "duckdb")]
use crate::adapters::duckdb::main::_variant_problems::variant_problems;
use crate::adapters::postgres::main::session::_write_problems::write_problems as postgres_write_problems;
use crate::loader::errors::LoaderError;

/// Why `mode` cannot run against `table` (`exists` when it does) (problems), and what it may fail on (warnings); `variants` when the table has `VARIANT` columns, which MotherDuck needs a recent enough DuckDB for. None where the warehouse's writes need nothing of the table or its server.
/// Why `mode` cannot run against `table` (`exists` when it does) (problems), and what it may fail on (warnings); `variants` when the table has `VARIANT` columns, which MotherDuck needs a recent enough DuckDB for; and a MotherDuck `external_stage` SQLLoad cannot write or MotherDuck cannot read. None where the warehouse's writes need nothing of the table or its server.
pub async fn write_problems(
destination: &Destination,
table: &TableName,
Expand All @@ -20,7 +22,14 @@ pub async fn write_problems(
postgres_write_problems(postgres, table, (mode, exists)).await
}
#[cfg(feature = "duckdb")]
Destination::DuckDb(duckdb) if variants => Ok(variant_problems(duckdb).await),
Destination::DuckDb(duckdb) => {
let (mut problems, warnings) = match variants {
true => variant_problems(duckdb).await,
false => (Vec::new(), Vec::new()),
};
problems.extend(stage_problems(duckdb).await);
Ok((problems, warnings))
}
_ => Ok((Vec::new(), Vec::new())),
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ pub fn open_table_writer(
Destination::DuckDb(duckdb) => Ok(Writer::DuckDb(DuckDbWriter {
destination: duckdb.clone(),
open_transaction: false,
staged: Default::default(),
})),
Destination::Snowflake(snowflake) => {
let external_folder = match &route {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,6 @@ pub(in crate::adapters::duckdb) fn shared_destination(
own_limit: None,
unflushed: Arc::default(),
secrets: Arc::default(),
stage: None,
})
}
138 changes: 135 additions & 3 deletions crates/sqlload/src/adapters/duckdb/_helpers/table_writer.rs
Original file line number Diff line number Diff line change
@@ -1,20 +1,27 @@
//! DuckDB's write session: Arrow appends, in one transaction per batch when the batch goes straight into the table or a build table, and checkpoints under the memory budget.
//! DuckDB's write session: Arrow appends (or a MotherDuck `external_stage`'s chunks), in one transaction per batch when the batch goes straight into the table or a build table, and checkpoints under the memory budget.

use arrow_array::RecordBatch;

use crate::adapters::contract::constants::{BEGIN_STATEMENT, COMMIT_STATEMENT};
use crate::adapters::contract::models::WriteTarget;
use crate::adapters::contract::types::SqlAdapter;
use crate::adapters::contract::types::TableWriter;
use crate::adapters::duckdb::constants::{
CHECKPOINT_BUSY, CHECKPOINT_RETRY, CHECKPOINT_STATEMENT, CHECKPOINT_WAIT,
};
use crate::adapters::duckdb::main::_append_batch::append_batch;
use crate::adapters::duckdb::main::_execute_statements::execute_statements;
use crate::adapters::duckdb::models::DuckDbWriter;
use crate::adapters::duckdb::main::_open_bucket::open_bucket;
use crate::adapters::duckdb::main::_pull_files::pull_files;
use crate::adapters::duckdb::models::{DuckDbSql, DuckDbWriter, ExternalStage};
use crate::adapters::motherduck_bridge::main::_chunk_parquet::chunk_parquet;
use crate::column_types::main::is_variant::is_variant;
use crate::file_store::main::delete_files::delete_files;
use crate::file_store::main::write_file::write_file;
use crate::loader::errors::LoaderError;
use crate::memory_budget::main::large_values::large_values;
use crate::memory_budget::models::DuckDbShare;
use std::sync::Arc;
use std::sync::atomic::Ordering;

impl TableWriter for DuckDbWriter {
Expand All @@ -27,6 +34,9 @@ impl TableWriter for DuckDbWriter {
target: &WriteTarget<'_>,
batch: RecordBatch,
) -> Result<(), LoaderError> {
if let Some(stage) = self.destination.stage.clone() {
return self.write_chunk(&stage, target, batch).await;
}
let into = target.staging.unwrap_or(target.table);
if self.destination.memory.is_some()
&& let Some(rows) = large_values(&batch)
Expand Down Expand Up @@ -62,17 +72,19 @@ impl TableWriter for DuckDbWriter {
}

async fn load_batch(&mut self, _target: &WriteTarget<'_>) -> Result<(), LoaderError> {
self.load_chunks().await?;
if self.open_transaction {
execute_statements(&self.destination, vec![COMMIT_STATEMENT.to_owned()]).await?;
self.open_transaction = false;
self.remove_chunks(true).await;
}
Ok(())
}

fn next_batch(&mut self) {}

async fn settle_rows(&mut self) -> Result<(), LoaderError> {
Ok(())
self.load_chunks().await
}

fn reopen_rows(&mut self) {}
Expand All @@ -84,6 +96,7 @@ impl TableWriter for DuckDbWriter {
{
tracing::warn!(%error, "cannot roll back the open batch");
}
self.remove_chunks(false).await;
}

async fn remove_batch_files(&mut self, _target: &WriteTarget<'_>) {}
Expand Down Expand Up @@ -126,3 +139,122 @@ impl TableWriter for DuckDbWriter {
Ok(())
}
}

impl DuckDbWriter {
/// Write `batch` to `stage` as one zstd Parquet chunk (`VARIANT` columns typed `JSON`), which waits there for [`Self::load_chunks`]; the batch's transaction, when it goes straight into the table or a build table, opens only when MotherDuck reads them, right after the bucket's secret is set, so it never reads with keys set before a long upload.
pub(in crate::adapters) async fn write_chunk(
&mut self,
stage: &Arc<ExternalStage>,
target: &WriteTarget<'_>,
batch: RecordBatch,
) -> Result<(), LoaderError> {
let into = target.staging.unwrap_or(target.table);
let names: Vec<String> = (batch.schema().fields().iter())
.map(|field| DuckDbSql.render_identifier(field.name()))
.collect();
let insert = format!(
"INSERT INTO {} ({list}) SELECT {list} FROM read_parquet(",
DuckDbSql.render_qualified_name(into),
list = names.join(", ")
);
if insert != self.staged.insert {
self.load_chunks().await?;
self.staged.insert = insert;
}
self.staged.transaction = target.direct || target.builds;
let json: Vec<String> = (target.write_columns.iter())
.filter(|column| is_variant(&column.column_type))
.map(|column| column.name.clone())
.collect();
let rows = batch.num_rows() as u64;
let body = tokio::task::spawn_blocking(move || {
let mut body: Vec<u8> = Vec::new();
chunk_parquet(&mut body, &batch, &json).map(|()| body)
})
.await
.map_err(|error| LoaderError::Invalid(format!("the Parquet thread failed: {error}")))?
.map_err(LoaderError::Invalid)?;
self.staged.written += 1;
let key = format!(
"{}/{}/{}-{}.parquet",
target.load_id,
key_safe(&format!("{}.{}", into.schema, into.table)),
target.batch,
self.staged.written
);
write_file(&stage.folder, &key, body.into())
.await
.map_err(|error| LoaderError::Invalid(error.to_string()))?;
self.staged.waiting.push((key, rows));
Ok(())
}

/// Have MotherDuck's servers read the waiting chunks into their table (`MD_RUN = REMOTE`), failing unless it reports as many rows as their footers hold. Outside a transaction the rows are in, and the files go now; inside one, once it commits ([`Self::remove_chunks`]).
pub(in crate::adapters) async fn load_chunks(&mut self) -> Result<(), LoaderError> {
let Some(stage) = self.destination.stage.clone() else {
return Ok(());
};
if self.staged.waiting.is_empty() {
return Ok(());
}
if !self.open_transaction {
open_bucket(&self.destination, &stage.folder.location.store).await?;
}
if self.staged.transaction && !self.open_transaction {
execute_statements(&self.destination, vec![BEGIN_STATEMENT.to_owned()]).await?;
self.open_transaction = true;
}
let waiting = std::mem::take(&mut self.staged.waiting);
let urls: Vec<String> = (waiting.iter())
.map(|(key, _)| {
DuckDbSql.render_loader_value_literal(&format!("{}{key}", stage.folder.location))
})
.collect();
let insert = format!(
"{}[{}], hive_partitioning = false, MD_RUN = REMOTE)",
self.staged.insert,
urls.join(", ")
);
let keys: Vec<String> = waiting.iter().map(|(key, _)| key.clone()).collect();
self.staged.loaded.extend(keys);
let rows = pull_files(&self.destination, insert).await?;
let expected: u64 = waiting.iter().map(|(_, rows)| rows).sum();
if rows != expected {
return Err(LoaderError::Invalid(format!(
"MotherDuck loaded {rows} rows from the chunks staged in {}, but their footers hold {expected}; the batch is not committed",
stage.folder.location
)));
}
if !self.open_transaction {
self.remove_chunks(true).await;
}
Ok(())
}

/// Remove the chunks MotherDuck read once they `committed`, unless the stage keeps them, and always those of a batch that failed (read or still waiting); a file that cannot be removed is only logged.
pub(in crate::adapters) async fn remove_chunks(&mut self, committed: bool) {
let Some(stage) = self.destination.stage.clone() else {
return;
};
let mut keys = std::mem::take(&mut self.staged.loaded);
keys.extend(self.staged.waiting.drain(..).map(|(key, _)| key));
if (stage.keep && committed) || keys.is_empty() {
return;
}
if let Err(error) = delete_files(&stage.folder, &keys).await {
tracing::warn!(stage = %stage.folder.location, %error, "staged chunks were left in the external_stage");
}
}
}

/// `name` as a part of an object key that `read_parquet` never takes for a glob: letters, digits, `_`, `-` and `.` as they are, any other byte as `~` and its hex.
fn key_safe(name: &str) -> String {
name.bytes()
.map(
|byte| match byte.is_ascii_alphanumeric() || b"_-.".contains(&byte) {
true => char::from(byte).to_string(),
false => format!("~{byte:02x}"),
},
)
.collect()
}
7 changes: 4 additions & 3 deletions crates/sqlload/src/adapters/duckdb/main/_append_batch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use crate::adapters::duckdb::errors::DuckDbError;
use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination};
use crate::adapters::motherduck_bridge::main::_append_batch::append_batch as bridge_append;

/// Insert `batch` into `table` by column name, inside the connection's open transaction if it has one: columns the batch lacks get their `DEFAULT`, and values are converted to the column's type, so bad JSON or UUIDs fail here; a column among `variants` (a `VARIANT` column) takes each value as JSON, so objects, arrays, numbers and booleans keep their types. MotherDuck's bridge reads the batch from a Parquet file instead.
/// Insert `batch` into `table` by column name, inside the connection's open transaction if it has one: columns the batch lacks get their `DEFAULT`, and values are converted to the column's type, so bad JSON or UUIDs fail here; a column among `variants` (a `VARIANT` column) takes each value as JSON, so objects, arrays, numbers and booleans keep their types. MotherDuck's bridge `COPY`s the batch from a zstd Parquet file instead, its `VARIANT` columns typed `JSON` there.
pub(in crate::adapters) async fn append_batch(
destination: &DuckDbDestination,
(table, variants): (&TableName, &[&str]),
Expand All @@ -37,8 +37,9 @@ pub(in crate::adapters) async fn append_batch(
.collect();
let (list, values) = (names.join(", "), values.join(", "));
if let DuckDbConnection::Bridge(bridge) = &destination.connection {
let insert = format!("INSERT INTO {target} ({list}) SELECT {values} FROM read_parquet(?)");
return Ok(bridge_append(bridge, insert, batch).await?);
let copy = format!("COPY {target} ({list}) FROM ? (FORMAT parquet)");
let json: Vec<String> = variants.iter().map(|name| (*name).to_owned()).collect();
return Ok(bridge_append(bridge, copy, (batch, json)).await?);
}
with_connection(destination, move |connection| {
connection
Expand Down
6 changes: 5 additions & 1 deletion crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ pub(crate) fn open_motherduck(
target_database: Option<&str>,
(python, memory): (Option<&str>, Option<DuckDbShare>),
) -> Result<DuckDbDestination, DuckDbError> {
let ConnectionConfig::MotherDuck { database, token } = connection else {
let ConnectionConfig::MotherDuck {
database, token, ..
} = connection
else {
return Err(DuckDbError::MotherDuck(format!(
"a {} connection is not a MotherDuck one",
connection.kind()
Expand Down Expand Up @@ -52,5 +55,6 @@ pub(crate) fn open_motherduck(
own_limit: None,
unflushed: Arc::default(),
secrets: Arc::default(),
stage: None,
})
}
38 changes: 38 additions & 0 deletions crates/sqlload/src/adapters/duckdb/main/_open_target.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
//! Open the DuckDB or MotherDuck database a target loads into.

use std::sync::Arc;

use crate::adapters::duckdb::main::_open_destination::open_destination;
use crate::adapters::duckdb::models::{DuckDbDestination, ExternalStage};
use crate::file_store::main::open_folder::open_folder;
use crate::loader::errors::LoaderError;
use crate::project::main::motherduck_stage::motherduck_stage;
use crate::project::models::{ProjectConfig, TargetConfig};

/// Open the database of `target`, whose connection is a `duckdb` or `motherduck` one, with the `external_stage` of a MotherDuck connection and its bucket's client.
pub(in crate::adapters) fn open_target(
project: &ProjectConfig,
target: &TargetConfig,
) -> Result<DuckDbDestination, LoaderError> {
let connection = (project.connections.get(&target.connection)).ok_or_else(|| {
LoaderError::Invalid(format!("connection {:?} does not exist", target.connection))
})?;
let destination = open_destination(
connection,
target.database.as_deref(),
project.memory.budget,
)?;
let Some((location, keep)) =
motherduck_stage(project, &target.connection).map_err(LoaderError::Invalid)?
else {
return Ok(destination);
};
Ok(DuckDbDestination {
stage: Some(Arc::new(ExternalStage {
folder: open_folder(&location)?,
keep,
checked: tokio::sync::OnceCell::new(),
})),
..destination
})
}
1 change: 1 addition & 0 deletions crates/sqlload/src/adapters/duckdb/main/_own_connection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,5 +60,6 @@ pub(in crate::adapters) fn own_connection(
own_limit: destination.own_limit,
unflushed: destination.unflushed.clone(),
secrets: destination.secrets.clone(),
stage: destination.stage.clone(),
})
}
Loading
Loading