diff --git a/crates/sqlload/src/adapters/dispatch/main/dialect/source_dialect.rs b/crates/sqlload/src/adapters/dispatch/main/dialect/source_dialect.rs index 134f10ca3..dc90a40a8 100644 --- a/crates/sqlload/src/adapters/dispatch/main/dialect/source_dialect.rs +++ b/crates/sqlload/src/adapters/dispatch/main/dialect/source_dialect.rs @@ -3,6 +3,7 @@ use crate::bigquery_source::constants::BIGQUERY_SOURCE; use crate::column_types::models::Warehouse; use crate::databricks_source::constants::DATABRICKS_SOURCE; +use crate::duckdb_source::constants::{DUCKDB_SOURCE, MOTHERDUCK_SOURCE}; use crate::loader::constants::SNOWFLAKE_SOURCE; /// The warehouse whose SQL a warehouse source of `system` (its kind as written) filters in; `None` for a system without one. @@ -11,6 +12,7 @@ pub fn source_dialect(system: &str) -> Option { SNOWFLAKE_SOURCE => Some(Warehouse::Snowflake), BIGQUERY_SOURCE => Some(Warehouse::BigQuery), DATABRICKS_SOURCE => Some(Warehouse::Databricks), + DUCKDB_SOURCE | MOTHERDUCK_SOURCE => Some(Warehouse::DuckDb), _ => None, } } diff --git a/crates/sqlload/src/adapters/dispatch/main/session/mod.rs b/crates/sqlload/src/adapters/dispatch/main/session/mod.rs index 156123185..ebd17fce5 100644 --- a/crates/sqlload/src/adapters/dispatch/main/session/mod.rs +++ b/crates/sqlload/src/adapters/dispatch/main/session/mod.rs @@ -7,6 +7,8 @@ pub mod execute; pub mod execute_ddl; pub mod helper_process; pub mod live_columns; +#[cfg(feature = "duckdb")] +pub mod open_source_bridge; pub mod recover_unfinished; pub mod single_text; pub mod table_destination; diff --git a/crates/sqlload/src/adapters/dispatch/main/session/open_source_bridge.rs b/crates/sqlload/src/adapters/dispatch/main/session/open_source_bridge.rs new file mode 100644 index 000000000..8822305bc --- /dev/null +++ b/crates/sqlload/src/adapters/dispatch/main/session/open_source_bridge.rs @@ -0,0 +1,18 @@ +//! Open a MotherDuck database a source reads. + +use crate::adapters::duckdb::errors::DuckDbError; +use crate::adapters::duckdb::main::_open_motherduck::open_motherduck; +use crate::adapters::duckdb::models::DuckDbConnection; +use crate::adapters::motherduck_bridge::models::BridgeConnection; +use crate::project::models::ConnectionConfig; + +/// The bridge connection to the database of `connection`, a `motherduck` one, signed in as a MotherDuck target is. +pub fn open_source_bridge(connection: &ConnectionConfig) -> Result { + let opened = open_motherduck(connection, None, (None, None))?; + match opened.connection { + DuckDbConnection::Bridge(bridge) => Ok(bridge), + DuckDbConnection::Local(_) => Err(DuckDbError::MotherDuck( + "a MotherDuck database opens on the MotherDuck bridge".to_owned(), + )), + } +} diff --git a/crates/sqlload/src/adapters/duckdb/main/_describe_table.rs b/crates/sqlload/src/adapters/duckdb/main/_describe_table.rs index e5d8d4f5d..02928e596 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_describe_table.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_describe_table.rs @@ -6,7 +6,7 @@ use crate::adapters::duckdb::_helpers::blocking::with_connection; use crate::adapters::duckdb::constants::DESCRIBE_TABLE_SQL; use crate::adapters::duckdb::errors::DuckDbError; use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination, DuckDbSql}; -use crate::adapters::motherduck_bridge::main::_run_statement::run_statement; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; use crate::column_types::models::LiveColumnType; use crate::schema_plan::models::LiveColumn; diff --git a/crates/sqlload/src/adapters/duckdb/main/_execute_statements.rs b/crates/sqlload/src/adapters/duckdb/main/_execute_statements.rs index ae256a6ff..79952edf0 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_execute_statements.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_execute_statements.rs @@ -3,7 +3,7 @@ use crate::adapters::duckdb::_helpers::blocking::with_connection; use crate::adapters::duckdb::errors::DuckDbError; use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination}; -use crate::adapters::motherduck_bridge::main::_run_statement::run_statement; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; /// Run each statement in order, stopping at the first failure (an open transaction is rolled back). pub(in crate::adapters) async fn execute_statements( diff --git a/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs b/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs index bdec532ad..456faa708 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs @@ -12,7 +12,7 @@ use crate::memory_budget::models::DuckDbShare; use crate::project::models::ConnectionConfig; /// Open the MotherDuck database of `connection` (`target_database` when the target names one) through DuckDB's `md:` connection on the MotherDuck bridge, signed in with its token, else the token in the `motherduck_token` or `MOTHERDUCK_TOKEN` variable. Without a token MotherDuck would wait for a browser sign-in, so it is refused instead. `python` runs the bridge; `None` picks it as the bridge does. Under `memory`, its part of the run's memory budget, the bridge's DuckDB gets that `memory_limit` and thread count, and commits are cut into pieces that fit it; a table's row groups set no limit, as MotherDuck checkpoints them on its servers. -pub(in crate::adapters) fn open_motherduck( +pub(crate) fn open_motherduck( connection: &ConnectionConfig, target_database: Option<&str>, (python, memory): (Option<&str>, Option), diff --git a/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs b/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs index 67bdfed6a..60224db4c 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs @@ -4,7 +4,7 @@ use std::sync::{Arc, Mutex}; use crate::adapters::duckdb::errors::DuckDbError; use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination}; -use crate::adapters::motherduck_bridge::main::_fork_connection::fork_connection; +use crate::adapters::motherduck_bridge::main::fork_connection::fork_connection; use crate::memory_budget::main::table_memory::table_memory; /// A new connection to the same database, so one table's transaction never mixes with another table's loading at the same time. Under the memory watchdog, the table it loads can have the connection's running statement interrupted. diff --git a/crates/sqlload/src/adapters/duckdb/main/_query_text.rs b/crates/sqlload/src/adapters/duckdb/main/_query_text.rs index 5abf9ec71..d653bea09 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_query_text.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_query_text.rs @@ -3,7 +3,7 @@ use crate::adapters::duckdb::_helpers::blocking::with_connection; use crate::adapters::duckdb::errors::DuckDbError; use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination}; -use crate::adapters::motherduck_bridge::main::_run_statement::run_statement; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; /// The first column of the first row of `query`, as text; `None` for SQL `NULL`. pub(in crate::adapters) async fn query_text( diff --git a/crates/sqlload/src/adapters/duckdb/main/_query_texts.rs b/crates/sqlload/src/adapters/duckdb/main/_query_texts.rs index c4e9a75de..65108b942 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_query_texts.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_query_texts.rs @@ -3,7 +3,7 @@ use crate::adapters::duckdb::_helpers::blocking::with_connection; use crate::adapters::duckdb::errors::DuckDbError; use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination}; -use crate::adapters::motherduck_bridge::main::_run_statement::run_statement; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; /// The first column of every row of `query`, as text; SQL `NULL`s are skipped. pub(in crate::adapters) async fn query_texts( diff --git a/crates/sqlload/src/adapters/duckdb/main/_table_shape.rs b/crates/sqlload/src/adapters/duckdb/main/_table_shape.rs index 1425cbba2..16d92e763 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_table_shape.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_table_shape.rs @@ -6,7 +6,7 @@ use crate::adapters::duckdb::_helpers::blocking::with_connection; use crate::adapters::duckdb::constants::{DEFINITION_QUERY, INDEXES_QUERY}; use crate::adapters::duckdb::errors::DuckDbError; use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination, DuckDbSql}; -use crate::adapters::motherduck_bridge::main::_run_statement::run_statement; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; /// `table`'s columns and constraints as DuckDB would create them, and its indexes' statements on `table` by its full name (DuckDB stores them on `schema.table`, which MotherDuck's other attached databases can make ambiguous); `None` when there is no such table. pub(in crate::adapters) async fn table_shape( diff --git a/crates/sqlload/src/adapters/motherduck_bridge/_helpers/bridge.py b/crates/sqlload/src/adapters/motherduck_bridge/_helpers/bridge.py index 359800a76..15dea9c41 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/_helpers/bridge.py +++ b/crates/sqlload/src/adapters/motherduck_bridge/_helpers/bridge.py @@ -6,6 +6,10 @@ -> {"id": 1, "ok": true, "duckdb": "1.5.6", "python": "3.12.3"} {"id": 2, "op": "run", "cursor": 3, "sql": "SELECT ...", "params": ["raw"]} -> {"id": 2, "ok": true, "rows": [["id", "BIGINT"], ...]} + {"id": 3, "op": "stream", "cursor": 4, "sql": "SELECT ...", "rows": 100000} + -> {"id": 3, "ok": true} + {"id": 4, "op": "next", "cursor": 4, "path": "/tmp/batch.arrow"} + -> {"id": 4, "ok": true, "rows": 100000}, the next batch of at most that many rows written there as an Arrow IPC file; 0 rows once all are read {"op": "close", "cursor": 3} -> no reply {"op": "interrupt", "cursor": 3} @@ -41,6 +45,22 @@ def plain(value): return str(value) +def write_next(batches, path): + """The next batch `batches` holds, written to the existing file `path` as an Arrow IPC file; its row count, 0 once every batch is read. A missing `path` fails.""" + import pyarrow.ipc + + try: + batch = batches.read_next_batch() + except StopIteration: + return 0 + # SQLLoad created the file, readable by its user only; opening it rather than creating it means a file SQLLoad has already removed (its read stopped) is never written again. + with open(path, "r+b") as sink: + sink.truncate() + with pyarrow.ipc.new_file(sink, batch.schema) as writer: + writer.write_batch(batch) + return batch.num_rows + + def main(): # Replies go to a private copy of stdout; anything else that prints (DuckDB, an extension) lands on stderr instead. replies = os.fdopen(os.dup(1), "wb", buffering=0) @@ -64,6 +84,7 @@ def failed(request, error): def serve(work, number): cursor = None + batches = None while True: request = work.get() if request is None: @@ -71,7 +92,17 @@ def serve(work, number): try: if cursor is None: cursor = opened[number] = database.cursor() + if request["op"] == "next": + reply({"id": request["id"], "ok": True, "rows": write_next(batches, request["path"])}) + continue cursor.execute(request["sql"], request.get("params") or None) + if request["op"] == "stream": + # Arrow batches straight from DuckDB's streaming result, one at a time as `next` asks, so the bridge never holds them all. + # duckdb 1.4 names the reader fetch_record_batch; 1.5 adds to_arrow_reader, and 2.0 keeps only that. + reader = getattr(cursor, "to_arrow_reader", None) or cursor.fetch_record_batch + batches = reader(request["rows"]) + reply({"id": request["id"], "ok": True}) + continue rows = cursor.fetchall() if cursor.description else [] reply({"id": request["id"], "ok": True, "rows": [[plain(value) for value in row] for row in rows]}) except BaseException as error: # noqa: BLE001 - every failure goes back to SQLLoad as its reply, or it would wait forever @@ -91,7 +122,7 @@ def serve(work, number): reply({"id": request["id"], "ok": True, "duckdb": duckdb.__version__, "python": sys.version.split()[0]}) except Exception as error: # noqa: BLE001 failed(request, error) - elif op == "run": + elif op in ("run", "stream", "next"): work = cursors.get(request["cursor"]) if work is None: work = cursors[request["cursor"]] = queue.Queue() diff --git a/crates/sqlload/src/adapters/motherduck_bridge/_helpers/process.rs b/crates/sqlload/src/adapters/motherduck_bridge/_helpers/process.rs index fb35234a0..b5da57011 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/_helpers/process.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/_helpers/process.rs @@ -10,7 +10,7 @@ use crate::adapters::motherduck_bridge::constants::{ BRIDGE_SCRIPT, PYTHON_VARIABLE, STDERR_TAIL_BYTES, SUPPORTED_DUCKDB, }; use crate::adapters::motherduck_bridge::errors::BridgeError; -use crate::adapters::motherduck_bridge::models::{BridgeProcess, Replies}; +use crate::adapters::motherduck_bridge::models::{BridgeCursor, BridgeProcess, Replies}; /// Start the bridge on `python`: a writer thread feeding its stdin (closing it, and reaping the process, once every handle is gone), a reader thread routing its replies and, once stdout and stderr have both closed or a reply is unreadable, failing what still waits, and one keeping its stderr's tail with each of `hidden` taken out. pub(crate) fn start(python: &str, hidden: Vec) -> Result, BridgeError> { @@ -100,6 +100,19 @@ pub(crate) fn send( Ok(received) } +/// Send `request` on `cursor` and wait for its reply, failed or not, or why the bridge stopped first. +pub(crate) async fn ask( + cursor: &BridgeCursor, + request: serde_json::Value, +) -> Result { + let received = send(&cursor.process, request)?; + let python = cursor.process.python.clone(); + tokio::task::spawn_blocking(move || received.recv()) + .await + .map_err(|error| BridgeError::Stopped(error.to_string()))? + .map_err(|_| BridgeError::Stopped(format!("the MotherDuck bridge on {python} stopped")))? +} + /// `reply`'s failure, as its message and its Python error type; `None` when it succeeded. pub(crate) fn failure(reply: &serde_json::Value) -> Option<(String, String)> { if reply["ok"].as_bool() == Some(true) { diff --git a/crates/sqlload/src/adapters/motherduck_bridge/errors.rs b/crates/sqlload/src/adapters/motherduck_bridge/errors.rs index d12113b1f..0de186a1b 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/errors.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/errors.rs @@ -10,4 +10,7 @@ pub enum BridgeError { /// DuckDB refused the request, in its own words. #[error("{0}")] Engine(String), + /// What the caller's batch consumer failed with, as it said it. + #[error("{0}")] + Consumer(String), } diff --git a/crates/sqlload/src/adapters/motherduck_bridge/main/_append_batch.rs b/crates/sqlload/src/adapters/motherduck_bridge/main/_append_batch.rs index 44ce7faca..63bee0f53 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/main/_append_batch.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/main/_append_batch.rs @@ -4,7 +4,7 @@ use arrow_array::RecordBatch; use parquet::arrow::ArrowWriter; use crate::adapters::motherduck_bridge::errors::BridgeError; -use crate::adapters::motherduck_bridge::main::_run_statement::run_statement; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; use crate::adapters::motherduck_bridge::models::BridgeConnection; /// Run `insert`, whose one `?` is the path of a Parquet file holding `batch` (such as `INSERT INTO t (a) SELECT a FROM read_parquet(?)`), on `connection`. The file is a temporary one on this machine, which DuckDB reads there and removed once the statement ends: the bridge's DuckDB is another version than SQLLoad's, so Arrow cannot cross between them in memory. diff --git a/crates/sqlload/src/adapters/motherduck_bridge/main/_fork_connection.rs b/crates/sqlload/src/adapters/motherduck_bridge/main/fork_connection.rs similarity index 85% rename from crates/sqlload/src/adapters/motherduck_bridge/main/_fork_connection.rs rename to crates/sqlload/src/adapters/motherduck_bridge/main/fork_connection.rs index 9e73a37fd..6602d926d 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/main/_fork_connection.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/main/fork_connection.rs @@ -6,7 +6,7 @@ use std::sync::atomic::Ordering; use crate::adapters::motherduck_bridge::models::{BridgeConnection, BridgeCursor}; /// A new connection to the database `connection` opened, on the same bridge: its own DuckDB cursor, so its transactions never mix with another connection's. -pub(in crate::adapters) fn fork_connection(connection: &BridgeConnection) -> BridgeConnection { +pub(crate) fn fork_connection(connection: &BridgeConnection) -> BridgeConnection { let process = connection.cursor.process.clone(); let id = process.next_cursor.fetch_add(1, Ordering::Relaxed); BridgeConnection { diff --git a/crates/sqlload/src/adapters/motherduck_bridge/main/mod.rs b/crates/sqlload/src/adapters/motherduck_bridge/main/mod.rs index 1322dca74..8f3a94a78 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/main/mod.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/main/mod.rs @@ -1,4 +1,5 @@ pub(in crate::adapters) mod _append_batch; -pub(in crate::adapters) mod _fork_connection; pub(in crate::adapters) mod _open_bridge; -pub(in crate::adapters) mod _run_statement; +pub(crate) mod fork_connection; +pub(crate) mod run_statement; +pub(crate) mod stream_rows; diff --git a/crates/sqlload/src/adapters/motherduck_bridge/main/_run_statement.rs b/crates/sqlload/src/adapters/motherduck_bridge/main/run_statement.rs similarity index 75% rename from crates/sqlload/src/adapters/motherduck_bridge/main/_run_statement.rs rename to crates/sqlload/src/adapters/motherduck_bridge/main/run_statement.rs index 08a6b6049..a2eb727c5 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/main/_run_statement.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/main/run_statement.rs @@ -1,11 +1,11 @@ //! Run SQL on a bridge connection. -use crate::adapters::motherduck_bridge::_helpers::process::{failure, send}; +use crate::adapters::motherduck_bridge::_helpers::process::{ask, failure}; use crate::adapters::motherduck_bridge::errors::BridgeError; use crate::adapters::motherduck_bridge::models::BridgeConnection; /// Run `sql` (one statement or several, separated by `;`) with text `params` bound to its `?`s, after every earlier statement on `connection`, and return the last statement's rows: each value as text (numbers and booleans as their SQL text), `None` for `NULL`. -pub(in crate::adapters) async fn run_statement( +pub(crate) async fn run_statement( connection: &BridgeConnection, sql: String, params: Vec, @@ -13,14 +13,7 @@ pub(in crate::adapters) async fn run_statement( let cursor = &connection.cursor; let request = serde_json::json!({"op": "run", "cursor": cursor.id, "sql": sql, "params": params}); - let received = send(&cursor.process, request)?; - let python = cursor.process.python.clone(); - let reply = tokio::task::spawn_blocking(move || received.recv()) - .await - .map_err(|error| BridgeError::Stopped(error.to_string()))? - .map_err(|_| { - BridgeError::Stopped(format!("the MotherDuck bridge on {python} stopped")) - })??; + let reply = ask(cursor, request).await?; if let Some((error, _)) = failure(&reply) { return Err(BridgeError::Engine(error)); } diff --git a/crates/sqlload/src/adapters/motherduck_bridge/main/stream_rows.rs b/crates/sqlload/src/adapters/motherduck_bridge/main/stream_rows.rs new file mode 100644 index 000000000..bbdc3cd4a --- /dev/null +++ b/crates/sqlload/src/adapters/motherduck_bridge/main/stream_rows.rs @@ -0,0 +1,63 @@ +//! Stream a query's rows from the bridge as Arrow batches. + +use arrow_array::RecordBatch; +use arrow_ipc::reader::FileReader; + +use crate::adapters::motherduck_bridge::_helpers::process::{ask, failure}; +use crate::adapters::motherduck_bridge::constants::{MISSING_PACKAGE_ERRORS, PYTHON_VARIABLE}; +use crate::adapters::motherduck_bridge::errors::BridgeError; +use crate::adapters::motherduck_bridge::models::BridgeConnection; + +/// Run `sql` on `connection` and pass its rows to `on_batch` in batches of at most `batch_rows`; returns how many rows there were. The bridge reads one batch at a time from DuckDB's streaming result and hands it over in a temporary Arrow IPC file on this machine, removed once read: its DuckDB is another version than SQLLoad's, so Arrow cannot cross between them in memory. +pub(crate) async fn stream_rows( + connection: &BridgeConnection, + sql: String, + batch_rows: usize, + mut on_batch: impl AsyncFnMut(RecordBatch) -> Result<(), String>, +) -> Result { + let cursor = &connection.cursor; + let python = &cursor.process.python; + let stream = + serde_json::json!({"op": "stream", "cursor": cursor.id, "sql": sql, "rows": batch_rows}); + if let Some((error, _)) = failure(&ask(cursor, stream).await?) { + return Err(BridgeError::Engine(error)); + } + let mut rows: u64 = 0; + loop { + let file = tempfile::Builder::new() + .prefix("sqlload-motherduck-") + .suffix(".arrow") + .tempfile() + .map_err(|error| { + BridgeError::Setup(format!( + "cannot make a file for a MotherDuck batch: {error}" + )) + })?; + let next = serde_json::json!({"op": "next", "cursor": cursor.id, "path": file.path().display().to_string()}); + let reply = ask(cursor, next).await?; + if let Some((error, kind)) = failure(&reply) { + return Err(match MISSING_PACKAGE_ERRORS.contains(&kind.as_str()) { + true => BridgeError::Setup(format!( + "reading from MotherDuck needs Python's pyarrow package beside duckdb, which {python} lacks ({error}): install sqlload[motherduck] with pip or uv, or set {PYTHON_VARIABLE} to a Python that has both" + )), + false => BridgeError::Engine(error), + }); + } + if reply["rows"].as_u64().unwrap_or_default() == 0 { + return Ok(rows); + } + let batches = tokio::task::spawn_blocking(move || { + let reader = FileReader::try_new(file.reopen()?, None)?; + reader.collect::, _>>() + }) + .await + .map_err(|error| BridgeError::Stopped(error.to_string()))? + .map_err(|error: arrow_schema::ArrowError| { + BridgeError::Engine(format!("cannot read a MotherDuck batch: {error}")) + })?; + for batch in batches { + rows += batch.num_rows() as u64; + on_batch(batch).await.map_err(BridgeError::Consumer)?; + } + } +} diff --git a/crates/sqlload/src/duckdb_source/_helpers/catalog.rs b/crates/sqlload/src/duckdb_source/_helpers/catalog.rs new file mode 100644 index 000000000..cf6158dc8 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/_helpers/catalog.rs @@ -0,0 +1,94 @@ +//! The tables and views `include` matches, as source tables from DuckDB's catalog functions. + +use std::collections::BTreeMap; + +use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; + +use crate::duckdb_source::_helpers::connection::TextRows; +use crate::duckdb_source::_helpers::type_map::source_column; +use crate::duckdb_source::constants::{CATALOG_QUERIES, NOT_NULL, TRUE_TEXT}; +use crate::duckdb_source::errors::DuckDbSourceError; +use crate::postgres_source::models::SourceTable; + +/// The database's tables and views whose `schema.table` matches `include` and not `exclude` (exactly, as DuckDB keeps names), sorted, from the answers to `CATALOG_QUERIES` in order: the tables and views, their columns, and their primary keys. +pub(crate) fn catalog_tables( + answers: &[TextRows], + (include, exclude): (&[String], &[String]), +) -> Result, DuckDbSourceError> { + let (included, excluded) = (matcher(include)?, matcher(exclude)?); + let [listed, columns, keys] = answers else { + return Err(DuckDbSourceError::Invalid(format!( + "the catalog answered {} of its {} queries", + answers.len(), + CATALOG_QUERIES.len() + ))); + }; + let text = |row: &[Option], index: usize| { + row.get(index).cloned().flatten().unwrap_or_default() + }; + let mut tables: BTreeMap<(String, String), SourceTable> = BTreeMap::new(); + for row in listed { + let (schema, name) = (text(row, 0), text(row, 1)); + let qualified = format!("{schema}.{name}"); + if !included.is_match(&qualified) || excluded.is_match(&qualified) { + continue; + } + tables.insert( + (schema.clone(), name.clone()), + SourceTable { + split_pages: None, + schema, + name, + columns: Vec::new(), + primary_key: Vec::new(), + indexed_columns: Vec::new(), + estimated_rows: text(row, 3).parse().unwrap_or(-1), + estimated_bytes: -1, + comment: Some(text(row, 2)).filter(|comment| !comment.is_empty()), + listed_keys: Vec::new(), + }, + ); + } + for row in columns { + if let Some(table) = tables.get_mut(&(text(row, 0), text(row, 1))) { + let nullable = text(row, 4) == TRUE_TEXT; + table + .columns + .push(source_column(&text(row, 2), &text(row, 3), nullable)); + } + } + for row in keys { + if let Some(table) = tables.get_mut(&(text(row, 0), text(row, 1))) { + table.primary_key.push(text(row, 2)); + } + } + Ok(tables.into_values().collect()) +} + +/// Each result column a `DESCRIBE` answered: its name, DuckDB type and whether it may be NULL. +pub(crate) fn described_columns(rows: &TextRows) -> Vec<(String, String, bool)> { + rows.iter() + .map(|row| { + let text = |index: usize| row.get(index).cloned().flatten(); + ( + text(0).unwrap_or_default(), + text(1).unwrap_or_default(), + text(2).is_none_or(|null| null != NOT_NULL), + ) + }) + .collect() +} + +/// One glob set for `schema.table` patterns, matched exactly. +fn matcher(patterns: &[String]) -> Result { + let mut builder = GlobSetBuilder::new(); + for pattern in patterns { + let glob = GlobBuilder::new(pattern).build().map_err(|error| { + DuckDbSourceError::Invalid(format!("invalid table pattern {pattern:?}: {error}")) + })?; + builder.add(glob); + } + builder + .build() + .map_err(|error| DuckDbSourceError::Invalid(error.to_string())) +} diff --git a/crates/sqlload/src/duckdb_source/_helpers/connection.rs b/crates/sqlload/src/duckdb_source/_helpers/connection.rs new file mode 100644 index 000000000..57fcf2aee --- /dev/null +++ b/crates/sqlload/src/duckdb_source/_helpers/connection.rs @@ -0,0 +1,176 @@ +//! Connections to a DuckDB source's database: a file read-only, or a MotherDuck database on the MotherDuck bridge. + +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::path::PathBuf; +use std::sync::{Mutex, OnceLock, PoisonError}; + +use crate::adapters::dispatch::main::session::open_source_bridge::open_source_bridge; +use crate::adapters::motherduck_bridge::main::fork_connection::fork_connection; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; +use crate::adapters::motherduck_bridge::models::BridgeConnection; +use crate::duckdb_source::errors::DuckDbSourceError; +use crate::duckdb_source::models::{DuckDbReader, FileHold, OpenFile, ReadOnlyFile, SourceBridge}; +use crate::memory_budget::main::watch_child::watch_child; + +/// Rows of text values, `None` for `NULL`. +pub(crate) type TextRows = Vec>>; + +/// Each database file a source reads now, by its canonical path, with the one instance this process holds on it. DuckDB locks a file with POSIX locks, which belong to the process: closing a second instance on the file would drop the first one's lock too, letting another process write the file mid-read. +static OPEN_FILES: OnceLock>> = OnceLock::new(); + +/// A connection to the reader's database file, opened read-only, so the file is never changed and SQLLoad holds no write lock on it. Reads open at the same time share one instance, with a temporary folder of its own to spill into (not the default folder beside the file) and a `memory_limit` of the reader's limit for each of them, as separate instances would have had. +pub(crate) fn open_read_only(reader: &DuckDbReader) -> Result { + let path = std::fs::canonicalize(&reader.database) + .map_err(|error| DuckDbSourceError::Invalid(format!("{}: {error}", reader.database)))?; + let files = OPEN_FILES.get_or_init(Mutex::default); + let mut open = files.lock().unwrap_or_else(PoisonError::into_inner); + let (file, connection) = match open.entry(path.clone()) { + Entry::Occupied(entry) => { + let file = entry.into_mut(); + let connection = file.connection.try_clone()?; + (file, connection) + } + Entry::Vacant(entry) => { + let spill = tempfile::Builder::new() + .prefix("sqlload-duckdb-source-") + .tempdir() + .map_err(|error| DuckDbSourceError::Invalid(error.to_string()))?; + let config = duckdb::Config::default() + .access_mode(duckdb::AccessMode::ReadOnly)? + .with("temp_directory", spill.path().display().to_string())?; + let instance = duckdb::Connection::open_with_flags(&reader.database, config)?; + let connection = instance.try_clone()?; + let file = entry.insert(OpenFile { + reads: 0, + read_limit: reader.memory_limit, + connection: instance, + _spill: spill, + }); + (file, connection) + } + }; + file.reads += 1; + let limited = share_limit(file); + drop(open); + let read = ReadOnlyFile { + connection, + _hold: FileHold { path }, + }; + limited?; + Ok(read) +} + +/// Give `file`'s instance its read limit for each of its open reads; nothing without a limit. +fn share_limit(file: &OpenFile) -> Result<(), duckdb::Error> { + let Some(limit) = file.read_limit else { + return Ok(()); + }; + let total = limit.saturating_mul(file.reads as u64) / 1024; + file.connection + .execute_batch(&format!("SET memory_limit = '{total}KiB'")) +} + +/// A read ending lowers its file's shared `memory_limit` (a limit DuckDB can't lower yet stays as it was); the last read to end closes the instance, while it holds the open files' lock, so no new instance on the file opens before this one has closed. +impl Drop for FileHold { + fn drop(&mut self) { + let files = OPEN_FILES.get_or_init(Mutex::default); + let mut open = files.lock().unwrap_or_else(PoisonError::into_inner); + if let Entry::Occupied(mut entry) = open.entry(self.path.clone()) { + entry.get_mut().reads -= 1; + if entry.get().reads == 0 { + drop(entry.remove()); + } else { + let _ = share_limit(entry.get()); + } + } + } +} + +/// A connection of its own to a `motherduck` reader's database, on the reader's bridge: opened on its first use, signed in as a MotherDuck target is, its DuckDB given the reader's `memory_limit`, and its memory counted by the load's watchdog while it is open. +pub(crate) async fn open_bridge( + reader: &DuckDbReader, +) -> Result { + let Some(connection) = reader.motherduck.clone() else { + return Err(DuckDbSourceError::Invalid(format!( + "{} is a file, not a MotherDuck database", + reader.database + ))); + }; + let (memory_limit, guard) = (reader.memory_limit, reader.guard.clone()); + let shared = reader + .bridge + .get_or_try_init(|| async move { + let bridge = tokio::task::spawn_blocking(move || open_source_bridge(&connection)) + .await + .map_err(|error| { + DuckDbSourceError::Invalid(format!("opening MotherDuck stopped: {error}")) + })? + .map_err(|error| DuckDbSourceError::Invalid(error.to_string()))?; + if let Some(limit) = memory_limit { + let set = format!("SET memory_limit = '{}KiB'", limit / 1024); + run_statement(&bridge, set, Vec::new()) + .await + .map_err(DuckDbSourceError::Bridge)?; + } + let watched = guard.map(|guard| watch_child(&guard, bridge.cursor.process.pid)); + Ok::<_, DuckDbSourceError>(SourceBridge { + connection: bridge, + _watched: watched, + }) + }) + .await?; + Ok(fork_connection(&shared.connection)) +} + +/// Each of `queries`' rows, run in order on one connection to the reader's database, every value as text; a failure names the index of the query that failed. Each query casts its columns to `VARCHAR`. +pub(crate) async fn text_rows( + reader: &DuckDbReader, + queries: Vec, +) -> Result, (usize, DuckDbSourceError)> { + if reader.motherduck.is_none() { + let reader = reader.clone(); + return tokio::task::spawn_blocking(move || { + let connection = open_read_only(&reader).map_err(|error| (0, error))?; + queries + .iter() + .enumerate() + .map(|(index, sql)| { + local_rows(&connection.connection, sql).map_err(|error| (index, error.into())) + }) + .collect() + }) + .await + .map_err(|error| { + ( + 0, + DuckDbSourceError::Invalid(format!("the query stopped: {error}")), + ) + })?; + }; + let bridge = open_bridge(reader).await.map_err(|error| (0, error))?; + let mut answers = Vec::with_capacity(queries.len()); + for (index, sql) in queries.into_iter().enumerate() { + let rows = run_statement(&bridge, sql, Vec::new()) + .await + .map_err(|error| (index, DuckDbSourceError::Bridge(error)))?; + answers.push(rows); + } + Ok(answers) +} + +/// `sql`'s rows on a file's connection, each value as text. +fn local_rows(connection: &duckdb::Connection, sql: &str) -> Result { + let mut statement = connection.prepare(sql)?; + let mut rows = statement.query([])?; + let mut answer: TextRows = Vec::new(); + while let Some(row) = rows.next()? { + let width = row.as_ref().column_count(); + answer.push( + (0..width) + .map(|index| row.get::<_, Option>(index)) + .collect::, _>>()?, + ); + } + Ok(answer) +} diff --git a/crates/sqlload/src/duckdb_source/_helpers/convert.rs b/crates/sqlload/src/duckdb_source/_helpers/convert.rs new file mode 100644 index 000000000..cb12bf9d1 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/_helpers/convert.rs @@ -0,0 +1,82 @@ +//! Result batches in the read columns' types. + +use std::sync::Arc; + +use arrow_array::{ArrayRef, RecordBatch}; +use arrow_schema::{DataType, Schema, SchemaRef}; + +use crate::column_types::main::arrow_field::arrow_field; +use crate::column_types::main::exact_cast::exact_cast; +use crate::column_types::models::ColumnType; +use crate::postgres_source::models::ReadColumn; +use crate::row_batches::main::normalise_batch::normalise_batch; + +/// The Arrow schema the read columns are handed on in, with the columns `json` reads from lists, structs and maps. +pub(crate) fn output_schema(columns: &[ReadColumn]) -> Result<(SchemaRef, Vec), String> { + let fields = columns + .iter() + .map(|column| { + arrow_field(&column.name, &column.output) + .ok_or_else(|| format!("{} has no Arrow representation", column.name)) + }) + .collect::, String>>()?; + let json: Vec = columns + .iter() + .map(|column| column.output == ColumnType::Json) + .collect(); + Ok((Arc::new(Schema::new(fields)), json)) +} + +/// Whether DuckDB sent `array` nested (a list, array, struct or map) rather than as text. +fn nested(array: &ArrayRef) -> bool { + matches!( + array.data_type(), + DataType::Struct(_) + | DataType::List(_) + | DataType::LargeList(_) + | DataType::FixedSizeList(..) + | DataType::Map(..) + ) +} + +/// A result batch in `schema`: a nested `json` column as its JSON text, every other column cast to its type; a value that does not convert (such as a `DECIMAL` too wide for a pinned one) is an error, never NULL. +pub(crate) fn convert_batch( + batch: &RecordBatch, + (schema, json): (&SchemaRef, &[bool]), +) -> Result { + let texts: Vec<(&str, ArrayRef)> = schema + .fields() + .iter() + .zip(json) + .filter(|(_, json)| **json) + .filter_map(|(field, _)| { + Some(( + field.name().as_str(), + batch.column_by_name(field.name())?.clone(), + )) + }) + .filter(|(_, array)| nested(array)) + .collect(); + let texts = match texts.is_empty() { + true => None, + false => Some( + RecordBatch::try_from_iter(texts) + .map_err(|error| error.to_string()) + .and_then(|nested| normalise_batch(&nested, &[]))?, + ), + }; + let columns: Vec = schema + .fields() + .iter() + .map(|field| { + let read = texts + .as_ref() + .and_then(|texts| texts.column_by_name(field.name())) + .or_else(|| batch.column_by_name(field.name())) + .ok_or_else(|| format!("the result has no column {}", field.name()))?; + exact_cast(read, field.data_type()) + .map_err(|error| format!("{}: {error}", field.name())) + }) + .collect::>()?; + RecordBatch::try_new(schema.clone(), columns).map_err(|error| error.to_string()) +} diff --git a/crates/sqlload/src/duckdb_source/_helpers/mod.rs b/crates/sqlload/src/duckdb_source/_helpers/mod.rs new file mode 100644 index 000000000..3150e68d0 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod catalog; +pub(crate) mod connection; +pub(crate) mod convert; +pub(crate) mod query; +pub(crate) mod type_map; diff --git a/crates/sqlload/src/duckdb_source/_helpers/query.rs b/crates/sqlload/src/duckdb_source/_helpers/query.rs new file mode 100644 index 000000000..39a722b1c --- /dev/null +++ b/crates/sqlload/src/duckdb_source/_helpers/query.rs @@ -0,0 +1,53 @@ +//! DuckDB SQL for reading a table or query. + +use crate::duckdb_source::models::DuckDbReadRequest; +use crate::postgres_source::models::SourceTable; + +/// `name` in double quotes, with `"` doubled, as DuckDB reads a quoted identifier. +pub(crate) fn quote_identifier(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +/// `"schema"."table"` for a catalog table, or the query as a `subquery`. +fn from_item((table, sql): (&SourceTable, Option<&str>)) -> String { + match sql { + Some(sql) => subquery(sql), + None => [&table.schema, &table.name] + .map(|name| quote_identifier(name)) + .join("."), + } +} + +/// `SELECT FROM [WHERE ]`. +pub(crate) fn select_query(request: &DuckDbReadRequest<'_>) -> String { + let columns: Vec<&str> = request + .columns + .iter() + .map(|column| column.expression.as_str()) + .collect(); + let filter = request + .filter + .map(|filter| format!(" WHERE {filter}")) + .unwrap_or_default(); + format!( + "SELECT {} FROM {}{filter}", + columns.join(", "), + from_item((request.table, request.sql)) + ) +} + +/// `SELECT COUNT(*) FROM WHERE `, the table and query written as `select_query` writes them: how many rows `filter` selects. +pub(crate) fn count_query((table, sql): (&SourceTable, Option<&str>), filter: &str) -> String { + format!( + "SELECT CAST(COUNT(*) AS VARCHAR) FROM {} WHERE {filter}", + from_item((table, sql)) + ) +} + +/// Query `sql` in parentheses, as a `FROM` item: without its final semicolons, and closed on a line of its own so a trailing `--` comment ends first. `plan` describes it in this form, so a query it can't wrap fails there, not at load. +pub(crate) fn subquery(sql: &str) -> String { + let body = sql + .trim_start() + .trim_end_matches(|character: char| character == ';' || character.is_whitespace()); + format!("({body}\n)") +} diff --git a/crates/sqlload/src/duckdb_source/_helpers/type_map.rs b/crates/sqlload/src/duckdb_source/_helpers/type_map.rs new file mode 100644 index 000000000..04b8463ce --- /dev/null +++ b/crates/sqlload/src/duckdb_source/_helpers/type_map.rs @@ -0,0 +1,63 @@ +//! DuckDB column types as portable ones. + +use crate::column_types::constants::MAX_DECIMAL_PRECISION; +use crate::column_types::models::ColumnType; +use crate::postgres_source::models::{PgType, SourceColumn}; + +/// A column from its name, its type as DuckDB writes it (`duckdb_columns().data_type`, or a result's column type) and whether it may be NULL: lists, arrays, structs and maps as `json`; a type with no portable one (`HUGEINT`, `UBIGINT`, `INTERVAL`, `BIT`, `UNION`, `TIME WITH TIME ZONE`, a nanosecond timestamp, a decimal wider than 38 digits) must be pinned or excluded. +pub(crate) fn source_column(name: &str, data_type: &str, nullable: bool) -> SourceColumn { + let shown = data_type.trim().to_owned(); + let column_type = scalar_type(&shown.to_ascii_uppercase()).ok_or_else(|| { + format!( + "DuckDB {shown} has no portable type; pin one in columns, such as text, or exclude the column" + ) + }); + SourceColumn { + name: name.to_owned(), + postgres_type: shown, + pg_type: PgType::Other, + column_type, + nullable, + source_spelling: None, + repeat_of: None, + } +} + +/// Whether an upper-cased DuckDB type is a list or fixed-size array, which a pinned `array(T)` reads as a list. +pub(crate) fn is_list(shown: &str) -> bool { + shown.ends_with(']') +} + +/// The portable type of an upper-cased DuckDB type, or `None` for one with none. +fn scalar_type(shown: &str) -> Option { + if is_list(shown) { + return Some(ColumnType::Json); + } + let end = shown.find('(').unwrap_or(shown.len()); + let base = shown[..end].trim(); + Some(match base { + "BOOLEAN" => ColumnType::Bool, + "TINYINT" | "SMALLINT" | "UTINYINT" => ColumnType::SmallInt, + "INTEGER" | "BIGINT" | "USMALLINT" | "UINTEGER" => ColumnType::Int, + "FLOAT" | "DOUBLE" => ColumnType::Float, + "DECIMAL" => { + let sizes = shown[end..].trim_start_matches('(').trim_end_matches(')'); + let mut parts = sizes.split(',').map(|part| part.trim().parse::()); + let (Some(Ok(precision)), Some(Ok(scale))) = (parts.next(), parts.next()) else { + return None; + }; + (precision <= MAX_DECIMAL_PRECISION) + .then_some(ColumnType::Decimal { precision, scale })? + } + "VARCHAR" | "ENUM" => ColumnType::Text, + "BLOB" => ColumnType::Binary, + "DATE" => ColumnType::Date, + "TIME" => ColumnType::Time, + "TIMESTAMP" | "TIMESTAMP_S" | "TIMESTAMP_MS" => ColumnType::TimestampNtz, + "TIMESTAMP WITH TIME ZONE" => ColumnType::TimestampTz, + "UUID" => ColumnType::Uuid, + "JSON" => ColumnType::Json, + "STRUCT" | "MAP" => ColumnType::Json, + _ => return None, + }) +} diff --git a/crates/sqlload/src/duckdb_source/constants.rs b/crates/sqlload/src/duckdb_source/constants.rs new file mode 100644 index 000000000..f26dbe036 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/constants.rs @@ -0,0 +1,28 @@ +/// The system name a DuckDB source's tables carry in plans. +pub const DUCKDB_SOURCE: &str = "duckdb"; + +/// The system name a MotherDuck source's tables carry in plans. +pub const MOTHERDUCK_SOURCE: &str = "motherduck"; + +/// The database path DuckDB opens in memory, which holds no tables to read. +pub const MEMORY_DATABASE: &str = ":memory:"; + +/// What `DESCRIBE` writes in its `null` column for a column that is never NULL. +pub const NOT_NULL: &str = "NO"; + +/// How many result batches the read holds ahead of the consumer. +pub const BATCHES_AHEAD: usize = 4; + +/// How DuckDB writes `true` cast to `VARCHAR`. +pub const TRUE_TEXT: &str = "true"; + +/// The catalog queries, every value cast to `VARCHAR` so a file and the MotherDuck bridge answer alike, all of the opened database's own: its tables and views (schema, name, comment, estimated rows), their columns in order (schema, table, name, type, nullable), and their primary keys (schema, table, one key column a row, in key order). They use only DuckDB's own functions, none of an extension it would have to download first. +pub const CATALOG_QUERIES: [&str; 3] = [ + "SELECT schema_name, table_name, comment, CAST(estimated_size AS VARCHAR) FROM duckdb_tables() WHERE database_name = current_database() \ + UNION ALL SELECT schema_name, view_name, comment, '-1' FROM duckdb_views() WHERE database_name = current_database() AND NOT internal", + "SELECT schema_name, table_name, column_name, data_type, CAST(is_nullable AS VARCHAR) FROM duckdb_columns() WHERE database_name = current_database() ORDER BY schema_name, table_name, column_index", + "SELECT schema_name, table_name, column_name FROM (\ + SELECT schema_name, table_name, unnest(constraint_column_names) AS column_name, generate_subscripts(constraint_column_names, 1) AS position \ + FROM duckdb_constraints() WHERE database_name = current_database() AND constraint_type = 'PRIMARY KEY'\ + ) ORDER BY schema_name, table_name, position", +]; diff --git a/crates/sqlload/src/duckdb_source/errors.rs b/crates/sqlload/src/duckdb_source/errors.rs new file mode 100644 index 000000000..3efb14891 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/errors.rs @@ -0,0 +1,13 @@ +/// Why reading a DuckDB source failed. +#[derive(Debug, thiserror::Error)] +pub enum DuckDbSourceError { + #[error("DuckDB: {0}")] + DuckDb(#[from] duckdb::Error), + #[error("{0}")] + Invalid(String), + #[error("MotherDuck: {0}")] + Bridge(crate::adapters::motherduck_bridge::errors::BridgeError), + /// What the caller's batch consumer failed with, as it said it. + #[error("{0}")] + Consumer(String), +} diff --git a/crates/sqlload/src/duckdb_source/main/count_rows.rs b/crates/sqlload/src/duckdb_source/main/count_rows.rs new file mode 100644 index 000000000..1bff89b72 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/main/count_rows.rs @@ -0,0 +1,27 @@ +//! Count a table's rows that a condition selects. + +use crate::duckdb_source::_helpers::connection::text_rows; +use crate::duckdb_source::_helpers::query::count_query; +use crate::duckdb_source::errors::DuckDbSourceError; +use crate::duckdb_source::models::DuckDbReader; +use crate::postgres_source::models::SourceTable; + +/// How many rows of `table` (or of its query `sql`) `filter` selects. +pub async fn count_rows( + reader: &DuckDbReader, + (table, sql): (&SourceTable, Option<&str>), + filter: &str, +) -> Result { + let query = count_query((table, sql), filter); + let answers = text_rows(reader, vec![query.clone()]) + .await + .map_err(|(_, error)| error)?; + let count = answers + .first() + .and_then(|rows| rows.first()) + .and_then(|row| row.first().cloned().flatten()) + .unwrap_or_default(); + count.parse::().map_err(|error| { + DuckDbSourceError::Invalid(format!("{query} returned the count {count:?}: {error}")) + }) +} diff --git a/crates/sqlload/src/duckdb_source/main/describe_source.rs b/crates/sqlload/src/duckdb_source/main/describe_source.rs new file mode 100644 index 000000000..888722fdf --- /dev/null +++ b/crates/sqlload/src/duckdb_source/main/describe_source.rs @@ -0,0 +1,24 @@ +//! What a DuckDB or MotherDuck source reads, for `debug`. + +use crate::duckdb_source::_helpers::connection::text_rows; +use crate::duckdb_source::errors::DuckDbSourceError; +use crate::duckdb_source::models::DuckDbReader; + +/// The reader's database and the DuckDB version reading it, once the database has opened: a file read-only, a MotherDuck database on the bridge. +pub async fn describe_source(reader: &DuckDbReader) -> Result { + let answers = text_rows(reader, vec!["SELECT version()".to_owned()]) + .await + .map_err(|(_, error)| error)?; + let version = answers + .first() + .and_then(|rows| rows.first()) + .and_then(|row| row.first().cloned().flatten()) + .unwrap_or_default(); + Ok(match reader.motherduck { + Some(_) => format!( + "{} on MotherDuck, through DuckDB {version}", + reader.database + ), + None => format!("{}, read-only, DuckDB {version}", reader.database), + }) +} diff --git a/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs b/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs new file mode 100644 index 000000000..a57cc3f88 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs @@ -0,0 +1,59 @@ +//! A reader for a `duckdb` or `motherduck` source. + +use crate::duckdb_source::constants::MEMORY_DATABASE; +use crate::duckdb_source::errors::DuckDbSourceError; +use crate::duckdb_source::models::DuckDbReader; +use crate::project::main::motherduck_path::motherduck_path; +use crate::project::models::{ConnectionConfig, ProjectConfig, SourceConfig}; + +/// A reader for `source`'s connection: a `duckdb` connection's database file, which each read opens read-only with a `memory_limit` of half a table's share of the project's memory budget (DuckDB's own default without one), or a `motherduck` connection's database, which each read opens on the MotherDuck bridge. Refused: `:memory:`, which holds no tables to read; a MotherDuck path on a `duckdb` connection, which is a `motherduck` source; and a `motherduck` connection naming no database, as a source reads one. Nothing is opened yet. +pub fn duckdb_reader( + project: &ProjectConfig, + source: &SourceConfig, +) -> Result { + let memory_limit = project.memory.budget.map(|budget| budget.table_share / 2); + let database = match project.connections.get(&source.connection) { + Some(ConnectionConfig::DuckDb { database }) => database, + Some(connection @ ConnectionConfig::MotherDuck { database, .. }) => { + let database = database + .as_deref() + .filter(|database| motherduck_path(database).name.is_some()) + .ok_or_else(|| { + DuckDbSourceError::Invalid(format!( + "a motherduck source reads one database: name it in connection {:?}'s database", + source.connection + )) + })?; + return Ok(DuckDbReader { + database: database.to_owned(), + memory_limit, + motherduck: Some(connection.clone()), + bridge: std::sync::Arc::default(), + guard: project.memory.guard.clone(), + }); + } + _ => { + return Err(DuckDbSourceError::Invalid(format!( + "connection {:?} is not a duckdb or motherduck connection", + source.connection + ))); + } + }; + if database == MEMORY_DATABASE { + return Err(DuckDbSourceError::Invalid( + "a duckdb source reads a database file; :memory: holds no tables to read".to_owned(), + )); + } + if motherduck_path(database).prefixed { + return Err(DuckDbSourceError::Invalid( + "a duckdb connection's database is a file; for a MotherDuck database use kind = \"motherduck\"".to_owned(), + )); + } + Ok(DuckDbReader { + database: database.clone(), + memory_limit, + motherduck: None, + bridge: std::sync::Arc::default(), + guard: project.memory.guard.clone(), + }) +} diff --git a/crates/sqlload/src/duckdb_source/main/mod.rs b/crates/sqlload/src/duckdb_source/main/mod.rs new file mode 100644 index 000000000..dc4075b94 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/main/mod.rs @@ -0,0 +1,6 @@ +pub mod count_rows; +pub mod describe_source; +pub mod duckdb_reader; +pub mod read_catalog; +pub mod read_columns; +pub mod read_rows; diff --git a/crates/sqlload/src/duckdb_source/main/read_catalog.rs b/crates/sqlload/src/duckdb_source/main/read_catalog.rs new file mode 100644 index 000000000..7b87ab0f8 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/main/read_catalog.rs @@ -0,0 +1,68 @@ +//! Discover a DuckDB or MotherDuck source's tables: tables and views matching `include`, and query tables described by their result's columns. + +use crate::duckdb_source::_helpers::catalog::{catalog_tables, described_columns}; +use crate::duckdb_source::_helpers::connection::text_rows; +use crate::duckdb_source::_helpers::query::subquery; +use crate::duckdb_source::_helpers::type_map::source_column; +use crate::duckdb_source::constants::CATALOG_QUERIES; +use crate::duckdb_source::errors::DuckDbSourceError; +use crate::duckdb_source::models::DuckDbReader; +use crate::postgres_source::models::SourceTable; +use crate::project::models::SourceConfig; + +/// Every table and view of the reader's database whose `schema.table` matches `source`'s `include` and not its `exclude`; then one table per query table of `source`, its columns those of the query as a load wraps it, described with `DESCRIBE` so it reads no rows. Every query runs on one connection. +pub async fn read_catalog( + reader: &DuckDbReader, + source: &SourceConfig, +) -> Result, DuckDbSourceError> { + let named: Vec<(&String, &str)> = source + .tables + .iter() + .filter_map(|(name, table)| Some((name, table.sql.as_deref()?))) + .collect(); + let catalog = match source.include.is_empty() { + true => 0, + false => CATALOG_QUERIES.len(), + }; + let queries: Vec = CATALOG_QUERIES[..catalog] + .iter() + .map(|query| (*query).to_owned()) + .chain( + named + .iter() + .map(|(_, sql)| format!("DESCRIBE SELECT * FROM {}", subquery(sql))), + ) + .collect(); + let answers = text_rows(reader, queries).await.map_err(|(index, error)| { + match index + .checked_sub(catalog) + .and_then(|query| named.get(query)) + { + Some((name, _)) => DuckDbSourceError::Invalid(format!("query table {name}: {error}")), + None => error, + } + })?; + let (listed, described) = answers.split_at(catalog); + let mut tables: Vec = match catalog { + 0 => Vec::new(), + _ => catalog_tables(listed, (&source.include, &source.exclude))?, + }; + for ((name, _), rows) in named.iter().zip(described) { + tables.push(SourceTable { + split_pages: None, + schema: String::new(), + name: (*name).clone(), + columns: described_columns(rows) + .iter() + .map(|(column, data_type, nullable)| source_column(column, data_type, *nullable)) + .collect(), + primary_key: Vec::new(), + indexed_columns: Vec::new(), + estimated_rows: -1, + estimated_bytes: -1, + comment: None, + listed_keys: Vec::new(), + }); + } + Ok(tables) +} diff --git a/crates/sqlload/src/duckdb_source/main/read_columns.rs b/crates/sqlload/src/duckdb_source/main/read_columns.rs new file mode 100644 index 000000000..f229ec54d --- /dev/null +++ b/crates/sqlload/src/duckdb_source/main/read_columns.rs @@ -0,0 +1,47 @@ +//! Decide how each column of a DuckDB table or query is read. + +use std::collections::BTreeMap; + +use crate::column_types::main::arrow_field::arrow_field; +use crate::column_types::models::ColumnType; +use crate::duckdb_source::_helpers::query::quote_identifier; +use crate::duckdb_source::_helpers::type_map::is_list; +use crate::postgres_source::models::{ReadColumn, SourceTable, WireType}; + +/// How to read every column of `table`, using `pinned` types where given and the natural mapping otherwise; each read column's expression is its quoted name, which cursor filters compare against. A pin with no Arrow type, or an `array(T)` pin over a column that isn't a list, is a problem. +pub fn read_columns( + table: &SourceTable, + pinned: &BTreeMap, +) -> Result, Vec> { + let mut columns: Vec = Vec::with_capacity(table.columns.len()); + let mut problems: Vec = Vec::new(); + for column in &table.columns { + let output = match (pinned.get(&column.name), &column.column_type) { + (Some(pinned), _) => pinned.clone(), + (None, Ok(natural)) => natural.clone(), + (None, Err(reason)) => { + problems.push(format!("{}: {reason}", column.name)); + continue; + } + }; + let readable = arrow_field(&column.name, &output).is_some() + && (!matches!(output, ColumnType::Array(_)) || is_list(&column.postgres_type)); + if !readable { + problems.push(format!( + "{} has type {}, which cannot be read as {output}", + column.name, column.postgres_type + )); + continue; + } + columns.push(ReadColumn { + name: column.name.clone(), + expression: quote_identifier(column.source_spelling.as_deref().unwrap_or(&column.name)), + wire: WireType::Text, + output, + }); + } + match problems.is_empty() { + true => Ok(columns), + false => Err(problems), + } +} diff --git a/crates/sqlload/src/duckdb_source/main/read_rows.rs b/crates/sqlload/src/duckdb_source/main/read_rows.rs new file mode 100644 index 000000000..ade3880d1 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/main/read_rows.rs @@ -0,0 +1,93 @@ +//! Read a table or query as Arrow batches from a read-only connection. + +use arrow_array::RecordBatch; + +use crate::adapters::motherduck_bridge::main::stream_rows::stream_rows; +use crate::duckdb_source::_helpers::connection::{open_bridge, open_read_only}; +use crate::duckdb_source::_helpers::convert::{convert_batch, output_schema}; +use crate::duckdb_source::_helpers::query::select_query; +use crate::duckdb_source::constants::BATCHES_AHEAD; +use crate::duckdb_source::errors::DuckDbSourceError; +use crate::duckdb_source::models::{DuckDbReadRequest, DuckDbReader}; +use crate::row_batches::main::loaded_batch::loaded_batch; + +/// Read `request`'s rows and pass them to `on_batch` in batches of about `batch_rows`, each converted to the read columns' types; returns the row count. One statement streams the rows, so the read is consistent on its own. From a file, a thread reads at most a few batches ahead of `on_batch`; from MotherDuck, the bridge reads one batch at a time as it is asked. Either stops when `on_batch` fails. +pub async fn read_rows( + reader: &DuckDbReader, + request: &DuckDbReadRequest<'_>, + batch_rows: usize, + mut on_batch: impl AsyncFnMut(RecordBatch) -> Result<(), String>, +) -> Result { + let shape = output_schema(request.columns).map_err(DuckDbSourceError::Invalid)?; + if reader.motherduck.is_some() { + let bridge = open_bridge(reader).await?; + let converted = async |batch: RecordBatch| { + let batch = loaded_batch(&batch, request.table) + .and_then(|batch| convert_batch(&batch, (&shape.0, &shape.1)))?; + on_batch(batch).await + }; + return stream_rows(&bridge, select_query(request), batch_rows, converted) + .await + .map_err(|error| match error { + crate::adapters::motherduck_bridge::errors::BridgeError::Consumer(said) => { + DuckDbSourceError::Consumer(said) + } + other => DuckDbSourceError::Bridge(other), + }); + } + let (reader, query) = (reader.clone(), select_query(request)); + let (sender, mut receiver) = + tokio::sync::mpsc::channel::>(BATCHES_AHEAD); + let streaming = tokio::task::spawn_blocking(move || { + let read = || -> Result<(), DuckDbSourceError> { + let connection = open_read_only(&reader)?; + let mut statement = connection.connection.prepare(&query)?; + let mut held: Vec = Vec::new(); + let mut held_rows = 0; + for chunk in statement.stream_arrow([])? { + let chunk = chunk?; + held_rows += chunk.num_rows(); + held.push(chunk); + if held_rows >= batch_rows { + if !send(&sender, &std::mem::take(&mut held))? { + return Ok(()); + } + held_rows = 0; + } + } + send(&sender, &held)?; + Ok(()) + }; + if let Err(error) = read() { + let _ = sender.blocking_send(Err(error)); + } + }); + let mut rows: u64 = 0; + while let Some(batch) = receiver.recv().await { + let batch = loaded_batch(&batch?, request.table) + .and_then(|batch| convert_batch(&batch, (&shape.0, &shape.1))) + .map_err(DuckDbSourceError::Invalid)?; + rows += batch.num_rows() as u64; + on_batch(batch).await.map_err(DuckDbSourceError::Consumer)?; + } + streaming + .await + .map_err(|error| DuckDbSourceError::Invalid(format!("the read stopped: {error}")))?; + Ok(rows) +} + +/// Send the `held` chunks as one batch; `false` when the consumer has gone. +fn send( + sender: &tokio::sync::mpsc::Sender>, + held: &[RecordBatch], +) -> Result { + let Some(first) = held.first() else { + return Ok(true); + }; + let batch = arrow_select::concat::concat_batches(&first.schema(), held.iter()) + .map_err(|error| DuckDbSourceError::Invalid(error.to_string()))?; + Ok(match sender.blocking_send(Ok(batch)) { + Ok(()) => true, + Err(tokio::sync::mpsc::error::SendError(_)) => false, + }) +} diff --git a/crates/sqlload/src/duckdb_source/mod.rs b/crates/sqlload/src/duckdb_source/mod.rs new file mode 100644 index 000000000..873e29d5c --- /dev/null +++ b/crates/sqlload/src/duckdb_source/mod.rs @@ -0,0 +1,14 @@ +//! DuckDB as a source: tables and views of a database file that `include` finds, and query tables, read as Arrow from a read-only connection. Only the constants are compiled without the `duckdb` feature, so plans can name the system. + +#[cfg(feature = "duckdb")] +pub(crate) mod _helpers; +pub mod constants; +#[cfg(feature = "duckdb")] +pub mod errors; +#[cfg(feature = "duckdb")] +pub mod main; +#[cfg(feature = "duckdb")] +pub mod models; + +#[cfg(all(test, feature = "duckdb"))] +mod tests; diff --git a/crates/sqlload/src/duckdb_source/models.rs b/crates/sqlload/src/duckdb_source/models.rs new file mode 100644 index 000000000..a57a4dae9 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/models.rs @@ -0,0 +1,60 @@ +use crate::adapters::motherduck_bridge::models::BridgeConnection; +use crate::memory_budget::models::{MemoryGuard, WatchedChild}; +use crate::postgres_source::models::{ReadColumn, SourceTable}; +use crate::project::models::ConnectionConfig; + +/// A DuckDB or MotherDuck source: the database it reads, opened for each read (a file read-only, a MotherDuck database on the MotherDuck bridge), and the `memory_limit` each connection to a file gets (half of a table's share of the memory budget; DuckDB's own default without a budget). +#[derive(Debug, Clone)] +pub struct DuckDbReader { + /// The database file, or a MotherDuck database's `md:` path as written. + pub(crate) database: String, + pub(crate) memory_limit: Option, + /// A `motherduck` source's connection; `None` for a database file. + pub(crate) motherduck: Option, + /// A `motherduck` source's bridge, shared by the reader's clones and opened on its first read; each read takes a connection of its own on it. + pub(crate) bridge: std::sync::Arc>, + /// The load's memory watchdog, which counts the bridge's memory once it opens; `None` without a budget. + pub(crate) guard: Option>, +} + +/// A `motherduck` source's open bridge, its memory counted by the load's watchdog while it is open. +#[derive(Debug)] +pub(crate) struct SourceBridge { + pub(crate) connection: BridgeConnection, + pub(crate) _watched: Option, +} + +/// What one read selects: a table or a query, its columns, and the rows' filter. +#[derive(Debug, Clone, Copy)] +pub struct DuckDbReadRequest<'a> { + pub table: &'a SourceTable, + /// The query whose rows are the table; `None` reads the table itself. + pub sql: Option<&'a str>, + pub columns: &'a [ReadColumn], + /// A DuckDB condition on the read columns, already rendered. + pub filter: Option<&'a str>, +} + +/// A read-only connection to a source's database file, on the one DuckDB instance this process holds on the file while any read of it is open. +#[derive(Debug)] +pub(crate) struct ReadOnlyFile { + /// Declared first, so it closes before `_hold` lets the instance go. + pub(crate) connection: duckdb::Connection, + pub(crate) _hold: FileHold, +} + +/// One open read's share of the instance on a database file, by the file's canonical path; the last share to end closes the instance. +#[derive(Debug)] +pub(crate) struct FileHold { + pub(crate) path: std::path::PathBuf, +} + +/// The instance this process holds on a database file: the reads sharing it, its own connection (closed last), and the folder its queries spill into. +#[derive(Debug)] +pub(crate) struct OpenFile { + pub(crate) reads: usize, + /// Each read's `memory_limit`; the instance gets it once for each open read. + pub(crate) read_limit: Option, + pub(crate) connection: duckdb::Connection, + pub(crate) _spill: tempfile::TempDir, +} diff --git a/crates/sqlload/src/duckdb_source/tests.rs b/crates/sqlload/src/duckdb_source/tests.rs new file mode 100644 index 000000000..adc2b3bd3 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_read_source.rs"] +mod read_source; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_type_map.rs"] +mod type_map; diff --git a/crates/sqlload/src/duckdb_source/tests/helpers.rs b/crates/sqlload/src/duckdb_source/tests/helpers.rs new file mode 100644 index 000000000..07aa18bdf --- /dev/null +++ b/crates/sqlload/src/duckdb_source/tests/helpers.rs @@ -0,0 +1,82 @@ +use crate::project::main::parse_project::parse_project; +use crate::project::models::ProjectConfig; + +/// A shop database: a commented table with a primary key and a type that must be pinned, a view, and a scratch table. +pub(crate) const SHOP: &str = " +CREATE SCHEMA sales; +CREATE TABLE sales.orders ( + id INTEGER PRIMARY KEY, + amount DECIMAL(12,2), + name VARCHAR, + placed_at TIMESTAMPTZ, + tags VARCHAR[], + meta STRUCT(rank INTEGER), + total HUGEINT +); +COMMENT ON TABLE sales.orders IS 'Orders, one row each.'; +INSERT INTO sales.orders VALUES + (1, 10.50, 'ann', '2026-01-01 00:00:00+00', ['a', 'b'], {'rank': 1}, 170141183460469231731687303715884105727), + (2, NULL, 'bo\"b', '2026-01-02 12:30:00+02', [], {'rank': 2}, 2), + (3, 99999999.99, NULL, NULL, NULL, NULL, 3); +CREATE VIEW sales.big_orders AS SELECT id, amount FROM sales.orders WHERE amount > 50; +CREATE TABLE sales.tmp_scratch (x INTEGER); +"; + +/// A runtime for one test's reads. +pub(crate) fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("a runtime") +} + +/// A project whose `shop` source reads `database` with `source_lines` in its table, and a `recent` query table. +pub(crate) fn shop_project(database: &str, source_lines: &str) -> Result { + parse_project( + &format!( + "[connections.shop]\nkind = \"duckdb\"\ndatabase = {database:?}\n[connections.wh]\nkind = \"duckdb\"\ndatabase = \":memory:\"\n[targets.dev]\nconnection = \"wh\"\nloader_schema = \"raw\"\n[sources.shop]\nkind = \"duckdb\"\nconnection = \"shop\"\ndescription = \"Shop.\"\nwrite_strategy = \"replace\"\n{source_lines}\n[sources.shop.tables.recent]\ndescription = \"Recent orders.\"\nsql = \"SELECT id, amount FROM sales.orders WHERE id > 1 -- newest\\n;\\n\"\n" + ), + &|_| None, + ) + .map_err(|error| error.to_string()) +} + +/// Set to a database file in the process [`write_from_another_process`] starts, which then tries to write that file. +const WRITE_FROM_ANOTHER_PROCESS: &str = "SQLLOAD_TEST_DUCKDB_WRITE_FILE"; + +/// The test that calls [`write_when_asked`] first, run again in a process of its own, says whether it could write the file at `path`. +pub(crate) fn write_from_another_process(path: &std::path::Path) -> Result { + let exe = std::env::current_exe().map_err(|error| error.to_string())?; + let test = "duckdb_source::tests::read_source::given_open_reads_of_a_file_when_another_process_writes_it_then_it_is_locked_until_the_last_read_ends"; + let output = std::process::Command::new(exe) + .args(["--exact", test, "--nocapture"]) + .env(WRITE_FROM_ANOTHER_PROCESS, path) + .output() + .map_err(|error| error.to_string())?; + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +/// In the process [`write_from_another_process`] starts: try to write the file it names, say whether that worked, and end the process. +pub(crate) fn write_when_asked() { + let Some(path) = std::env::var_os(WRITE_FROM_ANOTHER_PROCESS) else { + return; + }; + let written = duckdb::Connection::open(path) + .and_then(|connection| connection.execute_batch("CREATE TABLE written (x INTEGER)")); + println!( + "{}", + written.map_or_else( + |error| format!( + "the file was {}: {error}", + // DuckDB says "lock" on Unix; on Windows the lock is a sharing violation. + ["not opened", "locked"][usize::from( + ["lock", "being used by another process"] + .iter() + .any(|said| error.to_string().contains(said)) + )] + ), + |()| "the file was written".to_owned() + ) + ); + std::process::exit(0); +} diff --git a/crates/sqlload/src/duckdb_source/tests/test_read_source.rs b/crates/sqlload/src/duckdb_source/tests/test_read_source.rs new file mode 100644 index 000000000..7e33b578d --- /dev/null +++ b/crates/sqlload/src/duckdb_source/tests/test_read_source.rs @@ -0,0 +1,362 @@ +use std::collections::BTreeMap; + +use arrow_cast::display::array_value_to_string; +use arrow_select::concat::concat_batches; + +use crate::column_types::models::ColumnType; +use crate::duckdb_source::_helpers::catalog::catalog_tables; +use crate::duckdb_source::_helpers::connection::{TextRows, open_read_only}; +use crate::duckdb_source::constants::CATALOG_QUERIES; +use crate::duckdb_source::main::duckdb_reader::duckdb_reader; +use crate::duckdb_source::main::read_catalog::read_catalog; +use crate::duckdb_source::main::read_columns::read_columns; +use crate::duckdb_source::main::read_rows::read_rows; +use crate::duckdb_source::models::{DuckDbReadRequest, DuckDbReader}; +use crate::duckdb_source::tests::helpers::{ + SHOP, runtime, shop_project, write_from_another_process, write_when_asked, +}; +use crate::duckdb_source::tests::test_types::{ + LockedFileTestCase, PrimaryKeyTestCase, ReadCatalogTestCase, ReadRowsTestCase, + RefusedReaderTestCase, SharedLimitTestCase, +}; + +#[test] +fn given_a_duckdb_file_when_listing_its_catalog_then_include_and_exclude_pick_tables_and_views_before_query_tables() +-> Result<(), String> { + let folder = tempfile::tempdir().map_err(|error| error.to_string())?; + let path = folder.path().join("shop.duckdb"); + duckdb::Connection::open(&path) + .and_then(|connection| connection.execute_batch(SHOP)) + .map_err(|error| error.to_string())?; + let test_cases = [ + ReadCatalogTestCase { + description: "tables and views by include, less exclude, then the query table", + source_lines: "include = [\"sales.*\"]\nexclude = [\"sales.tmp_*\"]", + expected_tables: &["sales.big_orders", "sales.orders", "recent"], + }, + ReadCatalogTestCase { + description: "without include only the query table", + source_lines: "", + expected_tables: &["recent"], + }, + ]; + for test_case in test_cases { + let project = shop_project(&path.display().to_string(), test_case.source_lines)?; + let source = &project.sources["shop"]; + let reader = duckdb_reader(&project, source).map_err(|error| error.to_string())?; + let tables = runtime() + .block_on(read_catalog(&reader, source)) + .map_err(|error| format!("{}: {error}", test_case.description))?; + let names: Vec = tables.iter().map(|table| table.qualified_name()).collect(); + assert_eq!( + names, test_case.expected_tables, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_duckdb_without_extensions_when_listing_its_catalog_then_every_query_answers_and_keys_keep_their_order() +-> Result<(), String> { + let config = duckdb::Config::default() + .with("autoinstall_known_extensions", "false") + .and_then(|config| config.with("autoload_known_extensions", "false")) + .map_err(|error| error.to_string())?; + let connection = + duckdb::Connection::open_in_memory_with_flags(config).map_err(|error| error.to_string())?; + connection + .execute_batch(&format!( + "{SHOP} CREATE TABLE sales.pairs (a INTEGER, b INTEGER, c INTEGER, PRIMARY KEY (c, a));" + )) + .map_err(|error| error.to_string())?; + let mut answers: Vec = Vec::new(); + for query in CATALOG_QUERIES { + let mut statement = connection + .prepare(query) + .map_err(|error| format!("{query}: {error}"))?; + let width = statement.column_count(); + let rows = statement + .query_map([], |row| { + (0..width) + .map(|index| row.get::<_, Option>(index)) + .collect() + }) + .and_then(Iterator::collect) + .map_err(|error| format!("{query}: {error}"))?; + answers.push(rows); + } + let tables = catalog_tables(&answers, (&["sales.*".to_owned()], &[])) + .map_err(|error| error.to_string())?; + let test_cases = [ + PrimaryKeyTestCase { + description: "a one-column key", + table: "sales.orders", + expected_key: &["id"], + }, + PrimaryKeyTestCase { + description: "a key declared out of column order keeps its declared order", + table: "sales.pairs", + expected_key: &["c", "a"], + }, + PrimaryKeyTestCase { + description: "a table without a key", + table: "sales.tmp_scratch", + expected_key: &[], + }, + ]; + for test_case in test_cases { + let key = tables + .iter() + .find(|table| table.qualified_name() == test_case.table) + .map(|table| table.primary_key.clone()); + assert_eq!( + key.as_deref(), + Some( + test_case + .expected_key + .iter() + .map(|column| (*column).to_owned()) + .collect::>() + .as_slice() + ), + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_a_duckdb_file_when_reading_a_table_or_query_then_rows_load_typed_and_the_file_is_left_as_it_was() +-> Result<(), String> { + let folder = tempfile::tempdir().map_err(|error| error.to_string())?; + let path = folder.path().join("shop.duckdb"); + duckdb::Connection::open(&path) + .and_then(|connection| connection.execute_batch(SHOP)) + .map_err(|error| error.to_string())?; + let before = std::fs::read(&path).map_err(|error| error.to_string())?; + let project = shop_project(&path.display().to_string(), "include = [\"sales.orders\"]")?; + let source = &project.sources["shop"]; + let reader = duckdb_reader(&project, source).map_err(|error| error.to_string())?; + let runtime = runtime(); + let tables = runtime + .block_on(read_catalog(&reader, source)) + .map_err(|error| error.to_string())?; + let orders = &tables[0]; + assert_eq!( + (orders.primary_key.as_slice(), orders.comment.as_deref()), + (["id".to_owned()].as_slice(), Some("Orders, one row each.")), + "the primary key and comment come from the catalog" + ); + assert!( + read_columns(orders, &BTreeMap::new()).is_err(), + "an unpinned HUGEINT is a problem" + ); + let pinned = BTreeMap::from([("total".to_owned(), ColumnType::Text)]); + let test_cases = [ + ReadRowsTestCase { + description: "the table, one row per batch, filtered: decimals, instants in UTC, lists and structs as JSON, a pinned HUGEINT as text", + table: "sales.orders", + filter: Some("\"id\" <= 2"), + batch_rows: 1, + expected_values: &[ + "id=1|2", + "amount=10.50|", + "name=ann|bo\"b", + "placed_at=2026-01-01T00:00:00Z|2026-01-02T10:30:00Z", + r#"tags=["a","b"]|[]"#, + r#"meta={"rank":1}|{"rank":2}"#, + "total=170141183460469231731687303715884105727|2", + ], + }, + ReadRowsTestCase { + description: "the query table, its filter applied outside the query after its trailing comment", + table: "recent", + filter: Some("\"id\" > 2"), + batch_rows: 100, + expected_values: &["id=3", "amount=99999999.99"], + }, + ]; + for test_case in test_cases { + let table = tables + .iter() + .find(|table| table.qualified_name() == test_case.table) + .ok_or(test_case.description)?; + let columns = read_columns(table, &pinned).map_err(|problems| problems.join("; "))?; + let mut batches = Vec::new(); + let request = DuckDbReadRequest { + table, + sql: source + .tables + .get(test_case.table) + .and_then(|table| table.sql.as_deref()), + columns: &columns, + filter: test_case.filter, + }; + runtime + .block_on(read_rows( + &reader, + &request, + test_case.batch_rows, + async |batch| { + batches.push(batch); + Ok(()) + }, + )) + .map_err(|error| format!("{}: {error}", test_case.description))?; + let batch = + concat_batches(&batches[0].schema(), &batches).map_err(|error| error.to_string())?; + let values = test_case + .expected_values + .iter() + .map(|expected| { + let name = expected.split_once('=').map_or(*expected, |(name, _)| name); + let column = batch + .column_by_name(name) + .ok_or_else(|| format!("no column {name}"))?; + let values = (0..column.len()) + .map(|row| array_value_to_string(column, row)) + .collect::, _>>() + .map_err(|error| error.to_string())?; + Ok(format!("{name}={}", values.join("|"))) + }) + .collect::, String>>()?; + assert_eq!( + values, test_case.expected_values, + "{}", + test_case.description + ); + } + assert_eq!( + std::fs::read(&path).map_err(|error| error.to_string())?, + before, + "the file is read, never written" + ); + Ok(()) +} + +#[test] +fn given_a_memory_or_motherduck_database_when_making_a_reader_then_it_is_refused() +-> Result<(), String> { + let test_cases = [ + RefusedReaderTestCase { + description: "an in-memory database holds nothing to read", + database: ":memory:", + expected_error: ":memory: holds no tables to read", + }, + RefusedReaderTestCase { + description: "a MotherDuck database is a motherduck source", + database: "md:shop", + expected_error: "kind = \"motherduck\"", + }, + ]; + for test_case in test_cases { + let project = shop_project(test_case.database, "")?; + let refused = duckdb_reader(&project, &project.sources["shop"]) + .err() + .map(|error| error.to_string()) + .unwrap_or_default(); + assert!( + refused.contains(test_case.expected_error), + "{}: {refused}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_open_reads_of_a_file_when_another_process_writes_it_then_it_is_locked_until_the_last_read_ends() +-> Result<(), String> { + write_when_asked(); + let test_cases = [ + LockedFileTestCase { + description: "one of two reads ended: the other's lock still holds", + reads: 2, + ended: 1, + expected_said: "the file was locked", + }, + LockedFileTestCase { + description: "every read ended: the file is free to write", + reads: 1, + ended: 1, + expected_said: "the file was written", + }, + ]; + for test_case in test_cases { + let folder = tempfile::tempdir().map_err(|error| error.to_string())?; + let path = folder.path().join("shop.duckdb"); + duckdb::Connection::open(&path) + .and_then(|connection| connection.execute_batch(SHOP)) + .map_err(|error| error.to_string())?; + let project = shop_project(&path.display().to_string(), "")?; + let reader = + duckdb_reader(&project, &project.sources["shop"]).map_err(|error| error.to_string())?; + let mut reads = (0..test_case.reads) + .map(|_| open_read_only(&reader)) + .collect::, _>>() + .map_err(|error| error.to_string())?; + reads.drain(..test_case.ended); + let said = write_from_another_process(&path)?; + drop(reads); + assert!( + said.contains(test_case.expected_said), + "{}: {said}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_reads_sharing_a_file_when_they_open_and_end_then_its_memory_limit_is_one_read_limit_each() +-> Result<(), String> { + let test_cases = [ + SharedLimitTestCase { + description: "one read gets its own limit", + reads: 1, + ended: 0, + expected_limit: "128.0 MiB", + }, + SharedLimitTestCase { + description: "three reads at once get three limits", + reads: 3, + ended: 0, + expected_limit: "384.0 MiB", + }, + SharedLimitTestCase { + description: "two of three ended: one limit is left", + reads: 3, + ended: 2, + expected_limit: "128.0 MiB", + }, + ]; + for test_case in test_cases { + let folder = tempfile::tempdir().map_err(|error| error.to_string())?; + let path = folder.path().join("shop.duckdb"); + duckdb::Connection::open(&path) + .and_then(|connection| connection.execute_batch(SHOP)) + .map_err(|error| error.to_string())?; + let project = shop_project(&path.display().to_string(), "")?; + let reader = DuckDbReader { + memory_limit: Some(128 * 1024 * 1024), + ..duckdb_reader(&project, &project.sources["shop"]) + .map_err(|error| error.to_string())? + }; + let mut reads = (0..test_case.reads) + .map(|_| open_read_only(&reader)) + .collect::, _>>() + .map_err(|error| error.to_string())?; + reads.drain(..test_case.ended); + let limit: String = reads[0] + .connection + .query_row("SELECT current_setting('memory_limit')", [], |row| { + row.get(0) + }) + .map_err(|error| error.to_string())?; + assert_eq!(limit, test_case.expected_limit, "{}", test_case.description); + } + Ok(()) +} diff --git a/crates/sqlload/src/duckdb_source/tests/test_type_map.rs b/crates/sqlload/src/duckdb_source/tests/test_type_map.rs new file mode 100644 index 000000000..e2faba4c7 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/tests/test_type_map.rs @@ -0,0 +1,100 @@ +use crate::column_types::models::ColumnType; +use crate::duckdb_source::_helpers::type_map::source_column; +use crate::duckdb_source::tests::test_types::SourceColumnTestCase; + +#[test] +fn given_duckdb_column_types_when_mapping_them_then_each_gets_its_portable_type_or_must_be_pinned() +{ + let test_cases = [ + SourceColumnTestCase { + description: "unsigned 8-bit fits smallint", + data_type: "UTINYINT", + expected_type: Some(ColumnType::SmallInt), + }, + SourceColumnTestCase { + description: "unsigned 32-bit fits a 64-bit int", + data_type: "UINTEGER", + expected_type: Some(ColumnType::Int), + }, + SourceColumnTestCase { + description: "unsigned 64-bit does not fit int", + data_type: "UBIGINT", + expected_type: None, + }, + SourceColumnTestCase { + description: "HUGEINT does not fit int", + data_type: "HUGEINT", + expected_type: None, + }, + SourceColumnTestCase { + description: "a decimal keeps precision and scale", + data_type: "DECIMAL(12,2)", + expected_type: Some(ColumnType::Decimal { + precision: 12, + scale: 2, + }), + }, + SourceColumnTestCase { + description: "a 38-digit decimal is the widest", + data_type: "DECIMAL(38,10)", + expected_type: Some(ColumnType::Decimal { + precision: 38, + scale: 10, + }), + }, + SourceColumnTestCase { + description: "an enum is its label", + data_type: "ENUM('a', 'b')", + expected_type: Some(ColumnType::Text), + }, + SourceColumnTestCase { + description: "TIMESTAMPTZ is an instant", + data_type: "TIMESTAMP WITH TIME ZONE", + expected_type: Some(ColumnType::TimestampTz), + }, + SourceColumnTestCase { + description: "millisecond timestamps widen to microseconds", + data_type: "TIMESTAMP_MS", + expected_type: Some(ColumnType::TimestampNtz), + }, + SourceColumnTestCase { + description: "nanosecond timestamps would lose digits", + data_type: "TIMESTAMP_NS", + expected_type: None, + }, + SourceColumnTestCase { + description: "a list is json", + data_type: "VARCHAR[]", + expected_type: Some(ColumnType::Json), + }, + SourceColumnTestCase { + description: "a fixed-size array is json", + data_type: "INTEGER[3]", + expected_type: Some(ColumnType::Json), + }, + SourceColumnTestCase { + description: "a struct is json", + data_type: "STRUCT(a INTEGER, b VARCHAR)", + expected_type: Some(ColumnType::Json), + }, + SourceColumnTestCase { + description: "an interval has no portable type", + data_type: "INTERVAL", + expected_type: None, + }, + ]; + for test_case in test_cases { + let column = source_column("c", test_case.data_type, true); + assert_eq!( + column.column_type.ok(), + test_case.expected_type, + "{}", + test_case.description + ); + assert_eq!( + column.postgres_type, test_case.data_type, + "{}: plan shows DuckDB's type", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/duckdb_source/tests/test_types.rs b/crates/sqlload/src/duckdb_source/tests/test_types.rs new file mode 100644 index 000000000..39bc949cf --- /dev/null +++ b/crates/sqlload/src/duckdb_source/tests/test_types.rs @@ -0,0 +1,65 @@ +use crate::column_types::models::ColumnType; + +pub(crate) struct SourceColumnTestCase { + pub(crate) description: &'static str, + /// The type as `duckdb_columns().data_type` or a result's column type writes it. + pub(crate) data_type: &'static str, + /// The portable type; `None` when a type must be pinned. + pub(crate) expected_type: Option, +} + +/// Which tables a shop database's catalog lists for a source's settings. +pub(crate) struct ReadCatalogTestCase { + pub(crate) description: &'static str, + /// Lines of the `[sources.shop]` table: `include`, `exclude` and the like. + pub(crate) source_lines: &'static str, + /// The tables listed, by qualified name, in order. + pub(crate) expected_tables: &'static [&'static str], +} + +/// The primary key the catalog reports for one table, in key order. +pub(crate) struct PrimaryKeyTestCase { + pub(crate) description: &'static str, + /// The table's qualified name as the catalog lists it. + pub(crate) table: &'static str, + pub(crate) expected_key: &'static [&'static str], +} + +/// One read of a table or query table the shop source lists. +pub(crate) struct ReadRowsTestCase { + pub(crate) description: &'static str, + /// The table's qualified name as the catalog lists it. + pub(crate) table: &'static str, + /// A DuckDB condition on the read columns. + pub(crate) filter: Option<&'static str>, + pub(crate) batch_rows: usize, + /// Each named column's values in row order as Arrow displays them (`""` for NULL), as `name=value|value`. + pub(crate) expected_values: &'static [&'static str], +} + +/// A duckdb connection whose database the source cannot read. +pub(crate) struct RefusedReaderTestCase { + pub(crate) description: &'static str, + pub(crate) database: &'static str, + /// Part of the refusal. + pub(crate) expected_error: &'static str, +} + +/// Reads of a database file open in this process, some of them ended, while another process tries to write the file. +pub(crate) struct LockedFileTestCase { + pub(crate) description: &'static str, + pub(crate) reads: usize, + /// How many of the reads have ended when the other process writes. + pub(crate) ended: usize, + /// What the other process says. + pub(crate) expected_said: &'static str, +} + +/// Reads of one database file open in this process, some of them ended, each with a 128 MiB `memory_limit`, and the limit their shared instance has. +pub(crate) struct SharedLimitTestCase { + pub(crate) description: &'static str, + pub(crate) reads: usize, + pub(crate) ended: usize, + /// `current_setting('memory_limit')` as DuckDB prints it. + pub(crate) expected_limit: &'static str, +} diff --git a/crates/sqlload/src/lib.rs b/crates/sqlload/src/lib.rs index 311053888..eb8b2985e 100644 --- a/crates/sqlload/src/lib.rs +++ b/crates/sqlload/src/lib.rs @@ -21,6 +21,7 @@ pub mod copy; pub mod databricks_api; pub mod databricks_source; pub mod delivery; +pub mod duckdb_source; pub mod envelope; pub mod exporter; pub mod file_format; diff --git a/crates/sqlload/src/loader/_helpers/resolve.rs b/crates/sqlload/src/loader/_helpers/resolve.rs index 0a00180c8..bc6d45291 100644 --- a/crates/sqlload/src/loader/_helpers/resolve.rs +++ b/crates/sqlload/src/loader/_helpers/resolve.rs @@ -1702,42 +1702,48 @@ async fn failed_count( table: &SourceTable, filter: &str, ) -> Option { - match reader? { - SourceReader::Postgres(client, _) => { - crate::postgres_source::main::count_rows::count_rows(client, table, filter) + let failed = match reader? { + SourceReader::BigQuery(bigquery) => { + crate::bigquery_source::main::dry_count::dry_count(bigquery, (table, None), filter) .await .err() - .map(|error| error.to_string()) + .map(LoaderError::from) + } + reader => count_rows(reader, (table, None), filter).await.err(), + }; + failed.map(|error| error.to_string()) +} + +/// How many rows of `table` (or of its query `sql`) `filter` selects, counted by the source's database; `None` for a files or Google Sheets source, which has no database to ask. +pub(crate) async fn count_rows( + reader: &SourceReader, + (table, sql): (&SourceTable, Option<&str>), + filter: &str, +) -> Result, LoaderError> { + Ok(Some(match reader { + SourceReader::Postgres(client, _) => { + crate::postgres_source::main::count_rows::count_rows(client, table, filter).await? } SourceReader::Snowflake(snowflake) => { - crate::snowflake_source::main::count_rows::count_rows(snowflake, (table, None), filter) - .await - .err() - .map(|error| error.to_string()) + crate::snowflake_source::main::count_rows::count_rows(snowflake, (table, sql), filter) + .await? } SourceReader::BigQuery(bigquery) => { - crate::bigquery_source::main::dry_count::dry_count(bigquery, (table, None), filter) - .await - .err() - .map(|error| error.to_string()) + crate::bigquery_source::main::count_rows::count_rows(bigquery, (table, sql), filter) + .await? } SourceReader::Databricks(databricks) => { - crate::databricks_source::main::count_rows::count_rows( - databricks, - (table, None), - filter, - ) - .await - .err() - .map(|error| error.to_string()) + crate::databricks_source::main::count_rows::count_rows(databricks, (table, sql), filter) + .await? } #[cfg(feature = "odbc")] SourceReader::Odbc(odbc) => { - crate::odbc_source::main::count_rows::count_rows(odbc, (table, None), filter) - .await - .err() - .map(|error| error.to_string()) + crate::odbc_source::main::count_rows::count_rows(odbc, (table, sql), filter).await? } - SourceReader::Files(_) | SourceReader::Sheets(..) => None, - } + #[cfg(feature = "duckdb")] + SourceReader::DuckDb(duckdb) => { + crate::duckdb_source::main::count_rows::count_rows(duckdb, (table, sql), filter).await? + } + SourceReader::Files(_) | SourceReader::Sheets(..) => return Ok(None), + })) } diff --git a/crates/sqlload/src/loader/_helpers/sources.rs b/crates/sqlload/src/loader/_helpers/sources.rs index be6481bef..3d7d11de2 100644 --- a/crates/sqlload/src/loader/_helpers/sources.rs +++ b/crates/sqlload/src/loader/_helpers/sources.rs @@ -25,6 +25,13 @@ use crate::databricks_source::main::databricks_reader::databricks_reader; use crate::databricks_source::main::read_catalog::read_catalog as read_databricks_catalog; use crate::databricks_source::main::read_rows::read_rows as read_databricks_rows; use crate::databricks_source::models::DatabricksReadRequest; +#[cfg(feature = "duckdb")] +use crate::duckdb_source::main::{ + duckdb_reader::duckdb_reader, read_catalog::read_catalog as read_duckdb_catalog, + read_rows::read_rows as read_duckdb_rows, +}; +#[cfg(feature = "duckdb")] +use crate::duckdb_source::models::DuckDbReadRequest; use crate::file_store::main::open_folder::open_folder; use crate::file_store::models::{OpenFolder, StoredFile}; use crate::files_source::constants::{ @@ -65,6 +72,8 @@ use crate::loader::_helpers::specs::{ }; use crate::loader::_helpers::specs::{path_values, pinned_unsampled}; use crate::loader::_helpers::specs::{returned_columns, runnable_transform, transform_input}; +#[cfg(not(feature = "duckdb"))] +use crate::loader::constants::NO_DUCKDB_SOURCE; use crate::loader::constants::NO_TRANSFORM_HOST; use crate::loader::constants::{ DRAINED_REASON, KEPT_TABLES_COLUMN_FIX, LOCALHOST, PRIMARY_KEY_PLACEHOLDER, RELOAD_FIX, @@ -77,6 +86,8 @@ use crate::loader::models::{ DrainedFiles, FilesSource, PlanOptions, ResolvedTables, SkippedTable, SourceEnd, SourceProblem, SourceReader, TablePlan, TableSpec, }; +#[cfg(feature = "odbc")] +use crate::odbc_source::main::source_reader::source_reader; use crate::postgres_source::constants::DEFAULT_BATCH_BYTES; use crate::postgres_source::main::connect_postgres::connect_postgres; use crate::postgres_source::main::copy_table::copy_table; @@ -226,6 +237,14 @@ pub(crate) async fn open_source( SourceKind::Databricks => Ok(SourceReader::Databricks(Box::new(databricks_reader( project, source, )?))), + #[cfg(feature = "duckdb")] + SourceKind::DuckDb | SourceKind::MotherDuck => Ok(SourceReader::DuckDb(Box::new( + duckdb_reader(project, source)?, + ))), + #[cfg(not(feature = "duckdb"))] + SourceKind::DuckDb | SourceKind::MotherDuck => { + Err(LoaderError::Invalid(NO_DUCKDB_SOURCE.to_owned())) + } SourceKind::Python => Err(LoaderError::Invalid( "a Python source is read by calling its table functions".to_owned(), )), @@ -370,6 +389,19 @@ pub(crate) async fn source_tables( .map(|tables| (tables, None, Vec::new())) .map_err(|error| LoaderError::from(error).into())) } + #[cfg(feature = "duckdb")] + SourceKind::DuckDb | SourceKind::MotherDuck => { + let read = + async { read_duckdb_catalog(&duckdb_reader(project, source)?, source).await }; + Ok(read + .await + .map(|tables| (tables, None, Vec::new())) + .map_err(|error| LoaderError::from(error).into())) + } + #[cfg(not(feature = "duckdb"))] + SourceKind::DuckDb | SourceKind::MotherDuck => { + Ok(Err(LoaderError::Invalid(NO_DUCKDB_SOURCE.to_owned()).into())) + } SourceKind::Python => Ok(Ok((python_tables(source), None, Vec::new()))), SourceKind::GoogleSheets => Ok(match sheets_tables(project, source).await { Ok((tables, warnings)) => Ok((tables, None, warnings)), @@ -378,39 +410,6 @@ pub(crate) async fn source_tables( } } -/// An `odbc` source's reader, for its connection's settings and the source's value caps. -#[cfg(feature = "odbc")] -pub(crate) fn odbc_source_reader( - project: &ProjectConfig, - source: &SourceConfig, -) -> Result { - let Some(ConnectionConfig::Odbc { - connection_string, - dsn, - user, - password, - }) = project.connections.get(&source.connection) - else { - return Err(LoaderError::Invalid(format!( - "connection {:?} is not an odbc connection", - source.connection - ))); - }; - let limits = crate::odbc_source::models::ValueLimits { - max_text_size: source.max_text_size, - max_binary_size: source.max_binary_size, - long_column_threshold: source - .long_column_threshold - .map(|size| usize::try_from(size.0).unwrap_or(usize::MAX)), - }; - crate::odbc_source::main::odbc_reader::odbc_reader( - (connection_string.as_ref(), dsn.as_deref()), - (user.as_deref(), password.as_ref()), - limits, - ) - .map_err(LoaderError::Invalid) -} - /// `planned`, each unblocked table of ODBC source `source` given how its rows are fetched on this machine: its budget, and in bulk or value by value (see [`plan_fetches`](plan_fetches)). #[cfg(feature = "odbc")] pub(crate) async fn with_fetch_memory( @@ -418,7 +417,7 @@ pub(crate) async fn with_fetch_memory( source: &SourceConfig, mut planned: Vec, ) -> Result, LoaderError> { - let reader = odbc_source_reader(project, source)?; + let reader = source_reader(project, source).map_err(LoaderError::Invalid)?; let open: Vec = (0..planned.len()) .filter(|&index| planned[index].schema.blocked.is_none()) .collect(); @@ -458,9 +457,9 @@ pub(crate) async fn with_fetch_memory( #[cfg(feature = "odbc")] fn open_odbc(project: &ProjectConfig, source: &SourceConfig) -> Result { - Ok(SourceReader::Odbc(Box::new(odbc_source_reader( - project, source, - )?))) + Ok(SourceReader::Odbc(Box::new( + source_reader(project, source).map_err(LoaderError::Invalid)?, + ))) } #[cfg(not(feature = "odbc"))] @@ -477,7 +476,7 @@ async fn discover_odbc( source: &SourceConfig, queries: &[(String, String)], ) -> Result, String> { - let reader = odbc_source_reader(project, source).map_err(|error| error.to_string())?; + let reader = source_reader(project, source)?; crate::odbc_source::main::read_catalog::read_catalog( &reader, ( @@ -1596,6 +1595,19 @@ async fn read_each( }; Ok(read_databricks_rows(reader, &request, on_batch).await?) } + #[cfg(feature = "duckdb")] + (SourceReader::DuckDb(reader), _) => { + let filter = table + .spec + .read_filter(part.filter.or_else(|| table.source_filter.clone())); + let request = DuckDbReadRequest { + table: &table.spec.source_table, + sql: table.spec.query_sql(), + columns: &table.read_columns, + filter: filter.as_deref(), + }; + Ok(read_duckdb_rows(reader, &request, SOURCE_BATCH_ROWS, on_batch).await?) + } (SourceReader::Sheets(client, spreadsheet_id), _) => { let batches = sheet_rows((client, spreadsheet_id), table, part.transform.as_ref()).await?; diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs index 5e03949d2..e9e0d0ca8 100644 --- a/crates/sqlload/src/loader/_helpers/specs.rs +++ b/crates/sqlload/src/loader/_helpers/specs.rs @@ -7,7 +7,6 @@ use crate::adapters::contract::models::{ChangeApply, TableName, WriteMode}; use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter; use crate::adapters::files::constants::{CURSOR_PLACEHOLDER_START, TABLE_PLACEHOLDER}; use crate::bigquery_source::constants::BIGQUERY_SOURCE; -use crate::bigquery_source::main::count_rows::count_rows as count_bigquery_rows; use crate::cdc::constants::{ BATCH_INTERVAL, CDC_FILES_PROBLEM, CDC_TARGETS_PROBLEM, FILES_BATCH_INTERVAL, }; @@ -19,7 +18,7 @@ use crate::column_types::main::is_variant::is_variant; use crate::column_types::main::parse_type::parse_type; use crate::column_types::models::{ColumnType, Warehouse}; use crate::databricks_source::constants::DATABRICKS_SOURCE; -use crate::databricks_source::main::count_rows::count_rows as count_databricks_rows; +use crate::duckdb_source::constants::{DUCKDB_SOURCE, MOTHERDUCK_SOURCE}; use crate::files_source::main::constant_columns::constant_columns; use crate::files_source::main::path_template::path_template; use crate::files_source::models::FileColumn; @@ -43,7 +42,6 @@ use crate::loader::models::{ }; use crate::loader::models::{SourceReader, TablePlan}; use crate::masking::models::MaskRules; -use crate::postgres_source::main::count_rows::count_rows as count_postgres_rows; use crate::postgres_source::models::{PgType, SourceColumn, SourceTable}; use crate::postgres_source::models::{ReadColumn, WireType}; use crate::project::constants::DEFAULT_MAX_DEPTH; @@ -66,7 +64,6 @@ use crate::sheets_source::models::SheetRead; use crate::similar_names::main::close_name::close_name; use crate::similar_names::main::nearest_name::nearest_name; use crate::snowflake_source::constants::USER_STAGE_FOLDER; -use crate::snowflake_source::main::count_rows::count_rows as count_snowflake_rows; use globset::{Glob, GlobMatcher, GlobSet, GlobSetBuilder}; use std::collections::{BTreeMap, BTreeSet}; @@ -532,17 +529,21 @@ pub(crate) fn build_spec( deletes, tombstones_of: overrides.tombstones_of.clone(), warehouse_source: match context.source.kind { - kind @ (SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks) => { - Some(WarehouseSource { - system: match kind { - SourceKind::Odbc => ODBC_SOURCE, - SourceKind::BigQuery => BIGQUERY_SOURCE, - _ => DATABRICKS_SOURCE, - }, - sql: overrides.sql.clone(), - route: WarehouseRoute::Query, - }) - } + kind @ (SourceKind::Odbc + | SourceKind::BigQuery + | SourceKind::Databricks + | SourceKind::DuckDb + | SourceKind::MotherDuck) => Some(WarehouseSource { + system: match kind { + SourceKind::Odbc => ODBC_SOURCE, + SourceKind::BigQuery => BIGQUERY_SOURCE, + SourceKind::Databricks => DATABRICKS_SOURCE, + SourceKind::DuckDb => DUCKDB_SOURCE, + _ => MOTHERDUCK_SOURCE, + }, + sql: overrides.sql.clone(), + route: WarehouseRoute::Query, + }), _ => warehouse_source(context.source, overrides), }, row_filter: layered(&settings.row_filter, &defaults.row_filter) @@ -783,7 +784,9 @@ pub(crate) fn missing_description( | SourceKind::Snowflake | SourceKind::Odbc | SourceKind::BigQuery - | SourceKind::Databricks => format!( + | SourceKind::Databricks + | SourceKind::DuckDb + | SourceKind::MotherDuck => format!( "no description; add description = \"...\" under [sources.{source_name}.tables.\"{qualified}\"]" ), }; @@ -1384,31 +1387,7 @@ pub(crate) async fn null_cursor_rows( .warehouse_source .as_ref() .and_then(|source| source.sql.as_deref()); - let counted = match reader { - SourceReader::Postgres(client, _) => { - count_postgres_rows(client, &spec.source_table, &filter).await? - } - SourceReader::Snowflake(snowflake) => { - count_snowflake_rows(snowflake, (&spec.source_table, sql), &filter).await? - } - SourceReader::BigQuery(bigquery) => { - count_bigquery_rows(bigquery, (&spec.source_table, sql), &filter).await? - } - SourceReader::Databricks(databricks) => { - count_databricks_rows(databricks, (&spec.source_table, sql), &filter).await? - } - #[cfg(feature = "odbc")] - SourceReader::Odbc(odbc) => { - crate::odbc_source::main::count_rows::count_rows( - odbc, - (&spec.source_table, sql), - &filter, - ) - .await? - } - SourceReader::Files(_) | SourceReader::Sheets(..) => return Ok(None), - }; - Ok(Some(counted)) + crate::loader::_helpers::resolve::count_rows(reader, (&spec.source_table, sql), &filter).await } /// Why `spec`'s load fails: `rows` source rows have a NULL cursor, which its incremental read never selects, so they would never load; with the fixes `plan` accepts for the table: `null_cursor = "include"` on `merge`, a coalesced cursor where one may be listed (not a Python table, `delete_insert` or a files target), and `null_cursor = "ignore"`. diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index f6fcc9850..c8a357d47 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -432,6 +432,16 @@ pub(crate) fn read_columns(spec: &TableSpec) -> Result, Vec + { + crate::duckdb_source::main::read_columns::read_columns( + &selected_columns(spec), + &spec.pinned, + ) + } #[cfg(feature = "odbc")] (None, None) if spec @@ -1857,7 +1867,8 @@ pub(crate) async fn block_refused_bounds( (source_name, source): (&str, &crate::project::models::SourceConfig), planned: &mut [TablePlan], ) -> Result<(), LoaderError> { - let reader = crate::loader::_helpers::sources::odbc_source_reader(project, source)?; + let reader = crate::odbc_source::main::source_reader::source_reader(project, source) + .map_err(LoaderError::Invalid)?; for table in planned .iter_mut() .filter(|table| table.schema.blocked.is_none()) diff --git a/crates/sqlload/src/loader/constants.rs b/crates/sqlload/src/loader/constants.rs index 910eeb37e..f4ef5f8c6 100644 --- a/crates/sqlload/src/loader/constants.rs +++ b/crates/sqlload/src/loader/constants.rs @@ -114,6 +114,9 @@ pub const UNLOADED_TIMES: &str = "the warehouse writes times and timestamps as w /// Why an `odbc` source cannot be read by this build. pub const NO_ODBC: &str = "this build of SQLLoad has no ODBC support; build it with the odbc feature (cargo install sqlload --features odbc)"; +/// Why a build without DuckDB refuses a `duckdb` source. +pub const NO_DUCKDB_SOURCE: &str = "this build of SQLLoad has no DuckDB support; build it with the duckdb feature (cargo install sqlload --features duckdb)"; + /// The plan warning for a files table whose checks pull its files through SQLLoad instead of the destination's own read. pub const CHECKS_LOADER_WARNING: &str = "checks run on rows SQLLoad stages, so every file goes through the loader rather than the destination's own read, which is slower; without checks the table stays on the direct route."; diff --git a/crates/sqlload/src/loader/errors.rs b/crates/sqlload/src/loader/errors.rs index d69f90e2e..4c7ba5225 100644 --- a/crates/sqlload/src/loader/errors.rs +++ b/crates/sqlload/src/loader/errors.rs @@ -37,6 +37,9 @@ pub enum LoaderError { #[cfg(feature = "odbc")] #[error(transparent)] Odbc(#[from] crate::odbc_source::errors::OdbcSourceError), + #[cfg(feature = "duckdb")] + #[error(transparent)] + DuckDbSource(#[from] crate::duckdb_source::errors::DuckDbSourceError), #[error(transparent)] Store(#[from] StoreError), #[error(transparent)] diff --git a/crates/sqlload/src/loader/main/_check_source.rs b/crates/sqlload/src/loader/main/_check_source.rs index 64f58ca1e..b398a4754 100644 --- a/crates/sqlload/src/loader/main/_check_source.rs +++ b/crates/sqlload/src/loader/main/_check_source.rs @@ -9,7 +9,6 @@ use crate::loader::errors::LoaderError; use crate::loader::models::SourceReader; use crate::project::models::{ConnectionConfig, ProjectConfig, SourceConfig, SourceKind}; use crate::sheets_source::main::tab_ranges::tab_ranges; -use crate::sql_api::main::execute_sql::execute_sql; /// What source `source` reads as, or why it cannot connect; for Postgres, what protects the connection and the bastion it goes through (a failure names the hop: the SSH tunnel, or the database via it). pub(in crate::loader) async fn check_source( @@ -25,16 +24,7 @@ pub(in crate::loader) async fn check_source( "a snowflake source opens as a Snowflake reader".to_owned(), )); }; - let answer = execute_sql( - &snowflake.sql, - "SELECT CURRENT_USER() || '@' || CURRENT_ACCOUNT() || ', warehouse ' || COALESCE(CURRENT_WAREHOUSE(), 'none') || ', Snowflake ' || CURRENT_VERSION()", - ) - .await?; - Ok(answer - .rows - .first() - .and_then(|row| row.first().cloned().flatten()) - .unwrap_or_default()) + Ok(crate::snowflake_source::main::describe_source::describe_source(&snowflake).await?) } SourceKind::BigQuery => { let SourceReader::BigQuery(reader) = open_source(project, source, 1).await? else { @@ -57,9 +47,19 @@ pub(in crate::loader) async fn check_source( )) } SourceKind::Databricks => check_databricks(project, source).await, + #[cfg(feature = "duckdb")] + SourceKind::DuckDb | SourceKind::MotherDuck => { + let reader = crate::duckdb_source::main::duckdb_reader::duckdb_reader(project, source)?; + Ok(crate::duckdb_source::main::describe_source::describe_source(&reader).await?) + } + #[cfg(not(feature = "duckdb"))] + SourceKind::DuckDb | SourceKind::MotherDuck => Err(LoaderError::Invalid( + crate::loader::constants::NO_DUCKDB_SOURCE.to_owned(), + )), #[cfg(feature = "odbc")] SourceKind::Odbc => { - let reader = crate::loader::_helpers::sources::odbc_source_reader(project, source)?; + let reader = crate::odbc_source::main::source_reader::source_reader(project, source) + .map_err(LoaderError::Invalid)?; Ok(crate::odbc_source::main::describe_source::describe_source(&reader).await?) } #[cfg(not(feature = "odbc"))] diff --git a/crates/sqlload/src/loader/models.rs b/crates/sqlload/src/loader/models.rs index afd1580ee..b24678bba 100644 --- a/crates/sqlload/src/loader/models.rs +++ b/crates/sqlload/src/loader/models.rs @@ -1305,6 +1305,9 @@ pub enum SourceReader { /// A database read through its ODBC driver. #[cfg(feature = "odbc")] Odbc(Box), + /// A DuckDB database file, opened read-only for each read. + #[cfg(feature = "duckdb")] + DuckDb(Box), /// A Google Sheets client and the spreadsheet it reads. Sheets(crate::sheets_source::models::SheetsClient, String), } diff --git a/crates/sqlload/src/memory_budget/main/mod.rs b/crates/sqlload/src/memory_budget/main/mod.rs index 28d1f9637..d2ecdd468 100644 --- a/crates/sqlload/src/memory_budget/main/mod.rs +++ b/crates/sqlload/src/memory_budget/main/mod.rs @@ -5,4 +5,5 @@ pub mod run_memory; pub mod run_stopped; pub mod start_watchdog; pub mod table_memory; +pub mod watch_child; pub mod watch_table; diff --git a/crates/sqlload/src/memory_budget/main/watch_child.rs b/crates/sqlload/src/memory_budget/main/watch_child.rs new file mode 100644 index 000000000..7d0a2d17a --- /dev/null +++ b/crates/sqlload/src/memory_budget/main/watch_child.rs @@ -0,0 +1,18 @@ +//! Count a helper process's memory with the load's. + +use std::sync::Arc; + +use crate::memory_budget::models::{MemoryGuard, WatchedChild}; + +/// Have `guard` count process `pid`'s memory with SQLLoad's until the returned handle is dropped, so a process that has ended (and whose id may be reused) stops counting. +pub fn watch_child(guard: &Arc, pid: u32) -> WatchedChild { + guard + .children + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(pid); + WatchedChild { + guard: guard.clone(), + pid, + } +} diff --git a/crates/sqlload/src/memory_budget/models.rs b/crates/sqlload/src/memory_budget/models.rs index 30b7da4ee..577d92141 100644 --- a/crates/sqlload/src/memory_budget/models.rs +++ b/crates/sqlload/src/memory_budget/models.rs @@ -169,3 +169,23 @@ pub struct WatchedTable { pub guard: std::sync::Arc, pub table: std::sync::Arc, } + +/// A helper process whose memory a guard counts with SQLLoad's, until this is dropped. +#[derive(Debug)] +pub struct WatchedChild { + pub guard: std::sync::Arc, + pub pid: u32, +} + +impl Drop for WatchedChild { + fn drop(&mut self) { + let mut children = self + .guard + .children + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(index) = children.iter().position(|pid| *pid == self.pid) { + children.swap_remove(index); + } + } +} diff --git a/crates/sqlload/src/memory_budget/tests/test_types.rs b/crates/sqlload/src/memory_budget/tests/test_types.rs index 4bcc69443..e584a1d0b 100644 --- a/crates/sqlload/src/memory_budget/tests/test_types.rs +++ b/crates/sqlload/src/memory_budget/tests/test_types.rs @@ -47,3 +47,11 @@ pub(crate) struct MemorySettingTestCase { pub(crate) text: &'static str, pub(crate) expected_setting: Result, } + +/// Helper processes a guard watches, the first `dropped` of them ended, and the ids it still counts. +pub(crate) struct WatchedChildTestCase { + pub(crate) description: &'static str, + pub(crate) pids: &'static [u32], + pub(crate) dropped: usize, + pub(crate) expected_counted: &'static [u32], +} diff --git a/crates/sqlload/src/memory_budget/tests/test_watch.rs b/crates/sqlload/src/memory_budget/tests/test_watch.rs index bea3f75d5..629163907 100644 --- a/crates/sqlload/src/memory_budget/tests/test_watch.rs +++ b/crates/sqlload/src/memory_budget/tests/test_watch.rs @@ -1,5 +1,8 @@ use crate::memory_budget::_helpers::watch::run_stop_due; -use crate::memory_budget::tests::test_types::RunStopTestCase; +use crate::memory_budget::main::start_watchdog::start_watchdog; +use crate::memory_budget::main::watch_child::watch_child; +use crate::memory_budget::models::{BudgetOrigin, MemoryBudget}; +use crate::memory_budget::tests::test_types::{RunStopTestCase, WatchedChildTestCase}; #[test] fn given_readings_past_95_percent_when_watching_then_the_run_stops_only_when_no_table_stopping_frees_memory_in_time() @@ -63,3 +66,55 @@ fn given_readings_past_95_percent_when_watching_then_the_run_stops_only_when_no_ ); } } + +#[test] +fn given_watched_children_when_some_handles_drop_then_the_guard_counts_only_the_others() { + let test_cases = [ + WatchedChildTestCase { + description: "both children count while open", + pids: &[41, 42], + dropped: 0, + expected_counted: &[41, 42], + }, + WatchedChildTestCase { + description: "an ended child's id stops counting", + pids: &[41, 42], + dropped: 1, + expected_counted: &[42], + }, + WatchedChildTestCase { + description: "no child counts once all have ended", + pids: &[41, 42], + dropped: 2, + expected_counted: &[], + }, + ]; + for test_case in test_cases { + let guard = start_watchdog(MemoryBudget { + bytes: u64::MAX, + origin: BudgetOrigin::Setting, + limit: None, + overhead: 0, + duckdb: None, + duckdb_targets: 0, + table_share: 0, + tables_at_once: 1, + }); + let mut handles: Vec<_> = test_case + .pids + .iter() + .map(|pid| watch_child(&guard, *pid)) + .collect(); + handles.drain(..test_case.dropped); + let counted = guard + .children + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + assert_eq!( + counted, test_case.expected_counted, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/odbc_source/main/odbc_reader.rs b/crates/sqlload/src/odbc_source/main/_odbc_reader.rs similarity index 98% rename from crates/sqlload/src/odbc_source/main/odbc_reader.rs rename to crates/sqlload/src/odbc_source/main/_odbc_reader.rs index c76e9def0..d13135b99 100644 --- a/crates/sqlload/src/odbc_source/main/odbc_reader.rs +++ b/crates/sqlload/src/odbc_source/main/_odbc_reader.rs @@ -4,7 +4,7 @@ use crate::auth::models::Secret; use crate::odbc_source::models::{OdbcReader, ValueLimits}; /// A reader connecting with `connection_string` as given, or with `DSN=` and `UID=` from `dsn` and `user` as written (driver managers read them unbraced; validation refuses `;`, `{` and `}` in them) and `PWD=` from `password` in braces, so any character in it stays literal; nothing connects until the first read. -pub fn odbc_reader( +pub(crate) fn odbc_reader( (connection_string, dsn): (Option<&Secret>, Option<&str>), (user, password): (Option<&str>, Option<&Secret>), limits: ValueLimits, diff --git a/crates/sqlload/src/odbc_source/main/mod.rs b/crates/sqlload/src/odbc_source/main/mod.rs index 9fa4f531b..63e160acb 100644 --- a/crates/sqlload/src/odbc_source/main/mod.rs +++ b/crates/sqlload/src/odbc_source/main/mod.rs @@ -1,9 +1,10 @@ +pub(in crate::odbc_source) mod _odbc_reader; pub mod count_rows; pub mod describe_source; pub mod memory_warning; -pub mod odbc_reader; pub mod plan_fetches; pub mod read_catalog; pub mod read_columns; pub mod read_rows; pub mod run_query; +pub mod source_reader; diff --git a/crates/sqlload/src/odbc_source/main/source_reader.rs b/crates/sqlload/src/odbc_source/main/source_reader.rs new file mode 100644 index 000000000..a14ebcabf --- /dev/null +++ b/crates/sqlload/src/odbc_source/main/source_reader.rs @@ -0,0 +1,33 @@ +//! The reader for an `odbc` source. + +use crate::odbc_source::main::_odbc_reader::odbc_reader; +use crate::odbc_source::models::{OdbcReader, ValueLimits}; +use crate::project::models::{ConnectionConfig, ProjectConfig, SourceConfig}; + +/// A reader for `source`'s `odbc` connection, with the source's value caps; nothing connects until the first read. +pub fn source_reader(project: &ProjectConfig, source: &SourceConfig) -> Result { + let Some(ConnectionConfig::Odbc { + connection_string, + dsn, + user, + password, + }) = project.connections.get(&source.connection) + else { + return Err(format!( + "connection {:?} is not an odbc connection", + source.connection + )); + }; + let limits = ValueLimits { + max_text_size: source.max_text_size, + max_binary_size: source.max_binary_size, + long_column_threshold: source + .long_column_threshold + .map(|size| usize::try_from(size.0).unwrap_or(usize::MAX)), + }; + odbc_reader( + (connection_string.as_ref(), dsn.as_deref()), + (user.as_deref(), password.as_ref()), + limits, + ) +} diff --git a/crates/sqlload/src/odbc_source/tests/test_connect.rs b/crates/sqlload/src/odbc_source/tests/test_connect.rs index 895f2fc1b..ba5a7c78d 100644 --- a/crates/sqlload/src/odbc_source/tests/test_connect.rs +++ b/crates/sqlload/src/odbc_source/tests/test_connect.rs @@ -1,6 +1,6 @@ use crate::auth::models::Secret; use crate::odbc_source::_helpers::connect::connect; -use crate::odbc_source::main::odbc_reader::odbc_reader; +use crate::odbc_source::main::_odbc_reader::odbc_reader; use crate::odbc_source::tests::helpers::elf_with_flags; use crate::odbc_source::tests::test_types::ConnectErrorTestCase; diff --git a/crates/sqlload/src/odbc_source/tests/test_odbc_reader.rs b/crates/sqlload/src/odbc_source/tests/test_odbc_reader.rs index 5feda5ef4..060f4b5ad 100644 --- a/crates/sqlload/src/odbc_source/tests/test_odbc_reader.rs +++ b/crates/sqlload/src/odbc_source/tests/test_odbc_reader.rs @@ -1,5 +1,5 @@ use crate::auth::models::Secret; -use crate::odbc_source::main::odbc_reader::odbc_reader; +use crate::odbc_source::main::_odbc_reader::odbc_reader; use crate::odbc_source::tests::test_types::OdbcReaderTestCase; #[test] diff --git a/crates/sqlload/src/project/_helpers/references.rs b/crates/sqlload/src/project/_helpers/references.rs index a12c2d479..53acfba9a 100644 --- a/crates/sqlload/src/project/_helpers/references.rs +++ b/crates/sqlload/src/project/_helpers/references.rs @@ -106,6 +106,8 @@ fn source_problems(project: &ProjectConfig, name: &str, source: &SourceConfig) - SourceKind::Odbc => (&["odbc"], "odbc"), SourceKind::BigQuery => (&["bigquery"], "bigquery"), SourceKind::Databricks => (&["databricks"], "databricks"), + SourceKind::DuckDb => (&["duckdb"], "duckdb"), + SourceKind::MotherDuck => (&["motherduck"], "motherduck"), SourceKind::Files => (&["s3", "filesystem", "sftp"], "s3, filesystem or sftp"), SourceKind::Python => (&[], "any"), SourceKind::GoogleSheets => (&["google_sheets"], "google_sheets"), @@ -224,11 +226,16 @@ fn kind_problems(place: &str, source: &SourceConfig) -> Vec { let queries = source.tables.values().any(|table| table.sql.is_some()); if !matches!( source.kind, - SourceKind::Snowflake | SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks + SourceKind::Snowflake + | SourceKind::Odbc + | SourceKind::BigQuery + | SourceKind::Databricks + | SourceKind::DuckDb + | SourceKind::MotherDuck ) && queries { problems.push(format!( - "{place} sql is for kind = \"snowflake\", \"odbc\", \"bigquery\" and \"databricks\"" + "{place} sql is for kind = \"snowflake\", \"odbc\", \"bigquery\", \"databricks\", \"duckdb\" and \"motherduck\"" )); } if source.kind != SourceKind::BigQuery && source.project.is_some() { @@ -275,10 +282,12 @@ fn kind_problems(place: &str, source: &SourceConfig) -> Vec { | SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks + | SourceKind::DuckDb + | SourceKind::MotherDuck ); if !selects && sheet_settings(source).any(|settings| settings.row_filter.is_some()) { problems.push(format!( - "{place} where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\" and \"databricks\", whose tables SQLLoad reads with a SELECT" + "{place} where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\", \"databricks\", \"duckdb\" and \"motherduck\", whose tables SQLLoad reads with a SELECT" )); } problems.extend( @@ -304,13 +313,17 @@ fn kind_problems(place: &str, source: &SourceConfig) -> Vec { | SourceKind::Snowflake | SourceKind::Odbc | SourceKind::BigQuery - | SourceKind::Databricks => { + | SourceKind::Databricks + | SourceKind::DuckDb + | SourceKind::MotherDuck => { if source.include.is_empty() && !queries { let tables = match source.kind { SourceKind::Snowflake | SourceKind::Odbc | SourceKind::BigQuery - | SourceKind::Databricks => { + | SourceKind::Databricks + | SourceKind::DuckDb + | SourceKind::MotherDuck => { "; list tables in include, or give a table a sql query" } _ => "", diff --git a/crates/sqlload/src/project/models.rs b/crates/sqlload/src/project/models.rs index a4cc7b380..2869c819f 100644 --- a/crates/sqlload/src/project/models.rs +++ b/crates/sqlload/src/project/models.rs @@ -609,7 +609,7 @@ impl SourceConfig { .unwrap_or_default() } - /// Whether the source finds its tables in a database's catalog with `include` patterns (`postgres`, `postgres_cdc`, `snowflake`, `odbc` or `bigquery`). + /// Whether the source finds its tables in a database's catalog with `include` patterns (every database source kind). pub fn discovers_tables(&self) -> bool { matches!( self.kind, @@ -619,6 +619,8 @@ impl SourceConfig { | SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks + | SourceKind::DuckDb + | SourceKind::MotherDuck ) } @@ -677,6 +679,8 @@ impl SourceConfig { | SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks + | SourceKind::DuckDb + | SourceKind::MotherDuck | SourceKind::Python, ) => DEFAULT_TABLE_NAME, (None, SourceKind::Files) => DEFAULT_FILES_TABLE_NAME, @@ -703,6 +707,12 @@ pub enum SourceKind { BigQuery, /// Databricks tables, views and queries, read as Arrow through the Statement Execution API's result links. Databricks, + /// Tables, views and queries of a DuckDB database file, read as Arrow from a read-only connection. + #[serde(rename = "duckdb")] + DuckDb, + /// Tables, views and queries of a MotherDuck database, read as Arrow through the MotherDuck bridge. + #[serde(rename = "motherduck")] + MotherDuck, /// Declared in Python with `sl.source` or `@sl.table`; never written in `sqlload.toml`. Python, /// Tabs or ranges of a Google Sheets spreadsheet, one table each. @@ -720,6 +730,8 @@ impl SourceKind { Self::Odbc => "odbc", Self::BigQuery => "bigquery", Self::Databricks => "databricks", + Self::DuckDb => "duckdb", + Self::MotherDuck => "motherduck", Self::Python => "python", Self::GoogleSheets => "google_sheets", } diff --git a/crates/sqlload/src/project/tests/test_parse_project.rs b/crates/sqlload/src/project/tests/test_parse_project.rs index f3c36d002..3bc750fe0 100644 --- a/crates/sqlload/src/project/tests/test_parse_project.rs +++ b/crates/sqlload/src/project/tests/test_parse_project.rs @@ -435,9 +435,9 @@ fn given_an_invalid_project_when_parsing_then_every_problem_is_named() { expected_message: "unknown variant `avro`", }, ParseProjectErrorTestCase { - description: "a query table is for snowflake, odbc, bigquery and databricks sources", + description: "a query table is for snowflake, odbc, bigquery, databricks, duckdb and motherduck sources", text: "[sources.app]\nkind = \"postgres\"\nconnection = \"pg\"\ninclude = [\"*\"]\n[sources.app.tables.\"public.a\"]\nsql = \"SELECT 1\"\n", - expected_message: "[sources.app] sql is for kind = \"snowflake\", \"odbc\", \"bigquery\" and \"databricks\"", + expected_message: "[sources.app] sql is for kind = \"snowflake\", \"odbc\", \"bigquery\", \"databricks\", \"duckdb\" and \"motherduck\"", }, ParseProjectErrorTestCase { description: "a databricks connection a target loads into needs a volume to stage files in", @@ -447,12 +447,12 @@ fn given_an_invalid_project_when_parsing_then_every_problem_is_named() { ParseProjectErrorTestCase { description: "where narrows a database table's SELECT, so a files source refuses it", text: "[connections.lake]\nkind = \"filesystem\"\nroot = \"/data\"\n[sources.raw]\nkind = \"files\"\nconnection = \"lake\"\npath = \"raw/\"\nformat = \"csv\"\nwhere = \"a = 1\"\n", - expected_message: "[sources.raw] where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\" and \"databricks\"", + expected_message: "[sources.raw] where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\", \"databricks\", \"duckdb\" and \"motherduck\"", }, ParseProjectErrorTestCase { description: "a postgres_cdc source captures every change, so it refuses where on a table", text: "[sources.app]\nkind = \"postgres_cdc\"\nconnection = \"pg\"\ninclude = [\"public.a\"]\n[sources.app.tables.\"public.a\"]\nwhere = \"a = 1\"\n", - expected_message: "[sources.app] where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\" and \"databricks\"", + expected_message: "[sources.app] where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\", \"databricks\", \"duckdb\" and \"motherduck\"", }, ParseProjectErrorTestCase { description: "a query table takes its condition in its sql", diff --git a/crates/sqlload/src/row_deletes/main/deletes_problems.rs b/crates/sqlload/src/row_deletes/main/deletes_problems.rs index b9e99916e..9b2e291db 100644 --- a/crates/sqlload/src/row_deletes/main/deletes_problems.rs +++ b/crates/sqlload/src/row_deletes/main/deletes_problems.rs @@ -27,6 +27,8 @@ pub fn deletes_problems( | SourceKind::Snowflake | SourceKind::BigQuery | SourceKind::Databricks + | SourceKind::DuckDb + | SourceKind::MotherDuck ) { problems.push(format!( "deletes reads every key of the source table to find the ones it no longer holds, which only database and warehouse sources list; a {} source can't. Fix: remove deletes", diff --git a/crates/sqlload/src/snowflake_source/main/describe_source.rs b/crates/sqlload/src/snowflake_source/main/describe_source.rs new file mode 100644 index 000000000..4c9ca3cd9 --- /dev/null +++ b/crates/sqlload/src/snowflake_source/main/describe_source.rs @@ -0,0 +1,19 @@ +//! Who a Snowflake source reads as, for `debug`. + +use crate::snowflake_source::errors::SnowflakeSourceError; +use crate::snowflake_source::models::SnowflakeReader; +use crate::sql_api::main::execute_sql::execute_sql; + +/// The reader's user and account, its warehouse and the Snowflake version. +pub async fn describe_source(reader: &SnowflakeReader) -> Result { + let answer = execute_sql( + &reader.sql, + "SELECT CURRENT_USER() || '@' || CURRENT_ACCOUNT() || ', warehouse ' || COALESCE(CURRENT_WAREHOUSE(), 'none') || ', Snowflake ' || CURRENT_VERSION()", + ) + .await?; + Ok(answer + .rows + .first() + .and_then(|row| row.first().cloned().flatten()) + .unwrap_or_default()) +} diff --git a/crates/sqlload/src/snowflake_source/main/mod.rs b/crates/sqlload/src/snowflake_source/main/mod.rs index 89c7de7c3..0215ae2d6 100644 --- a/crates/sqlload/src/snowflake_source/main/mod.rs +++ b/crates/sqlload/src/snowflake_source/main/mod.rs @@ -1,5 +1,6 @@ pub mod connect_snowflake_source; pub mod count_rows; +pub mod describe_source; pub mod read_catalog; pub mod read_columns; pub mod unload_files; diff --git a/crates/sqlload/tests/loader.rs b/crates/sqlload/tests/loader.rs index a28fe11fd..094767b60 100644 --- a/crates/sqlload/tests/loader.rs +++ b/crates/sqlload/tests/loader.rs @@ -14,6 +14,9 @@ mod clickhouse_helpers; mod cursor_helpers; #[path = "loader/databricks/helpers.rs"] mod databricks_helpers; +#[cfg(feature = "duckdb")] +#[path = "loader/duckdb/helpers.rs"] +mod duckdb_helpers; #[path = "loader/files/helpers.rs"] mod files_helpers; #[path = "loader/files_target/helpers.rs"] @@ -49,6 +52,9 @@ mod test_databricks_loads; mod test_databricks_source; #[path = "loader/test_debug_project.rs"] mod test_debug_project; +#[cfg(feature = "duckdb")] +#[path = "loader/test_duckdb_source.rs"] +mod test_duckdb_source; #[path = "loader/test_file_store.rs"] mod test_file_store; #[path = "loader/test_files_round_trip.rs"] diff --git a/crates/sqlload/tests/loader/duckdb/helpers.rs b/crates/sqlload/tests/loader/duckdb/helpers.rs new file mode 100644 index 000000000..7102e009d --- /dev/null +++ b/crates/sqlload/tests/loader/duckdb/helpers.rs @@ -0,0 +1,45 @@ +//! Helpers for the DuckDB source loader test. + +/// The project: a duckdb source over `source` with include and exclude patterns, an excluded column, its orders merged after a cursor with `where` and hard deletes, a query table, and DuckDB file `target` as the target. +pub(crate) fn duckdb_source_project(source: &std::path::Path, target: &std::path::Path) -> String { + format!( + r#"default_target = "local" + +[connections.shop_file] +kind = "duckdb" +database = "{source}" + +[connections.warehouse] +kind = "duckdb" +database = "{target}" + +[targets.local] +connection = "warehouse" +loader_schema = "raw" + +[sources.shop] +kind = "duckdb" +connection = "shop_file" +description = "The shop's DuckDB file." +include = ["sales.*"] +exclude = ["sales.tmp_*"] +exclude_columns = ["card"] +write_strategy = "replace" + +[sources.shop.tables."sales.orders"] +description = "Orders, one row each." +write_strategy = "merge" +unique_key = ["id"] +cursor_column = "updated_at" +where = "region = 'EU'" +deletes = "hard" +checks = {{ max_row_drop = "100%" }} + +[sources.shop.tables.big_query] +description = "Orders over 25, by query." +sql = "SELECT id, amount FROM sales.orders WHERE amount > 25 -- big ones" +"#, + source = source.display(), + target = target.display() + ) +} diff --git a/crates/sqlload/tests/loader/test_duckdb_source.rs b/crates/sqlload/tests/loader/test_duckdb_source.rs new file mode 100644 index 000000000..0370cd15d --- /dev/null +++ b/crates/sqlload/tests/loader/test_duckdb_source.rs @@ -0,0 +1,89 @@ +use crate::duckdb_helpers::duckdb_source_project; +use crate::helpers::{duckdb_execute, duckdb_text, run_sqlload}; +use crate::test_types::DuckDbSourceTestCase; + +const SOURCE: &str = " +CREATE SCHEMA sales; +CREATE TABLE sales.orders ( + id INTEGER PRIMARY KEY, + amount DECIMAL(12,2), + updated_at TIMESTAMP, + tags VARCHAR[], + card VARCHAR, + region VARCHAR +); +INSERT INTO sales.orders VALUES + (1, 10.50, '2026-01-01 00:00:00', ['a'], '4111', 'EU'), + (2, 20.00, '2026-01-02 00:00:00', [], '4222', 'EU'), + (3, 30.25, '2026-01-03 00:00:00', NULL, '4333', 'US'); +CREATE VIEW sales.big_orders AS SELECT id, amount FROM sales.orders WHERE amount > 15; +CREATE TABLE sales.tmp_scratch (x INTEGER); +"; + +/// A second run's changes: order 2 updated, order 4 added, order 1 untouched. +const CHANGES: &str = " +UPDATE sales.orders SET amount = 21.00, updated_at = '2026-01-05 00:00:00' WHERE id = 2; +INSERT INTO sales.orders VALUES (4, 40.00, '2026-01-06 00:00:00', ['b', 'c'], '4444', 'EU'); +"; + +/// Rows in id order with amount and the tags' JSON, the target's columns, and the view's and query table's rows. +const CHECK: &str = "SELECT (SELECT string_agg(id || ':' || amount || ':' || coalesce(CAST(tags AS VARCHAR), '-'), ',' ORDER BY id) FROM raw.shop__orders) || '|' || (SELECT string_agg(column_name, ',' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_schema = 'raw' AND table_name = 'shop__orders' AND column_name NOT LIKE '\\_sqlload%' ESCAPE '\\') || '|' || (SELECT count(*) FROM raw.shop__big_orders) || '|' || (SELECT string_agg(CAST(id AS VARCHAR), ',' ORDER BY id) FROM raw.shop__big_query)"; + +#[test] +fn given_a_duckdb_source_when_loading_again_then_the_cursor_merges_only_changes_and_sync_deletes_removes_gone_rows() +-> Result<(), String> { + let folder = tempfile::tempdir().map_err(|error| error.to_string())?; + let (source, target) = ( + folder.path().join("shop.duckdb"), + folder.path().join("out.duckdb"), + ); + duckdb_execute(&source, SOURCE)?; + let config = folder.path().join("sqlload.toml"); + std::fs::write(&config, duckdb_source_project(&source, &target)) + .map_err(|error| error.to_string())?; + let (planned, plan) = run_sqlload(&config, &["plan"], ("", ""))?; + assert!(planned, "{plan}"); + for table in [ + "raw.shop__orders", + "raw.shop__big_orders", + "raw.shop__big_query", + ] { + assert!(plan.contains(table), "plan lists {table}: {plan}"); + } + assert!( + !plan.contains("tmp_scratch"), + "exclude drops the table: {plan}" + ); + let test_cases = [ + DuckDbSourceTestCase { + description: "the first load: only EU rows, card excluded, tags as JSON, the view and the query", + source_change: "", + arguments: &["load"], + expected_rows: r#"1:10.50:["a"],2:20.00:[]|id,amount,updated_at,tags,region|2|3"#, + }, + DuckDbSourceTestCase { + description: "the second load merges the updated and new rows the cursor finds", + source_change: CHANGES, + arguments: &["load"], + expected_rows: r#"1:10.50:["a"],2:21.00:[],4:40.00:["b","c"]|id,amount,updated_at,tags,region|3|3,4"#, + }, + DuckDbSourceTestCase { + description: "--sync-deletes removes the row the source no longer holds", + source_change: "DELETE FROM sales.orders WHERE id = 1", + arguments: &["load", "--sync-deletes"], + expected_rows: r#"2:21.00:[],4:40.00:["b","c"]|id,amount,updated_at,tags,region|3|3,4"#, + }, + ]; + for test_case in test_cases { + duckdb_execute(&source, test_case.source_change)?; + let (loaded, output) = run_sqlload(&config, test_case.arguments, ("", ""))?; + assert!(loaded, "{}:\n{output}", test_case.description); + assert_eq!( + duckdb_text(&target, CHECK)?, + test_case.expected_rows, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlload/tests/loader/test_types.rs b/crates/sqlload/tests/loader/test_types.rs index 834134e6a..3f8989b86 100644 --- a/crates/sqlload/tests/loader/test_types.rs +++ b/crates/sqlload/tests/loader/test_types.rs @@ -1682,3 +1682,13 @@ pub(crate) struct WideLoadTestCase { /// Phrases the last load's output holds (`{table}` is the table's name). pub(crate) expected_output: &'static [&'static str], } + +/// One load from a duckdb source file into DuckDB, after a change to the source. +pub(crate) struct DuckDbSourceTestCase { + pub(crate) description: &'static str, + /// SQL run on the source file before the load; empty for none. + pub(crate) source_change: &'static str, + pub(crate) arguments: &'static [&'static str], + /// The orders' rows, the target's columns, the view's row count and the query table's ids. + pub(crate) expected_rows: &'static str, +} diff --git a/packages/sqlload/pyproject.toml b/packages/sqlload/pyproject.toml index a72863c18..f5cc4ed52 100644 --- a/packages/sqlload/pyproject.toml +++ b/packages/sqlload/pyproject.toml @@ -21,10 +21,10 @@ classifiers = [ dependencies = ["orjson>=3.10", "tzdata; sys_platform == \"win32\""] [project.optional-dependencies] -# MotherDuck targets: MotherDuck accepts DuckDB 1.4.1 to 1.5.6 (raise the bound as MotherDuck does), not the DuckDB -# 2.0 SQLLoad bundles, so SQLLoad runs their SQL on this package in a separate Python process. Remove once MotherDuck -# accepts SQLLoad's DuckDB. -motherduck = ["duckdb>=1.4.1,<=1.5.6"] +# MotherDuck targets and sources: MotherDuck accepts DuckDB 1.4.1 to 1.5.6 (raise the bound as MotherDuck does), not the +# DuckDB 2.0 SQLLoad bundles, so SQLLoad runs their SQL on this package in a separate Python process; a source's rows +# stream from it as Arrow batches through pyarrow. Remove once MotherDuck accepts SQLLoad's DuckDB. +motherduck = ["duckdb>=1.4.1,<=1.5.6", "pyarrow>=14"] [project.scripts] sqlload = "sqlload._cli:main" diff --git a/packages/sqlload/tests/test_motherduck.py b/packages/sqlload/tests/test_motherduck.py index 8d2e27426..6d7e00388 100644 --- a/packages/sqlload/tests/test_motherduck.py +++ b/packages/sqlload/tests/test_motherduck.py @@ -45,7 +45,8 @@ #: after checking the token, with another database attached as MotherDuck attaches the account's others. MD_REFUSE makes #: it refuse as MotherDuck's extension does on a DuckDB version it doesn't support; MD_DIE_ON ends the bridge when a #: statement holds that text; MD_LOG records every statement. MD_CLIENT_VERSION is the duckdb version the bridge reports, -#: and MD_SERVER_VERSION what `md_version()` answers (without it, the function doesn't exist). +#: and MD_SERVER_VERSION what `md_version()` answers (without it, the function doesn't exist). MD_READ_ONLY opens each +#: database read-only, so a source's bridges (one per table) can hold it at once as MotherDuck allows. STAND_IN = r''' import os import sys @@ -102,7 +103,10 @@ def connect(database=":memory:", *args, config=None, **kwargs): 'threw an exception: "\nYour DuckDB version (v2.0.0) is not yet supported by MotherDuck. The latest supported ' f'version is v1.5.6. Please downgrade to use MotherDuck. {database}\n"' ) - inner = _connect(os.path.join(FOLDER, f"{name}.duckdb"), config=config or {}) + read_only = {"access_mode": "READ_ONLY"} if os.environ.get("MD_READ_ONLY") else {} + inner = _connect(os.path.join(FOLDER, f"{name}.duckdb"), config={**(config or {}), **read_only}) + if read_only: + return Database(inner) inner.execute(f"ATTACH '{os.path.join(FOLDER, 'other.duckdb')}' AS other") inner.execute("CREATE SCHEMA IF NOT EXISTS other.raw; CREATE TABLE IF NOT EXISTS other.raw.cloud__cloud_rows (stray INTEGER)") if os.environ.get("MD_SERVER_VERSION"): @@ -487,3 +491,83 @@ def test_given_json_type_variant_on_motherduck_when_planning_and_loading_then_th loaded = load(project, "cloud_rows", **env) assert ("unshredded" in loaded.stdout + loaded.stderr) == ("unshredded" in expected), loaded.stdout + loaded.stderr assert cloud(project, 'SELECT variant_typeof(payload), CAST(payload.user.id AS BIGINT) FROM raw.cloud__cloud_rows') == [("OBJECT(user)", 42)] + + +#: A project reading the stand-in's `shop` database as a motherduck source into the local DuckDB file. +SOURCE_PROJECT = """\ +[connections.wh] +kind = "duckdb" +database = "wh.duckdb" + +[connections.md] +kind = "motherduck" +database = "shop" +token = "${ENV:SHOP_MD_TOKEN}" + +[targets.local] +connection = "wh" +loader_schema = "raw" + +[sources.shop] +kind = "motherduck" +connection = "md" +destination = "local" +description = "The shop's MotherDuck database." +include = ["sales.*"] +exclude = ["sales.tmp_*"] +exclude_columns = ["card"] +write_strategy = "replace" + +[sources.shop.tables."sales.orders"] +description = "Orders, merged after their cursor." +write_strategy = "merge" +unique_key = ["id"] +cursor_column = "updated_at" +where = "region = 'EU'" + +[sources.shop.tables.big_query] +description = "Orders over 25, by query." +sql = "SELECT id, amount FROM sales.orders WHERE amount > 25 -- big ones" +""" + +SHOP = """ +CREATE SCHEMA sales; +CREATE TABLE sales.orders (id INTEGER PRIMARY KEY, amount DECIMAL(12,2), updated_at TIMESTAMP, tags VARCHAR[], card VARCHAR, region VARCHAR); +INSERT INTO sales.orders VALUES + (1, 10.50, '2026-01-01 00:00:00', ['a'], '4111', 'EU'), + (2, 20.00, '2026-01-02 00:00:00', [], '4222', 'EU'), + (3, 30.25, '2026-01-03 00:00:00', NULL, '4333', 'US'); +CREATE VIEW sales.big_orders AS SELECT id, amount FROM sales.orders WHERE amount > 15; +CREATE TABLE sales.tmp_scratch (x INTEGER); +""" + +ORDERS = "SELECT id, CAST(amount AS VARCHAR), CAST(tags AS VARCHAR) FROM raw.shop__orders ORDER BY id" + + +def test_given_a_motherduck_source_when_loading_again_then_its_tables_views_and_queries_load_through_the_bridge_and_the_cursor_merges_changes( + tmp_path: Path, +) -> None: + project = make_project(tmp_path, "") + (project / "sqlload.toml").write_text(SOURCE_PROJECT) + with duckdb.connect(str(project / "md" / "shop.duckdb")) as shop: + shop.execute(SHOP) + statements = project / "statements.log" + debug = sqlload(project, "debug", MD_LOG=str(statements), MD_READ_ONLY="1") + assert debug.returncode == 0 and "shop on MotherDuck, through DuckDB" in debug.stdout, debug.stdout + debug.stderr + assert TOKEN not in debug.stdout + debug.stderr + planned = sqlload(project, "plan", MD_READ_ONLY="1") + assert planned.returncode == 0, planned.stdout + planned.stderr + assert "tmp_scratch" not in planned.stdout, planned.stdout + load(project, MD_READ_ONLY="1") + assert local(project, ORDERS) == [(1, "10.50", "[\"a\"]"), (2, "20.00", "[]")] + assert local(project, "SELECT count(*) FROM raw.shop__big_orders") == [(2,)] + assert local(project, "SELECT id FROM raw.shop__big_query ORDER BY id") == [(3,)] + columns = "SELECT column_name FROM information_schema.columns WHERE table_name = 'shop__orders' AND column_name NOT LIKE '\\_sqlload%' ESCAPE '\\'" + assert sorted(row[0] for row in local(project, columns)) == ["amount", "id", "region", "tags", "updated_at"] + with duckdb.connect(str(project / "md" / "shop.duckdb")) as shop: + shop.execute("UPDATE sales.orders SET amount = 21.00, updated_at = '2026-01-05 00:00:00' WHERE id = 2") + shop.execute("INSERT INTO sales.orders VALUES (4, 40.00, '2026-01-06 00:00:00', ['b', 'c'], '4444', 'EU')") + load(project, MD_LOG=str(statements), MD_READ_ONLY="1") + assert local(project, ORDERS) == [(1, "10.50", "[\"a\"]"), (2, "21.00", "[]"), (4, "40.00", "[\"b\",\"c\"]")] + reads = [line for line in statements.read_text().splitlines() if line.startswith("SELECT \"id\"") and "sales\".\"orders\"" in line] + assert reads and all("\"updated_at\" >" in read for read in reads), reads diff --git a/packages/sqlload/tests/test_motherduck_live.py b/packages/sqlload/tests/test_motherduck_live.py index 326db1b73..709720ae3 100644 --- a/packages/sqlload/tests/test_motherduck_live.py +++ b/packages/sqlload/tests/test_motherduck_live.py @@ -12,6 +12,7 @@ from pathlib import Path from typing import Any +import duckdb import pytest TOKEN_VARIABLE = "SQLLOAD_TEST_MOTHERDUCK_TOKEN" @@ -80,13 +81,13 @@ def rows(ctx): @pytest.fixture(scope="module") def python(tmp_path_factory: pytest.TempPathFactory) -> str: - """A Python with duckdb 1.5.6, the newest version MotherDuck accepts.""" + """A Python with duckdb 1.5.6, the newest version MotherDuck accepts, and pyarrow, which a source's rows stream through.""" folder = tmp_path_factory.mktemp("duckdb15") uv = shutil.which("uv") assert uv, "the live MotherDuck tests need uv to build a duckdb 1.5.6 environment" subprocess.run([uv, "venv", "--quiet", "--python", "3.12", str(folder / "venv")], check=True) executable = folder / "venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python") - subprocess.run([uv, "pip", "install", "--quiet", "--python", str(executable), "duckdb==1.5.6"], check=True) + subprocess.run([uv, "pip", "install", "--quiet", "--python", str(executable), "duckdb==1.5.6", "pyarrow"], check=True) return str(executable) @@ -200,3 +201,67 @@ def test_given_json_type_variant_when_loading_into_live_motherduck_then_dicts_la assert admin( python, f"SELECT variant_typeof(payload), CAST(payload.user.id AS BIGINT) FROM {database}.raw.cloud__customers" ) == [["OBJECT(user)", 42]] + + +SOURCE_PROJECT = """\ +[connections.md] +kind = "motherduck" +database = "{database}" +token = "${{ENV:{variable}}}" + +[connections.wh] +kind = "duckdb" +database = "wh.duckdb" + +[targets.local] +connection = "wh" +loader_schema = "raw" + +[sources.shop] +kind = "motherduck" +connection = "md" +destination = "local" +description = "The shop's MotherDuck database." +include = ["sales.*"] +exclude_columns = ["card"] +write_strategy = "replace" + +[sources.shop.tables."sales.orders"] +description = "Orders, merged after their cursor." +write_strategy = "merge" +unique_key = ["id"] +cursor_column = "updated_at" +where = "region = 'EU'" + +[sources.shop.tables.big_query] +description = "Orders over 25, by query." +sql = "SELECT id, amount FROM sales.orders WHERE amount > 25" +""" + + +def test_given_a_motherduck_source_when_loading_again_then_its_tables_and_queries_land_in_duckdb_and_the_cursor_merges_changes( + tmp_path: Path, python: str, database: str +) -> None: + admin( + python, + f"CREATE SCHEMA {database}.sales", + f"CREATE TABLE {database}.sales.orders (id INTEGER PRIMARY KEY, amount DECIMAL(12,2), updated_at TIMESTAMP, tags VARCHAR[], card VARCHAR, region VARCHAR)", + f"INSERT INTO {database}.sales.orders VALUES (1, 10.50, '2026-01-01', ['a'], '4111', 'EU'), (2, 20.00, '2026-01-02', [], '4222', 'EU'), (3, 30.25, '2026-01-03', NULL, '4333', 'US')", + ) + (tmp_path / "sqlload.toml").write_text(SOURCE_PROJECT.format(database=database, variable=TOKEN_VARIABLE)) + debug = sqlload(tmp_path, python, "debug") + assert debug.returncode == 0 and f"{database} on MotherDuck, through DuckDB v1.5.6" in debug.stdout, debug.stdout + debug.stderr + load(tmp_path, python) + admin( + python, + f"UPDATE {database}.sales.orders SET amount = 21.00, updated_at = '2026-01-05' WHERE id = 2", + f"INSERT INTO {database}.sales.orders VALUES (4, 40.00, '2026-01-06', ['b', 'c'], '4444', 'EU')", + ) + load(tmp_path, python) + with duckdb.connect(str(tmp_path / "wh.duckdb"), read_only=True) as local: + orders = local.execute("SELECT id, CAST(amount AS VARCHAR), CAST(tags AS VARCHAR) FROM raw.shop__orders ORDER BY id").fetchall() + big = local.execute("SELECT id FROM raw.shop__big_query ORDER BY id").fetchall() + columns = local.execute("SELECT column_name FROM information_schema.columns WHERE table_name = 'shop__orders' ORDER BY ordinal_position").fetchall() + assert orders == [(1, "10.50", '["a"]'), (2, "21.00", "[]"), (4, "40.00", '["b","c"]')] + assert big == [(3,), (4,)] + assert "card" not in [column[0] for column in columns] diff --git a/website/src/content/docs/docs/configuration.mdx b/website/src/content/docs/docs/configuration.mdx index d06dffe4c..4bbd4ba6d 100644 --- a/website/src/content/docs/docs/configuration.mdx +++ b/website/src/content/docs/docs/configuration.mdx @@ -110,7 +110,7 @@ See [ClickHouse](/docs/targets/clickhouse/). ### `motherduck` -In beta. It connects through a separate +In beta, as a target and as a source. It connects through a separate DuckDB 1.5 process: install `sqlload[motherduck]`; see [MotherDuck](/docs/targets/motherduck/). | Key | Meaning | diff --git a/website/src/content/docs/docs/index.mdx b/website/src/content/docs/docs/index.mdx index 9afe05454..da89d098a 100644 --- a/website/src/content/docs/docs/index.mdx +++ b/website/src/content/docs/docs/index.mdx @@ -15,8 +15,8 @@ SQLLoad loads data into your warehouse, and shows you how before it changes anyt | BigQuery | beta | beta | | Databricks | beta | beta | | ClickHouse | | ✓ one self-hosted server¹ | -| DuckDB | | ✓ | -| MotherDuck | | beta² | +| DuckDB | ✓ | ✓ | +| MotherDuck | beta² | beta² | | SQL Server, MySQL, MariaDB, OpenEdge and other databases through ODBC³ | ✓ | | | Files | ✓ Parquet, CSV, JSON, NDJSON, Excel (xlsx), XML; gzip, zstd and ZIP; any file name through `pattern` (S3, local folder, SFTP) | ✓ Parquet, CSV (S3, local folder) | | Google Sheets | ✓ | | diff --git a/website/src/content/docs/docs/sources/duckdb.mdx b/website/src/content/docs/docs/sources/duckdb.mdx new file mode 100644 index 000000000..1854aef4e --- /dev/null +++ b/website/src/content/docs/docs/sources/duckdb.mdx @@ -0,0 +1,113 @@ +--- +title: "DuckDB" +description: "Load tables, views and query results from a DuckDB database file, read as Arrow from a read-only connection: connections, choosing tables, query tables, cursors, types and memory." +--- + +A `duckdb` source loads tables, views and query results from a DuckDB database file. SQLLoad opens +the file read-only for each read and streams the result of one `SELECT` as Arrow, so it never +writes to the file. + +```toml +[connections.shop_file] +kind = "duckdb" +database = "shop.duckdb" + +[sources.shop] +kind = "duckdb" +connection = "shop_file" +description = "Tables of the shop's DuckDB file." +include = ["sales.*"] +exclude = ["sales.tmp_*"] +write_strategy = "replace" + +[sources.shop.tables."sales.orders"] +description = "Orders, one row each." +cursor_column = "updated_at" +write_strategy = "merge" +unique_key = "{primary_key}" + +[sources.shop.tables.big_orders] +description = "Orders over 100, with their customer." +sql = "SELECT o.*, c.name FROM sales.orders o JOIN sales.customers c USING (customer_id) WHERE o.amount > 100" +cursor_column = "updated_at" +write_strategy = "append" +``` + +## Connection + +A source uses a [`duckdb` connection](/docs/configuration/#duckdb), the same kind a DuckDB +target uses, whose `database` is the file to read. An in-memory database (`:memory:`) holds nothing +to read and is refused, and so is a MotherDuck database (`md:`). + +DuckDB lets one process write a file, or any number read it. While another process has the file +open for writing, a read fails, naming the lock; run the load when the writer has closed it. + +## Choosing tables + +`include` and `exclude` are `schema.table` patterns, matched against the names as DuckDB keeps +them, such as `sales.*` or `main.orders`. SQLLoad lists the file's tables and views, their columns +and their primary keys from DuckDB's catalog functions. A +`[sources.shop.tables."schema.table"]` entry sets options for one table; on its own, it doesn't add +the table. + +A source needs at least one `include` pattern or one query table. A table's primary key is its +`{primary_key}` for `unique_key`, and its comment becomes its description. + +## How tables are read + +Each read is one `SELECT` of the read columns, with the cursor window and any `where` in its +`WHERE`. One statement sees one version of the file, so a read is consistent on its own. SQLLoad +holds at most a few batches ahead of the target, and each connection's DuckDB `memory_limit` is +half the table's share of the [memory budget](/docs/cli/load/#memory-budget). + +## Query tables + +A table entry with `sql` is a query table: its rows are the query's result, and the entry's key is +its name. At `plan`, SQLLoad describes the query's columns without reading its rows. + +`cursor_column` names a column of the query's result. SQLLoad wraps the query and filters on the +cursor outside it, so the query needs no placeholder. Write it in DuckDB SQL. + +## Filtering rows + +`where` narrows a table to the rows that match a condition, written in DuckDB SQL: + +```toml +[sources.shop.tables."sales.orders"] +where = "region = 'EU'" +``` + +SQLLoad ANDs it into every `SELECT` it reads the table with, and a query table refuses it (put the +condition in its `sql`). + +## Types + +| DuckDB type | Loads as | +|-------------|----------| +| `TINYINT`, `SMALLINT`, `UTINYINT` | `smallint` | +| `INTEGER`, `BIGINT`, `USMALLINT`, `UINTEGER` | `int` | +| `DECIMAL(p,s)`, p up to 38 | `decimal(p,s)` | +| `FLOAT`, `DOUBLE` | `float` | +| `BOOLEAN` | `bool` | +| `VARCHAR`, `ENUM` | `text` | +| `BLOB` | `binary` | +| `DATE` | `date` | +| `TIME` | `time` | +| `TIMESTAMP`, `TIMESTAMP_S`, `TIMESTAMP_MS` | `timestamp_ntz` | +| `TIMESTAMP WITH TIME ZONE` | `timestamp_tz` | +| `UUID` | `uuid` | +| `JSON`, lists, arrays, `STRUCT`, `MAP` | `json` | +| `HUGEINT`, `UBIGINT`, `TIMESTAMP_NS`, `INTERVAL`, `BIT`, `UNION` and others | none: pin a type such as `text`, or exclude the column | + +Pin a type to load a column another way, such as `columns = { total = "text" }`, or a list as +`array(T)`. A value that doesn't convert to the pinned type fails the read, naming the column, +rather than loading as NULL. See [Types](/docs/concepts/types/). + +## Tested + +Unit tests read a real DuckDB file: tables, a view and a query table found by `include` and +`exclude`, primary keys and comments, every type above, a pinned `HUGEINT`, reads in several +batches, a filter applied outside a query that ends in a comment, and the file's bytes unchanged +after the read. A loader test loads a file into DuckDB twice: tables, a view and a query table, +with `exclude_columns`, `where`, a cursor merging only the rows that changed, and `--sync-deletes` +removing a row the file no longer holds. diff --git a/website/src/content/docs/docs/sources/index.mdx b/website/src/content/docs/docs/sources/index.mdx index 4f57d77cc..1b4f45275 100644 --- a/website/src/content/docs/docs/sources/index.mdx +++ b/website/src/content/docs/docs/sources/index.mdx @@ -1,9 +1,9 @@ --- title: "Sources" -description: "Postgres tables, Postgres changes, Snowflake tables and queries, ODBC databases, BigQuery and Databricks tables and queries (beta), files in S3 or a folder, Google Sheets, and Python sources: the kinds of source and the settings they share." +description: "Postgres tables, Postgres changes, Snowflake tables and queries, ODBC databases, BigQuery, Databricks and MotherDuck tables and queries (beta), DuckDB files, files in S3 or a folder, Google Sheets, and Python sources: the kinds of source and the settings they share." --- -A source is where data comes from. SQLLoad has nine kinds: +A source is where data comes from. SQLLoad has eleven kinds: | `kind` | Reads | Tables | Declared in | |--------|-------|--------|-------------| @@ -13,6 +13,8 @@ A source is where data comes from. SQLLoad has nine kinds: | [`odbc`](/docs/sources/odbc/) | Tables, views and query results of any database with an ODBC driver, read as Arrow | One per matching table, and one per query | `sqlload.toml` | | [`bigquery`](/docs/sources/bigquery/) (beta) | Tables, views and query results of BigQuery, read as Arrow through the Storage Read API | One per matching table, and one per query | `sqlload.toml` | | [`databricks`](/docs/sources/databricks/) (beta) | Tables, views and query results of a Databricks SQL warehouse, read as Arrow through the Statement Execution API | One per matching table, and one per query | `sqlload.toml` | +| [`duckdb`](/docs/sources/duckdb/) | Tables, views and query results of a DuckDB database file, read as Arrow | One per matching table, and one per query | `sqlload.toml` | +| [`motherduck`](/docs/sources/motherduck/) (beta) | Tables, views and query results of a MotherDuck database, read as Arrow through a separate DuckDB 1.5 process | One per matching table, and one per query | `sqlload.toml` | | [`files`](/docs/sources/files/) | Parquet, CSV, JSON or NDJSON files in S3 or a local folder | One per source | `sqlload.toml` | | [`google_sheets`](/docs/sources/google-sheets/) | Tabs or ranges of a Google Sheets spreadsheet, typed | One per tab or range | `sqlload.toml` | | [Python](/docs/sources/python/) | Whatever a Python function yields: dicts, Arrow data, JSON bytes, HTTP jobs | One per decorated function | Python, under `python_paths` | @@ -36,6 +38,10 @@ Each kind has its own page: tables, types, cost and permissions. - [Databricks](/docs/sources/databricks/) (beta): choosing tables, query tables, types, cost and permissions. +- [DuckDB](/docs/sources/duckdb/): choosing tables, query tables, types, and reading a file another + process writes. +- [MotherDuck](/docs/sources/motherduck/) (beta): installing the separate DuckDB 1.5 process, and + what reads as for a DuckDB source. - [Files](/docs/sources/files/): formats, partitions, which files load, and files the destination reads itself. - [Google Sheets](/docs/sources/google-sheets/): tabs and ranges, headers, and how cells are typed. diff --git a/website/src/content/docs/docs/sources/motherduck.mdx b/website/src/content/docs/docs/sources/motherduck.mdx new file mode 100644 index 000000000..508c0a325 --- /dev/null +++ b/website/src/content/docs/docs/sources/motherduck.mdx @@ -0,0 +1,77 @@ +--- +title: "MotherDuck" +description: "Load tables, views and query results from a MotherDuck database (beta), read as Arrow through a separate DuckDB 1.5 process: connection, choosing tables, query tables, cursors and types." +sidebar: + badge: + text: Beta + variant: caution +--- + +This source is in beta: it is tested, including live against MotherDuck, but gets less everyday use than the other connectors, so expect rough edges, and please report them in [GitHub issues](https://github.com/sqlstack-io/sqlload/issues). + +A `motherduck` source loads tables, views and query results from a MotherDuck database. It reads +the way a [DuckDB source](/docs/sources/duckdb/) does: one `SELECT` per read, streamed as Arrow, +with the same table choice, cursors, `where` and types. + +```toml +[connections.cloud] +kind = "motherduck" +database = "shop" +token = "${ENV:MOTHERDUCK_TOKEN}" + +[sources.shop] +kind = "motherduck" +connection = "cloud" +description = "Tables of the shop's MotherDuck database." +include = ["sales.*"] +write_strategy = "replace" + +[sources.shop.tables."sales.orders"] +description = "Orders, one row each." +cursor_column = "updated_at" +write_strategy = "merge" +unique_key = "{primary_key}" + +[sources.shop.tables.big_orders] +description = "Orders over 100." +sql = "SELECT * FROM sales.orders WHERE amount > 100" +``` + +## Install + +MotherDuck doesn't accept the DuckDB 2.0 that SQLLoad bundles yet, so SQLLoad reads a MotherDuck +source through the same separate Python process a [MotherDuck target](/docs/targets/motherduck/#install) +uses, running Python's `duckdb` 1.4.1 to 1.5.6. That process hands each batch to SQLLoad as Arrow, so +it also needs `pyarrow`. The `motherduck` extra installs both: + +```bash +pip install 'sqlload[motherduck]' +``` + +With `sqlload-cli`, set `SQLLOAD_MOTHERDUCK_PYTHON` to a Python that has `duckdb` 1.4.1 to 1.5.6 and +`pyarrow`. + +## Connection + +A source uses a [`motherduck` connection](/docs/configuration/#motherduck), the same kind a +MotherDuck target uses. Its `database` names the one database the source reads; a connection +without one is refused, as MotherDuck would otherwise attach every database in the account. The +token comes from `token`, else the `motherduck_token` or `MOTHERDUCK_TOKEN` environment variable, +and never appears in messages. + +## Choosing tables and reading them + +`include`, `exclude`, query tables, `cursor_column`, `where`, `exclude_columns` and pinned +`columns` work as for a [DuckDB source](/docs/sources/duckdb/#choosing-tables): SQLLoad lists the +database's own tables and views from DuckDB's catalog functions, and writes every filter in DuckDB +SQL. Each read is one statement on its own connection; the process reads one batch at a time, as +SQLLoad asks for it, so neither holds the whole table. + +Types map as on a [DuckDB source](/docs/sources/duckdb/#types). + +## Tested + +Live against a MotherDuck account on DuckDB 1.5.6: a table, a cursor merge that reads only changed +rows, `where`, `exclude_columns`, a list column and a query table, loaded twice into DuckDB. +Offline, the same load runs through the separate process against a stand-in for MotherDuck that +keeps the database in a local DuckDB file, with the token checked and kept out of every message. diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx index f710a90c1..3fc7dee3c 100644 --- a/website/src/content/docs/docs/supported-systems.mdx +++ b/website/src/content/docs/docs/supported-systems.mdx @@ -13,6 +13,8 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes | Databases with an ODBC driver: SQL Server, MySQL, MariaDB, OpenEdge and others | Tables and views, by `include` patterns, and query results | The driver, as Arrow. Tested with psqlODBC, Microsoft ODBC Driver 18 for SQL Server and MySQL Connector/ODBC; MariaDB Connector/ODBC and OpenEdge are checked by hand. See [ODBC](/docs/sources/odbc/#drivers). | | BigQuery (beta) | Tables and views, by `include` patterns, and query results | The Storage Read API, as Arrow in parallel streams; views and queries through a query job. See [BigQuery](/docs/sources/bigquery/). | | Databricks (beta) | Tables and views, by `include` patterns, and query results | One `SELECT` on the SQL warehouse, its result downloaded as Arrow files in parallel through the Statement Execution API's links. See [Databricks](/docs/sources/databricks/). | +| DuckDB | Tables and views of a database file, by `include` patterns, and query results | One `SELECT` on a read-only connection, streamed as Arrow. See [DuckDB](/docs/sources/duckdb/). | +| MotherDuck (beta) | Tables and views of one database, by `include` patterns, and query results | One `SELECT` run by a separate DuckDB 1.5 process (`sqlload[motherduck]`), its Arrow batches handed over one at a time. See [MotherDuck](/docs/sources/motherduck/). | | Files in S3 | Parquet, CSV, JSON, NDJSON, Excel (xlsx) and XML; gzip or zstd for text formats; ZIP archives of any of them; any file name through `pattern`. See [Files](/docs/sources/files/#excel-xml-and-zip-files). | The destination itself where it can load them exactly, otherwise SQLLoad's reader | | Files in a local folder | As for S3 | As for S3 | | Files on an SFTP server | As for S3 | SQLLoad's reader, over SFTP. See [Files](/docs/sources/files/#files-on-an-sftp-server). | @@ -45,6 +47,8 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes | ODBC | Yes, untested | Yes, tested live | Yes, untested | Yes, untested | Yes, untested | Yes | Yes | | BigQuery (beta) | Yes, untested | Yes, tested live | Yes, untested | Yes, untested | Yes, untested | Yes | Yes, untested | | Databricks (beta) | Yes, untested | Yes, untested | Yes, untested | Yes, tested live | Yes, tested live | Yes, tested live | Yes, tested live | +| DuckDB | Yes, untested | Yes, untested | Yes, untested | Yes, untested | Yes, untested | Yes | Yes, untested | +| MotherDuck (beta) | Yes, untested | Yes, untested | Yes, untested | Yes, untested | Yes, untested | Yes, tested live | Yes, untested | | Files | Yes, read by Snowflake from an external stage | Yes, staged by SQLLoad in the volume | Yes, read by SQLLoad and streamed | Yes; Parquet in S3 read by ClickHouse | Yes, read by SQLLoad | Yes; Parquet read by DuckDB | Read and rewritten by SQLLoad | | Files on an SFTP server | Yes, read by SQLLoad, tested live | Yes, read by SQLLoad, tested live | Yes, untested | Yes, untested | Yes, untested | Yes | Yes, untested | | Google Sheets | Yes | Yes | Yes | Yes | Yes | Yes | Yes | @@ -64,8 +68,8 @@ them into DuckDB. |--------|-----------------| | Postgres | Real Postgres in containers, as a source and as a target. `postgres_cdc` on Postgres 15, 16 and 17, plus live loads into DuckDB, Snowflake and Databricks. See [Postgres](/docs/targets/postgres/#tested). | | ClickHouse | Real ClickHouse 25.3, 25.8 and 26.10 in containers, with an S3-compatible store for the files it reads itself. See [ClickHouse](/docs/targets/clickhouse/#tested). | -| DuckDB | Real, in process. | -| MotherDuck | Live against a MotherDuck account on DuckDB 1.5.6: every write strategy, an incremental load and rerun, a new column and every portable type. Offline, every change loads through the separate process into a stand-in for MotherDuck and compares with DuckDB. See [MotherDuck](/docs/targets/motherduck/#tested). | +| DuckDB | Real, in process, as a source and a target. | +| MotherDuck | Live against a MotherDuck account on DuckDB 1.5.6: every write strategy, an incremental load and rerun, a new column and every portable type. Offline, every change loads through the separate process into a stand-in for MotherDuck and compares with DuckDB. As a source, live and offline: tables, a query table, `where`, `exclude_columns` and a cursor merge into DuckDB. See [MotherDuck](/docs/targets/motherduck/#tested). | | ODBC | psqlODBC against real Postgres in containers, into DuckDB, plus a live round from SQL Server and MySQL into Databricks. | | Databricks source | A fake Statement Execution API in tests (catalog discovery, result files in several chunks, an expired link fetched again, cursor reads, no token on downloads), plus a live round from a real SQL warehouse into DuckDB, Parquet files, Postgres and ClickHouse: tables, a view, a windowed query table, merges and every column shape. See [Databricks](/docs/sources/databricks/#tested). | | BigQuery source | A fake Storage Read API in tests (parallel streams, a broken stream resumed, cursor filters), and live loads from a sandbox project into DuckDB: tables, views, query tables, and integer, timestamp, text, `NUMERIC` and `BIGNUMERIC` cursors, plus a live round into Databricks. | diff --git a/website/src/content/docs/docs/targets/motherduck.mdx b/website/src/content/docs/docs/targets/motherduck.mdx index 1af549127..bfedab1fa 100644 --- a/website/src/content/docs/docs/targets/motherduck.mdx +++ b/website/src/content/docs/docs/targets/motherduck.mdx @@ -20,7 +20,8 @@ So SQLLoad sends a MotherDuck target's SQL to a separate Python process running package at a version MotherDuck accepts. The two DuckDB versions never share a process, and every other target still uses SQLLoad's own DuckDB. -Install the `motherduck` extra, which adds `duckdb>=1.4.1,<=1.5.6`: +Install the `motherduck` extra, which adds `duckdb>=1.4.1,<=1.5.6`, and `pyarrow`, which a +[MotherDuck source](/docs/sources/motherduck/) reads through: ```bash uv tool install 'sqlload[motherduck]' diff --git a/website/src/sidebar.json b/website/src/sidebar.json index da3011bfa..ce78762c9 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -70,6 +70,8 @@ "docs/sources/snowflake", "docs/sources/bigquery", "docs/sources/databricks", + "docs/sources/duckdb", + "docs/sources/motherduck", "docs/sources/files", "docs/sources/google-sheets", "docs/sources/odbc",