Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
use crate::bigquery_source::constants::BIGQUERY_SOURCE;
use crate::column_types::models::Warehouse;
use crate::databricks_source::constants::DATABRICKS_SOURCE;
use crate::duckdb_source::constants::{DUCKDB_SOURCE, MOTHERDUCK_SOURCE};
use crate::loader::constants::SNOWFLAKE_SOURCE;

/// The warehouse whose SQL a warehouse source of `system` (its kind as written) filters in; `None` for a system without one.
Expand All @@ -11,6 +12,7 @@ pub fn source_dialect(system: &str) -> Option<Warehouse> {
SNOWFLAKE_SOURCE => Some(Warehouse::Snowflake),
BIGQUERY_SOURCE => Some(Warehouse::BigQuery),
DATABRICKS_SOURCE => Some(Warehouse::Databricks),
DUCKDB_SOURCE | MOTHERDUCK_SOURCE => Some(Warehouse::DuckDb),
_ => None,
}
}
2 changes: 2 additions & 0 deletions crates/sqlload/src/adapters/dispatch/main/session/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ pub mod execute;
pub mod execute_ddl;
pub mod helper_process;
pub mod live_columns;
#[cfg(feature = "duckdb")]
pub mod open_source_bridge;
pub mod recover_unfinished;
pub mod single_text;
pub mod table_destination;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
//! Open a MotherDuck database a source reads.

use crate::adapters::duckdb::errors::DuckDbError;
use crate::adapters::duckdb::main::_open_motherduck::open_motherduck;
use crate::adapters::duckdb::models::DuckDbConnection;
use crate::adapters::motherduck_bridge::models::BridgeConnection;
use crate::project::models::ConnectionConfig;

/// The bridge connection to the database of `connection`, a `motherduck` one, signed in as a MotherDuck target is.
pub fn open_source_bridge(connection: &ConnectionConfig) -> Result<BridgeConnection, DuckDbError> {
let opened = open_motherduck(connection, None, (None, None))?;
match opened.connection {
DuckDbConnection::Bridge(bridge) => Ok(bridge),
DuckDbConnection::Local(_) => Err(DuckDbError::MotherDuck(
"a MotherDuck database opens on the MotherDuck bridge".to_owned(),
)),
}
}
2 changes: 1 addition & 1 deletion crates/sqlload/src/adapters/duckdb/main/_describe_table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use crate::adapters::duckdb::_helpers::blocking::with_connection;
use crate::adapters::duckdb::constants::DESCRIBE_TABLE_SQL;
use crate::adapters::duckdb::errors::DuckDbError;
use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination, DuckDbSql};
use crate::adapters::motherduck_bridge::main::_run_statement::run_statement;
use crate::adapters::motherduck_bridge::main::run_statement::run_statement;
use crate::column_types::models::LiveColumnType;
use crate::schema_plan::models::LiveColumn;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use crate::adapters::duckdb::_helpers::blocking::with_connection;
use crate::adapters::duckdb::errors::DuckDbError;
use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination};
use crate::adapters::motherduck_bridge::main::_run_statement::run_statement;
use crate::adapters::motherduck_bridge::main::run_statement::run_statement;

/// Run each statement in order, stopping at the first failure (an open transaction is rolled back).
pub(in crate::adapters) async fn execute_statements(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use crate::memory_budget::models::DuckDbShare;
use crate::project::models::ConnectionConfig;

/// Open the MotherDuck database of `connection` (`target_database` when the target names one) through DuckDB's `md:` connection on the MotherDuck bridge, signed in with its token, else the token in the `motherduck_token` or `MOTHERDUCK_TOKEN` variable. Without a token MotherDuck would wait for a browser sign-in, so it is refused instead. `python` runs the bridge; `None` picks it as the bridge does. Under `memory`, its part of the run's memory budget, the bridge's DuckDB gets that `memory_limit` and thread count, and commits are cut into pieces that fit it; a table's row groups set no limit, as MotherDuck checkpoints them on its servers.
pub(in crate::adapters) fn open_motherduck(
pub(crate) fn open_motherduck(
connection: &ConnectionConfig,
target_database: Option<&str>,
(python, memory): (Option<&str>, Option<DuckDbShare>),
Expand Down
2 changes: 1 addition & 1 deletion crates/sqlload/src/adapters/duckdb/main/_own_connection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use std::sync::{Arc, Mutex};

use crate::adapters::duckdb::errors::DuckDbError;
use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination};
use crate::adapters::motherduck_bridge::main::_fork_connection::fork_connection;
use crate::adapters::motherduck_bridge::main::fork_connection::fork_connection;
use crate::memory_budget::main::table_memory::table_memory;

/// A new connection to the same database, so one table's transaction never mixes with another table's loading at the same time. Under the memory watchdog, the table it loads can have the connection's running statement interrupted.
Expand Down
2 changes: 1 addition & 1 deletion crates/sqlload/src/adapters/duckdb/main/_query_text.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use crate::adapters::duckdb::_helpers::blocking::with_connection;
use crate::adapters::duckdb::errors::DuckDbError;
use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination};
use crate::adapters::motherduck_bridge::main::_run_statement::run_statement;
use crate::adapters::motherduck_bridge::main::run_statement::run_statement;

/// The first column of the first row of `query`, as text; `None` for SQL `NULL`.
pub(in crate::adapters) async fn query_text(
Expand Down
2 changes: 1 addition & 1 deletion crates/sqlload/src/adapters/duckdb/main/_query_texts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use crate::adapters::duckdb::_helpers::blocking::with_connection;
use crate::adapters::duckdb::errors::DuckDbError;
use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination};
use crate::adapters::motherduck_bridge::main::_run_statement::run_statement;
use crate::adapters::motherduck_bridge::main::run_statement::run_statement;

/// The first column of every row of `query`, as text; SQL `NULL`s are skipped.
pub(in crate::adapters) async fn query_texts(
Expand Down
2 changes: 1 addition & 1 deletion crates/sqlload/src/adapters/duckdb/main/_table_shape.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use crate::adapters::duckdb::_helpers::blocking::with_connection;
use crate::adapters::duckdb::constants::{DEFINITION_QUERY, INDEXES_QUERY};
use crate::adapters::duckdb::errors::DuckDbError;
use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination, DuckDbSql};
use crate::adapters::motherduck_bridge::main::_run_statement::run_statement;
use crate::adapters::motherduck_bridge::main::run_statement::run_statement;

/// `table`'s columns and constraints as DuckDB would create them, and its indexes' statements on `table` by its full name (DuckDB stores them on `schema.table`, which MotherDuck's other attached databases can make ambiguous); `None` when there is no such table.
pub(in crate::adapters) async fn table_shape(
Expand Down
33 changes: 32 additions & 1 deletion crates/sqlload/src/adapters/motherduck_bridge/_helpers/bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
-> {"id": 1, "ok": true, "duckdb": "1.5.6", "python": "3.12.3"}
{"id": 2, "op": "run", "cursor": 3, "sql": "SELECT ...", "params": ["raw"]}
-> {"id": 2, "ok": true, "rows": [["id", "BIGINT"], ...]}
{"id": 3, "op": "stream", "cursor": 4, "sql": "SELECT ...", "rows": 100000}
-> {"id": 3, "ok": true}
{"id": 4, "op": "next", "cursor": 4, "path": "/tmp/batch.arrow"}
-> {"id": 4, "ok": true, "rows": 100000}, the next batch of at most that many rows written there as an Arrow IPC file; 0 rows once all are read
{"op": "close", "cursor": 3}
-> no reply
{"op": "interrupt", "cursor": 3}
Expand Down Expand Up @@ -41,6 +45,22 @@ def plain(value):
return str(value)


def write_next(batches, path):
"""The next batch `batches` holds, written to the existing file `path` as an Arrow IPC file; its row count, 0 once every batch is read. A missing `path` fails."""
import pyarrow.ipc

try:
batch = batches.read_next_batch()
except StopIteration:
return 0
# SQLLoad created the file, readable by its user only; opening it rather than creating it means a file SQLLoad has already removed (its read stopped) is never written again.
with open(path, "r+b") as sink:
sink.truncate()
with pyarrow.ipc.new_file(sink, batch.schema) as writer:
writer.write_batch(batch)
return batch.num_rows


def main():
# Replies go to a private copy of stdout; anything else that prints (DuckDB, an extension) lands on stderr instead.
replies = os.fdopen(os.dup(1), "wb", buffering=0)
Expand All @@ -64,14 +84,25 @@ def failed(request, error):

def serve(work, number):
cursor = None
batches = None
while True:
request = work.get()
if request is None:
break
try:
if cursor is None:
cursor = opened[number] = database.cursor()
if request["op"] == "next":
reply({"id": request["id"], "ok": True, "rows": write_next(batches, request["path"])})
continue
cursor.execute(request["sql"], request.get("params") or None)
if request["op"] == "stream":
# Arrow batches straight from DuckDB's streaming result, one at a time as `next` asks, so the bridge never holds them all.
# duckdb 1.4 names the reader fetch_record_batch; 1.5 adds to_arrow_reader, and 2.0 keeps only that.
reader = getattr(cursor, "to_arrow_reader", None) or cursor.fetch_record_batch
batches = reader(request["rows"])
reply({"id": request["id"], "ok": True})
continue
rows = cursor.fetchall() if cursor.description else []
reply({"id": request["id"], "ok": True, "rows": [[plain(value) for value in row] for row in rows]})
except BaseException as error: # noqa: BLE001 - every failure goes back to SQLLoad as its reply, or it would wait forever
Expand All @@ -91,7 +122,7 @@ def serve(work, number):
reply({"id": request["id"], "ok": True, "duckdb": duckdb.__version__, "python": sys.version.split()[0]})
except Exception as error: # noqa: BLE001
failed(request, error)
elif op == "run":
elif op in ("run", "stream", "next"):
work = cursors.get(request["cursor"])
if work is None:
work = cursors[request["cursor"]] = queue.Queue()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use crate::adapters::motherduck_bridge::constants::{
BRIDGE_SCRIPT, PYTHON_VARIABLE, STDERR_TAIL_BYTES, SUPPORTED_DUCKDB,
};
use crate::adapters::motherduck_bridge::errors::BridgeError;
use crate::adapters::motherduck_bridge::models::{BridgeProcess, Replies};
use crate::adapters::motherduck_bridge::models::{BridgeCursor, BridgeProcess, Replies};

/// Start the bridge on `python`: a writer thread feeding its stdin (closing it, and reaping the process, once every handle is gone), a reader thread routing its replies and, once stdout and stderr have both closed or a reply is unreadable, failing what still waits, and one keeping its stderr's tail with each of `hidden` taken out.
pub(crate) fn start(python: &str, hidden: Vec<String>) -> Result<Arc<BridgeProcess>, BridgeError> {
Expand Down Expand Up @@ -100,6 +100,19 @@ pub(crate) fn send(
Ok(received)
}

/// Send `request` on `cursor` and wait for its reply, failed or not, or why the bridge stopped first.
pub(crate) async fn ask(
cursor: &BridgeCursor,
request: serde_json::Value,
) -> Result<serde_json::Value, BridgeError> {
let received = send(&cursor.process, request)?;
let python = cursor.process.python.clone();
tokio::task::spawn_blocking(move || received.recv())
.await
.map_err(|error| BridgeError::Stopped(error.to_string()))?
.map_err(|_| BridgeError::Stopped(format!("the MotherDuck bridge on {python} stopped")))?
}

/// `reply`'s failure, as its message and its Python error type; `None` when it succeeded.
pub(crate) fn failure(reply: &serde_json::Value) -> Option<(String, String)> {
if reply["ok"].as_bool() == Some(true) {
Expand Down
3 changes: 3 additions & 0 deletions crates/sqlload/src/adapters/motherduck_bridge/errors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,7 @@ pub enum BridgeError {
/// DuckDB refused the request, in its own words.
#[error("{0}")]
Engine(String),
/// What the caller's batch consumer failed with, as it said it.
#[error("{0}")]
Consumer(String),
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use arrow_array::RecordBatch;
use parquet::arrow::ArrowWriter;

use crate::adapters::motherduck_bridge::errors::BridgeError;
use crate::adapters::motherduck_bridge::main::_run_statement::run_statement;
use crate::adapters::motherduck_bridge::main::run_statement::run_statement;
use crate::adapters::motherduck_bridge::models::BridgeConnection;

/// Run `insert`, whose one `?` is the path of a Parquet file holding `batch` (such as `INSERT INTO t (a) SELECT a FROM read_parquet(?)`), on `connection`. The file is a temporary one on this machine, which DuckDB reads there and removed once the statement ends: the bridge's DuckDB is another version than SQLLoad's, so Arrow cannot cross between them in memory.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use std::sync::atomic::Ordering;
use crate::adapters::motherduck_bridge::models::{BridgeConnection, BridgeCursor};

/// A new connection to the database `connection` opened, on the same bridge: its own DuckDB cursor, so its transactions never mix with another connection's.
pub(in crate::adapters) fn fork_connection(connection: &BridgeConnection) -> BridgeConnection {
pub(crate) fn fork_connection(connection: &BridgeConnection) -> BridgeConnection {
let process = connection.cursor.process.clone();
let id = process.next_cursor.fetch_add(1, Ordering::Relaxed);
BridgeConnection {
Expand Down
5 changes: 3 additions & 2 deletions crates/sqlload/src/adapters/motherduck_bridge/main/mod.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
pub(in crate::adapters) mod _append_batch;
pub(in crate::adapters) mod _fork_connection;
pub(in crate::adapters) mod _open_bridge;
pub(in crate::adapters) mod _run_statement;
pub(crate) mod fork_connection;
pub(crate) mod run_statement;
pub(crate) mod stream_rows;
Original file line number Diff line number Diff line change
@@ -1,26 +1,19 @@
//! Run SQL on a bridge connection.

use crate::adapters::motherduck_bridge::_helpers::process::{failure, send};
use crate::adapters::motherduck_bridge::_helpers::process::{ask, failure};
use crate::adapters::motherduck_bridge::errors::BridgeError;
use crate::adapters::motherduck_bridge::models::BridgeConnection;

/// Run `sql` (one statement or several, separated by `;`) with text `params` bound to its `?`s, after every earlier statement on `connection`, and return the last statement's rows: each value as text (numbers and booleans as their SQL text), `None` for `NULL`.
pub(in crate::adapters) async fn run_statement(
pub(crate) async fn run_statement(
connection: &BridgeConnection,
sql: String,
params: Vec<String>,
) -> Result<Vec<Vec<Option<String>>>, BridgeError> {
let cursor = &connection.cursor;
let request =
serde_json::json!({"op": "run", "cursor": cursor.id, "sql": sql, "params": params});
let received = send(&cursor.process, request)?;
let python = cursor.process.python.clone();
let reply = tokio::task::spawn_blocking(move || received.recv())
.await
.map_err(|error| BridgeError::Stopped(error.to_string()))?
.map_err(|_| {
BridgeError::Stopped(format!("the MotherDuck bridge on {python} stopped"))
})??;
let reply = ask(cursor, request).await?;
if let Some((error, _)) = failure(&reply) {
return Err(BridgeError::Engine(error));
}
Expand Down
63 changes: 63 additions & 0 deletions crates/sqlload/src/adapters/motherduck_bridge/main/stream_rows.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
//! Stream a query's rows from the bridge as Arrow batches.

use arrow_array::RecordBatch;
use arrow_ipc::reader::FileReader;

use crate::adapters::motherduck_bridge::_helpers::process::{ask, failure};
use crate::adapters::motherduck_bridge::constants::{MISSING_PACKAGE_ERRORS, PYTHON_VARIABLE};
use crate::adapters::motherduck_bridge::errors::BridgeError;
use crate::adapters::motherduck_bridge::models::BridgeConnection;

/// Run `sql` on `connection` and pass its rows to `on_batch` in batches of at most `batch_rows`; returns how many rows there were. The bridge reads one batch at a time from DuckDB's streaming result and hands it over in a temporary Arrow IPC file on this machine, removed once read: its DuckDB is another version than SQLLoad's, so Arrow cannot cross between them in memory.
pub(crate) async fn stream_rows(
connection: &BridgeConnection,
sql: String,
batch_rows: usize,
mut on_batch: impl AsyncFnMut(RecordBatch) -> Result<(), String>,
) -> Result<u64, BridgeError> {
let cursor = &connection.cursor;
let python = &cursor.process.python;
let stream =
serde_json::json!({"op": "stream", "cursor": cursor.id, "sql": sql, "rows": batch_rows});
if let Some((error, _)) = failure(&ask(cursor, stream).await?) {
return Err(BridgeError::Engine(error));
}
let mut rows: u64 = 0;
loop {
let file = tempfile::Builder::new()
.prefix("sqlload-motherduck-")
.suffix(".arrow")
.tempfile()
.map_err(|error| {
BridgeError::Setup(format!(
"cannot make a file for a MotherDuck batch: {error}"
))
})?;
let next = serde_json::json!({"op": "next", "cursor": cursor.id, "path": file.path().display().to_string()});
let reply = ask(cursor, next).await?;
if let Some((error, kind)) = failure(&reply) {
return Err(match MISSING_PACKAGE_ERRORS.contains(&kind.as_str()) {
true => BridgeError::Setup(format!(
"reading from MotherDuck needs Python's pyarrow package beside duckdb, which {python} lacks ({error}): install sqlload[motherduck] with pip or uv, or set {PYTHON_VARIABLE} to a Python that has both"
)),
false => BridgeError::Engine(error),
});
}
if reply["rows"].as_u64().unwrap_or_default() == 0 {
return Ok(rows);
}
let batches = tokio::task::spawn_blocking(move || {
let reader = FileReader::try_new(file.reopen()?, None)?;
reader.collect::<Result<Vec<RecordBatch>, _>>()
})
.await
.map_err(|error| BridgeError::Stopped(error.to_string()))?
.map_err(|error: arrow_schema::ArrowError| {
BridgeError::Engine(format!("cannot read a MotherDuck batch: {error}"))
})?;
for batch in batches {
rows += batch.num_rows() as u64;
on_batch(batch).await.map_err(BridgeError::Consumer)?;
}
}
}
Loading
Loading