From e7f2becd0b545c5f8e8d272253d79a95251c1f6d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 08:45:54 +0000 Subject: [PATCH 1/9] wip: DuckDB source module (not wired into the loader yet) --- .../src/duckdb_source/_helpers/catalog.rs | 108 ++++++++++++++++++ .../src/duckdb_source/_helpers/connection.rs | 14 +++ .../src/duckdb_source/_helpers/convert.rs | 82 +++++++++++++ .../sqlload/src/duckdb_source/_helpers/mod.rs | 5 + .../src/duckdb_source/_helpers/query.rs | 51 +++++++++ .../src/duckdb_source/_helpers/type_map.rs | 62 ++++++++++ crates/sqlload/src/duckdb_source/constants.rs | 5 + crates/sqlload/src/duckdb_source/errors.rs | 11 ++ .../src/duckdb_source/main/count_rows.rs | 24 ++++ .../src/duckdb_source/main/duckdb_reader.rs | 33 ++++++ crates/sqlload/src/duckdb_source/main/mod.rs | 5 + .../src/duckdb_source/main/read_catalog.rs | 66 +++++++++++ .../src/duckdb_source/main/read_columns.rs | 47 ++++++++ .../src/duckdb_source/main/read_rows.rs | 74 ++++++++++++ crates/sqlload/src/duckdb_source/mod.rs | 10 ++ crates/sqlload/src/duckdb_source/models.rs | 19 +++ crates/sqlload/src/lib.rs | 2 + 17 files changed, 618 insertions(+) create mode 100644 crates/sqlload/src/duckdb_source/_helpers/catalog.rs create mode 100644 crates/sqlload/src/duckdb_source/_helpers/connection.rs create mode 100644 crates/sqlload/src/duckdb_source/_helpers/convert.rs create mode 100644 crates/sqlload/src/duckdb_source/_helpers/mod.rs create mode 100644 crates/sqlload/src/duckdb_source/_helpers/query.rs create mode 100644 crates/sqlload/src/duckdb_source/_helpers/type_map.rs create mode 100644 crates/sqlload/src/duckdb_source/constants.rs create mode 100644 crates/sqlload/src/duckdb_source/errors.rs create mode 100644 crates/sqlload/src/duckdb_source/main/count_rows.rs create mode 100644 crates/sqlload/src/duckdb_source/main/duckdb_reader.rs create mode 100644 crates/sqlload/src/duckdb_source/main/mod.rs create mode 100644 crates/sqlload/src/duckdb_source/main/read_catalog.rs create mode 100644 crates/sqlload/src/duckdb_source/main/read_columns.rs create mode 100644 crates/sqlload/src/duckdb_source/main/read_rows.rs create mode 100644 crates/sqlload/src/duckdb_source/mod.rs create mode 100644 crates/sqlload/src/duckdb_source/models.rs diff --git a/crates/sqlload/src/duckdb_source/_helpers/catalog.rs b/crates/sqlload/src/duckdb_source/_helpers/catalog.rs new file mode 100644 index 000000000..a4434ba06 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/_helpers/catalog.rs @@ -0,0 +1,108 @@ +//! The tables and views `include` matches, as source tables from DuckDB's catalog functions. + +use std::collections::BTreeMap; + +use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; + +use crate::duckdb_source::_helpers::type_map::source_column; +use crate::duckdb_source::errors::DuckDbSourceError; +use crate::postgres_source::models::SourceTable; + +/// The database's tables and views whose `schema.table` matches `include` and not `exclude` (exactly, as DuckDB keeps names), sorted: from `duckdb_tables()` and `duckdb_views()`, their columns from `duckdb_columns()` and their primary keys from `duckdb_constraints()`, all of the file's own database. +pub(crate) fn catalog_tables( + connection: &duckdb::Connection, + (include, exclude): (&[String], &[String]), +) -> Result, DuckDbSourceError> { + let (included, excluded) = (matcher(include)?, matcher(exclude)?); + let mut tables: BTreeMap<(String, String), SourceTable> = BTreeMap::new(); + let mut listed = connection.prepare( + "SELECT schema_name, table_name, comment, estimated_size FROM duckdb_tables() WHERE database_name = current_database() \ + UNION ALL SELECT schema_name, view_name, comment, -1 FROM duckdb_views() WHERE database_name = current_database() AND NOT internal", + )?; + let rows = listed.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, i64>(3)?, + )) + })?; + for row in rows { + let (schema, name, comment, estimated_rows) = row?; + let qualified = format!("{schema}.{name}"); + if !included.is_match(&qualified) || excluded.is_match(&qualified) { + continue; + } + tables.insert( + (schema.clone(), name.clone()), + SourceTable { + split_pages: None, + schema, + name, + columns: Vec::new(), + primary_key: Vec::new(), + indexed_columns: Vec::new(), + estimated_rows, + estimated_bytes: -1, + comment: comment.filter(|comment| !comment.is_empty()), + listed_keys: Vec::new(), + }, + ); + } + if tables.is_empty() { + return Ok(Vec::new()); + } + let mut columns = connection.prepare( + "SELECT schema_name, table_name, column_name, data_type, is_nullable FROM duckdb_columns() WHERE database_name = current_database() ORDER BY schema_name, table_name, column_index", + )?; + let rows = columns.query_map([], |row| { + Ok(( + (row.get::<_, String>(0)?, row.get::<_, String>(1)?), + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, bool>(4)?, + )) + })?; + for row in rows { + let (key, name, data_type, nullable) = row?; + if let Some(table) = tables.get_mut(&key) { + table.columns.push(source_column(&name, &data_type, nullable)); + } + } + let mut keys = connection.prepare( + "SELECT schema_name, table_name, constraint_column_names FROM duckdb_constraints() WHERE database_name = current_database() AND constraint_type = 'PRIMARY KEY'", + )?; + let rows = keys.query_map([], |row| { + Ok(( + (row.get::<_, String>(0)?, row.get::<_, String>(1)?), + row.get::<_, duckdb::types::Value>(2)?, + )) + })?; + for row in rows { + let (key, names) = row?; + if let (Some(table), duckdb::types::Value::List(names)) = (tables.get_mut(&key), names) { + table.primary_key = names + .into_iter() + .filter_map(|name| match name { + duckdb::types::Value::Text(name) => Some(name), + _ => None, + }) + .collect(); + } + } + Ok(tables.into_values().collect()) +} + +/// One glob set for `schema.table` patterns, matched exactly. +fn matcher(patterns: &[String]) -> Result { + let mut builder = GlobSetBuilder::new(); + for pattern in patterns { + let glob = GlobBuilder::new(pattern).build().map_err(|error| { + DuckDbSourceError::Invalid(format!("invalid table pattern {pattern:?}: {error}")) + })?; + builder.add(glob); + } + builder + .build() + .map_err(|error| DuckDbSourceError::Invalid(error.to_string())) +} diff --git a/crates/sqlload/src/duckdb_source/_helpers/connection.rs b/crates/sqlload/src/duckdb_source/_helpers/connection.rs new file mode 100644 index 000000000..98140ad14 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/_helpers/connection.rs @@ -0,0 +1,14 @@ +//! A read-only connection to a DuckDB source's database. + +use crate::duckdb_source::errors::DuckDbSourceError; +use crate::duckdb_source::models::DuckDbReader; + +/// The reader's database opened read-only, so a file another process writes is never changed and SQLLoad holds no write lock on it, with the reader's `memory_limit`. +pub(crate) fn open_read_only(reader: &DuckDbReader) -> Result { + let config = duckdb::Config::default().access_mode(duckdb::AccessMode::ReadOnly)?; + let connection = duckdb::Connection::open_with_flags(&reader.database, config)?; + if let Some(limit) = reader.memory_limit { + connection.execute_batch(&format!("SET memory_limit = '{}KiB'", limit / 1024))?; + } + Ok(connection) +} diff --git a/crates/sqlload/src/duckdb_source/_helpers/convert.rs b/crates/sqlload/src/duckdb_source/_helpers/convert.rs new file mode 100644 index 000000000..cb12bf9d1 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/_helpers/convert.rs @@ -0,0 +1,82 @@ +//! Result batches in the read columns' types. + +use std::sync::Arc; + +use arrow_array::{ArrayRef, RecordBatch}; +use arrow_schema::{DataType, Schema, SchemaRef}; + +use crate::column_types::main::arrow_field::arrow_field; +use crate::column_types::main::exact_cast::exact_cast; +use crate::column_types::models::ColumnType; +use crate::postgres_source::models::ReadColumn; +use crate::row_batches::main::normalise_batch::normalise_batch; + +/// The Arrow schema the read columns are handed on in, with the columns `json` reads from lists, structs and maps. +pub(crate) fn output_schema(columns: &[ReadColumn]) -> Result<(SchemaRef, Vec), String> { + let fields = columns + .iter() + .map(|column| { + arrow_field(&column.name, &column.output) + .ok_or_else(|| format!("{} has no Arrow representation", column.name)) + }) + .collect::, String>>()?; + let json: Vec = columns + .iter() + .map(|column| column.output == ColumnType::Json) + .collect(); + Ok((Arc::new(Schema::new(fields)), json)) +} + +/// Whether DuckDB sent `array` nested (a list, array, struct or map) rather than as text. +fn nested(array: &ArrayRef) -> bool { + matches!( + array.data_type(), + DataType::Struct(_) + | DataType::List(_) + | DataType::LargeList(_) + | DataType::FixedSizeList(..) + | DataType::Map(..) + ) +} + +/// A result batch in `schema`: a nested `json` column as its JSON text, every other column cast to its type; a value that does not convert (such as a `DECIMAL` too wide for a pinned one) is an error, never NULL. +pub(crate) fn convert_batch( + batch: &RecordBatch, + (schema, json): (&SchemaRef, &[bool]), +) -> Result { + let texts: Vec<(&str, ArrayRef)> = schema + .fields() + .iter() + .zip(json) + .filter(|(_, json)| **json) + .filter_map(|(field, _)| { + Some(( + field.name().as_str(), + batch.column_by_name(field.name())?.clone(), + )) + }) + .filter(|(_, array)| nested(array)) + .collect(); + let texts = match texts.is_empty() { + true => None, + false => Some( + RecordBatch::try_from_iter(texts) + .map_err(|error| error.to_string()) + .and_then(|nested| normalise_batch(&nested, &[]))?, + ), + }; + let columns: Vec = schema + .fields() + .iter() + .map(|field| { + let read = texts + .as_ref() + .and_then(|texts| texts.column_by_name(field.name())) + .or_else(|| batch.column_by_name(field.name())) + .ok_or_else(|| format!("the result has no column {}", field.name()))?; + exact_cast(read, field.data_type()) + .map_err(|error| format!("{}: {error}", field.name())) + }) + .collect::>()?; + RecordBatch::try_new(schema.clone(), columns).map_err(|error| error.to_string()) +} diff --git a/crates/sqlload/src/duckdb_source/_helpers/mod.rs b/crates/sqlload/src/duckdb_source/_helpers/mod.rs new file mode 100644 index 000000000..3150e68d0 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod catalog; +pub(crate) mod connection; +pub(crate) mod convert; +pub(crate) mod query; +pub(crate) mod type_map; diff --git a/crates/sqlload/src/duckdb_source/_helpers/query.rs b/crates/sqlload/src/duckdb_source/_helpers/query.rs new file mode 100644 index 000000000..9023dc512 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/_helpers/query.rs @@ -0,0 +1,51 @@ +//! DuckDB SQL for reading a table or query. + +use crate::duckdb_source::models::DuckDbReadRequest; +use crate::postgres_source::models::SourceTable; + +/// `name` in double quotes, with `"` doubled, as DuckDB reads a quoted identifier. +pub(crate) fn quote_identifier(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +/// `"schema"."table"` for a catalog table, or the query as a `subquery`. +fn from_item((table, sql): (&SourceTable, Option<&str>)) -> String { + match sql { + Some(sql) => subquery(sql), + None => [&table.schema, &table.name].map(|name| quote_identifier(name)).join("."), + } +} + +/// `SELECT FROM [WHERE ]`. +pub(crate) fn select_query(request: &DuckDbReadRequest<'_>) -> String { + let columns: Vec<&str> = request + .columns + .iter() + .map(|column| column.expression.as_str()) + .collect(); + let filter = request + .filter + .map(|filter| format!(" WHERE {filter}")) + .unwrap_or_default(); + format!( + "SELECT {} FROM {}{filter}", + columns.join(", "), + from_item((request.table, request.sql)) + ) +} + +/// `SELECT COUNT(*) FROM WHERE `, the table and query written as `select_query` writes them: how many rows `filter` selects. +pub(crate) fn count_query((table, sql): (&SourceTable, Option<&str>), filter: &str) -> String { + format!( + "SELECT COUNT(*) FROM {} WHERE {filter}", + from_item((table, sql)) + ) +} + +/// Query `sql` in parentheses, as a `FROM` item: without its final semicolons, and closed on a line of its own so a trailing `--` comment ends first. `plan` describes it in this form, so a query it can't wrap fails there, not at load. +pub(crate) fn subquery(sql: &str) -> String { + let body = sql + .trim_start() + .trim_end_matches(|character: char| character == ';' || character.is_whitespace()); + format!("({body}\n)") +} diff --git a/crates/sqlload/src/duckdb_source/_helpers/type_map.rs b/crates/sqlload/src/duckdb_source/_helpers/type_map.rs new file mode 100644 index 000000000..2cc72a0a6 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/_helpers/type_map.rs @@ -0,0 +1,62 @@ +//! DuckDB column types as portable ones. + +use crate::column_types::constants::MAX_DECIMAL_PRECISION; +use crate::column_types::models::ColumnType; +use crate::postgres_source::models::{PgType, SourceColumn}; + +/// A column from its name, its type as DuckDB writes it (`duckdb_columns().data_type`, or a result's column type) and whether it may be NULL: lists, arrays, structs and maps as `json`; a type with no portable one (`HUGEINT`, `UBIGINT`, `INTERVAL`, `BIT`, `UNION`, `TIME WITH TIME ZONE`, a nanosecond timestamp, a decimal wider than 38 digits) must be pinned or excluded. +pub(crate) fn source_column(name: &str, data_type: &str, nullable: bool) -> SourceColumn { + let shown = data_type.trim().to_owned(); + let column_type = scalar_type(&shown.to_ascii_uppercase()).ok_or_else(|| { + format!( + "DuckDB {shown} has no portable type; pin one in columns, such as text, or exclude the column" + ) + }); + SourceColumn { + name: name.to_owned(), + postgres_type: shown, + pg_type: PgType::Other, + column_type, + nullable, + source_spelling: None, + repeat_of: None, + } +} + +/// Whether an upper-cased DuckDB type is a list or fixed-size array, which a pinned `array(T)` reads as a list. +pub(crate) fn is_list(shown: &str) -> bool { + shown.ends_with(']') +} + +/// The portable type of an upper-cased DuckDB type, or `None` for one with none. +fn scalar_type(shown: &str) -> Option { + if is_list(shown) { + return Some(ColumnType::Json); + } + let end = shown.find('(').unwrap_or(shown.len()); + let base = shown[..end].trim(); + Some(match base { + "BOOLEAN" => ColumnType::Bool, + "TINYINT" | "SMALLINT" | "UTINYINT" => ColumnType::SmallInt, + "INTEGER" | "BIGINT" | "USMALLINT" | "UINTEGER" => ColumnType::Int, + "FLOAT" | "DOUBLE" => ColumnType::Float, + "DECIMAL" => { + let sizes = shown[end..].trim_start_matches('(').trim_end_matches(')'); + let mut parts = sizes.split(',').map(|part| part.trim().parse::()); + let (Some(Ok(precision)), Some(Ok(scale))) = (parts.next(), parts.next()) else { + return None; + }; + (precision <= MAX_DECIMAL_PRECISION).then_some(ColumnType::Decimal { precision, scale })? + } + "VARCHAR" | "ENUM" => ColumnType::Text, + "BLOB" => ColumnType::Binary, + "DATE" => ColumnType::Date, + "TIME" => ColumnType::Time, + "TIMESTAMP" | "TIMESTAMP_S" | "TIMESTAMP_MS" => ColumnType::TimestampNtz, + "TIMESTAMP WITH TIME ZONE" => ColumnType::TimestampTz, + "UUID" => ColumnType::Uuid, + "JSON" => ColumnType::Json, + "STRUCT" | "MAP" => ColumnType::Json, + _ => return None, + }) +} diff --git a/crates/sqlload/src/duckdb_source/constants.rs b/crates/sqlload/src/duckdb_source/constants.rs new file mode 100644 index 000000000..cf5f37c83 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/constants.rs @@ -0,0 +1,5 @@ +/// The system name a DuckDB source's tables carry in plans. +pub const DUCKDB_SOURCE: &str = "duckdb"; + +/// How many result batches the read holds ahead of the consumer. +pub const BATCHES_AHEAD: usize = 4; diff --git a/crates/sqlload/src/duckdb_source/errors.rs b/crates/sqlload/src/duckdb_source/errors.rs new file mode 100644 index 000000000..f270af918 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/errors.rs @@ -0,0 +1,11 @@ +/// Why reading a DuckDB source failed. +#[derive(Debug, thiserror::Error)] +pub enum DuckDbSourceError { + #[error("DuckDB: {0}")] + DuckDb(#[from] duckdb::Error), + #[error("{0}")] + Invalid(String), + /// What the caller's batch consumer failed with, as it said it. + #[error("{0}")] + Consumer(String), +} diff --git a/crates/sqlload/src/duckdb_source/main/count_rows.rs b/crates/sqlload/src/duckdb_source/main/count_rows.rs new file mode 100644 index 000000000..424c95a8e --- /dev/null +++ b/crates/sqlload/src/duckdb_source/main/count_rows.rs @@ -0,0 +1,24 @@ +//! Count a table's rows that a condition selects. + +use crate::duckdb_source::_helpers::connection::open_read_only; +use crate::duckdb_source::_helpers::query::count_query; +use crate::duckdb_source::errors::DuckDbSourceError; +use crate::duckdb_source::models::DuckDbReader; +use crate::postgres_source::models::SourceTable; + +/// How many rows of `table` (or of its query `sql`) `filter` selects. +pub async fn count_rows( + reader: &DuckDbReader, + (table, sql): (&SourceTable, Option<&str>), + filter: &str, +) -> Result { + let (reader, query) = (reader.clone(), count_query((table, sql), filter)); + tokio::task::spawn_blocking(move || { + let connection = open_read_only(&reader)?; + let count: i64 = connection.query_row(&query, [], |row| row.get(0))?; + u64::try_from(count) + .map_err(|error| DuckDbSourceError::Invalid(format!("{query} returned {count}: {error}"))) + }) + .await + .map_err(|error| DuckDbSourceError::Invalid(format!("counting rows stopped: {error}")))? +} diff --git a/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs b/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs new file mode 100644 index 000000000..fa2e0468b --- /dev/null +++ b/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs @@ -0,0 +1,33 @@ +//! A reader for a `duckdb` source's connection. + +use crate::duckdb_source::errors::DuckDbSourceError; +use crate::duckdb_source::models::DuckDbReader; +use crate::project::main::motherduck_path::motherduck_path; +use crate::project::models::ConnectionConfig; + +/// A reader for a `duckdb` connection's database file; each read opens it read-only, with `memory_limit` (half a table's share of the memory budget, or DuckDB's own default without one). `:memory:` and MotherDuck paths are refused: an empty in-memory database has nothing to read, and a MotherDuck database is a `motherduck` source. Nothing is opened yet. +pub fn duckdb_reader( + connection: &ConnectionConfig, + memory_limit: Option, +) -> Result { + let ConnectionConfig::DuckDb { database } = connection else { + return Err(DuckDbSourceError::Invalid(format!( + "a {} connection is not a duckdb one", + connection.kind() + ))); + }; + if database == ":memory:" { + return Err(DuckDbSourceError::Invalid( + "a duckdb source reads a database file; :memory: holds no tables to read".to_owned(), + )); + } + if motherduck_path(database).prefixed { + return Err(DuckDbSourceError::Invalid( + "a duckdb connection's database is a file; for a MotherDuck database use kind = \"motherduck\"".to_owned(), + )); + } + Ok(DuckDbReader { + database: database.clone(), + memory_limit, + }) +} diff --git a/crates/sqlload/src/duckdb_source/main/mod.rs b/crates/sqlload/src/duckdb_source/main/mod.rs new file mode 100644 index 000000000..fb2904c0b --- /dev/null +++ b/crates/sqlload/src/duckdb_source/main/mod.rs @@ -0,0 +1,5 @@ +pub mod count_rows; +pub mod duckdb_reader; +pub mod read_catalog; +pub mod read_columns; +pub mod read_rows; diff --git a/crates/sqlload/src/duckdb_source/main/read_catalog.rs b/crates/sqlload/src/duckdb_source/main/read_catalog.rs new file mode 100644 index 000000000..fe8ffec9c --- /dev/null +++ b/crates/sqlload/src/duckdb_source/main/read_catalog.rs @@ -0,0 +1,66 @@ +//! Discover a DuckDB source's tables: tables and views matching `include`, and query tables described by their result's columns. + +use crate::duckdb_source::_helpers::catalog::catalog_tables; +use crate::duckdb_source::_helpers::connection::open_read_only; +use crate::duckdb_source::_helpers::query::subquery; +use crate::duckdb_source::_helpers::type_map::source_column; +use crate::duckdb_source::errors::DuckDbSourceError; +use crate::duckdb_source::models::DuckDbReader; +use crate::postgres_source::models::SourceTable; + +/// Every table and view of the reader's database whose `schema.table` matches `include` and not `exclude`; then one table per query in `queries` (name and SQL), its columns those of the query as a load wraps it, described with `DESCRIBE` so it reads no rows. +pub async fn read_catalog( + reader: &DuckDbReader, + (include, exclude): (&[String], &[String]), + queries: &[(String, String)], +) -> Result, DuckDbSourceError> { + let (reader, include, exclude, queries) = + (reader.clone(), include.to_vec(), exclude.to_vec(), queries.to_vec()); + tokio::task::spawn_blocking(move || { + let connection = open_read_only(&reader)?; + let mut tables: Vec = match include.is_empty() { + true => Vec::new(), + false => catalog_tables(&connection, (&include, &exclude))?, + }; + for (name, sql) in &queries { + let described = describe(&connection, sql).map_err(|error| { + DuckDbSourceError::Invalid(format!("query table {name}: {error}")) + })?; + tables.push(SourceTable { + split_pages: None, + schema: String::new(), + name: name.clone(), + columns: described + .iter() + .map(|(column, data_type, nullable)| source_column(column, data_type, *nullable)) + .collect(), + primary_key: Vec::new(), + indexed_columns: Vec::new(), + estimated_rows: -1, + estimated_bytes: -1, + comment: None, + listed_keys: Vec::new(), + }); + } + Ok(tables) + }) + .await + .map_err(|error| DuckDbSourceError::Invalid(format!("reading the catalog stopped: {error}")))? +} + +/// Each result column of `sql`: its name, DuckDB type and whether it may be NULL. +fn describe( + connection: &duckdb::Connection, + sql: &str, +) -> Result, duckdb::Error> { + let mut statement = + connection.prepare(&format!("DESCRIBE SELECT * FROM {}", subquery(sql)))?; + let rows = statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?.is_none_or(|null| null != "NO"), + )) + })?; + rows.collect() +} diff --git a/crates/sqlload/src/duckdb_source/main/read_columns.rs b/crates/sqlload/src/duckdb_source/main/read_columns.rs new file mode 100644 index 000000000..f229ec54d --- /dev/null +++ b/crates/sqlload/src/duckdb_source/main/read_columns.rs @@ -0,0 +1,47 @@ +//! Decide how each column of a DuckDB table or query is read. + +use std::collections::BTreeMap; + +use crate::column_types::main::arrow_field::arrow_field; +use crate::column_types::models::ColumnType; +use crate::duckdb_source::_helpers::query::quote_identifier; +use crate::duckdb_source::_helpers::type_map::is_list; +use crate::postgres_source::models::{ReadColumn, SourceTable, WireType}; + +/// How to read every column of `table`, using `pinned` types where given and the natural mapping otherwise; each read column's expression is its quoted name, which cursor filters compare against. A pin with no Arrow type, or an `array(T)` pin over a column that isn't a list, is a problem. +pub fn read_columns( + table: &SourceTable, + pinned: &BTreeMap, +) -> Result, Vec> { + let mut columns: Vec = Vec::with_capacity(table.columns.len()); + let mut problems: Vec = Vec::new(); + for column in &table.columns { + let output = match (pinned.get(&column.name), &column.column_type) { + (Some(pinned), _) => pinned.clone(), + (None, Ok(natural)) => natural.clone(), + (None, Err(reason)) => { + problems.push(format!("{}: {reason}", column.name)); + continue; + } + }; + let readable = arrow_field(&column.name, &output).is_some() + && (!matches!(output, ColumnType::Array(_)) || is_list(&column.postgres_type)); + if !readable { + problems.push(format!( + "{} has type {}, which cannot be read as {output}", + column.name, column.postgres_type + )); + continue; + } + columns.push(ReadColumn { + name: column.name.clone(), + expression: quote_identifier(column.source_spelling.as_deref().unwrap_or(&column.name)), + wire: WireType::Text, + output, + }); + } + match problems.is_empty() { + true => Ok(columns), + false => Err(problems), + } +} diff --git a/crates/sqlload/src/duckdb_source/main/read_rows.rs b/crates/sqlload/src/duckdb_source/main/read_rows.rs new file mode 100644 index 000000000..3b7478575 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/main/read_rows.rs @@ -0,0 +1,74 @@ +//! Read a table or query as Arrow batches from a read-only connection. + +use arrow_array::RecordBatch; + +use crate::duckdb_source::_helpers::connection::open_read_only; +use crate::duckdb_source::_helpers::convert::{convert_batch, output_schema}; +use crate::duckdb_source::_helpers::query::select_query; +use crate::duckdb_source::constants::BATCHES_AHEAD; +use crate::duckdb_source::errors::DuckDbSourceError; +use crate::duckdb_source::models::{DuckDbReadRequest, DuckDbReader}; +use crate::row_batches::main::loaded_batch::loaded_batch; + +/// Read `request`'s rows and pass them to `on_batch` in batches of about `batch_rows`, each converted to the read columns' types; returns the row count. One statement streams the rows, so the read is consistent on its own; a thread reads at most a few batches ahead of `on_batch`, and stops when `on_batch` fails. +pub async fn read_rows( + reader: &DuckDbReader, + request: &DuckDbReadRequest<'_>, + batch_rows: usize, + mut on_batch: impl AsyncFnMut(RecordBatch) -> Result<(), String>, +) -> Result { + let shape = output_schema(request.columns).map_err(DuckDbSourceError::Invalid)?; + let (reader, query) = (reader.clone(), select_query(request)); + let (sender, mut receiver) = + tokio::sync::mpsc::channel::>(BATCHES_AHEAD); + let streaming = tokio::task::spawn_blocking(move || { + let read = || -> Result<(), DuckDbSourceError> { + let connection = open_read_only(&reader)?; + let mut statement = connection.prepare(&query)?; + let mut held: Vec = Vec::new(); + let mut held_rows = 0; + for chunk in statement.stream_arrow([])? { + let chunk = chunk?; + held_rows += chunk.num_rows(); + held.push(chunk); + if held_rows >= batch_rows { + if !send(&sender, &mut held)? { + return Ok(()); + } + held_rows = 0; + } + } + send(&sender, &mut held)?; + Ok(()) + }; + if let Err(error) = read() { + let _ = sender.blocking_send(Err(error)); + } + }); + let mut rows: u64 = 0; + while let Some(batch) = receiver.recv().await { + let batch = loaded_batch(&batch?, request.table) + .and_then(|batch| convert_batch(&batch, (&shape.0, &shape.1))) + .map_err(DuckDbSourceError::Invalid)?; + rows += batch.num_rows() as u64; + on_batch(batch).await.map_err(DuckDbSourceError::Consumer)?; + } + streaming + .await + .map_err(|error| DuckDbSourceError::Invalid(format!("the read stopped: {error}")))?; + Ok(rows) +} + +/// Send the held chunks as one batch, emptying `held`; `false` when the consumer has gone. +fn send( + sender: &tokio::sync::mpsc::Sender>, + held: &mut Vec, +) -> Result { + let Some(first) = held.first() else { + return Ok(true); + }; + let batch = arrow_select::concat::concat_batches(&first.schema(), held.iter()) + .map_err(|error| DuckDbSourceError::Invalid(error.to_string()))?; + held.clear(); + Ok(sender.blocking_send(Ok(batch)).is_ok()) +} diff --git a/crates/sqlload/src/duckdb_source/mod.rs b/crates/sqlload/src/duckdb_source/mod.rs new file mode 100644 index 000000000..be753b643 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/mod.rs @@ -0,0 +1,10 @@ +//! DuckDB as a source: tables and views of a database file that `include` finds, and query tables, read as Arrow from a read-only connection. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlload/src/duckdb_source/models.rs b/crates/sqlload/src/duckdb_source/models.rs new file mode 100644 index 000000000..858bb8c2a --- /dev/null +++ b/crates/sqlload/src/duckdb_source/models.rs @@ -0,0 +1,19 @@ +use crate::postgres_source::models::{ReadColumn, SourceTable}; + +/// A DuckDB source: the database file it reads, opened read-only for each read, and the `memory_limit` each of its connections gets (half of a table's share of the memory budget; DuckDB's own default without a budget). +#[derive(Debug, Clone)] +pub struct DuckDbReader { + pub(crate) database: String, + pub(crate) memory_limit: Option, +} + +/// What one read selects: a table or a query, its columns, and the rows' filter. +#[derive(Debug, Clone, Copy)] +pub struct DuckDbReadRequest<'a> { + pub table: &'a SourceTable, + /// The query whose rows are the table; `None` reads the table itself. + pub sql: Option<&'a str>, + pub columns: &'a [ReadColumn], + /// A DuckDB condition on the read columns, already rendered. + pub filter: Option<&'a str>, +} diff --git a/crates/sqlload/src/lib.rs b/crates/sqlload/src/lib.rs index 311053888..f4ad04aa0 100644 --- a/crates/sqlload/src/lib.rs +++ b/crates/sqlload/src/lib.rs @@ -20,6 +20,8 @@ pub mod contract; pub mod copy; pub mod databricks_api; pub mod databricks_source; +#[cfg(feature = "duckdb")] +pub mod duckdb_source; pub mod delivery; pub mod envelope; pub mod exporter; From 9e72a37b0862a897cabefad120b2bbfd8343501f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 09:01:13 +0000 Subject: [PATCH 2/9] wip: duckdb source kind in project config (unbuilt) --- .../sqlload/src/project/_helpers/references.rs | 18 +++++++++++++----- crates/sqlload/src/project/models.rs | 8 +++++++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/sqlload/src/project/_helpers/references.rs b/crates/sqlload/src/project/_helpers/references.rs index a12c2d479..a08ee0d0a 100644 --- a/crates/sqlload/src/project/_helpers/references.rs +++ b/crates/sqlload/src/project/_helpers/references.rs @@ -106,6 +106,7 @@ fn source_problems(project: &ProjectConfig, name: &str, source: &SourceConfig) - SourceKind::Odbc => (&["odbc"], "odbc"), SourceKind::BigQuery => (&["bigquery"], "bigquery"), SourceKind::Databricks => (&["databricks"], "databricks"), + SourceKind::DuckDb => (&["duckdb"], "duckdb"), SourceKind::Files => (&["s3", "filesystem", "sftp"], "s3, filesystem or sftp"), SourceKind::Python => (&[], "any"), SourceKind::GoogleSheets => (&["google_sheets"], "google_sheets"), @@ -224,11 +225,15 @@ fn kind_problems(place: &str, source: &SourceConfig) -> Vec { let queries = source.tables.values().any(|table| table.sql.is_some()); if !matches!( source.kind, - SourceKind::Snowflake | SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks + SourceKind::Snowflake + | SourceKind::Odbc + | SourceKind::BigQuery + | SourceKind::Databricks + | SourceKind::DuckDb ) && queries { problems.push(format!( - "{place} sql is for kind = \"snowflake\", \"odbc\", \"bigquery\" and \"databricks\"" + "{place} sql is for kind = \"snowflake\", \"odbc\", \"bigquery\", \"databricks\" and \"duckdb\"" )); } if source.kind != SourceKind::BigQuery && source.project.is_some() { @@ -275,10 +280,11 @@ fn kind_problems(place: &str, source: &SourceConfig) -> Vec { | SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks + | SourceKind::DuckDb ); if !selects && sheet_settings(source).any(|settings| settings.row_filter.is_some()) { problems.push(format!( - "{place} where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\" and \"databricks\", whose tables SQLLoad reads with a SELECT" + "{place} where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\", \"databricks\" and \"duckdb\", whose tables SQLLoad reads with a SELECT" )); } problems.extend( @@ -304,13 +310,15 @@ fn kind_problems(place: &str, source: &SourceConfig) -> Vec { | SourceKind::Snowflake | SourceKind::Odbc | SourceKind::BigQuery - | SourceKind::Databricks => { + | SourceKind::Databricks + | SourceKind::DuckDb => { if source.include.is_empty() && !queries { let tables = match source.kind { SourceKind::Snowflake | SourceKind::Odbc | SourceKind::BigQuery - | SourceKind::Databricks => { + | SourceKind::Databricks + | SourceKind::DuckDb => { "; list tables in include, or give a table a sql query" } _ => "", diff --git a/crates/sqlload/src/project/models.rs b/crates/sqlload/src/project/models.rs index a4cc7b380..0025cfcb2 100644 --- a/crates/sqlload/src/project/models.rs +++ b/crates/sqlload/src/project/models.rs @@ -609,7 +609,7 @@ impl SourceConfig { .unwrap_or_default() } - /// Whether the source finds its tables in a database's catalog with `include` patterns (`postgres`, `postgres_cdc`, `snowflake`, `odbc` or `bigquery`). + /// Whether the source finds its tables in a database's catalog with `include` patterns (every database source kind). pub fn discovers_tables(&self) -> bool { matches!( self.kind, @@ -619,6 +619,7 @@ impl SourceConfig { | SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks + | SourceKind::DuckDb ) } @@ -677,6 +678,7 @@ impl SourceConfig { | SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks + | SourceKind::DuckDb | SourceKind::Python, ) => DEFAULT_TABLE_NAME, (None, SourceKind::Files) => DEFAULT_FILES_TABLE_NAME, @@ -703,6 +705,9 @@ pub enum SourceKind { BigQuery, /// Databricks tables, views and queries, read as Arrow through the Statement Execution API's result links. Databricks, + /// Tables, views and queries of a DuckDB database file, read as Arrow from a read-only connection. + #[serde(rename = "duckdb")] + DuckDb, /// Declared in Python with `sl.source` or `@sl.table`; never written in `sqlload.toml`. Python, /// Tabs or ranges of a Google Sheets spreadsheet, one table each. @@ -720,6 +725,7 @@ impl SourceKind { Self::Odbc => "odbc", Self::BigQuery => "bigquery", Self::Databricks => "databricks", + Self::DuckDb => "duckdb", Self::Python => "python", Self::GoogleSheets => "google_sheets", } From fa6d6a220d6c922e88e4675f3287526d9649c38c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:24:13 +0000 Subject: [PATCH 3/9] wip: wire the DuckDB source into the loader, with tests and docs (not yet compiled) --- .../dispatch/main/dialect/source_dialect.rs | 2 + .../src/duckdb_source/main/describe_source.rs | 17 +++ crates/sqlload/src/duckdb_source/main/mod.rs | 1 + crates/sqlload/src/duckdb_source/mod.rs | 8 +- crates/sqlload/src/duckdb_source/tests.rs | 4 + .../duckdb_source/tests/test_read_source.rs | 142 ++++++++++++++++++ .../src/duckdb_source/tests/test_type_map.rs | 28 ++++ crates/sqlload/src/lib.rs | 1 - crates/sqlload/src/loader/_helpers/resolve.rs | 7 + crates/sqlload/src/loader/_helpers/sources.rs | 12 ++ crates/sqlload/src/loader/_helpers/specs.rs | 54 ++----- .../sqlload/src/loader/_helpers/table_plan.rs | 7 + crates/sqlload/src/loader/constants.rs | 3 + crates/sqlload/src/loader/errors.rs | 3 + .../sqlload/src/loader/main/_check_source.rs | 3 + crates/sqlload/src/loader/main/_count_rows.rs | 44 ++++++ .../sqlload/src/loader/main/_duckdb_source.rs | 122 +++++++++++++++ crates/sqlload/src/loader/main/mod.rs | 2 + crates/sqlload/src/loader/models.rs | 3 + .../src/row_deletes/main/deletes_problems.rs | 1 + crates/sqlload/tests/loader.rs | 3 + .../tests/loader/test_duckdb_source.rs | 101 +++++++++++++ website/src/content/docs/docs/index.mdx | 2 +- .../src/content/docs/docs/sources/duckdb.mdx | 112 ++++++++++++++ .../src/content/docs/docs/sources/index.mdx | 7 +- .../content/docs/docs/supported-systems.mdx | 4 +- 26 files changed, 647 insertions(+), 46 deletions(-) create mode 100644 crates/sqlload/src/duckdb_source/main/describe_source.rs create mode 100644 crates/sqlload/src/duckdb_source/tests.rs create mode 100644 crates/sqlload/src/duckdb_source/tests/test_read_source.rs create mode 100644 crates/sqlload/src/duckdb_source/tests/test_type_map.rs create mode 100644 crates/sqlload/src/loader/main/_count_rows.rs create mode 100644 crates/sqlload/src/loader/main/_duckdb_source.rs create mode 100644 crates/sqlload/tests/loader/test_duckdb_source.rs create mode 100644 website/src/content/docs/docs/sources/duckdb.mdx diff --git a/crates/sqlload/src/adapters/dispatch/main/dialect/source_dialect.rs b/crates/sqlload/src/adapters/dispatch/main/dialect/source_dialect.rs index 134f10ca3..90b91612a 100644 --- a/crates/sqlload/src/adapters/dispatch/main/dialect/source_dialect.rs +++ b/crates/sqlload/src/adapters/dispatch/main/dialect/source_dialect.rs @@ -3,6 +3,7 @@ use crate::bigquery_source::constants::BIGQUERY_SOURCE; use crate::column_types::models::Warehouse; use crate::databricks_source::constants::DATABRICKS_SOURCE; +use crate::duckdb_source::constants::DUCKDB_SOURCE; use crate::loader::constants::SNOWFLAKE_SOURCE; /// The warehouse whose SQL a warehouse source of `system` (its kind as written) filters in; `None` for a system without one. @@ -11,6 +12,7 @@ pub fn source_dialect(system: &str) -> Option { SNOWFLAKE_SOURCE => Some(Warehouse::Snowflake), BIGQUERY_SOURCE => Some(Warehouse::BigQuery), DATABRICKS_SOURCE => Some(Warehouse::Databricks), + DUCKDB_SOURCE => Some(Warehouse::DuckDb), _ => None, } } diff --git a/crates/sqlload/src/duckdb_source/main/describe_source.rs b/crates/sqlload/src/duckdb_source/main/describe_source.rs new file mode 100644 index 000000000..f39435a6f --- /dev/null +++ b/crates/sqlload/src/duckdb_source/main/describe_source.rs @@ -0,0 +1,17 @@ +//! What a DuckDB source reads, for `debug`. + +use crate::duckdb_source::_helpers::connection::open_read_only; +use crate::duckdb_source::errors::DuckDbSourceError; +use crate::duckdb_source::models::DuckDbReader; + +/// The reader's database file and the DuckDB version reading it, once the file has opened read-only. +pub async fn describe_source(reader: &DuckDbReader) -> Result { + let reader = reader.clone(); + tokio::task::spawn_blocking(move || { + let connection = open_read_only(&reader)?; + let version: String = connection.query_row("SELECT version()", [], |row| row.get(0))?; + Ok(format!("{}, read-only, DuckDB {version}", reader.database)) + }) + .await + .map_err(|error| DuckDbSourceError::Invalid(format!("opening the database stopped: {error}")))? +} diff --git a/crates/sqlload/src/duckdb_source/main/mod.rs b/crates/sqlload/src/duckdb_source/main/mod.rs index fb2904c0b..dc4075b94 100644 --- a/crates/sqlload/src/duckdb_source/main/mod.rs +++ b/crates/sqlload/src/duckdb_source/main/mod.rs @@ -1,4 +1,5 @@ pub mod count_rows; +pub mod describe_source; pub mod duckdb_reader; pub mod read_catalog; pub mod read_columns; diff --git a/crates/sqlload/src/duckdb_source/mod.rs b/crates/sqlload/src/duckdb_source/mod.rs index be753b643..873e29d5c 100644 --- a/crates/sqlload/src/duckdb_source/mod.rs +++ b/crates/sqlload/src/duckdb_source/mod.rs @@ -1,10 +1,14 @@ -//! DuckDB as a source: tables and views of a database file that `include` finds, and query tables, read as Arrow from a read-only connection. +//! DuckDB as a source: tables and views of a database file that `include` finds, and query tables, read as Arrow from a read-only connection. Only the constants are compiled without the `duckdb` feature, so plans can name the system. +#[cfg(feature = "duckdb")] pub(crate) mod _helpers; pub mod constants; +#[cfg(feature = "duckdb")] pub mod errors; +#[cfg(feature = "duckdb")] pub mod main; +#[cfg(feature = "duckdb")] pub mod models; -#[cfg(test)] +#[cfg(all(test, feature = "duckdb"))] mod tests; diff --git a/crates/sqlload/src/duckdb_source/tests.rs b/crates/sqlload/src/duckdb_source/tests.rs new file mode 100644 index 000000000..d8f5eb86b --- /dev/null +++ b/crates/sqlload/src/duckdb_source/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_read_source.rs"] +mod read_source; +#[path = "tests/test_type_map.rs"] +mod type_map; diff --git a/crates/sqlload/src/duckdb_source/tests/test_read_source.rs b/crates/sqlload/src/duckdb_source/tests/test_read_source.rs new file mode 100644 index 000000000..6d7d47a6c --- /dev/null +++ b/crates/sqlload/src/duckdb_source/tests/test_read_source.rs @@ -0,0 +1,142 @@ +use std::collections::BTreeMap; + +use arrow_array::{Array, Decimal128Array, Int64Array, RecordBatch, StringArray, TimestampMicrosecondArray}; + +use crate::column_types::models::ColumnType; +use crate::duckdb_source::main::duckdb_reader::duckdb_reader; +use crate::duckdb_source::main::read_catalog::read_catalog; +use crate::duckdb_source::main::read_columns::read_columns; +use crate::duckdb_source::main::read_rows::read_rows; +use crate::duckdb_source::models::DuckDbReadRequest; +use crate::project::models::ConnectionConfig; + +const SETUP: &str = " +CREATE SCHEMA sales; +CREATE TABLE sales.orders ( + id INTEGER PRIMARY KEY, + amount DECIMAL(12,2), + name VARCHAR, + placed_at TIMESTAMPTZ, + tags VARCHAR[], + meta STRUCT(rank INTEGER), + total HUGEINT +); +COMMENT ON TABLE sales.orders IS 'Orders, one row each.'; +INSERT INTO sales.orders VALUES + (1, 10.50, 'ann', '2026-01-01 00:00:00+00', ['a', 'b'], {'rank': 1}, 170141183460469231731687303715884105727), + (2, NULL, 'bo\"b', '2026-01-02 12:30:00+02', [], {'rank': 2}, 2), + (3, 99999999.99, NULL, NULL, NULL, NULL, 3); +CREATE VIEW sales.big_orders AS SELECT id, amount FROM sales.orders WHERE amount > 50; +CREATE TABLE sales.tmp_scratch (x INTEGER); +"; + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("a runtime") +} + +#[test] +fn given_a_duckdb_file_when_reading_its_tables_and_a_query_then_rows_load_typed_and_the_file_is_left_as_it_was() + { + let folder = tempfile::tempdir().expect("a folder"); + let path = folder.path().join("shop.duckdb"); + duckdb::Connection::open(&path) + .and_then(|connection| connection.execute_batch(SETUP)) + .expect("the database"); + let before = std::fs::read(&path).expect("the file"); + let reader = duckdb_reader( + &ConnectionConfig::DuckDb { + database: path.display().to_string(), + }, + Some(256 << 20), + ) + .expect("a reader"); + let queries = [( + "recent".to_owned(), + "SELECT id, amount FROM sales.orders WHERE id > 1; -- newest".to_owned(), + )]; + let runtime = runtime(); + let tables = runtime + .block_on(read_catalog( + &reader, + (&["sales.*".to_owned()], &["sales.tmp_*".to_owned()]), + &queries, + )) + .expect("the catalog"); + let names: Vec = tables.iter().map(|table| table.qualified_name()).collect(); + assert_eq!(names, ["sales.big_orders", "sales.orders", "recent"], "tables and views by include, less exclude, then queries"); + let orders = &tables[1]; + assert_eq!(orders.primary_key, ["id"]); + assert_eq!(orders.comment.as_deref(), Some("Orders, one row each.")); + let total = orders.columns.iter().find(|column| column.name == "total").expect("total"); + assert!(total.column_type.is_err(), "HUGEINT must be pinned"); + assert!(read_columns(orders, &BTreeMap::new()).is_err(), "an unpinned HUGEINT is a problem"); + let pinned = BTreeMap::from([("total".to_owned(), ColumnType::Text)]); + let columns = read_columns(orders, &pinned).expect("read columns"); + let mut batches: Vec = Vec::new(); + let rows = runtime + .block_on(read_rows( + &reader, + &DuckDbReadRequest { + table: orders, + sql: None, + columns: &columns, + filter: Some("\"id\" <= 2"), + }, + 1, + async |batch| { + batches.push(batch); + Ok(()) + }, + )) + .expect("the rows"); + assert_eq!(rows, 2); + assert_eq!(batches.len(), 2, "one batch per batch_rows"); + let batch = arrow_select::concat::concat_batches(&batches[0].schema(), &batches).expect("one batch"); + let column = |name: &str| batch.column_by_name(name).expect(name).clone(); + let ids = column("id"); + let ids = ids.as_any().downcast_ref::().expect("int64 ids"); + assert_eq!(ids.values(), &[1, 2]); + let amounts = column("amount"); + let amounts = amounts.as_any().downcast_ref::().expect("decimals"); + assert_eq!((amounts.value(0), amounts.is_null(1), amounts.scale()), (1050, true, 2)); + let placed = column("placed_at"); + let placed = placed.as_any().downcast_ref::().expect("timestamps"); + assert_eq!(placed.value(1), 1_767_349_800_000_000, "2026-01-02 10:30 UTC"); + let text = |name: &str| { + let array = column(name); + let strings = array.as_any().downcast_ref::().expect(name); + (0..strings.len()).map(|row| strings.is_valid(row).then(|| strings.value(row).to_owned())).collect::>() + }; + assert_eq!(text("name"), [Some("ann".to_owned()), Some("bo\"b".to_owned())]); + assert_eq!(text("tags"), [Some(r#"["a","b"]"#.to_owned()), Some("[]".to_owned())]); + assert_eq!(text("meta"), [Some(r#"{"rank":1}"#.to_owned()), Some(r#"{"rank":2}"#.to_owned())]); + assert_eq!(text("total")[0].as_deref(), Some("170141183460469231731687303715884105727")); + let recent = &tables[2]; + let columns = read_columns(recent, &BTreeMap::new()).expect("query columns"); + let rows = runtime + .block_on(read_rows( + &reader, + &DuckDbReadRequest { + table: recent, + sql: Some(&queries[0].1), + columns: &columns, + filter: Some("\"id\" > 2"), + }, + 100, + async |_| Ok(()), + )) + .expect("the query's rows"); + assert_eq!(rows, 1, "the filter applies outside the query, after its trailing comment"); + assert_eq!(std::fs::read(&path).expect("the file"), before, "the file is read, never written"); +} + +#[test] +fn given_a_memory_or_motherduck_database_when_making_a_reader_then_it_is_refused() { + for database in [":memory:", "md:shop"] { + let refused = duckdb_reader(&ConnectionConfig::DuckDb { database: database.to_owned() }, None); + assert!(refused.is_err(), "{database}"); + } +} diff --git a/crates/sqlload/src/duckdb_source/tests/test_type_map.rs b/crates/sqlload/src/duckdb_source/tests/test_type_map.rs new file mode 100644 index 000000000..cbd22cd12 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/tests/test_type_map.rs @@ -0,0 +1,28 @@ +use crate::column_types::models::ColumnType; +use crate::duckdb_source::_helpers::type_map::source_column; + +#[test] +fn given_duckdb_column_types_when_mapping_them_then_each_gets_its_portable_type_or_must_be_pinned() { + let decimal = |precision, scale| ColumnType::Decimal { precision, scale }; + let test_cases: [(&str, &str, Option); 14] = [ + ("unsigned 8-bit fits smallint", "UTINYINT", Some(ColumnType::SmallInt)), + ("unsigned 32-bit fits a 64-bit int", "UINTEGER", Some(ColumnType::Int)), + ("unsigned 64-bit does not fit int", "UBIGINT", None), + ("HUGEINT does not fit int", "HUGEINT", None), + ("a decimal keeps precision and scale", "DECIMAL(12,2)", Some(decimal(12, 2))), + ("a 38-digit decimal is the widest", "DECIMAL(38,10)", Some(decimal(38, 10))), + ("an enum is its label", "ENUM('a', 'b')", Some(ColumnType::Text)), + ("TIMESTAMPTZ is an instant", "TIMESTAMP WITH TIME ZONE", Some(ColumnType::TimestampTz)), + ("millisecond timestamps widen to microseconds", "TIMESTAMP_MS", Some(ColumnType::TimestampNtz)), + ("nanosecond timestamps would lose digits", "TIMESTAMP_NS", None), + ("a list is json", "VARCHAR[]", Some(ColumnType::Json)), + ("a fixed-size array is json", "INTEGER[3]", Some(ColumnType::Json)), + ("a struct is json", "STRUCT(a INTEGER, b VARCHAR)", Some(ColumnType::Json)), + ("an interval has no portable type", "INTERVAL", None), + ]; + for (description, data_type, expected) in test_cases { + let column = source_column("c", data_type, true); + assert_eq!(column.column_type.ok(), expected, "{description}"); + assert_eq!(column.postgres_type, data_type, "{description}: plan shows DuckDB's type"); + } +} diff --git a/crates/sqlload/src/lib.rs b/crates/sqlload/src/lib.rs index f4ad04aa0..875e56649 100644 --- a/crates/sqlload/src/lib.rs +++ b/crates/sqlload/src/lib.rs @@ -20,7 +20,6 @@ pub mod contract; pub mod copy; pub mod databricks_api; pub mod databricks_source; -#[cfg(feature = "duckdb")] pub mod duckdb_source; pub mod delivery; pub mod envelope; diff --git a/crates/sqlload/src/loader/_helpers/resolve.rs b/crates/sqlload/src/loader/_helpers/resolve.rs index 0a00180c8..ea66aebdd 100644 --- a/crates/sqlload/src/loader/_helpers/resolve.rs +++ b/crates/sqlload/src/loader/_helpers/resolve.rs @@ -1738,6 +1738,13 @@ async fn failed_count( .err() .map(|error| error.to_string()) } + #[cfg(feature = "duckdb")] + SourceReader::DuckDb(duckdb) => { + crate::duckdb_source::main::count_rows::count_rows(duckdb, (table, None), filter) + .await + .err() + .map(|error| error.to_string()) + } SourceReader::Files(_) | SourceReader::Sheets(..) => None, } } diff --git a/crates/sqlload/src/loader/_helpers/sources.rs b/crates/sqlload/src/loader/_helpers/sources.rs index be6481bef..5c26aaf24 100644 --- a/crates/sqlload/src/loader/_helpers/sources.rs +++ b/crates/sqlload/src/loader/_helpers/sources.rs @@ -226,6 +226,7 @@ pub(crate) async fn open_source( SourceKind::Databricks => Ok(SourceReader::Databricks(Box::new(databricks_reader( project, source, )?))), + SourceKind::DuckDb => crate::loader::main::_duckdb_source::open_duckdb(project, source), SourceKind::Python => Err(LoaderError::Invalid( "a Python source is read by calling its table functions".to_owned(), )), @@ -370,6 +371,12 @@ pub(crate) async fn source_tables( .map(|tables| (tables, None, Vec::new())) .map_err(|error| LoaderError::from(error).into())) } + SourceKind::DuckDb => Ok( + crate::loader::main::_duckdb_source::duckdb_tables(project, source) + .await + .map(|tables| (tables, None, Vec::new())) + .map_err(|error| error.into()), + ), SourceKind::Python => Ok(Ok((python_tables(source), None, Vec::new()))), SourceKind::GoogleSheets => Ok(match sheets_tables(project, source).await { Ok((tables, warnings)) => Ok((tables, None, warnings)), @@ -1596,6 +1603,11 @@ async fn read_each( }; Ok(read_databricks_rows(reader, &request, on_batch).await?) } + #[cfg(feature = "duckdb")] + (SourceReader::DuckDb(reader), _) => { + crate::loader::main::_duckdb_source::read_duckdb(reader, table, part.filter, on_batch) + .await + } (SourceReader::Sheets(client, spreadsheet_id), _) => { let batches = sheet_rows((client, spreadsheet_id), table, part.transform.as_ref()).await?; diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs index 5e03949d2..359151e2d 100644 --- a/crates/sqlload/src/loader/_helpers/specs.rs +++ b/crates/sqlload/src/loader/_helpers/specs.rs @@ -7,7 +7,6 @@ use crate::adapters::contract::models::{ChangeApply, TableName, WriteMode}; use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter; use crate::adapters::files::constants::{CURSOR_PLACEHOLDER_START, TABLE_PLACEHOLDER}; use crate::bigquery_source::constants::BIGQUERY_SOURCE; -use crate::bigquery_source::main::count_rows::count_rows as count_bigquery_rows; use crate::cdc::constants::{ BATCH_INTERVAL, CDC_FILES_PROBLEM, CDC_TARGETS_PROBLEM, FILES_BATCH_INTERVAL, }; @@ -19,7 +18,7 @@ use crate::column_types::main::is_variant::is_variant; use crate::column_types::main::parse_type::parse_type; use crate::column_types::models::{ColumnType, Warehouse}; use crate::databricks_source::constants::DATABRICKS_SOURCE; -use crate::databricks_source::main::count_rows::count_rows as count_databricks_rows; +use crate::duckdb_source::constants::DUCKDB_SOURCE; use crate::files_source::main::constant_columns::constant_columns; use crate::files_source::main::path_template::path_template; use crate::files_source::models::FileColumn; @@ -43,7 +42,6 @@ use crate::loader::models::{ }; use crate::loader::models::{SourceReader, TablePlan}; use crate::masking::models::MaskRules; -use crate::postgres_source::main::count_rows::count_rows as count_postgres_rows; use crate::postgres_source::models::{PgType, SourceColumn, SourceTable}; use crate::postgres_source::models::{ReadColumn, WireType}; use crate::project::constants::DEFAULT_MAX_DEPTH; @@ -66,7 +64,6 @@ use crate::sheets_source::models::SheetRead; use crate::similar_names::main::close_name::close_name; use crate::similar_names::main::nearest_name::nearest_name; use crate::snowflake_source::constants::USER_STAGE_FOLDER; -use crate::snowflake_source::main::count_rows::count_rows as count_snowflake_rows; use globset::{Glob, GlobMatcher, GlobSet, GlobSetBuilder}; use std::collections::{BTreeMap, BTreeSet}; @@ -532,17 +529,19 @@ pub(crate) fn build_spec( deletes, tombstones_of: overrides.tombstones_of.clone(), warehouse_source: match context.source.kind { - kind @ (SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks) => { - Some(WarehouseSource { - system: match kind { - SourceKind::Odbc => ODBC_SOURCE, - SourceKind::BigQuery => BIGQUERY_SOURCE, - _ => DATABRICKS_SOURCE, - }, + kind @ (SourceKind::Odbc + | SourceKind::BigQuery + | SourceKind::Databricks + | SourceKind::DuckDb) => Some(WarehouseSource { + system: match kind { + SourceKind::Odbc => ODBC_SOURCE, + SourceKind::BigQuery => BIGQUERY_SOURCE, + SourceKind::Databricks => DATABRICKS_SOURCE, + _ => DUCKDB_SOURCE, + }, sql: overrides.sql.clone(), route: WarehouseRoute::Query, - }) - } + }), _ => warehouse_source(context.source, overrides), }, row_filter: layered(&settings.row_filter, &defaults.row_filter) @@ -783,7 +782,8 @@ pub(crate) fn missing_description( | SourceKind::Snowflake | SourceKind::Odbc | SourceKind::BigQuery - | SourceKind::Databricks => format!( + | SourceKind::Databricks + | SourceKind::DuckDb => format!( "no description; add description = \"...\" under [sources.{source_name}.tables.\"{qualified}\"]" ), }; @@ -1384,31 +1384,7 @@ pub(crate) async fn null_cursor_rows( .warehouse_source .as_ref() .and_then(|source| source.sql.as_deref()); - let counted = match reader { - SourceReader::Postgres(client, _) => { - count_postgres_rows(client, &spec.source_table, &filter).await? - } - SourceReader::Snowflake(snowflake) => { - count_snowflake_rows(snowflake, (&spec.source_table, sql), &filter).await? - } - SourceReader::BigQuery(bigquery) => { - count_bigquery_rows(bigquery, (&spec.source_table, sql), &filter).await? - } - SourceReader::Databricks(databricks) => { - count_databricks_rows(databricks, (&spec.source_table, sql), &filter).await? - } - #[cfg(feature = "odbc")] - SourceReader::Odbc(odbc) => { - crate::odbc_source::main::count_rows::count_rows( - odbc, - (&spec.source_table, sql), - &filter, - ) - .await? - } - SourceReader::Files(_) | SourceReader::Sheets(..) => return Ok(None), - }; - Ok(Some(counted)) + crate::loader::main::_count_rows::count_rows(reader, (&spec.source_table, sql), &filter).await } /// Why `spec`'s load fails: `rows` source rows have a NULL cursor, which its incremental read never selects, so they would never load; with the fixes `plan` accepts for the table: `null_cursor = "include"` on `merge`, a coalesced cursor where one may be listed (not a Python table, `delete_insert` or a files target), and `null_cursor = "ignore"`. diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index f6fcc9850..98b412928 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -432,6 +432,13 @@ pub(crate) fn read_columns(spec: &TableSpec) -> Result, Vec { + crate::duckdb_source::main::read_columns::read_columns( + &selected_columns(spec), + &spec.pinned, + ) + } #[cfg(feature = "odbc")] (None, None) if spec diff --git a/crates/sqlload/src/loader/constants.rs b/crates/sqlload/src/loader/constants.rs index 910eeb37e..f4ef5f8c6 100644 --- a/crates/sqlload/src/loader/constants.rs +++ b/crates/sqlload/src/loader/constants.rs @@ -114,6 +114,9 @@ pub const UNLOADED_TIMES: &str = "the warehouse writes times and timestamps as w /// Why an `odbc` source cannot be read by this build. pub const NO_ODBC: &str = "this build of SQLLoad has no ODBC support; build it with the odbc feature (cargo install sqlload --features odbc)"; +/// Why a build without DuckDB refuses a `duckdb` source. +pub const NO_DUCKDB_SOURCE: &str = "this build of SQLLoad has no DuckDB support; build it with the duckdb feature (cargo install sqlload --features duckdb)"; + /// The plan warning for a files table whose checks pull its files through SQLLoad instead of the destination's own read. pub const CHECKS_LOADER_WARNING: &str = "checks run on rows SQLLoad stages, so every file goes through the loader rather than the destination's own read, which is slower; without checks the table stays on the direct route."; diff --git a/crates/sqlload/src/loader/errors.rs b/crates/sqlload/src/loader/errors.rs index d69f90e2e..4c7ba5225 100644 --- a/crates/sqlload/src/loader/errors.rs +++ b/crates/sqlload/src/loader/errors.rs @@ -37,6 +37,9 @@ pub enum LoaderError { #[cfg(feature = "odbc")] #[error(transparent)] Odbc(#[from] crate::odbc_source::errors::OdbcSourceError), + #[cfg(feature = "duckdb")] + #[error(transparent)] + DuckDbSource(#[from] crate::duckdb_source::errors::DuckDbSourceError), #[error(transparent)] Store(#[from] StoreError), #[error(transparent)] diff --git a/crates/sqlload/src/loader/main/_check_source.rs b/crates/sqlload/src/loader/main/_check_source.rs index 64f58ca1e..7167c8d34 100644 --- a/crates/sqlload/src/loader/main/_check_source.rs +++ b/crates/sqlload/src/loader/main/_check_source.rs @@ -57,6 +57,9 @@ pub(in crate::loader) async fn check_source( )) } SourceKind::Databricks => check_databricks(project, source).await, + SourceKind::DuckDb => { + crate::loader::main::_duckdb_source::check_duckdb(project, source).await + } #[cfg(feature = "odbc")] SourceKind::Odbc => { let reader = crate::loader::_helpers::sources::odbc_source_reader(project, source)?; diff --git a/crates/sqlload/src/loader/main/_count_rows.rs b/crates/sqlload/src/loader/main/_count_rows.rs new file mode 100644 index 000000000..2af76a15c --- /dev/null +++ b/crates/sqlload/src/loader/main/_count_rows.rs @@ -0,0 +1,44 @@ +//! How many rows of a source table a condition selects, counted by the database. + +use crate::loader::errors::LoaderError; +use crate::loader::models::SourceReader; +use crate::postgres_source::models::SourceTable; + +/// How many rows of `table` (or of its query `sql`) `filter` selects, counted by the source's database; `None` for a files or Google Sheets source, which has no database to ask. +pub(in crate::loader) async fn count_rows( + reader: &SourceReader, + (table, sql): (&SourceTable, Option<&str>), + filter: &str, +) -> Result, LoaderError> { + Ok(Some(match reader { + SourceReader::Postgres(client, _) => { + crate::postgres_source::main::count_rows::count_rows(client, table, filter).await? + } + SourceReader::Snowflake(snowflake) => { + crate::snowflake_source::main::count_rows::count_rows(snowflake, (table, sql), filter) + .await? + } + SourceReader::BigQuery(bigquery) => { + crate::bigquery_source::main::count_rows::count_rows(bigquery, (table, sql), filter) + .await? + } + SourceReader::Databricks(databricks) => { + crate::databricks_source::main::count_rows::count_rows( + databricks, + (table, sql), + filter, + ) + .await? + } + #[cfg(feature = "odbc")] + SourceReader::Odbc(odbc) => { + crate::odbc_source::main::count_rows::count_rows(odbc, (table, sql), filter).await? + } + #[cfg(feature = "duckdb")] + SourceReader::DuckDb(duckdb) => { + crate::duckdb_source::main::count_rows::count_rows(duckdb, (table, sql), filter) + .await? + } + SourceReader::Files(_) | SourceReader::Sheets(..) => return Ok(None), + })) +} diff --git a/crates/sqlload/src/loader/main/_duckdb_source.rs b/crates/sqlload/src/loader/main/_duckdb_source.rs new file mode 100644 index 000000000..38f4f24c4 --- /dev/null +++ b/crates/sqlload/src/loader/main/_duckdb_source.rs @@ -0,0 +1,122 @@ +//! A `duckdb` source in the loader: its reader, its tables, its rows and its check; a build without the `duckdb` feature refuses it. + +use crate::loader::errors::LoaderError; +use crate::loader::models::SourceReader; +use crate::postgres_source::models::SourceTable; +use crate::project::models::{ProjectConfig, SourceConfig}; + +/// The source's reader, each of whose connections gets half a table's share of the memory budget. +#[cfg(feature = "duckdb")] +fn duckdb_source_reader( + project: &ProjectConfig, + source: &SourceConfig, +) -> Result { + let Some(connection) = project.connections.get(&source.connection) else { + return Err(LoaderError::Invalid(format!( + "connection {:?} is not defined", + source.connection + ))); + }; + let memory_limit = project.memory.budget.map(|budget| budget.table_share / 2); + Ok(crate::duckdb_source::main::duckdb_reader::duckdb_reader( + connection, + memory_limit, + )?) +} + +/// The source's reader; nothing is opened until a read. +#[cfg(feature = "duckdb")] +pub(in crate::loader) fn open_duckdb( + project: &ProjectConfig, + source: &SourceConfig, +) -> Result { + Ok(SourceReader::DuckDb(Box::new(duckdb_source_reader( + project, source, + )?))) +} + +#[cfg(not(feature = "duckdb"))] +pub(in crate::loader) fn open_duckdb( + _: &ProjectConfig, + _: &SourceConfig, +) -> Result { + Err(LoaderError::Invalid( + crate::loader::constants::NO_DUCKDB_SOURCE.to_owned(), + )) +} + +/// The source's tables and views that `include` finds, then one table per query table. +#[cfg(feature = "duckdb")] +pub(in crate::loader) async fn duckdb_tables( + project: &ProjectConfig, + source: &SourceConfig, +) -> Result, LoaderError> { + let queries: Vec<(String, String)> = source + .tables + .iter() + .filter_map(|(name, table)| Some((name.clone(), table.sql.clone()?))) + .collect(); + let reader = duckdb_source_reader(project, source)?; + Ok(crate::duckdb_source::main::read_catalog::read_catalog( + &reader, + (&source.include, &source.exclude), + &queries, + ) + .await?) +} + +#[cfg(not(feature = "duckdb"))] +pub(in crate::loader) async fn duckdb_tables( + _: &ProjectConfig, + _: &SourceConfig, +) -> Result, LoaderError> { + Err(LoaderError::Invalid( + crate::loader::constants::NO_DUCKDB_SOURCE.to_owned(), + )) +} + +/// Read `table`'s rows that `filter` selects (the plan's filter when `None`), passing each batch to `on_batch`; returns the row count. +#[cfg(feature = "duckdb")] +pub(in crate::loader) async fn read_duckdb( + reader: &crate::duckdb_source::models::DuckDbReader, + table: &crate::loader::models::TablePlan, + filter: Option, + on_batch: impl AsyncFnMut(arrow_array::RecordBatch) -> Result<(), String>, +) -> Result { + let filter = table + .spec + .read_filter(filter.or_else(|| table.source_filter.clone())); + let request = crate::duckdb_source::models::DuckDbReadRequest { + table: &table.spec.source_table, + sql: table.spec.query_sql(), + columns: &table.read_columns, + filter: filter.as_deref(), + }; + Ok(crate::duckdb_source::main::read_rows::read_rows( + reader, + &request, + crate::loader::constants::SOURCE_BATCH_ROWS, + on_batch, + ) + .await?) +} + +/// The database file the source reads and the DuckDB version reading it, for `debug`. +#[cfg(feature = "duckdb")] +pub(in crate::loader) async fn check_duckdb( + project: &ProjectConfig, + source: &SourceConfig, +) -> Result { + let reader = duckdb_source_reader(project, source)?; + Ok(crate::duckdb_source::main::describe_source::describe_source(&reader).await?) +} + +#[cfg(not(feature = "duckdb"))] +pub(in crate::loader) async fn check_duckdb( + _: &ProjectConfig, + _: &SourceConfig, +) -> Result { + Err(LoaderError::Invalid( + crate::loader::constants::NO_DUCKDB_SOURCE.to_owned(), + )) +} diff --git a/crates/sqlload/src/loader/main/mod.rs b/crates/sqlload/src/loader/main/mod.rs index 9abf6747b..d7b0672da 100644 --- a/crates/sqlload/src/loader/main/mod.rs +++ b/crates/sqlload/src/loader/main/mod.rs @@ -1,4 +1,6 @@ pub(in crate::loader) mod _check_source; +pub(in crate::loader) mod _count_rows; +pub(in crate::loader) mod _duckdb_source; pub(in crate::loader) mod _files_repair; pub(in crate::loader) mod _files_schema; pub(in crate::loader) mod _files_window; diff --git a/crates/sqlload/src/loader/models.rs b/crates/sqlload/src/loader/models.rs index afd1580ee..b24678bba 100644 --- a/crates/sqlload/src/loader/models.rs +++ b/crates/sqlload/src/loader/models.rs @@ -1305,6 +1305,9 @@ pub enum SourceReader { /// A database read through its ODBC driver. #[cfg(feature = "odbc")] Odbc(Box), + /// A DuckDB database file, opened read-only for each read. + #[cfg(feature = "duckdb")] + DuckDb(Box), /// A Google Sheets client and the spreadsheet it reads. Sheets(crate::sheets_source::models::SheetsClient, String), } diff --git a/crates/sqlload/src/row_deletes/main/deletes_problems.rs b/crates/sqlload/src/row_deletes/main/deletes_problems.rs index b9e99916e..3083df54f 100644 --- a/crates/sqlload/src/row_deletes/main/deletes_problems.rs +++ b/crates/sqlload/src/row_deletes/main/deletes_problems.rs @@ -27,6 +27,7 @@ pub fn deletes_problems( | SourceKind::Snowflake | SourceKind::BigQuery | SourceKind::Databricks + | SourceKind::DuckDb ) { problems.push(format!( "deletes reads every key of the source table to find the ones it no longer holds, which only database and warehouse sources list; a {} source can't. Fix: remove deletes", diff --git a/crates/sqlload/tests/loader.rs b/crates/sqlload/tests/loader.rs index a28fe11fd..a3fb71b24 100644 --- a/crates/sqlload/tests/loader.rs +++ b/crates/sqlload/tests/loader.rs @@ -47,6 +47,9 @@ mod test_column_names; mod test_databricks_loads; #[path = "loader/test_databricks_source.rs"] mod test_databricks_source; +#[cfg(feature = "duckdb")] +#[path = "loader/test_duckdb_source.rs"] +mod test_duckdb_source; #[path = "loader/test_debug_project.rs"] mod test_debug_project; #[path = "loader/test_file_store.rs"] diff --git a/crates/sqlload/tests/loader/test_duckdb_source.rs b/crates/sqlload/tests/loader/test_duckdb_source.rs new file mode 100644 index 000000000..24ee1804a --- /dev/null +++ b/crates/sqlload/tests/loader/test_duckdb_source.rs @@ -0,0 +1,101 @@ +use crate::helpers::{duckdb_execute, duckdb_text, run_sqlload}; + +const SOURCE: &str = " +CREATE SCHEMA sales; +CREATE TABLE sales.orders ( + id INTEGER PRIMARY KEY, + amount DECIMAL(12,2), + updated_at TIMESTAMP, + tags VARCHAR[], + card VARCHAR, + region VARCHAR +); +INSERT INTO sales.orders VALUES + (1, 10.50, '2026-01-01 00:00:00', ['a'], '4111', 'EU'), + (2, 20.00, '2026-01-02 00:00:00', [], '4222', 'EU'), + (3, 30.25, '2026-01-03 00:00:00', NULL, '4333', 'US'); +CREATE VIEW sales.big_orders AS SELECT id, amount FROM sales.orders WHERE amount > 15; +CREATE TABLE sales.tmp_scratch (x INTEGER); +"; + +/// A second run's changes: order 2 updated, order 4 added, order 1 untouched. +const CHANGES: &str = " +UPDATE sales.orders SET amount = 21.00, updated_at = '2026-01-05 00:00:00' WHERE id = 2; +INSERT INTO sales.orders VALUES (4, 40.00, '2026-01-06 00:00:00', ['b', 'c'], '4444', 'EU'); +"; + +fn project(source: &std::path::Path, target: &std::path::Path) -> String { + format!( + r#"default_target = "local" + +[connections.shop_file] +kind = "duckdb" +database = "{source}" + +[connections.warehouse] +kind = "duckdb" +database = "{target}" + +[targets.local] +connection = "warehouse" +loader_schema = "raw" + +[sources.shop] +kind = "duckdb" +connection = "shop_file" +description = "The shop's DuckDB file." +include = ["sales.*"] +exclude = ["sales.tmp_*"] +exclude_columns = ["card"] +write_strategy = "replace" + +[sources.shop.tables."sales.orders"] +description = "Orders, one row each." +write_strategy = "merge" +unique_key = ["id"] +cursor_column = "updated_at" +where = "region = 'EU'" + +[sources.shop.tables.big_query] +description = "Orders over 25, by query." +sql = "SELECT id, amount FROM sales.orders WHERE amount > 25 -- big ones" +"#, + source = source.display(), + target = target.display() + ) +} + +/// Rows in id order with amount and the tags' JSON, the target's columns, and the view's and query table's rows. +const CHECK: &str = "SELECT (SELECT string_agg(id || ':' || amount || ':' || coalesce(CAST(tags AS VARCHAR), '-'), ',' ORDER BY id) FROM raw.shop__orders) || '|' || (SELECT string_agg(column_name, ',' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_schema = 'raw' AND table_name = 'shop__orders' AND column_name NOT LIKE '\\_sqlload%' ESCAPE '\\') || '|' || (SELECT count(*) FROM raw.shop__big_orders) || '|' || (SELECT string_agg(CAST(id AS VARCHAR), ',' ORDER BY id) FROM raw.shop__big_query)"; + +#[test] +fn given_a_duckdb_source_when_loading_twice_then_tables_views_and_queries_load_and_the_cursor_merges_only_changes() +-> Result<(), String> { + let folder = tempfile::tempdir().map_err(|error| error.to_string())?; + let (source, target) = (folder.path().join("shop.duckdb"), folder.path().join("out.duckdb")); + duckdb_execute(&source, SOURCE)?; + let config = folder.path().join("sqlload.toml"); + std::fs::write(&config, project(&source, &target)).map_err(|error| error.to_string())?; + let (planned, plan) = run_sqlload(&config, &["plan"], ("", ""))?; + assert!(planned, "{plan}"); + for table in ["raw.shop__orders", "raw.shop__big_orders", "raw.shop__big_query"] { + assert!(plan.contains(table), "plan lists {table}: {plan}"); + } + assert!(!plan.contains("tmp_scratch"), "exclude drops the table: {plan}"); + let (loaded, output) = run_sqlload(&config, &["load"], ("", ""))?; + assert!(loaded, "{output}"); + assert_eq!( + duckdb_text(&target, CHECK)?, + r#"1:10.50:["a"],2:20.00:[]|id,amount,updated_at,tags,region|2|3"#, + "the first load: only EU rows, card excluded, tags as JSON, the view and the query" + ); + duckdb_execute(&source, CHANGES)?; + let (loaded, output) = run_sqlload(&config, &["load"], ("", ""))?; + assert!(loaded, "{output}"); + assert_eq!( + duckdb_text(&target, CHECK)?, + r#"1:10.50:["a"],2:21.00:[],4:40.00:["b","c"]|id,amount,updated_at,tags,region|3|3,4"#, + "the second load merges the updated and new rows the cursor finds" + ); + Ok(()) +} diff --git a/website/src/content/docs/docs/index.mdx b/website/src/content/docs/docs/index.mdx index 9afe05454..56bdb5dd6 100644 --- a/website/src/content/docs/docs/index.mdx +++ b/website/src/content/docs/docs/index.mdx @@ -15,7 +15,7 @@ SQLLoad loads data into your warehouse, and shows you how before it changes anyt | BigQuery | beta | beta | | Databricks | beta | beta | | ClickHouse | | ✓ one self-hosted server¹ | -| DuckDB | | ✓ | +| DuckDB | ✓ | ✓ | | MotherDuck | | beta² | | SQL Server, MySQL, MariaDB, OpenEdge and other databases through ODBC³ | ✓ | | | Files | ✓ Parquet, CSV, JSON, NDJSON, Excel (xlsx), XML; gzip, zstd and ZIP; any file name through `pattern` (S3, local folder, SFTP) | ✓ Parquet, CSV (S3, local folder) | diff --git a/website/src/content/docs/docs/sources/duckdb.mdx b/website/src/content/docs/docs/sources/duckdb.mdx new file mode 100644 index 000000000..509999ffa --- /dev/null +++ b/website/src/content/docs/docs/sources/duckdb.mdx @@ -0,0 +1,112 @@ +--- +title: "DuckDB" +description: "Load tables, views and query results from a DuckDB database file, read as Arrow from a read-only connection: connections, choosing tables, query tables, cursors, types and memory." +--- + +A `duckdb` source loads tables, views and query results from a DuckDB database file. SQLLoad opens +the file read-only for each read and streams the result of one `SELECT` as Arrow, so it never +writes to the file. + +```toml +[connections.shop_file] +kind = "duckdb" +database = "shop.duckdb" + +[sources.shop] +kind = "duckdb" +connection = "shop_file" +description = "Tables of the shop's DuckDB file." +include = ["sales.*"] +exclude = ["sales.tmp_*"] +write_strategy = "replace" + +[sources.shop.tables."sales.orders"] +description = "Orders, one row each." +cursor_column = "updated_at" +write_strategy = "merge" +unique_key = "{primary_key}" + +[sources.shop.tables.big_orders] +description = "Orders over 100, with their customer." +sql = "SELECT o.*, c.name FROM sales.orders o JOIN sales.customers c USING (customer_id) WHERE o.amount > 100" +cursor_column = "updated_at" +write_strategy = "append" +``` + +## Connection + +A source uses a [`duckdb` connection](/docs/configuration/#duckdb), the same kind a DuckDB +target uses, whose `database` is the file to read. An in-memory database (`:memory:`) holds nothing +to read and is refused, and so is a MotherDuck database (`md:`). + +DuckDB lets one process write a file, or any number read it. While another process has the file +open for writing, a read fails, naming the lock; run the load when the writer has closed it. + +## Choosing tables + +`include` and `exclude` are `schema.table` patterns, matched against the names as DuckDB keeps +them, such as `sales.*` or `main.orders`. SQLLoad lists the file's tables and views, their columns +and their primary keys from DuckDB's catalog functions. A +`[sources.shop.tables."schema.table"]` entry sets options for one table; on its own, it doesn't add +the table. + +A source needs at least one `include` pattern or one query table. A table's primary key is its +`{primary_key}` for `unique_key`, and its comment becomes its description. + +## How tables are read + +Each read is one `SELECT` of the read columns, with the cursor window and any `where` in its +`WHERE`. One statement sees one version of the file, so a read is consistent on its own. SQLLoad +holds at most a few batches ahead of the target, and each connection's DuckDB `memory_limit` is +half the table's share of the [memory budget](/docs/cli/load/#memory-budget). + +## Query tables + +A table entry with `sql` is a query table: its rows are the query's result, and the entry's key is +its name. At `plan`, SQLLoad describes the query's columns without reading its rows. + +`cursor_column` names a column of the query's result. SQLLoad wraps the query and filters on the +cursor outside it, so the query needs no placeholder. Write it in DuckDB SQL. + +## Filtering rows + +`where` narrows a table to the rows that match a condition, written in DuckDB SQL: + +```toml +[sources.shop.tables."sales.orders"] +where = "region = 'EU'" +``` + +SQLLoad ANDs it into every `SELECT` it reads the table with, and a query table refuses it (put the +condition in its `sql`). + +## Types + +| DuckDB type | Loads as | +|-------------|----------| +| `TINYINT`, `SMALLINT`, `UTINYINT` | `smallint` | +| `INTEGER`, `BIGINT`, `USMALLINT`, `UINTEGER` | `int` | +| `DECIMAL(p,s)`, p up to 38 | `decimal(p,s)` | +| `FLOAT`, `DOUBLE` | `float` | +| `BOOLEAN` | `bool` | +| `VARCHAR`, `ENUM` | `text` | +| `BLOB` | `binary` | +| `DATE` | `date` | +| `TIME` | `time` | +| `TIMESTAMP`, `TIMESTAMP_S`, `TIMESTAMP_MS` | `timestamp_ntz` | +| `TIMESTAMP WITH TIME ZONE` | `timestamp_tz` | +| `UUID` | `uuid` | +| `JSON`, lists, arrays, `STRUCT`, `MAP` | `json` | +| `HUGEINT`, `UBIGINT`, `TIMESTAMP_NS`, `INTERVAL`, `BIT`, `UNION` and others | none: pin a type such as `text`, or exclude the column | + +Pin a type to load a column another way, such as `columns = { total = "text" }`, or a list as +`array(T)`. A value that doesn't convert to the pinned type fails the read, naming the column, +rather than loading as NULL. See [Types](/docs/concepts/types/). + +## Tested + +Unit tests read a real DuckDB file: tables, a view and a query table found by `include` and +`exclude`, primary keys and comments, every type above, a pinned `HUGEINT`, reads in several +batches, a filter applied outside a query that ends in a comment, and the file's bytes unchanged +after the read. A loader test loads a file into DuckDB twice: tables, a view and a query table, +with `exclude_columns`, `where`, and a cursor merging only the rows that changed. diff --git a/website/src/content/docs/docs/sources/index.mdx b/website/src/content/docs/docs/sources/index.mdx index 4f57d77cc..b20231d85 100644 --- a/website/src/content/docs/docs/sources/index.mdx +++ b/website/src/content/docs/docs/sources/index.mdx @@ -1,9 +1,9 @@ --- title: "Sources" -description: "Postgres tables, Postgres changes, Snowflake tables and queries, ODBC databases, BigQuery and Databricks tables and queries (beta), files in S3 or a folder, Google Sheets, and Python sources: the kinds of source and the settings they share." +description: "Postgres tables, Postgres changes, Snowflake tables and queries, ODBC databases, BigQuery and Databricks tables and queries (beta), DuckDB files, files in S3 or a folder, Google Sheets, and Python sources: the kinds of source and the settings they share." --- -A source is where data comes from. SQLLoad has nine kinds: +A source is where data comes from. SQLLoad has ten kinds: | `kind` | Reads | Tables | Declared in | |--------|-------|--------|-------------| @@ -13,6 +13,7 @@ A source is where data comes from. SQLLoad has nine kinds: | [`odbc`](/docs/sources/odbc/) | Tables, views and query results of any database with an ODBC driver, read as Arrow | One per matching table, and one per query | `sqlload.toml` | | [`bigquery`](/docs/sources/bigquery/) (beta) | Tables, views and query results of BigQuery, read as Arrow through the Storage Read API | One per matching table, and one per query | `sqlload.toml` | | [`databricks`](/docs/sources/databricks/) (beta) | Tables, views and query results of a Databricks SQL warehouse, read as Arrow through the Statement Execution API | One per matching table, and one per query | `sqlload.toml` | +| [`duckdb`](/docs/sources/duckdb/) | Tables, views and query results of a DuckDB database file, read as Arrow | One per matching table, and one per query | `sqlload.toml` | | [`files`](/docs/sources/files/) | Parquet, CSV, JSON or NDJSON files in S3 or a local folder | One per source | `sqlload.toml` | | [`google_sheets`](/docs/sources/google-sheets/) | Tabs or ranges of a Google Sheets spreadsheet, typed | One per tab or range | `sqlload.toml` | | [Python](/docs/sources/python/) | Whatever a Python function yields: dicts, Arrow data, JSON bytes, HTTP jobs | One per decorated function | Python, under `python_paths` | @@ -36,6 +37,8 @@ Each kind has its own page: tables, types, cost and permissions. - [Databricks](/docs/sources/databricks/) (beta): choosing tables, query tables, types, cost and permissions. +- [DuckDB](/docs/sources/duckdb/): choosing tables, query tables, types, and reading a file another + process writes. - [Files](/docs/sources/files/): formats, partitions, which files load, and files the destination reads itself. - [Google Sheets](/docs/sources/google-sheets/): tabs and ranges, headers, and how cells are typed. diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx index f710a90c1..09b3d759e 100644 --- a/website/src/content/docs/docs/supported-systems.mdx +++ b/website/src/content/docs/docs/supported-systems.mdx @@ -13,6 +13,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes | Databases with an ODBC driver: SQL Server, MySQL, MariaDB, OpenEdge and others | Tables and views, by `include` patterns, and query results | The driver, as Arrow. Tested with psqlODBC, Microsoft ODBC Driver 18 for SQL Server and MySQL Connector/ODBC; MariaDB Connector/ODBC and OpenEdge are checked by hand. See [ODBC](/docs/sources/odbc/#drivers). | | BigQuery (beta) | Tables and views, by `include` patterns, and query results | The Storage Read API, as Arrow in parallel streams; views and queries through a query job. See [BigQuery](/docs/sources/bigquery/). | | Databricks (beta) | Tables and views, by `include` patterns, and query results | One `SELECT` on the SQL warehouse, its result downloaded as Arrow files in parallel through the Statement Execution API's links. See [Databricks](/docs/sources/databricks/). | +| DuckDB | Tables and views of a database file, by `include` patterns, and query results | One `SELECT` on a read-only connection, streamed as Arrow. See [DuckDB](/docs/sources/duckdb/). | | Files in S3 | Parquet, CSV, JSON, NDJSON, Excel (xlsx) and XML; gzip or zstd for text formats; ZIP archives of any of them; any file name through `pattern`. See [Files](/docs/sources/files/#excel-xml-and-zip-files). | The destination itself where it can load them exactly, otherwise SQLLoad's reader | | Files in a local folder | As for S3 | As for S3 | | Files on an SFTP server | As for S3 | SQLLoad's reader, over SFTP. See [Files](/docs/sources/files/#files-on-an-sftp-server). | @@ -45,6 +46,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes | ODBC | Yes, untested | Yes, tested live | Yes, untested | Yes, untested | Yes, untested | Yes | Yes | | BigQuery (beta) | Yes, untested | Yes, tested live | Yes, untested | Yes, untested | Yes, untested | Yes | Yes, untested | | Databricks (beta) | Yes, untested | Yes, untested | Yes, untested | Yes, tested live | Yes, tested live | Yes, tested live | Yes, tested live | +| DuckDB | Yes, untested | Yes, untested | Yes, untested | Yes, untested | Yes, untested | Yes | Yes, untested | | Files | Yes, read by Snowflake from an external stage | Yes, staged by SQLLoad in the volume | Yes, read by SQLLoad and streamed | Yes; Parquet in S3 read by ClickHouse | Yes, read by SQLLoad | Yes; Parquet read by DuckDB | Read and rewritten by SQLLoad | | Files on an SFTP server | Yes, read by SQLLoad, tested live | Yes, read by SQLLoad, tested live | Yes, untested | Yes, untested | Yes, untested | Yes | Yes, untested | | Google Sheets | Yes | Yes | Yes | Yes | Yes | Yes | Yes | @@ -64,7 +66,7 @@ them into DuckDB. |--------|-----------------| | Postgres | Real Postgres in containers, as a source and as a target. `postgres_cdc` on Postgres 15, 16 and 17, plus live loads into DuckDB, Snowflake and Databricks. See [Postgres](/docs/targets/postgres/#tested). | | ClickHouse | Real ClickHouse 25.3, 25.8 and 26.10 in containers, with an S3-compatible store for the files it reads itself. See [ClickHouse](/docs/targets/clickhouse/#tested). | -| DuckDB | Real, in process. | +| DuckDB | Real, in process, as a source and a target. | | MotherDuck | Live against a MotherDuck account on DuckDB 1.5.6: every write strategy, an incremental load and rerun, a new column and every portable type. Offline, every change loads through the separate process into a stand-in for MotherDuck and compares with DuckDB. See [MotherDuck](/docs/targets/motherduck/#tested). | | ODBC | psqlODBC against real Postgres in containers, into DuckDB, plus a live round from SQL Server and MySQL into Databricks. | | Databricks source | A fake Statement Execution API in tests (catalog discovery, result files in several chunks, an expired link fetched again, cursor reads, no token on downloads), plus a live round from a real SQL warehouse into DuckDB, Parquet files, Postgres and ClickHouse: tables, a view, a windowed query table, merges and every column shape. See [Databricks](/docs/sources/databricks/#tested). | From c93f18405093b41eab9c3a1206c5cd6eb9c8c131 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:29:24 +0000 Subject: [PATCH 4/9] wip: DuckDB source reader built from the project, tests in case tables, odbc reader moved to its source --- .../src/duckdb_source/_helpers/catalog.rs | 4 +- .../src/duckdb_source/_helpers/connection.rs | 4 +- .../src/duckdb_source/_helpers/query.rs | 4 +- .../src/duckdb_source/_helpers/type_map.rs | 3 +- crates/sqlload/src/duckdb_source/constants.rs | 6 + .../src/duckdb_source/main/count_rows.rs | 5 +- .../src/duckdb_source/main/duckdb_reader.rs | 22 +- .../src/duckdb_source/main/read_catalog.rs | 29 +- .../src/duckdb_source/main/read_rows.rs | 14 +- crates/sqlload/src/duckdb_source/tests.rs | 4 + .../src/duckdb_source/tests/helpers.rs | 42 +++ .../duckdb_source/tests/test_read_source.rs | 287 ++++++++++-------- .../src/duckdb_source/tests/test_type_map.rs | 114 +++++-- .../src/duckdb_source/tests/test_types.rs | 38 +++ crates/sqlload/src/lib.rs | 2 +- crates/sqlload/src/loader/_helpers/resolve.rs | 59 ++-- crates/sqlload/src/loader/_helpers/sources.rs | 86 +++--- crates/sqlload/src/loader/_helpers/specs.rs | 8 +- .../sqlload/src/loader/_helpers/table_plan.rs | 3 +- .../sqlload/src/loader/main/_check_source.rs | 23 +- crates/sqlload/src/loader/main/_count_rows.rs | 44 --- .../sqlload/src/loader/main/_duckdb_source.rs | 122 -------- crates/sqlload/src/loader/main/mod.rs | 2 - .../main/{odbc_reader.rs => _odbc_reader.rs} | 2 +- crates/sqlload/src/odbc_source/main/mod.rs | 3 +- .../src/odbc_source/main/source_reader.rs | 33 ++ .../src/odbc_source/tests/test_connect.rs | 2 +- .../src/odbc_source/tests/test_odbc_reader.rs | 2 +- .../src/project/_helpers/references.rs | 4 +- .../snowflake_source/main/describe_source.rs | 19 ++ .../sqlload/src/snowflake_source/main/mod.rs | 1 + crates/sqlload/tests/loader.rs | 7 +- crates/sqlload/tests/loader/duckdb/helpers.rs | 45 +++ .../tests/loader/test_duckdb_source.rs | 108 +++---- crates/sqlload/tests/loader/test_types.rs | 10 + .../src/content/docs/docs/sources/duckdb.mdx | 3 +- 36 files changed, 661 insertions(+), 503 deletions(-) create mode 100644 crates/sqlload/src/duckdb_source/tests/helpers.rs create mode 100644 crates/sqlload/src/duckdb_source/tests/test_types.rs delete mode 100644 crates/sqlload/src/loader/main/_count_rows.rs delete mode 100644 crates/sqlload/src/loader/main/_duckdb_source.rs rename crates/sqlload/src/odbc_source/main/{odbc_reader.rs => _odbc_reader.rs} (98%) create mode 100644 crates/sqlload/src/odbc_source/main/source_reader.rs create mode 100644 crates/sqlload/src/snowflake_source/main/describe_source.rs create mode 100644 crates/sqlload/tests/loader/duckdb/helpers.rs diff --git a/crates/sqlload/src/duckdb_source/_helpers/catalog.rs b/crates/sqlload/src/duckdb_source/_helpers/catalog.rs index a4434ba06..18a7d13a7 100644 --- a/crates/sqlload/src/duckdb_source/_helpers/catalog.rs +++ b/crates/sqlload/src/duckdb_source/_helpers/catalog.rs @@ -66,7 +66,9 @@ pub(crate) fn catalog_tables( for row in rows { let (key, name, data_type, nullable) = row?; if let Some(table) = tables.get_mut(&key) { - table.columns.push(source_column(&name, &data_type, nullable)); + table + .columns + .push(source_column(&name, &data_type, nullable)); } } let mut keys = connection.prepare( diff --git a/crates/sqlload/src/duckdb_source/_helpers/connection.rs b/crates/sqlload/src/duckdb_source/_helpers/connection.rs index 98140ad14..be2146337 100644 --- a/crates/sqlload/src/duckdb_source/_helpers/connection.rs +++ b/crates/sqlload/src/duckdb_source/_helpers/connection.rs @@ -4,7 +4,9 @@ use crate::duckdb_source::errors::DuckDbSourceError; use crate::duckdb_source::models::DuckDbReader; /// The reader's database opened read-only, so a file another process writes is never changed and SQLLoad holds no write lock on it, with the reader's `memory_limit`. -pub(crate) fn open_read_only(reader: &DuckDbReader) -> Result { +pub(crate) fn open_read_only( + reader: &DuckDbReader, +) -> Result { let config = duckdb::Config::default().access_mode(duckdb::AccessMode::ReadOnly)?; let connection = duckdb::Connection::open_with_flags(&reader.database, config)?; if let Some(limit) = reader.memory_limit { diff --git a/crates/sqlload/src/duckdb_source/_helpers/query.rs b/crates/sqlload/src/duckdb_source/_helpers/query.rs index 9023dc512..575111f14 100644 --- a/crates/sqlload/src/duckdb_source/_helpers/query.rs +++ b/crates/sqlload/src/duckdb_source/_helpers/query.rs @@ -12,7 +12,9 @@ pub(crate) fn quote_identifier(name: &str) -> String { fn from_item((table, sql): (&SourceTable, Option<&str>)) -> String { match sql { Some(sql) => subquery(sql), - None => [&table.schema, &table.name].map(|name| quote_identifier(name)).join("."), + None => [&table.schema, &table.name] + .map(|name| quote_identifier(name)) + .join("."), } } diff --git a/crates/sqlload/src/duckdb_source/_helpers/type_map.rs b/crates/sqlload/src/duckdb_source/_helpers/type_map.rs index 2cc72a0a6..04b8463ce 100644 --- a/crates/sqlload/src/duckdb_source/_helpers/type_map.rs +++ b/crates/sqlload/src/duckdb_source/_helpers/type_map.rs @@ -46,7 +46,8 @@ fn scalar_type(shown: &str) -> Option { let (Some(Ok(precision)), Some(Ok(scale))) = (parts.next(), parts.next()) else { return None; }; - (precision <= MAX_DECIMAL_PRECISION).then_some(ColumnType::Decimal { precision, scale })? + (precision <= MAX_DECIMAL_PRECISION) + .then_some(ColumnType::Decimal { precision, scale })? } "VARCHAR" | "ENUM" => ColumnType::Text, "BLOB" => ColumnType::Binary, diff --git a/crates/sqlload/src/duckdb_source/constants.rs b/crates/sqlload/src/duckdb_source/constants.rs index cf5f37c83..58d1a1412 100644 --- a/crates/sqlload/src/duckdb_source/constants.rs +++ b/crates/sqlload/src/duckdb_source/constants.rs @@ -1,5 +1,11 @@ /// The system name a DuckDB source's tables carry in plans. pub const DUCKDB_SOURCE: &str = "duckdb"; +/// The database path DuckDB opens in memory, which holds no tables to read. +pub const MEMORY_DATABASE: &str = ":memory:"; + +/// What `DESCRIBE` writes in its `null` column for a column that is never NULL. +pub const NOT_NULL: &str = "NO"; + /// How many result batches the read holds ahead of the consumer. pub const BATCHES_AHEAD: usize = 4; diff --git a/crates/sqlload/src/duckdb_source/main/count_rows.rs b/crates/sqlload/src/duckdb_source/main/count_rows.rs index 424c95a8e..c4c7ca4f3 100644 --- a/crates/sqlload/src/duckdb_source/main/count_rows.rs +++ b/crates/sqlload/src/duckdb_source/main/count_rows.rs @@ -16,8 +16,9 @@ pub async fn count_rows( tokio::task::spawn_blocking(move || { let connection = open_read_only(&reader)?; let count: i64 = connection.query_row(&query, [], |row| row.get(0))?; - u64::try_from(count) - .map_err(|error| DuckDbSourceError::Invalid(format!("{query} returned {count}: {error}"))) + u64::try_from(count).map_err(|error| { + DuckDbSourceError::Invalid(format!("{query} returned {count}: {error}")) + }) }) .await .map_err(|error| DuckDbSourceError::Invalid(format!("counting rows stopped: {error}")))? diff --git a/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs b/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs index fa2e0468b..c95f96f9b 100644 --- a/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs +++ b/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs @@ -1,22 +1,24 @@ -//! A reader for a `duckdb` source's connection. +//! A reader for a `duckdb` source. +use crate::duckdb_source::constants::MEMORY_DATABASE; use crate::duckdb_source::errors::DuckDbSourceError; use crate::duckdb_source::models::DuckDbReader; use crate::project::main::motherduck_path::motherduck_path; -use crate::project::models::ConnectionConfig; +use crate::project::models::{ConnectionConfig, ProjectConfig, SourceConfig}; -/// A reader for a `duckdb` connection's database file; each read opens it read-only, with `memory_limit` (half a table's share of the memory budget, or DuckDB's own default without one). `:memory:` and MotherDuck paths are refused: an empty in-memory database has nothing to read, and a MotherDuck database is a `motherduck` source. Nothing is opened yet. +/// A reader for `source`'s `duckdb` connection's database file; each read opens it read-only, with a `memory_limit` of half a table's share of the project's memory budget (DuckDB's own default without one). `:memory:` and MotherDuck paths are refused: an empty in-memory database has nothing to read, and a MotherDuck database is a `motherduck` source. Nothing is opened yet. pub fn duckdb_reader( - connection: &ConnectionConfig, - memory_limit: Option, + project: &ProjectConfig, + source: &SourceConfig, ) -> Result { - let ConnectionConfig::DuckDb { database } = connection else { + let Some(ConnectionConfig::DuckDb { database }) = project.connections.get(&source.connection) + else { return Err(DuckDbSourceError::Invalid(format!( - "a {} connection is not a duckdb one", - connection.kind() + "connection {:?} is not a duckdb connection", + source.connection ))); }; - if database == ":memory:" { + if database == MEMORY_DATABASE { return Err(DuckDbSourceError::Invalid( "a duckdb source reads a database file; :memory: holds no tables to read".to_owned(), )); @@ -28,6 +30,6 @@ pub fn duckdb_reader( } Ok(DuckDbReader { database: database.clone(), - memory_limit, + memory_limit: project.memory.budget.map(|budget| budget.table_share / 2), }) } diff --git a/crates/sqlload/src/duckdb_source/main/read_catalog.rs b/crates/sqlload/src/duckdb_source/main/read_catalog.rs index fe8ffec9c..a0caa5cd6 100644 --- a/crates/sqlload/src/duckdb_source/main/read_catalog.rs +++ b/crates/sqlload/src/duckdb_source/main/read_catalog.rs @@ -4,18 +4,27 @@ use crate::duckdb_source::_helpers::catalog::catalog_tables; use crate::duckdb_source::_helpers::connection::open_read_only; use crate::duckdb_source::_helpers::query::subquery; use crate::duckdb_source::_helpers::type_map::source_column; +use crate::duckdb_source::constants::NOT_NULL; use crate::duckdb_source::errors::DuckDbSourceError; use crate::duckdb_source::models::DuckDbReader; use crate::postgres_source::models::SourceTable; +use crate::project::models::SourceConfig; -/// Every table and view of the reader's database whose `schema.table` matches `include` and not `exclude`; then one table per query in `queries` (name and SQL), its columns those of the query as a load wraps it, described with `DESCRIBE` so it reads no rows. +/// Every table and view of the reader's database whose `schema.table` matches `source`'s `include` and not its `exclude`; then one table per query table of `source`, its columns those of the query as a load wraps it, described with `DESCRIBE` so it reads no rows. pub async fn read_catalog( reader: &DuckDbReader, - (include, exclude): (&[String], &[String]), - queries: &[(String, String)], + source: &SourceConfig, ) -> Result, DuckDbSourceError> { - let (reader, include, exclude, queries) = - (reader.clone(), include.to_vec(), exclude.to_vec(), queries.to_vec()); + let queries: Vec<(String, String)> = source + .tables + .iter() + .filter_map(|(name, table)| Some((name.clone(), table.sql.clone()?))) + .collect(); + let (reader, include, exclude) = ( + reader.clone(), + source.include.clone(), + source.exclude.clone(), + ); tokio::task::spawn_blocking(move || { let connection = open_read_only(&reader)?; let mut tables: Vec = match include.is_empty() { @@ -32,7 +41,9 @@ pub async fn read_catalog( name: name.clone(), columns: described .iter() - .map(|(column, data_type, nullable)| source_column(column, data_type, *nullable)) + .map(|(column, data_type, nullable)| { + source_column(column, data_type, *nullable) + }) .collect(), primary_key: Vec::new(), indexed_columns: Vec::new(), @@ -53,13 +64,13 @@ fn describe( connection: &duckdb::Connection, sql: &str, ) -> Result, duckdb::Error> { - let mut statement = - connection.prepare(&format!("DESCRIBE SELECT * FROM {}", subquery(sql)))?; + let mut statement = connection.prepare(&format!("DESCRIBE SELECT * FROM {}", subquery(sql)))?; let rows = statement.query_map([], |row| { Ok(( row.get::<_, String>(0)?, row.get::<_, String>(1)?, - row.get::<_, Option>(2)?.is_none_or(|null| null != "NO"), + row.get::<_, Option>(2)? + .is_none_or(|null| null != NOT_NULL), )) })?; rows.collect() diff --git a/crates/sqlload/src/duckdb_source/main/read_rows.rs b/crates/sqlload/src/duckdb_source/main/read_rows.rs index 3b7478575..798be486b 100644 --- a/crates/sqlload/src/duckdb_source/main/read_rows.rs +++ b/crates/sqlload/src/duckdb_source/main/read_rows.rs @@ -32,13 +32,13 @@ pub async fn read_rows( held_rows += chunk.num_rows(); held.push(chunk); if held_rows >= batch_rows { - if !send(&sender, &mut held)? { + if !send(&sender, &std::mem::take(&mut held))? { return Ok(()); } held_rows = 0; } } - send(&sender, &mut held)?; + send(&sender, &held)?; Ok(()) }; if let Err(error) = read() { @@ -59,16 +59,18 @@ pub async fn read_rows( Ok(rows) } -/// Send the held chunks as one batch, emptying `held`; `false` when the consumer has gone. +/// Send the `held` chunks as one batch; `false` when the consumer has gone. fn send( sender: &tokio::sync::mpsc::Sender>, - held: &mut Vec, + held: &[RecordBatch], ) -> Result { let Some(first) = held.first() else { return Ok(true); }; let batch = arrow_select::concat::concat_batches(&first.schema(), held.iter()) .map_err(|error| DuckDbSourceError::Invalid(error.to_string()))?; - held.clear(); - Ok(sender.blocking_send(Ok(batch)).is_ok()) + Ok(match sender.blocking_send(Ok(batch)) { + Ok(()) => true, + Err(tokio::sync::mpsc::error::SendError(_)) => false, + }) } diff --git a/crates/sqlload/src/duckdb_source/tests.rs b/crates/sqlload/src/duckdb_source/tests.rs index d8f5eb86b..adc2b3bd3 100644 --- a/crates/sqlload/src/duckdb_source/tests.rs +++ b/crates/sqlload/src/duckdb_source/tests.rs @@ -1,4 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_read_source.rs"] mod read_source; +#[path = "tests/test_types.rs"] +mod test_types; #[path = "tests/test_type_map.rs"] mod type_map; diff --git a/crates/sqlload/src/duckdb_source/tests/helpers.rs b/crates/sqlload/src/duckdb_source/tests/helpers.rs new file mode 100644 index 000000000..7b8706e20 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/tests/helpers.rs @@ -0,0 +1,42 @@ +use crate::project::main::parse_project::parse_project; +use crate::project::models::ProjectConfig; + +/// A shop database: a commented table with a primary key and a type that must be pinned, a view, and a scratch table. +pub(crate) const SHOP: &str = " +CREATE SCHEMA sales; +CREATE TABLE sales.orders ( + id INTEGER PRIMARY KEY, + amount DECIMAL(12,2), + name VARCHAR, + placed_at TIMESTAMPTZ, + tags VARCHAR[], + meta STRUCT(rank INTEGER), + total HUGEINT +); +COMMENT ON TABLE sales.orders IS 'Orders, one row each.'; +INSERT INTO sales.orders VALUES + (1, 10.50, 'ann', '2026-01-01 00:00:00+00', ['a', 'b'], {'rank': 1}, 170141183460469231731687303715884105727), + (2, NULL, 'bo\"b', '2026-01-02 12:30:00+02', [], {'rank': 2}, 2), + (3, 99999999.99, NULL, NULL, NULL, NULL, 3); +CREATE VIEW sales.big_orders AS SELECT id, amount FROM sales.orders WHERE amount > 50; +CREATE TABLE sales.tmp_scratch (x INTEGER); +"; + +/// A runtime for one test's reads. +pub(crate) fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("a runtime") +} + +/// A project whose `shop` source reads `database` with `source_lines` in its table, and a `recent` query table. +pub(crate) fn shop_project(database: &str, source_lines: &str) -> Result { + parse_project( + &format!( + "[connections.shop]\nkind = \"duckdb\"\ndatabase = {database:?}\n[connections.wh]\nkind = \"duckdb\"\ndatabase = \":memory:\"\n[targets.dev]\nconnection = \"wh\"\nloader_schema = \"raw\"\n[sources.shop]\nkind = \"duckdb\"\nconnection = \"shop\"\ndescription = \"Shop.\"\nwrite_strategy = \"replace\"\n{source_lines}\n[sources.shop.tables.recent]\ndescription = \"Recent orders.\"\nsql = \"SELECT id, amount FROM sales.orders WHERE id > 1 -- newest\\n;\\n\"\n" + ), + &|_| None, + ) + .map_err(|error| error.to_string()) +} diff --git a/crates/sqlload/src/duckdb_source/tests/test_read_source.rs b/crates/sqlload/src/duckdb_source/tests/test_read_source.rs index 6d7d47a6c..24685b3b8 100644 --- a/crates/sqlload/src/duckdb_source/tests/test_read_source.rs +++ b/crates/sqlload/src/duckdb_source/tests/test_read_source.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; -use arrow_array::{Array, Decimal128Array, Int64Array, RecordBatch, StringArray, TimestampMicrosecondArray}; +use arrow_cast::display::array_value_to_string; +use arrow_select::concat::concat_batches; use crate::column_types::models::ColumnType; use crate::duckdb_source::main::duckdb_reader::duckdb_reader; @@ -8,135 +9,183 @@ use crate::duckdb_source::main::read_catalog::read_catalog; use crate::duckdb_source::main::read_columns::read_columns; use crate::duckdb_source::main::read_rows::read_rows; use crate::duckdb_source::models::DuckDbReadRequest; -use crate::project::models::ConnectionConfig; +use crate::duckdb_source::tests::helpers::{SHOP, runtime, shop_project}; +use crate::duckdb_source::tests::test_types::{ + ReadCatalogTestCase, ReadRowsTestCase, RefusedReaderTestCase, +}; -const SETUP: &str = " -CREATE SCHEMA sales; -CREATE TABLE sales.orders ( - id INTEGER PRIMARY KEY, - amount DECIMAL(12,2), - name VARCHAR, - placed_at TIMESTAMPTZ, - tags VARCHAR[], - meta STRUCT(rank INTEGER), - total HUGEINT -); -COMMENT ON TABLE sales.orders IS 'Orders, one row each.'; -INSERT INTO sales.orders VALUES - (1, 10.50, 'ann', '2026-01-01 00:00:00+00', ['a', 'b'], {'rank': 1}, 170141183460469231731687303715884105727), - (2, NULL, 'bo\"b', '2026-01-02 12:30:00+02', [], {'rank': 2}, 2), - (3, 99999999.99, NULL, NULL, NULL, NULL, 3); -CREATE VIEW sales.big_orders AS SELECT id, amount FROM sales.orders WHERE amount > 50; -CREATE TABLE sales.tmp_scratch (x INTEGER); -"; - -fn runtime() -> tokio::runtime::Runtime { - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("a runtime") +#[test] +fn given_a_duckdb_file_when_listing_its_catalog_then_include_and_exclude_pick_tables_and_views_before_query_tables() +-> Result<(), String> { + let folder = tempfile::tempdir().map_err(|error| error.to_string())?; + let path = folder.path().join("shop.duckdb"); + duckdb::Connection::open(&path) + .and_then(|connection| connection.execute_batch(SHOP)) + .map_err(|error| error.to_string())?; + let test_cases = [ + ReadCatalogTestCase { + description: "tables and views by include, less exclude, then the query table", + source_lines: "include = [\"sales.*\"]\nexclude = [\"sales.tmp_*\"]", + expected_tables: &["sales.big_orders", "sales.orders", "recent"], + }, + ReadCatalogTestCase { + description: "without include only the query table", + source_lines: "", + expected_tables: &["recent"], + }, + ]; + for test_case in test_cases { + let project = shop_project(&path.display().to_string(), test_case.source_lines)?; + let source = &project.sources["shop"]; + let reader = duckdb_reader(&project, source).map_err(|error| error.to_string())?; + let tables = runtime() + .block_on(read_catalog(&reader, source)) + .map_err(|error| format!("{}: {error}", test_case.description))?; + let names: Vec = tables.iter().map(|table| table.qualified_name()).collect(); + assert_eq!( + names, test_case.expected_tables, + "{}", + test_case.description + ); + } + Ok(()) } #[test] -fn given_a_duckdb_file_when_reading_its_tables_and_a_query_then_rows_load_typed_and_the_file_is_left_as_it_was() - { - let folder = tempfile::tempdir().expect("a folder"); +fn given_a_duckdb_file_when_reading_a_table_or_query_then_rows_load_typed_and_the_file_is_left_as_it_was() +-> Result<(), String> { + let folder = tempfile::tempdir().map_err(|error| error.to_string())?; let path = folder.path().join("shop.duckdb"); duckdb::Connection::open(&path) - .and_then(|connection| connection.execute_batch(SETUP)) - .expect("the database"); - let before = std::fs::read(&path).expect("the file"); - let reader = duckdb_reader( - &ConnectionConfig::DuckDb { - database: path.display().to_string(), - }, - Some(256 << 20), - ) - .expect("a reader"); - let queries = [( - "recent".to_owned(), - "SELECT id, amount FROM sales.orders WHERE id > 1; -- newest".to_owned(), - )]; + .and_then(|connection| connection.execute_batch(SHOP)) + .map_err(|error| error.to_string())?; + let before = std::fs::read(&path).map_err(|error| error.to_string())?; + let project = shop_project(&path.display().to_string(), "include = [\"sales.orders\"]")?; + let source = &project.sources["shop"]; + let reader = duckdb_reader(&project, source).map_err(|error| error.to_string())?; let runtime = runtime(); let tables = runtime - .block_on(read_catalog( - &reader, - (&["sales.*".to_owned()], &["sales.tmp_*".to_owned()]), - &queries, - )) - .expect("the catalog"); - let names: Vec = tables.iter().map(|table| table.qualified_name()).collect(); - assert_eq!(names, ["sales.big_orders", "sales.orders", "recent"], "tables and views by include, less exclude, then queries"); - let orders = &tables[1]; - assert_eq!(orders.primary_key, ["id"]); - assert_eq!(orders.comment.as_deref(), Some("Orders, one row each.")); - let total = orders.columns.iter().find(|column| column.name == "total").expect("total"); - assert!(total.column_type.is_err(), "HUGEINT must be pinned"); - assert!(read_columns(orders, &BTreeMap::new()).is_err(), "an unpinned HUGEINT is a problem"); + .block_on(read_catalog(&reader, source)) + .map_err(|error| error.to_string())?; + let orders = &tables[0]; + assert_eq!( + (orders.primary_key.as_slice(), orders.comment.as_deref()), + (["id".to_owned()].as_slice(), Some("Orders, one row each.")), + "the primary key and comment come from the catalog" + ); + assert!( + read_columns(orders, &BTreeMap::new()).is_err(), + "an unpinned HUGEINT is a problem" + ); let pinned = BTreeMap::from([("total".to_owned(), ColumnType::Text)]); - let columns = read_columns(orders, &pinned).expect("read columns"); - let mut batches: Vec = Vec::new(); - let rows = runtime - .block_on(read_rows( - &reader, - &DuckDbReadRequest { - table: orders, - sql: None, - columns: &columns, - filter: Some("\"id\" <= 2"), - }, - 1, - async |batch| { - batches.push(batch); - Ok(()) - }, - )) - .expect("the rows"); - assert_eq!(rows, 2); - assert_eq!(batches.len(), 2, "one batch per batch_rows"); - let batch = arrow_select::concat::concat_batches(&batches[0].schema(), &batches).expect("one batch"); - let column = |name: &str| batch.column_by_name(name).expect(name).clone(); - let ids = column("id"); - let ids = ids.as_any().downcast_ref::().expect("int64 ids"); - assert_eq!(ids.values(), &[1, 2]); - let amounts = column("amount"); - let amounts = amounts.as_any().downcast_ref::().expect("decimals"); - assert_eq!((amounts.value(0), amounts.is_null(1), amounts.scale()), (1050, true, 2)); - let placed = column("placed_at"); - let placed = placed.as_any().downcast_ref::().expect("timestamps"); - assert_eq!(placed.value(1), 1_767_349_800_000_000, "2026-01-02 10:30 UTC"); - let text = |name: &str| { - let array = column(name); - let strings = array.as_any().downcast_ref::().expect(name); - (0..strings.len()).map(|row| strings.is_valid(row).then(|| strings.value(row).to_owned())).collect::>() - }; - assert_eq!(text("name"), [Some("ann".to_owned()), Some("bo\"b".to_owned())]); - assert_eq!(text("tags"), [Some(r#"["a","b"]"#.to_owned()), Some("[]".to_owned())]); - assert_eq!(text("meta"), [Some(r#"{"rank":1}"#.to_owned()), Some(r#"{"rank":2}"#.to_owned())]); - assert_eq!(text("total")[0].as_deref(), Some("170141183460469231731687303715884105727")); - let recent = &tables[2]; - let columns = read_columns(recent, &BTreeMap::new()).expect("query columns"); - let rows = runtime - .block_on(read_rows( - &reader, - &DuckDbReadRequest { - table: recent, - sql: Some(&queries[0].1), - columns: &columns, - filter: Some("\"id\" > 2"), - }, - 100, - async |_| Ok(()), - )) - .expect("the query's rows"); - assert_eq!(rows, 1, "the filter applies outside the query, after its trailing comment"); - assert_eq!(std::fs::read(&path).expect("the file"), before, "the file is read, never written"); + let test_cases = [ + ReadRowsTestCase { + description: "the table, one row per batch, filtered: decimals, instants in UTC, lists and structs as JSON, a pinned HUGEINT as text", + table: "sales.orders", + filter: Some("\"id\" <= 2"), + batch_rows: 1, + expected_values: &[ + "id=1|2", + "amount=10.50|", + "name=ann|bo\"b", + "placed_at=2026-01-01T00:00:00Z|2026-01-02T10:30:00Z", + r#"tags=["a","b"]|[]"#, + r#"meta={"rank":1}|{"rank":2}"#, + "total=170141183460469231731687303715884105727|2", + ], + }, + ReadRowsTestCase { + description: "the query table, its filter applied outside the query after its trailing comment", + table: "recent", + filter: Some("\"id\" > 2"), + batch_rows: 100, + expected_values: &["id=3", "amount=99999999.99"], + }, + ]; + for test_case in test_cases { + let table = tables + .iter() + .find(|table| table.qualified_name() == test_case.table) + .ok_or(test_case.description)?; + let columns = read_columns(table, &pinned).map_err(|problems| problems.join("; "))?; + let mut batches = Vec::new(); + let request = DuckDbReadRequest { + table, + sql: source + .tables + .get(test_case.table) + .and_then(|table| table.sql.as_deref()), + columns: &columns, + filter: test_case.filter, + }; + runtime + .block_on(read_rows( + &reader, + &request, + test_case.batch_rows, + async |batch| { + batches.push(batch); + Ok(()) + }, + )) + .map_err(|error| format!("{}: {error}", test_case.description))?; + let batch = + concat_batches(&batches[0].schema(), &batches).map_err(|error| error.to_string())?; + let values = test_case + .expected_values + .iter() + .map(|expected| { + let name = expected.split_once('=').map_or(*expected, |(name, _)| name); + let column = batch + .column_by_name(name) + .ok_or_else(|| format!("no column {name}"))?; + let values = (0..column.len()) + .map(|row| array_value_to_string(column, row)) + .collect::, _>>() + .map_err(|error| error.to_string())?; + Ok(format!("{name}={}", values.join("|"))) + }) + .collect::, String>>()?; + assert_eq!( + values, test_case.expected_values, + "{}", + test_case.description + ); + } + assert_eq!( + std::fs::read(&path).map_err(|error| error.to_string())?, + before, + "the file is read, never written" + ); + Ok(()) } #[test] -fn given_a_memory_or_motherduck_database_when_making_a_reader_then_it_is_refused() { - for database in [":memory:", "md:shop"] { - let refused = duckdb_reader(&ConnectionConfig::DuckDb { database: database.to_owned() }, None); - assert!(refused.is_err(), "{database}"); +fn given_a_memory_or_motherduck_database_when_making_a_reader_then_it_is_refused() +-> Result<(), String> { + let test_cases = [ + RefusedReaderTestCase { + description: "an in-memory database holds nothing to read", + database: ":memory:", + expected_error: ":memory: holds no tables to read", + }, + RefusedReaderTestCase { + description: "a MotherDuck database is a motherduck source", + database: "md:shop", + expected_error: "kind = \"motherduck\"", + }, + ]; + for test_case in test_cases { + let project = shop_project(test_case.database, "")?; + let refused = duckdb_reader(&project, &project.sources["shop"]) + .err() + .map(|error| error.to_string()) + .unwrap_or_default(); + assert!( + refused.contains(test_case.expected_error), + "{}: {refused}", + test_case.description + ); } + Ok(()) } diff --git a/crates/sqlload/src/duckdb_source/tests/test_type_map.rs b/crates/sqlload/src/duckdb_source/tests/test_type_map.rs index cbd22cd12..e2faba4c7 100644 --- a/crates/sqlload/src/duckdb_source/tests/test_type_map.rs +++ b/crates/sqlload/src/duckdb_source/tests/test_type_map.rs @@ -1,28 +1,100 @@ use crate::column_types::models::ColumnType; use crate::duckdb_source::_helpers::type_map::source_column; +use crate::duckdb_source::tests::test_types::SourceColumnTestCase; #[test] -fn given_duckdb_column_types_when_mapping_them_then_each_gets_its_portable_type_or_must_be_pinned() { - let decimal = |precision, scale| ColumnType::Decimal { precision, scale }; - let test_cases: [(&str, &str, Option); 14] = [ - ("unsigned 8-bit fits smallint", "UTINYINT", Some(ColumnType::SmallInt)), - ("unsigned 32-bit fits a 64-bit int", "UINTEGER", Some(ColumnType::Int)), - ("unsigned 64-bit does not fit int", "UBIGINT", None), - ("HUGEINT does not fit int", "HUGEINT", None), - ("a decimal keeps precision and scale", "DECIMAL(12,2)", Some(decimal(12, 2))), - ("a 38-digit decimal is the widest", "DECIMAL(38,10)", Some(decimal(38, 10))), - ("an enum is its label", "ENUM('a', 'b')", Some(ColumnType::Text)), - ("TIMESTAMPTZ is an instant", "TIMESTAMP WITH TIME ZONE", Some(ColumnType::TimestampTz)), - ("millisecond timestamps widen to microseconds", "TIMESTAMP_MS", Some(ColumnType::TimestampNtz)), - ("nanosecond timestamps would lose digits", "TIMESTAMP_NS", None), - ("a list is json", "VARCHAR[]", Some(ColumnType::Json)), - ("a fixed-size array is json", "INTEGER[3]", Some(ColumnType::Json)), - ("a struct is json", "STRUCT(a INTEGER, b VARCHAR)", Some(ColumnType::Json)), - ("an interval has no portable type", "INTERVAL", None), +fn given_duckdb_column_types_when_mapping_them_then_each_gets_its_portable_type_or_must_be_pinned() +{ + let test_cases = [ + SourceColumnTestCase { + description: "unsigned 8-bit fits smallint", + data_type: "UTINYINT", + expected_type: Some(ColumnType::SmallInt), + }, + SourceColumnTestCase { + description: "unsigned 32-bit fits a 64-bit int", + data_type: "UINTEGER", + expected_type: Some(ColumnType::Int), + }, + SourceColumnTestCase { + description: "unsigned 64-bit does not fit int", + data_type: "UBIGINT", + expected_type: None, + }, + SourceColumnTestCase { + description: "HUGEINT does not fit int", + data_type: "HUGEINT", + expected_type: None, + }, + SourceColumnTestCase { + description: "a decimal keeps precision and scale", + data_type: "DECIMAL(12,2)", + expected_type: Some(ColumnType::Decimal { + precision: 12, + scale: 2, + }), + }, + SourceColumnTestCase { + description: "a 38-digit decimal is the widest", + data_type: "DECIMAL(38,10)", + expected_type: Some(ColumnType::Decimal { + precision: 38, + scale: 10, + }), + }, + SourceColumnTestCase { + description: "an enum is its label", + data_type: "ENUM('a', 'b')", + expected_type: Some(ColumnType::Text), + }, + SourceColumnTestCase { + description: "TIMESTAMPTZ is an instant", + data_type: "TIMESTAMP WITH TIME ZONE", + expected_type: Some(ColumnType::TimestampTz), + }, + SourceColumnTestCase { + description: "millisecond timestamps widen to microseconds", + data_type: "TIMESTAMP_MS", + expected_type: Some(ColumnType::TimestampNtz), + }, + SourceColumnTestCase { + description: "nanosecond timestamps would lose digits", + data_type: "TIMESTAMP_NS", + expected_type: None, + }, + SourceColumnTestCase { + description: "a list is json", + data_type: "VARCHAR[]", + expected_type: Some(ColumnType::Json), + }, + SourceColumnTestCase { + description: "a fixed-size array is json", + data_type: "INTEGER[3]", + expected_type: Some(ColumnType::Json), + }, + SourceColumnTestCase { + description: "a struct is json", + data_type: "STRUCT(a INTEGER, b VARCHAR)", + expected_type: Some(ColumnType::Json), + }, + SourceColumnTestCase { + description: "an interval has no portable type", + data_type: "INTERVAL", + expected_type: None, + }, ]; - for (description, data_type, expected) in test_cases { - let column = source_column("c", data_type, true); - assert_eq!(column.column_type.ok(), expected, "{description}"); - assert_eq!(column.postgres_type, data_type, "{description}: plan shows DuckDB's type"); + for test_case in test_cases { + let column = source_column("c", test_case.data_type, true); + assert_eq!( + column.column_type.ok(), + test_case.expected_type, + "{}", + test_case.description + ); + assert_eq!( + column.postgres_type, test_case.data_type, + "{}: plan shows DuckDB's type", + test_case.description + ); } } diff --git a/crates/sqlload/src/duckdb_source/tests/test_types.rs b/crates/sqlload/src/duckdb_source/tests/test_types.rs new file mode 100644 index 000000000..a4cd40312 --- /dev/null +++ b/crates/sqlload/src/duckdb_source/tests/test_types.rs @@ -0,0 +1,38 @@ +use crate::column_types::models::ColumnType; + +pub(crate) struct SourceColumnTestCase { + pub(crate) description: &'static str, + /// The type as `duckdb_columns().data_type` or a result's column type writes it. + pub(crate) data_type: &'static str, + /// The portable type; `None` when a type must be pinned. + pub(crate) expected_type: Option, +} + +/// Which tables a shop database's catalog lists for a source's settings. +pub(crate) struct ReadCatalogTestCase { + pub(crate) description: &'static str, + /// Lines of the `[sources.shop]` table: `include`, `exclude` and the like. + pub(crate) source_lines: &'static str, + /// The tables listed, by qualified name, in order. + pub(crate) expected_tables: &'static [&'static str], +} + +/// One read of a table or query table the shop source lists. +pub(crate) struct ReadRowsTestCase { + pub(crate) description: &'static str, + /// The table's qualified name as the catalog lists it. + pub(crate) table: &'static str, + /// A DuckDB condition on the read columns. + pub(crate) filter: Option<&'static str>, + pub(crate) batch_rows: usize, + /// Each named column's values in row order as Arrow displays them (`""` for NULL), as `name=value|value`. + pub(crate) expected_values: &'static [&'static str], +} + +/// A duckdb connection whose database the source cannot read. +pub(crate) struct RefusedReaderTestCase { + pub(crate) description: &'static str, + pub(crate) database: &'static str, + /// Part of the refusal. + pub(crate) expected_error: &'static str, +} diff --git a/crates/sqlload/src/lib.rs b/crates/sqlload/src/lib.rs index 875e56649..eb8b2985e 100644 --- a/crates/sqlload/src/lib.rs +++ b/crates/sqlload/src/lib.rs @@ -20,8 +20,8 @@ pub mod contract; pub mod copy; pub mod databricks_api; pub mod databricks_source; -pub mod duckdb_source; pub mod delivery; +pub mod duckdb_source; pub mod envelope; pub mod exporter; pub mod file_format; diff --git a/crates/sqlload/src/loader/_helpers/resolve.rs b/crates/sqlload/src/loader/_helpers/resolve.rs index ea66aebdd..bc6d45291 100644 --- a/crates/sqlload/src/loader/_helpers/resolve.rs +++ b/crates/sqlload/src/loader/_helpers/resolve.rs @@ -1702,49 +1702,48 @@ async fn failed_count( table: &SourceTable, filter: &str, ) -> Option { - match reader? { - SourceReader::Postgres(client, _) => { - crate::postgres_source::main::count_rows::count_rows(client, table, filter) + let failed = match reader? { + SourceReader::BigQuery(bigquery) => { + crate::bigquery_source::main::dry_count::dry_count(bigquery, (table, None), filter) .await .err() - .map(|error| error.to_string()) + .map(LoaderError::from) + } + reader => count_rows(reader, (table, None), filter).await.err(), + }; + failed.map(|error| error.to_string()) +} + +/// How many rows of `table` (or of its query `sql`) `filter` selects, counted by the source's database; `None` for a files or Google Sheets source, which has no database to ask. +pub(crate) async fn count_rows( + reader: &SourceReader, + (table, sql): (&SourceTable, Option<&str>), + filter: &str, +) -> Result, LoaderError> { + Ok(Some(match reader { + SourceReader::Postgres(client, _) => { + crate::postgres_source::main::count_rows::count_rows(client, table, filter).await? } SourceReader::Snowflake(snowflake) => { - crate::snowflake_source::main::count_rows::count_rows(snowflake, (table, None), filter) - .await - .err() - .map(|error| error.to_string()) + crate::snowflake_source::main::count_rows::count_rows(snowflake, (table, sql), filter) + .await? } SourceReader::BigQuery(bigquery) => { - crate::bigquery_source::main::dry_count::dry_count(bigquery, (table, None), filter) - .await - .err() - .map(|error| error.to_string()) + crate::bigquery_source::main::count_rows::count_rows(bigquery, (table, sql), filter) + .await? } SourceReader::Databricks(databricks) => { - crate::databricks_source::main::count_rows::count_rows( - databricks, - (table, None), - filter, - ) - .await - .err() - .map(|error| error.to_string()) + crate::databricks_source::main::count_rows::count_rows(databricks, (table, sql), filter) + .await? } #[cfg(feature = "odbc")] SourceReader::Odbc(odbc) => { - crate::odbc_source::main::count_rows::count_rows(odbc, (table, None), filter) - .await - .err() - .map(|error| error.to_string()) + crate::odbc_source::main::count_rows::count_rows(odbc, (table, sql), filter).await? } #[cfg(feature = "duckdb")] SourceReader::DuckDb(duckdb) => { - crate::duckdb_source::main::count_rows::count_rows(duckdb, (table, None), filter) - .await - .err() - .map(|error| error.to_string()) + crate::duckdb_source::main::count_rows::count_rows(duckdb, (table, sql), filter).await? } - SourceReader::Files(_) | SourceReader::Sheets(..) => None, - } + SourceReader::Files(_) | SourceReader::Sheets(..) => return Ok(None), + })) } diff --git a/crates/sqlload/src/loader/_helpers/sources.rs b/crates/sqlload/src/loader/_helpers/sources.rs index 5c26aaf24..8aecd2a45 100644 --- a/crates/sqlload/src/loader/_helpers/sources.rs +++ b/crates/sqlload/src/loader/_helpers/sources.rs @@ -25,6 +25,13 @@ use crate::databricks_source::main::databricks_reader::databricks_reader; use crate::databricks_source::main::read_catalog::read_catalog as read_databricks_catalog; use crate::databricks_source::main::read_rows::read_rows as read_databricks_rows; use crate::databricks_source::models::DatabricksReadRequest; +#[cfg(feature = "duckdb")] +use crate::duckdb_source::main::{ + duckdb_reader::duckdb_reader, read_catalog::read_catalog as read_duckdb_catalog, + read_rows::read_rows as read_duckdb_rows, +}; +#[cfg(feature = "duckdb")] +use crate::duckdb_source::models::DuckDbReadRequest; use crate::file_store::main::open_folder::open_folder; use crate::file_store::models::{OpenFolder, StoredFile}; use crate::files_source::constants::{ @@ -65,6 +72,8 @@ use crate::loader::_helpers::specs::{ }; use crate::loader::_helpers::specs::{path_values, pinned_unsampled}; use crate::loader::_helpers::specs::{returned_columns, runnable_transform, transform_input}; +#[cfg(not(feature = "duckdb"))] +use crate::loader::constants::NO_DUCKDB_SOURCE; use crate::loader::constants::NO_TRANSFORM_HOST; use crate::loader::constants::{ DRAINED_REASON, KEPT_TABLES_COLUMN_FIX, LOCALHOST, PRIMARY_KEY_PLACEHOLDER, RELOAD_FIX, @@ -77,6 +86,8 @@ use crate::loader::models::{ DrainedFiles, FilesSource, PlanOptions, ResolvedTables, SkippedTable, SourceEnd, SourceProblem, SourceReader, TablePlan, TableSpec, }; +#[cfg(feature = "odbc")] +use crate::odbc_source::main::source_reader::source_reader; use crate::postgres_source::constants::DEFAULT_BATCH_BYTES; use crate::postgres_source::main::connect_postgres::connect_postgres; use crate::postgres_source::main::copy_table::copy_table; @@ -226,7 +237,12 @@ pub(crate) async fn open_source( SourceKind::Databricks => Ok(SourceReader::Databricks(Box::new(databricks_reader( project, source, )?))), - SourceKind::DuckDb => crate::loader::main::_duckdb_source::open_duckdb(project, source), + #[cfg(feature = "duckdb")] + SourceKind::DuckDb => Ok(SourceReader::DuckDb(Box::new(duckdb_reader( + project, source, + )?))), + #[cfg(not(feature = "duckdb"))] + SourceKind::DuckDb => Err(LoaderError::Invalid(NO_DUCKDB_SOURCE.to_owned())), SourceKind::Python => Err(LoaderError::Invalid( "a Python source is read by calling its table functions".to_owned(), )), @@ -371,12 +387,17 @@ pub(crate) async fn source_tables( .map(|tables| (tables, None, Vec::new())) .map_err(|error| LoaderError::from(error).into())) } - SourceKind::DuckDb => Ok( - crate::loader::main::_duckdb_source::duckdb_tables(project, source) + #[cfg(feature = "duckdb")] + SourceKind::DuckDb => { + let read = + async { read_duckdb_catalog(&duckdb_reader(project, source)?, source).await }; + Ok(read .await .map(|tables| (tables, None, Vec::new())) - .map_err(|error| error.into()), - ), + .map_err(|error| LoaderError::from(error).into())) + } + #[cfg(not(feature = "duckdb"))] + SourceKind::DuckDb => Ok(Err(LoaderError::Invalid(NO_DUCKDB_SOURCE.to_owned()).into())), SourceKind::Python => Ok(Ok((python_tables(source), None, Vec::new()))), SourceKind::GoogleSheets => Ok(match sheets_tables(project, source).await { Ok((tables, warnings)) => Ok((tables, None, warnings)), @@ -385,39 +406,6 @@ pub(crate) async fn source_tables( } } -/// An `odbc` source's reader, for its connection's settings and the source's value caps. -#[cfg(feature = "odbc")] -pub(crate) fn odbc_source_reader( - project: &ProjectConfig, - source: &SourceConfig, -) -> Result { - let Some(ConnectionConfig::Odbc { - connection_string, - dsn, - user, - password, - }) = project.connections.get(&source.connection) - else { - return Err(LoaderError::Invalid(format!( - "connection {:?} is not an odbc connection", - source.connection - ))); - }; - let limits = crate::odbc_source::models::ValueLimits { - max_text_size: source.max_text_size, - max_binary_size: source.max_binary_size, - long_column_threshold: source - .long_column_threshold - .map(|size| usize::try_from(size.0).unwrap_or(usize::MAX)), - }; - crate::odbc_source::main::odbc_reader::odbc_reader( - (connection_string.as_ref(), dsn.as_deref()), - (user.as_deref(), password.as_ref()), - limits, - ) - .map_err(LoaderError::Invalid) -} - /// `planned`, each unblocked table of ODBC source `source` given how its rows are fetched on this machine: its budget, and in bulk or value by value (see [`plan_fetches`](plan_fetches)). #[cfg(feature = "odbc")] pub(crate) async fn with_fetch_memory( @@ -425,7 +413,7 @@ pub(crate) async fn with_fetch_memory( source: &SourceConfig, mut planned: Vec, ) -> Result, LoaderError> { - let reader = odbc_source_reader(project, source)?; + let reader = source_reader(project, source).map_err(LoaderError::Invalid)?; let open: Vec = (0..planned.len()) .filter(|&index| planned[index].schema.blocked.is_none()) .collect(); @@ -465,9 +453,9 @@ pub(crate) async fn with_fetch_memory( #[cfg(feature = "odbc")] fn open_odbc(project: &ProjectConfig, source: &SourceConfig) -> Result { - Ok(SourceReader::Odbc(Box::new(odbc_source_reader( - project, source, - )?))) + Ok(SourceReader::Odbc(Box::new( + source_reader(project, source).map_err(LoaderError::Invalid)?, + ))) } #[cfg(not(feature = "odbc"))] @@ -484,7 +472,7 @@ async fn discover_odbc( source: &SourceConfig, queries: &[(String, String)], ) -> Result, String> { - let reader = odbc_source_reader(project, source).map_err(|error| error.to_string())?; + let reader = source_reader(project, source)?; crate::odbc_source::main::read_catalog::read_catalog( &reader, ( @@ -1605,8 +1593,16 @@ async fn read_each( } #[cfg(feature = "duckdb")] (SourceReader::DuckDb(reader), _) => { - crate::loader::main::_duckdb_source::read_duckdb(reader, table, part.filter, on_batch) - .await + let filter = table + .spec + .read_filter(part.filter.or_else(|| table.source_filter.clone())); + let request = DuckDbReadRequest { + table: &table.spec.source_table, + sql: table.spec.query_sql(), + columns: &table.read_columns, + filter: filter.as_deref(), + }; + Ok(read_duckdb_rows(reader, &request, SOURCE_BATCH_ROWS, on_batch).await?) } (SourceReader::Sheets(client, spreadsheet_id), _) => { let batches = diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs index 359151e2d..729ba03d2 100644 --- a/crates/sqlload/src/loader/_helpers/specs.rs +++ b/crates/sqlload/src/loader/_helpers/specs.rs @@ -539,9 +539,9 @@ pub(crate) fn build_spec( SourceKind::Databricks => DATABRICKS_SOURCE, _ => DUCKDB_SOURCE, }, - sql: overrides.sql.clone(), - route: WarehouseRoute::Query, - }), + sql: overrides.sql.clone(), + route: WarehouseRoute::Query, + }), _ => warehouse_source(context.source, overrides), }, row_filter: layered(&settings.row_filter, &defaults.row_filter) @@ -1384,7 +1384,7 @@ pub(crate) async fn null_cursor_rows( .warehouse_source .as_ref() .and_then(|source| source.sql.as_deref()); - crate::loader::main::_count_rows::count_rows(reader, (&spec.source_table, sql), &filter).await + crate::loader::_helpers::resolve::count_rows(reader, (&spec.source_table, sql), &filter).await } /// Why `spec`'s load fails: `rows` source rows have a NULL cursor, which its incremental read never selects, so they would never load; with the fixes `plan` accepts for the table: `null_cursor = "include"` on `merge`, a coalesced cursor where one may be listed (not a Python table, `delete_insert` or a files target), and `null_cursor = "ignore"`. diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index 98b412928..35b08230f 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -1864,7 +1864,8 @@ pub(crate) async fn block_refused_bounds( (source_name, source): (&str, &crate::project::models::SourceConfig), planned: &mut [TablePlan], ) -> Result<(), LoaderError> { - let reader = crate::loader::_helpers::sources::odbc_source_reader(project, source)?; + let reader = crate::odbc_source::main::source_reader::source_reader(project, source) + .map_err(LoaderError::Invalid)?; for table in planned .iter_mut() .filter(|table| table.schema.blocked.is_none()) diff --git a/crates/sqlload/src/loader/main/_check_source.rs b/crates/sqlload/src/loader/main/_check_source.rs index 7167c8d34..abb71ec9b 100644 --- a/crates/sqlload/src/loader/main/_check_source.rs +++ b/crates/sqlload/src/loader/main/_check_source.rs @@ -9,7 +9,6 @@ use crate::loader::errors::LoaderError; use crate::loader::models::SourceReader; use crate::project::models::{ConnectionConfig, ProjectConfig, SourceConfig, SourceKind}; use crate::sheets_source::main::tab_ranges::tab_ranges; -use crate::sql_api::main::execute_sql::execute_sql; /// What source `source` reads as, or why it cannot connect; for Postgres, what protects the connection and the bastion it goes through (a failure names the hop: the SSH tunnel, or the database via it). pub(in crate::loader) async fn check_source( @@ -25,16 +24,7 @@ pub(in crate::loader) async fn check_source( "a snowflake source opens as a Snowflake reader".to_owned(), )); }; - let answer = execute_sql( - &snowflake.sql, - "SELECT CURRENT_USER() || '@' || CURRENT_ACCOUNT() || ', warehouse ' || COALESCE(CURRENT_WAREHOUSE(), 'none') || ', Snowflake ' || CURRENT_VERSION()", - ) - .await?; - Ok(answer - .rows - .first() - .and_then(|row| row.first().cloned().flatten()) - .unwrap_or_default()) + Ok(crate::snowflake_source::main::describe_source::describe_source(&snowflake).await?) } SourceKind::BigQuery => { let SourceReader::BigQuery(reader) = open_source(project, source, 1).await? else { @@ -57,12 +47,19 @@ pub(in crate::loader) async fn check_source( )) } SourceKind::Databricks => check_databricks(project, source).await, + #[cfg(feature = "duckdb")] SourceKind::DuckDb => { - crate::loader::main::_duckdb_source::check_duckdb(project, source).await + let reader = crate::duckdb_source::main::duckdb_reader::duckdb_reader(project, source)?; + Ok(crate::duckdb_source::main::describe_source::describe_source(&reader).await?) } + #[cfg(not(feature = "duckdb"))] + SourceKind::DuckDb => Err(LoaderError::Invalid( + crate::loader::constants::NO_DUCKDB_SOURCE.to_owned(), + )), #[cfg(feature = "odbc")] SourceKind::Odbc => { - let reader = crate::loader::_helpers::sources::odbc_source_reader(project, source)?; + let reader = crate::odbc_source::main::source_reader::source_reader(project, source) + .map_err(LoaderError::Invalid)?; Ok(crate::odbc_source::main::describe_source::describe_source(&reader).await?) } #[cfg(not(feature = "odbc"))] diff --git a/crates/sqlload/src/loader/main/_count_rows.rs b/crates/sqlload/src/loader/main/_count_rows.rs deleted file mode 100644 index 2af76a15c..000000000 --- a/crates/sqlload/src/loader/main/_count_rows.rs +++ /dev/null @@ -1,44 +0,0 @@ -//! How many rows of a source table a condition selects, counted by the database. - -use crate::loader::errors::LoaderError; -use crate::loader::models::SourceReader; -use crate::postgres_source::models::SourceTable; - -/// How many rows of `table` (or of its query `sql`) `filter` selects, counted by the source's database; `None` for a files or Google Sheets source, which has no database to ask. -pub(in crate::loader) async fn count_rows( - reader: &SourceReader, - (table, sql): (&SourceTable, Option<&str>), - filter: &str, -) -> Result, LoaderError> { - Ok(Some(match reader { - SourceReader::Postgres(client, _) => { - crate::postgres_source::main::count_rows::count_rows(client, table, filter).await? - } - SourceReader::Snowflake(snowflake) => { - crate::snowflake_source::main::count_rows::count_rows(snowflake, (table, sql), filter) - .await? - } - SourceReader::BigQuery(bigquery) => { - crate::bigquery_source::main::count_rows::count_rows(bigquery, (table, sql), filter) - .await? - } - SourceReader::Databricks(databricks) => { - crate::databricks_source::main::count_rows::count_rows( - databricks, - (table, sql), - filter, - ) - .await? - } - #[cfg(feature = "odbc")] - SourceReader::Odbc(odbc) => { - crate::odbc_source::main::count_rows::count_rows(odbc, (table, sql), filter).await? - } - #[cfg(feature = "duckdb")] - SourceReader::DuckDb(duckdb) => { - crate::duckdb_source::main::count_rows::count_rows(duckdb, (table, sql), filter) - .await? - } - SourceReader::Files(_) | SourceReader::Sheets(..) => return Ok(None), - })) -} diff --git a/crates/sqlload/src/loader/main/_duckdb_source.rs b/crates/sqlload/src/loader/main/_duckdb_source.rs deleted file mode 100644 index 38f4f24c4..000000000 --- a/crates/sqlload/src/loader/main/_duckdb_source.rs +++ /dev/null @@ -1,122 +0,0 @@ -//! A `duckdb` source in the loader: its reader, its tables, its rows and its check; a build without the `duckdb` feature refuses it. - -use crate::loader::errors::LoaderError; -use crate::loader::models::SourceReader; -use crate::postgres_source::models::SourceTable; -use crate::project::models::{ProjectConfig, SourceConfig}; - -/// The source's reader, each of whose connections gets half a table's share of the memory budget. -#[cfg(feature = "duckdb")] -fn duckdb_source_reader( - project: &ProjectConfig, - source: &SourceConfig, -) -> Result { - let Some(connection) = project.connections.get(&source.connection) else { - return Err(LoaderError::Invalid(format!( - "connection {:?} is not defined", - source.connection - ))); - }; - let memory_limit = project.memory.budget.map(|budget| budget.table_share / 2); - Ok(crate::duckdb_source::main::duckdb_reader::duckdb_reader( - connection, - memory_limit, - )?) -} - -/// The source's reader; nothing is opened until a read. -#[cfg(feature = "duckdb")] -pub(in crate::loader) fn open_duckdb( - project: &ProjectConfig, - source: &SourceConfig, -) -> Result { - Ok(SourceReader::DuckDb(Box::new(duckdb_source_reader( - project, source, - )?))) -} - -#[cfg(not(feature = "duckdb"))] -pub(in crate::loader) fn open_duckdb( - _: &ProjectConfig, - _: &SourceConfig, -) -> Result { - Err(LoaderError::Invalid( - crate::loader::constants::NO_DUCKDB_SOURCE.to_owned(), - )) -} - -/// The source's tables and views that `include` finds, then one table per query table. -#[cfg(feature = "duckdb")] -pub(in crate::loader) async fn duckdb_tables( - project: &ProjectConfig, - source: &SourceConfig, -) -> Result, LoaderError> { - let queries: Vec<(String, String)> = source - .tables - .iter() - .filter_map(|(name, table)| Some((name.clone(), table.sql.clone()?))) - .collect(); - let reader = duckdb_source_reader(project, source)?; - Ok(crate::duckdb_source::main::read_catalog::read_catalog( - &reader, - (&source.include, &source.exclude), - &queries, - ) - .await?) -} - -#[cfg(not(feature = "duckdb"))] -pub(in crate::loader) async fn duckdb_tables( - _: &ProjectConfig, - _: &SourceConfig, -) -> Result, LoaderError> { - Err(LoaderError::Invalid( - crate::loader::constants::NO_DUCKDB_SOURCE.to_owned(), - )) -} - -/// Read `table`'s rows that `filter` selects (the plan's filter when `None`), passing each batch to `on_batch`; returns the row count. -#[cfg(feature = "duckdb")] -pub(in crate::loader) async fn read_duckdb( - reader: &crate::duckdb_source::models::DuckDbReader, - table: &crate::loader::models::TablePlan, - filter: Option, - on_batch: impl AsyncFnMut(arrow_array::RecordBatch) -> Result<(), String>, -) -> Result { - let filter = table - .spec - .read_filter(filter.or_else(|| table.source_filter.clone())); - let request = crate::duckdb_source::models::DuckDbReadRequest { - table: &table.spec.source_table, - sql: table.spec.query_sql(), - columns: &table.read_columns, - filter: filter.as_deref(), - }; - Ok(crate::duckdb_source::main::read_rows::read_rows( - reader, - &request, - crate::loader::constants::SOURCE_BATCH_ROWS, - on_batch, - ) - .await?) -} - -/// The database file the source reads and the DuckDB version reading it, for `debug`. -#[cfg(feature = "duckdb")] -pub(in crate::loader) async fn check_duckdb( - project: &ProjectConfig, - source: &SourceConfig, -) -> Result { - let reader = duckdb_source_reader(project, source)?; - Ok(crate::duckdb_source::main::describe_source::describe_source(&reader).await?) -} - -#[cfg(not(feature = "duckdb"))] -pub(in crate::loader) async fn check_duckdb( - _: &ProjectConfig, - _: &SourceConfig, -) -> Result { - Err(LoaderError::Invalid( - crate::loader::constants::NO_DUCKDB_SOURCE.to_owned(), - )) -} diff --git a/crates/sqlload/src/loader/main/mod.rs b/crates/sqlload/src/loader/main/mod.rs index d7b0672da..9abf6747b 100644 --- a/crates/sqlload/src/loader/main/mod.rs +++ b/crates/sqlload/src/loader/main/mod.rs @@ -1,6 +1,4 @@ pub(in crate::loader) mod _check_source; -pub(in crate::loader) mod _count_rows; -pub(in crate::loader) mod _duckdb_source; pub(in crate::loader) mod _files_repair; pub(in crate::loader) mod _files_schema; pub(in crate::loader) mod _files_window; diff --git a/crates/sqlload/src/odbc_source/main/odbc_reader.rs b/crates/sqlload/src/odbc_source/main/_odbc_reader.rs similarity index 98% rename from crates/sqlload/src/odbc_source/main/odbc_reader.rs rename to crates/sqlload/src/odbc_source/main/_odbc_reader.rs index c76e9def0..d13135b99 100644 --- a/crates/sqlload/src/odbc_source/main/odbc_reader.rs +++ b/crates/sqlload/src/odbc_source/main/_odbc_reader.rs @@ -4,7 +4,7 @@ use crate::auth::models::Secret; use crate::odbc_source::models::{OdbcReader, ValueLimits}; /// A reader connecting with `connection_string` as given, or with `DSN=` and `UID=` from `dsn` and `user` as written (driver managers read them unbraced; validation refuses `;`, `{` and `}` in them) and `PWD=` from `password` in braces, so any character in it stays literal; nothing connects until the first read. -pub fn odbc_reader( +pub(crate) fn odbc_reader( (connection_string, dsn): (Option<&Secret>, Option<&str>), (user, password): (Option<&str>, Option<&Secret>), limits: ValueLimits, diff --git a/crates/sqlload/src/odbc_source/main/mod.rs b/crates/sqlload/src/odbc_source/main/mod.rs index 9fa4f531b..63e160acb 100644 --- a/crates/sqlload/src/odbc_source/main/mod.rs +++ b/crates/sqlload/src/odbc_source/main/mod.rs @@ -1,9 +1,10 @@ +pub(in crate::odbc_source) mod _odbc_reader; pub mod count_rows; pub mod describe_source; pub mod memory_warning; -pub mod odbc_reader; pub mod plan_fetches; pub mod read_catalog; pub mod read_columns; pub mod read_rows; pub mod run_query; +pub mod source_reader; diff --git a/crates/sqlload/src/odbc_source/main/source_reader.rs b/crates/sqlload/src/odbc_source/main/source_reader.rs new file mode 100644 index 000000000..a14ebcabf --- /dev/null +++ b/crates/sqlload/src/odbc_source/main/source_reader.rs @@ -0,0 +1,33 @@ +//! The reader for an `odbc` source. + +use crate::odbc_source::main::_odbc_reader::odbc_reader; +use crate::odbc_source::models::{OdbcReader, ValueLimits}; +use crate::project::models::{ConnectionConfig, ProjectConfig, SourceConfig}; + +/// A reader for `source`'s `odbc` connection, with the source's value caps; nothing connects until the first read. +pub fn source_reader(project: &ProjectConfig, source: &SourceConfig) -> Result { + let Some(ConnectionConfig::Odbc { + connection_string, + dsn, + user, + password, + }) = project.connections.get(&source.connection) + else { + return Err(format!( + "connection {:?} is not an odbc connection", + source.connection + )); + }; + let limits = ValueLimits { + max_text_size: source.max_text_size, + max_binary_size: source.max_binary_size, + long_column_threshold: source + .long_column_threshold + .map(|size| usize::try_from(size.0).unwrap_or(usize::MAX)), + }; + odbc_reader( + (connection_string.as_ref(), dsn.as_deref()), + (user.as_deref(), password.as_ref()), + limits, + ) +} diff --git a/crates/sqlload/src/odbc_source/tests/test_connect.rs b/crates/sqlload/src/odbc_source/tests/test_connect.rs index 895f2fc1b..ba5a7c78d 100644 --- a/crates/sqlload/src/odbc_source/tests/test_connect.rs +++ b/crates/sqlload/src/odbc_source/tests/test_connect.rs @@ -1,6 +1,6 @@ use crate::auth::models::Secret; use crate::odbc_source::_helpers::connect::connect; -use crate::odbc_source::main::odbc_reader::odbc_reader; +use crate::odbc_source::main::_odbc_reader::odbc_reader; use crate::odbc_source::tests::helpers::elf_with_flags; use crate::odbc_source::tests::test_types::ConnectErrorTestCase; diff --git a/crates/sqlload/src/odbc_source/tests/test_odbc_reader.rs b/crates/sqlload/src/odbc_source/tests/test_odbc_reader.rs index 5feda5ef4..060f4b5ad 100644 --- a/crates/sqlload/src/odbc_source/tests/test_odbc_reader.rs +++ b/crates/sqlload/src/odbc_source/tests/test_odbc_reader.rs @@ -1,5 +1,5 @@ use crate::auth::models::Secret; -use crate::odbc_source::main::odbc_reader::odbc_reader; +use crate::odbc_source::main::_odbc_reader::odbc_reader; use crate::odbc_source::tests::test_types::OdbcReaderTestCase; #[test] diff --git a/crates/sqlload/src/project/_helpers/references.rs b/crates/sqlload/src/project/_helpers/references.rs index a08ee0d0a..b18cc4592 100644 --- a/crates/sqlload/src/project/_helpers/references.rs +++ b/crates/sqlload/src/project/_helpers/references.rs @@ -318,9 +318,7 @@ fn kind_problems(place: &str, source: &SourceConfig) -> Vec { | SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks - | SourceKind::DuckDb => { - "; list tables in include, or give a table a sql query" - } + | SourceKind::DuckDb => "; list tables in include, or give a table a sql query", _ => "", }; problems.push(format!( diff --git a/crates/sqlload/src/snowflake_source/main/describe_source.rs b/crates/sqlload/src/snowflake_source/main/describe_source.rs new file mode 100644 index 000000000..4c9ca3cd9 --- /dev/null +++ b/crates/sqlload/src/snowflake_source/main/describe_source.rs @@ -0,0 +1,19 @@ +//! Who a Snowflake source reads as, for `debug`. + +use crate::snowflake_source::errors::SnowflakeSourceError; +use crate::snowflake_source::models::SnowflakeReader; +use crate::sql_api::main::execute_sql::execute_sql; + +/// The reader's user and account, its warehouse and the Snowflake version. +pub async fn describe_source(reader: &SnowflakeReader) -> Result { + let answer = execute_sql( + &reader.sql, + "SELECT CURRENT_USER() || '@' || CURRENT_ACCOUNT() || ', warehouse ' || COALESCE(CURRENT_WAREHOUSE(), 'none') || ', Snowflake ' || CURRENT_VERSION()", + ) + .await?; + Ok(answer + .rows + .first() + .and_then(|row| row.first().cloned().flatten()) + .unwrap_or_default()) +} diff --git a/crates/sqlload/src/snowflake_source/main/mod.rs b/crates/sqlload/src/snowflake_source/main/mod.rs index 89c7de7c3..0215ae2d6 100644 --- a/crates/sqlload/src/snowflake_source/main/mod.rs +++ b/crates/sqlload/src/snowflake_source/main/mod.rs @@ -1,5 +1,6 @@ pub mod connect_snowflake_source; pub mod count_rows; +pub mod describe_source; pub mod read_catalog; pub mod read_columns; pub mod unload_files; diff --git a/crates/sqlload/tests/loader.rs b/crates/sqlload/tests/loader.rs index a3fb71b24..094767b60 100644 --- a/crates/sqlload/tests/loader.rs +++ b/crates/sqlload/tests/loader.rs @@ -14,6 +14,9 @@ mod clickhouse_helpers; mod cursor_helpers; #[path = "loader/databricks/helpers.rs"] mod databricks_helpers; +#[cfg(feature = "duckdb")] +#[path = "loader/duckdb/helpers.rs"] +mod duckdb_helpers; #[path = "loader/files/helpers.rs"] mod files_helpers; #[path = "loader/files_target/helpers.rs"] @@ -47,11 +50,11 @@ mod test_column_names; mod test_databricks_loads; #[path = "loader/test_databricks_source.rs"] mod test_databricks_source; +#[path = "loader/test_debug_project.rs"] +mod test_debug_project; #[cfg(feature = "duckdb")] #[path = "loader/test_duckdb_source.rs"] mod test_duckdb_source; -#[path = "loader/test_debug_project.rs"] -mod test_debug_project; #[path = "loader/test_file_store.rs"] mod test_file_store; #[path = "loader/test_files_round_trip.rs"] diff --git a/crates/sqlload/tests/loader/duckdb/helpers.rs b/crates/sqlload/tests/loader/duckdb/helpers.rs new file mode 100644 index 000000000..7102e009d --- /dev/null +++ b/crates/sqlload/tests/loader/duckdb/helpers.rs @@ -0,0 +1,45 @@ +//! Helpers for the DuckDB source loader test. + +/// The project: a duckdb source over `source` with include and exclude patterns, an excluded column, its orders merged after a cursor with `where` and hard deletes, a query table, and DuckDB file `target` as the target. +pub(crate) fn duckdb_source_project(source: &std::path::Path, target: &std::path::Path) -> String { + format!( + r#"default_target = "local" + +[connections.shop_file] +kind = "duckdb" +database = "{source}" + +[connections.warehouse] +kind = "duckdb" +database = "{target}" + +[targets.local] +connection = "warehouse" +loader_schema = "raw" + +[sources.shop] +kind = "duckdb" +connection = "shop_file" +description = "The shop's DuckDB file." +include = ["sales.*"] +exclude = ["sales.tmp_*"] +exclude_columns = ["card"] +write_strategy = "replace" + +[sources.shop.tables."sales.orders"] +description = "Orders, one row each." +write_strategy = "merge" +unique_key = ["id"] +cursor_column = "updated_at" +where = "region = 'EU'" +deletes = "hard" +checks = {{ max_row_drop = "100%" }} + +[sources.shop.tables.big_query] +description = "Orders over 25, by query." +sql = "SELECT id, amount FROM sales.orders WHERE amount > 25 -- big ones" +"#, + source = source.display(), + target = target.display() + ) +} diff --git a/crates/sqlload/tests/loader/test_duckdb_source.rs b/crates/sqlload/tests/loader/test_duckdb_source.rs index 24ee1804a..0370cd15d 100644 --- a/crates/sqlload/tests/loader/test_duckdb_source.rs +++ b/crates/sqlload/tests/loader/test_duckdb_source.rs @@ -1,4 +1,6 @@ +use crate::duckdb_helpers::duckdb_source_project; use crate::helpers::{duckdb_execute, duckdb_text, run_sqlload}; +use crate::test_types::DuckDbSourceTestCase; const SOURCE: &str = " CREATE SCHEMA sales; @@ -24,78 +26,64 @@ UPDATE sales.orders SET amount = 21.00, updated_at = '2026-01-05 00:00:00' WHERE INSERT INTO sales.orders VALUES (4, 40.00, '2026-01-06 00:00:00', ['b', 'c'], '4444', 'EU'); "; -fn project(source: &std::path::Path, target: &std::path::Path) -> String { - format!( - r#"default_target = "local" - -[connections.shop_file] -kind = "duckdb" -database = "{source}" - -[connections.warehouse] -kind = "duckdb" -database = "{target}" - -[targets.local] -connection = "warehouse" -loader_schema = "raw" - -[sources.shop] -kind = "duckdb" -connection = "shop_file" -description = "The shop's DuckDB file." -include = ["sales.*"] -exclude = ["sales.tmp_*"] -exclude_columns = ["card"] -write_strategy = "replace" - -[sources.shop.tables."sales.orders"] -description = "Orders, one row each." -write_strategy = "merge" -unique_key = ["id"] -cursor_column = "updated_at" -where = "region = 'EU'" - -[sources.shop.tables.big_query] -description = "Orders over 25, by query." -sql = "SELECT id, amount FROM sales.orders WHERE amount > 25 -- big ones" -"#, - source = source.display(), - target = target.display() - ) -} - /// Rows in id order with amount and the tags' JSON, the target's columns, and the view's and query table's rows. const CHECK: &str = "SELECT (SELECT string_agg(id || ':' || amount || ':' || coalesce(CAST(tags AS VARCHAR), '-'), ',' ORDER BY id) FROM raw.shop__orders) || '|' || (SELECT string_agg(column_name, ',' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_schema = 'raw' AND table_name = 'shop__orders' AND column_name NOT LIKE '\\_sqlload%' ESCAPE '\\') || '|' || (SELECT count(*) FROM raw.shop__big_orders) || '|' || (SELECT string_agg(CAST(id AS VARCHAR), ',' ORDER BY id) FROM raw.shop__big_query)"; #[test] -fn given_a_duckdb_source_when_loading_twice_then_tables_views_and_queries_load_and_the_cursor_merges_only_changes() +fn given_a_duckdb_source_when_loading_again_then_the_cursor_merges_only_changes_and_sync_deletes_removes_gone_rows() -> Result<(), String> { let folder = tempfile::tempdir().map_err(|error| error.to_string())?; - let (source, target) = (folder.path().join("shop.duckdb"), folder.path().join("out.duckdb")); + let (source, target) = ( + folder.path().join("shop.duckdb"), + folder.path().join("out.duckdb"), + ); duckdb_execute(&source, SOURCE)?; let config = folder.path().join("sqlload.toml"); - std::fs::write(&config, project(&source, &target)).map_err(|error| error.to_string())?; + std::fs::write(&config, duckdb_source_project(&source, &target)) + .map_err(|error| error.to_string())?; let (planned, plan) = run_sqlload(&config, &["plan"], ("", ""))?; assert!(planned, "{plan}"); - for table in ["raw.shop__orders", "raw.shop__big_orders", "raw.shop__big_query"] { + for table in [ + "raw.shop__orders", + "raw.shop__big_orders", + "raw.shop__big_query", + ] { assert!(plan.contains(table), "plan lists {table}: {plan}"); } - assert!(!plan.contains("tmp_scratch"), "exclude drops the table: {plan}"); - let (loaded, output) = run_sqlload(&config, &["load"], ("", ""))?; - assert!(loaded, "{output}"); - assert_eq!( - duckdb_text(&target, CHECK)?, - r#"1:10.50:["a"],2:20.00:[]|id,amount,updated_at,tags,region|2|3"#, - "the first load: only EU rows, card excluded, tags as JSON, the view and the query" - ); - duckdb_execute(&source, CHANGES)?; - let (loaded, output) = run_sqlload(&config, &["load"], ("", ""))?; - assert!(loaded, "{output}"); - assert_eq!( - duckdb_text(&target, CHECK)?, - r#"1:10.50:["a"],2:21.00:[],4:40.00:["b","c"]|id,amount,updated_at,tags,region|3|3,4"#, - "the second load merges the updated and new rows the cursor finds" + assert!( + !plan.contains("tmp_scratch"), + "exclude drops the table: {plan}" ); + let test_cases = [ + DuckDbSourceTestCase { + description: "the first load: only EU rows, card excluded, tags as JSON, the view and the query", + source_change: "", + arguments: &["load"], + expected_rows: r#"1:10.50:["a"],2:20.00:[]|id,amount,updated_at,tags,region|2|3"#, + }, + DuckDbSourceTestCase { + description: "the second load merges the updated and new rows the cursor finds", + source_change: CHANGES, + arguments: &["load"], + expected_rows: r#"1:10.50:["a"],2:21.00:[],4:40.00:["b","c"]|id,amount,updated_at,tags,region|3|3,4"#, + }, + DuckDbSourceTestCase { + description: "--sync-deletes removes the row the source no longer holds", + source_change: "DELETE FROM sales.orders WHERE id = 1", + arguments: &["load", "--sync-deletes"], + expected_rows: r#"2:21.00:[],4:40.00:["b","c"]|id,amount,updated_at,tags,region|3|3,4"#, + }, + ]; + for test_case in test_cases { + duckdb_execute(&source, test_case.source_change)?; + let (loaded, output) = run_sqlload(&config, test_case.arguments, ("", ""))?; + assert!(loaded, "{}:\n{output}", test_case.description); + assert_eq!( + duckdb_text(&target, CHECK)?, + test_case.expected_rows, + "{}", + test_case.description + ); + } Ok(()) } diff --git a/crates/sqlload/tests/loader/test_types.rs b/crates/sqlload/tests/loader/test_types.rs index 834134e6a..3f8989b86 100644 --- a/crates/sqlload/tests/loader/test_types.rs +++ b/crates/sqlload/tests/loader/test_types.rs @@ -1682,3 +1682,13 @@ pub(crate) struct WideLoadTestCase { /// Phrases the last load's output holds (`{table}` is the table's name). pub(crate) expected_output: &'static [&'static str], } + +/// One load from a duckdb source file into DuckDB, after a change to the source. +pub(crate) struct DuckDbSourceTestCase { + pub(crate) description: &'static str, + /// SQL run on the source file before the load; empty for none. + pub(crate) source_change: &'static str, + pub(crate) arguments: &'static [&'static str], + /// The orders' rows, the target's columns, the view's row count and the query table's ids. + pub(crate) expected_rows: &'static str, +} diff --git a/website/src/content/docs/docs/sources/duckdb.mdx b/website/src/content/docs/docs/sources/duckdb.mdx index 509999ffa..1854aef4e 100644 --- a/website/src/content/docs/docs/sources/duckdb.mdx +++ b/website/src/content/docs/docs/sources/duckdb.mdx @@ -109,4 +109,5 @@ Unit tests read a real DuckDB file: tables, a view and a query table found by `i `exclude`, primary keys and comments, every type above, a pinned `HUGEINT`, reads in several batches, a filter applied outside a query that ends in a comment, and the file's bytes unchanged after the read. A loader test loads a file into DuckDB twice: tables, a view and a query table, -with `exclude_columns`, `where`, and a cursor merging only the rows that changed. +with `exclude_columns`, `where`, a cursor merging only the rows that changed, and `--sync-deletes` +removing a row the file no longer holds. From c51996229707f8075ef9764681ac3541752a0cee Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:03:07 +0000 Subject: [PATCH 5/9] feat: read DuckDB files and MotherDuck databases as sources A duckdb connection's file is read read-only. A motherduck connection is read through the MotherDuck bridge, which streams query results as Arrow. --- .../dispatch/main/dialect/source_dialect.rs | 4 +- .../src/adapters/dispatch/main/session/mod.rs | 2 + .../main/session/open_source_bridge.rs | 18 +++ .../adapters/duckdb/main/_describe_table.rs | 2 +- .../duckdb/main/_execute_statements.rs | 2 +- .../adapters/duckdb/main/_open_motherduck.rs | 2 +- .../adapters/duckdb/main/_own_connection.rs | 2 +- .../src/adapters/duckdb/main/_query_text.rs | 2 +- .../src/adapters/duckdb/main/_query_texts.rs | 2 +- .../src/adapters/duckdb/main/_table_shape.rs | 2 +- .../motherduck_bridge/_helpers/bridge.py | 30 ++++- .../motherduck_bridge/_helpers/process.rs | 15 ++- .../src/adapters/motherduck_bridge/errors.rs | 3 + .../motherduck_bridge/main/_append_batch.rs | 2 +- ..._fork_connection.rs => fork_connection.rs} | 2 +- .../adapters/motherduck_bridge/main/mod.rs | 5 +- .../{_run_statement.rs => run_statement.rs} | 13 +-- .../motherduck_bridge/main/stream_rows.rs | 63 ++++++++++ .../src/duckdb_source/_helpers/catalog.rs | 96 +++++++-------- .../src/duckdb_source/_helpers/connection.rs | 93 ++++++++++++++- .../src/duckdb_source/_helpers/query.rs | 2 +- crates/sqlload/src/duckdb_source/constants.rs | 14 +++ crates/sqlload/src/duckdb_source/errors.rs | 2 + .../src/duckdb_source/main/count_rows.rs | 22 ++-- .../src/duckdb_source/main/describe_source.rs | 27 +++-- .../src/duckdb_source/main/duckdb_reader.rs | 40 +++++-- .../src/duckdb_source/main/read_catalog.rs | 109 ++++++++---------- .../src/duckdb_source/main/read_rows.rs | 21 +++- crates/sqlload/src/duckdb_source/models.rs | 9 +- crates/sqlload/src/loader/_helpers/sources.rs | 16 ++- crates/sqlload/src/loader/_helpers/specs.rs | 11 +- .../sqlload/src/loader/_helpers/table_plan.rs | 5 +- .../sqlload/src/loader/main/_check_source.rs | 4 +- .../src/project/_helpers/references.rs | 15 ++- crates/sqlload/src/project/models.rs | 6 + .../src/row_deletes/main/deletes_problems.rs | 1 + packages/sqlload/pyproject.toml | 8 +- packages/sqlload/tests/test_motherduck.py | 88 +++++++++++++- .../sqlload/tests/test_motherduck_live.py | 69 ++++++++++- .../src/content/docs/docs/configuration.mdx | 2 +- website/src/content/docs/docs/index.mdx | 2 +- .../src/content/docs/docs/sources/index.mdx | 7 +- .../content/docs/docs/sources/motherduck.mdx | 77 +++++++++++++ .../content/docs/docs/supported-systems.mdx | 4 +- .../content/docs/docs/targets/motherduck.mdx | 3 +- website/src/sidebar.json | 2 + 46 files changed, 722 insertions(+), 204 deletions(-) create mode 100644 crates/sqlload/src/adapters/dispatch/main/session/open_source_bridge.rs rename crates/sqlload/src/adapters/motherduck_bridge/main/{_fork_connection.rs => fork_connection.rs} (85%) rename crates/sqlload/src/adapters/motherduck_bridge/main/{_run_statement.rs => run_statement.rs} (75%) create mode 100644 crates/sqlload/src/adapters/motherduck_bridge/main/stream_rows.rs create mode 100644 website/src/content/docs/docs/sources/motherduck.mdx diff --git a/crates/sqlload/src/adapters/dispatch/main/dialect/source_dialect.rs b/crates/sqlload/src/adapters/dispatch/main/dialect/source_dialect.rs index 90b91612a..dc90a40a8 100644 --- a/crates/sqlload/src/adapters/dispatch/main/dialect/source_dialect.rs +++ b/crates/sqlload/src/adapters/dispatch/main/dialect/source_dialect.rs @@ -3,7 +3,7 @@ use crate::bigquery_source::constants::BIGQUERY_SOURCE; use crate::column_types::models::Warehouse; use crate::databricks_source::constants::DATABRICKS_SOURCE; -use crate::duckdb_source::constants::DUCKDB_SOURCE; +use crate::duckdb_source::constants::{DUCKDB_SOURCE, MOTHERDUCK_SOURCE}; use crate::loader::constants::SNOWFLAKE_SOURCE; /// The warehouse whose SQL a warehouse source of `system` (its kind as written) filters in; `None` for a system without one. @@ -12,7 +12,7 @@ pub fn source_dialect(system: &str) -> Option { SNOWFLAKE_SOURCE => Some(Warehouse::Snowflake), BIGQUERY_SOURCE => Some(Warehouse::BigQuery), DATABRICKS_SOURCE => Some(Warehouse::Databricks), - DUCKDB_SOURCE => Some(Warehouse::DuckDb), + DUCKDB_SOURCE | MOTHERDUCK_SOURCE => Some(Warehouse::DuckDb), _ => None, } } diff --git a/crates/sqlload/src/adapters/dispatch/main/session/mod.rs b/crates/sqlload/src/adapters/dispatch/main/session/mod.rs index 156123185..ebd17fce5 100644 --- a/crates/sqlload/src/adapters/dispatch/main/session/mod.rs +++ b/crates/sqlload/src/adapters/dispatch/main/session/mod.rs @@ -7,6 +7,8 @@ pub mod execute; pub mod execute_ddl; pub mod helper_process; pub mod live_columns; +#[cfg(feature = "duckdb")] +pub mod open_source_bridge; pub mod recover_unfinished; pub mod single_text; pub mod table_destination; diff --git a/crates/sqlload/src/adapters/dispatch/main/session/open_source_bridge.rs b/crates/sqlload/src/adapters/dispatch/main/session/open_source_bridge.rs new file mode 100644 index 000000000..8822305bc --- /dev/null +++ b/crates/sqlload/src/adapters/dispatch/main/session/open_source_bridge.rs @@ -0,0 +1,18 @@ +//! Open a MotherDuck database a source reads. + +use crate::adapters::duckdb::errors::DuckDbError; +use crate::adapters::duckdb::main::_open_motherduck::open_motherduck; +use crate::adapters::duckdb::models::DuckDbConnection; +use crate::adapters::motherduck_bridge::models::BridgeConnection; +use crate::project::models::ConnectionConfig; + +/// The bridge connection to the database of `connection`, a `motherduck` one, signed in as a MotherDuck target is. +pub fn open_source_bridge(connection: &ConnectionConfig) -> Result { + let opened = open_motherduck(connection, None, (None, None))?; + match opened.connection { + DuckDbConnection::Bridge(bridge) => Ok(bridge), + DuckDbConnection::Local(_) => Err(DuckDbError::MotherDuck( + "a MotherDuck database opens on the MotherDuck bridge".to_owned(), + )), + } +} diff --git a/crates/sqlload/src/adapters/duckdb/main/_describe_table.rs b/crates/sqlload/src/adapters/duckdb/main/_describe_table.rs index e5d8d4f5d..02928e596 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_describe_table.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_describe_table.rs @@ -6,7 +6,7 @@ use crate::adapters::duckdb::_helpers::blocking::with_connection; use crate::adapters::duckdb::constants::DESCRIBE_TABLE_SQL; use crate::adapters::duckdb::errors::DuckDbError; use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination, DuckDbSql}; -use crate::adapters::motherduck_bridge::main::_run_statement::run_statement; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; use crate::column_types::models::LiveColumnType; use crate::schema_plan::models::LiveColumn; diff --git a/crates/sqlload/src/adapters/duckdb/main/_execute_statements.rs b/crates/sqlload/src/adapters/duckdb/main/_execute_statements.rs index ae256a6ff..79952edf0 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_execute_statements.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_execute_statements.rs @@ -3,7 +3,7 @@ use crate::adapters::duckdb::_helpers::blocking::with_connection; use crate::adapters::duckdb::errors::DuckDbError; use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination}; -use crate::adapters::motherduck_bridge::main::_run_statement::run_statement; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; /// Run each statement in order, stopping at the first failure (an open transaction is rolled back). pub(in crate::adapters) async fn execute_statements( diff --git a/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs b/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs index bdec532ad..456faa708 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_open_motherduck.rs @@ -12,7 +12,7 @@ use crate::memory_budget::models::DuckDbShare; use crate::project::models::ConnectionConfig; /// Open the MotherDuck database of `connection` (`target_database` when the target names one) through DuckDB's `md:` connection on the MotherDuck bridge, signed in with its token, else the token in the `motherduck_token` or `MOTHERDUCK_TOKEN` variable. Without a token MotherDuck would wait for a browser sign-in, so it is refused instead. `python` runs the bridge; `None` picks it as the bridge does. Under `memory`, its part of the run's memory budget, the bridge's DuckDB gets that `memory_limit` and thread count, and commits are cut into pieces that fit it; a table's row groups set no limit, as MotherDuck checkpoints them on its servers. -pub(in crate::adapters) fn open_motherduck( +pub(crate) fn open_motherduck( connection: &ConnectionConfig, target_database: Option<&str>, (python, memory): (Option<&str>, Option), diff --git a/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs b/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs index 67bdfed6a..60224db4c 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_own_connection.rs @@ -4,7 +4,7 @@ use std::sync::{Arc, Mutex}; use crate::adapters::duckdb::errors::DuckDbError; use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination}; -use crate::adapters::motherduck_bridge::main::_fork_connection::fork_connection; +use crate::adapters::motherduck_bridge::main::fork_connection::fork_connection; use crate::memory_budget::main::table_memory::table_memory; /// A new connection to the same database, so one table's transaction never mixes with another table's loading at the same time. Under the memory watchdog, the table it loads can have the connection's running statement interrupted. diff --git a/crates/sqlload/src/adapters/duckdb/main/_query_text.rs b/crates/sqlload/src/adapters/duckdb/main/_query_text.rs index 5abf9ec71..d653bea09 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_query_text.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_query_text.rs @@ -3,7 +3,7 @@ use crate::adapters::duckdb::_helpers::blocking::with_connection; use crate::adapters::duckdb::errors::DuckDbError; use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination}; -use crate::adapters::motherduck_bridge::main::_run_statement::run_statement; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; /// The first column of the first row of `query`, as text; `None` for SQL `NULL`. pub(in crate::adapters) async fn query_text( diff --git a/crates/sqlload/src/adapters/duckdb/main/_query_texts.rs b/crates/sqlload/src/adapters/duckdb/main/_query_texts.rs index c4e9a75de..65108b942 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_query_texts.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_query_texts.rs @@ -3,7 +3,7 @@ use crate::adapters::duckdb::_helpers::blocking::with_connection; use crate::adapters::duckdb::errors::DuckDbError; use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination}; -use crate::adapters::motherduck_bridge::main::_run_statement::run_statement; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; /// The first column of every row of `query`, as text; SQL `NULL`s are skipped. pub(in crate::adapters) async fn query_texts( diff --git a/crates/sqlload/src/adapters/duckdb/main/_table_shape.rs b/crates/sqlload/src/adapters/duckdb/main/_table_shape.rs index 1425cbba2..16d92e763 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_table_shape.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_table_shape.rs @@ -6,7 +6,7 @@ use crate::adapters::duckdb::_helpers::blocking::with_connection; use crate::adapters::duckdb::constants::{DEFINITION_QUERY, INDEXES_QUERY}; use crate::adapters::duckdb::errors::DuckDbError; use crate::adapters::duckdb::models::{DuckDbConnection, DuckDbDestination, DuckDbSql}; -use crate::adapters::motherduck_bridge::main::_run_statement::run_statement; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; /// `table`'s columns and constraints as DuckDB would create them, and its indexes' statements on `table` by its full name (DuckDB stores them on `schema.table`, which MotherDuck's other attached databases can make ambiguous); `None` when there is no such table. pub(in crate::adapters) async fn table_shape( diff --git a/crates/sqlload/src/adapters/motherduck_bridge/_helpers/bridge.py b/crates/sqlload/src/adapters/motherduck_bridge/_helpers/bridge.py index 359800a76..1d937a934 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/_helpers/bridge.py +++ b/crates/sqlload/src/adapters/motherduck_bridge/_helpers/bridge.py @@ -6,6 +6,10 @@ -> {"id": 1, "ok": true, "duckdb": "1.5.6", "python": "3.12.3"} {"id": 2, "op": "run", "cursor": 3, "sql": "SELECT ...", "params": ["raw"]} -> {"id": 2, "ok": true, "rows": [["id", "BIGINT"], ...]} + {"id": 3, "op": "stream", "cursor": 4, "sql": "SELECT ...", "rows": 100000} + -> {"id": 3, "ok": true} + {"id": 4, "op": "next", "cursor": 4, "path": "/tmp/batch.arrow"} + -> {"id": 4, "ok": true, "rows": 100000}, the next batch of at most that many rows written there as an Arrow IPC file; 0 rows once all are read {"op": "close", "cursor": 3} -> no reply {"op": "interrupt", "cursor": 3} @@ -41,6 +45,19 @@ def plain(value): return str(value) +def write_next(batches, path): + """The next batch `batches` holds, written to `path` as an Arrow IPC file; its row count, 0 once every batch is read.""" + import pyarrow.ipc + + try: + batch = batches.read_next_batch() + except StopIteration: + return 0 + with pyarrow.ipc.new_file(path, batch.schema) as writer: + writer.write_batch(batch) + return batch.num_rows + + def main(): # Replies go to a private copy of stdout; anything else that prints (DuckDB, an extension) lands on stderr instead. replies = os.fdopen(os.dup(1), "wb", buffering=0) @@ -64,6 +81,7 @@ def failed(request, error): def serve(work, number): cursor = None + batches = None while True: request = work.get() if request is None: @@ -71,7 +89,17 @@ def serve(work, number): try: if cursor is None: cursor = opened[number] = database.cursor() + if request["op"] == "next": + reply({"id": request["id"], "ok": True, "rows": write_next(batches, request["path"])}) + continue cursor.execute(request["sql"], request.get("params") or None) + if request["op"] == "stream": + # Arrow batches straight from DuckDB's streaming result, one at a time as `next` asks, so the bridge never holds them all. + # duckdb 1.4 names the reader fetch_record_batch; 1.5 adds to_arrow_reader, and 2.0 keeps only that. + reader = getattr(cursor, "to_arrow_reader", None) or cursor.fetch_record_batch + batches = reader(request["rows"]) + reply({"id": request["id"], "ok": True}) + continue rows = cursor.fetchall() if cursor.description else [] reply({"id": request["id"], "ok": True, "rows": [[plain(value) for value in row] for row in rows]}) except BaseException as error: # noqa: BLE001 - every failure goes back to SQLLoad as its reply, or it would wait forever @@ -91,7 +119,7 @@ def serve(work, number): reply({"id": request["id"], "ok": True, "duckdb": duckdb.__version__, "python": sys.version.split()[0]}) except Exception as error: # noqa: BLE001 failed(request, error) - elif op == "run": + elif op in ("run", "stream", "next"): work = cursors.get(request["cursor"]) if work is None: work = cursors[request["cursor"]] = queue.Queue() diff --git a/crates/sqlload/src/adapters/motherduck_bridge/_helpers/process.rs b/crates/sqlload/src/adapters/motherduck_bridge/_helpers/process.rs index fb35234a0..b5da57011 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/_helpers/process.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/_helpers/process.rs @@ -10,7 +10,7 @@ use crate::adapters::motherduck_bridge::constants::{ BRIDGE_SCRIPT, PYTHON_VARIABLE, STDERR_TAIL_BYTES, SUPPORTED_DUCKDB, }; use crate::adapters::motherduck_bridge::errors::BridgeError; -use crate::adapters::motherduck_bridge::models::{BridgeProcess, Replies}; +use crate::adapters::motherduck_bridge::models::{BridgeCursor, BridgeProcess, Replies}; /// Start the bridge on `python`: a writer thread feeding its stdin (closing it, and reaping the process, once every handle is gone), a reader thread routing its replies and, once stdout and stderr have both closed or a reply is unreadable, failing what still waits, and one keeping its stderr's tail with each of `hidden` taken out. pub(crate) fn start(python: &str, hidden: Vec) -> Result, BridgeError> { @@ -100,6 +100,19 @@ pub(crate) fn send( Ok(received) } +/// Send `request` on `cursor` and wait for its reply, failed or not, or why the bridge stopped first. +pub(crate) async fn ask( + cursor: &BridgeCursor, + request: serde_json::Value, +) -> Result { + let received = send(&cursor.process, request)?; + let python = cursor.process.python.clone(); + tokio::task::spawn_blocking(move || received.recv()) + .await + .map_err(|error| BridgeError::Stopped(error.to_string()))? + .map_err(|_| BridgeError::Stopped(format!("the MotherDuck bridge on {python} stopped")))? +} + /// `reply`'s failure, as its message and its Python error type; `None` when it succeeded. pub(crate) fn failure(reply: &serde_json::Value) -> Option<(String, String)> { if reply["ok"].as_bool() == Some(true) { diff --git a/crates/sqlload/src/adapters/motherduck_bridge/errors.rs b/crates/sqlload/src/adapters/motherduck_bridge/errors.rs index d12113b1f..0de186a1b 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/errors.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/errors.rs @@ -10,4 +10,7 @@ pub enum BridgeError { /// DuckDB refused the request, in its own words. #[error("{0}")] Engine(String), + /// What the caller's batch consumer failed with, as it said it. + #[error("{0}")] + Consumer(String), } diff --git a/crates/sqlload/src/adapters/motherduck_bridge/main/_append_batch.rs b/crates/sqlload/src/adapters/motherduck_bridge/main/_append_batch.rs index 44ce7faca..63bee0f53 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/main/_append_batch.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/main/_append_batch.rs @@ -4,7 +4,7 @@ use arrow_array::RecordBatch; use parquet::arrow::ArrowWriter; use crate::adapters::motherduck_bridge::errors::BridgeError; -use crate::adapters::motherduck_bridge::main::_run_statement::run_statement; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; use crate::adapters::motherduck_bridge::models::BridgeConnection; /// Run `insert`, whose one `?` is the path of a Parquet file holding `batch` (such as `INSERT INTO t (a) SELECT a FROM read_parquet(?)`), on `connection`. The file is a temporary one on this machine, which DuckDB reads there and removed once the statement ends: the bridge's DuckDB is another version than SQLLoad's, so Arrow cannot cross between them in memory. diff --git a/crates/sqlload/src/adapters/motherduck_bridge/main/_fork_connection.rs b/crates/sqlload/src/adapters/motherduck_bridge/main/fork_connection.rs similarity index 85% rename from crates/sqlload/src/adapters/motherduck_bridge/main/_fork_connection.rs rename to crates/sqlload/src/adapters/motherduck_bridge/main/fork_connection.rs index 9e73a37fd..6602d926d 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/main/_fork_connection.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/main/fork_connection.rs @@ -6,7 +6,7 @@ use std::sync::atomic::Ordering; use crate::adapters::motherduck_bridge::models::{BridgeConnection, BridgeCursor}; /// A new connection to the database `connection` opened, on the same bridge: its own DuckDB cursor, so its transactions never mix with another connection's. -pub(in crate::adapters) fn fork_connection(connection: &BridgeConnection) -> BridgeConnection { +pub(crate) fn fork_connection(connection: &BridgeConnection) -> BridgeConnection { let process = connection.cursor.process.clone(); let id = process.next_cursor.fetch_add(1, Ordering::Relaxed); BridgeConnection { diff --git a/crates/sqlload/src/adapters/motherduck_bridge/main/mod.rs b/crates/sqlload/src/adapters/motherduck_bridge/main/mod.rs index 1322dca74..8f3a94a78 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/main/mod.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/main/mod.rs @@ -1,4 +1,5 @@ pub(in crate::adapters) mod _append_batch; -pub(in crate::adapters) mod _fork_connection; pub(in crate::adapters) mod _open_bridge; -pub(in crate::adapters) mod _run_statement; +pub(crate) mod fork_connection; +pub(crate) mod run_statement; +pub(crate) mod stream_rows; diff --git a/crates/sqlload/src/adapters/motherduck_bridge/main/_run_statement.rs b/crates/sqlload/src/adapters/motherduck_bridge/main/run_statement.rs similarity index 75% rename from crates/sqlload/src/adapters/motherduck_bridge/main/_run_statement.rs rename to crates/sqlload/src/adapters/motherduck_bridge/main/run_statement.rs index 08a6b6049..a2eb727c5 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/main/_run_statement.rs +++ b/crates/sqlload/src/adapters/motherduck_bridge/main/run_statement.rs @@ -1,11 +1,11 @@ //! Run SQL on a bridge connection. -use crate::adapters::motherduck_bridge::_helpers::process::{failure, send}; +use crate::adapters::motherduck_bridge::_helpers::process::{ask, failure}; use crate::adapters::motherduck_bridge::errors::BridgeError; use crate::adapters::motherduck_bridge::models::BridgeConnection; /// Run `sql` (one statement or several, separated by `;`) with text `params` bound to its `?`s, after every earlier statement on `connection`, and return the last statement's rows: each value as text (numbers and booleans as their SQL text), `None` for `NULL`. -pub(in crate::adapters) async fn run_statement( +pub(crate) async fn run_statement( connection: &BridgeConnection, sql: String, params: Vec, @@ -13,14 +13,7 @@ pub(in crate::adapters) async fn run_statement( let cursor = &connection.cursor; let request = serde_json::json!({"op": "run", "cursor": cursor.id, "sql": sql, "params": params}); - let received = send(&cursor.process, request)?; - let python = cursor.process.python.clone(); - let reply = tokio::task::spawn_blocking(move || received.recv()) - .await - .map_err(|error| BridgeError::Stopped(error.to_string()))? - .map_err(|_| { - BridgeError::Stopped(format!("the MotherDuck bridge on {python} stopped")) - })??; + let reply = ask(cursor, request).await?; if let Some((error, _)) = failure(&reply) { return Err(BridgeError::Engine(error)); } diff --git a/crates/sqlload/src/adapters/motherduck_bridge/main/stream_rows.rs b/crates/sqlload/src/adapters/motherduck_bridge/main/stream_rows.rs new file mode 100644 index 000000000..bbdc3cd4a --- /dev/null +++ b/crates/sqlload/src/adapters/motherduck_bridge/main/stream_rows.rs @@ -0,0 +1,63 @@ +//! Stream a query's rows from the bridge as Arrow batches. + +use arrow_array::RecordBatch; +use arrow_ipc::reader::FileReader; + +use crate::adapters::motherduck_bridge::_helpers::process::{ask, failure}; +use crate::adapters::motherduck_bridge::constants::{MISSING_PACKAGE_ERRORS, PYTHON_VARIABLE}; +use crate::adapters::motherduck_bridge::errors::BridgeError; +use crate::adapters::motherduck_bridge::models::BridgeConnection; + +/// Run `sql` on `connection` and pass its rows to `on_batch` in batches of at most `batch_rows`; returns how many rows there were. The bridge reads one batch at a time from DuckDB's streaming result and hands it over in a temporary Arrow IPC file on this machine, removed once read: its DuckDB is another version than SQLLoad's, so Arrow cannot cross between them in memory. +pub(crate) async fn stream_rows( + connection: &BridgeConnection, + sql: String, + batch_rows: usize, + mut on_batch: impl AsyncFnMut(RecordBatch) -> Result<(), String>, +) -> Result { + let cursor = &connection.cursor; + let python = &cursor.process.python; + let stream = + serde_json::json!({"op": "stream", "cursor": cursor.id, "sql": sql, "rows": batch_rows}); + if let Some((error, _)) = failure(&ask(cursor, stream).await?) { + return Err(BridgeError::Engine(error)); + } + let mut rows: u64 = 0; + loop { + let file = tempfile::Builder::new() + .prefix("sqlload-motherduck-") + .suffix(".arrow") + .tempfile() + .map_err(|error| { + BridgeError::Setup(format!( + "cannot make a file for a MotherDuck batch: {error}" + )) + })?; + let next = serde_json::json!({"op": "next", "cursor": cursor.id, "path": file.path().display().to_string()}); + let reply = ask(cursor, next).await?; + if let Some((error, kind)) = failure(&reply) { + return Err(match MISSING_PACKAGE_ERRORS.contains(&kind.as_str()) { + true => BridgeError::Setup(format!( + "reading from MotherDuck needs Python's pyarrow package beside duckdb, which {python} lacks ({error}): install sqlload[motherduck] with pip or uv, or set {PYTHON_VARIABLE} to a Python that has both" + )), + false => BridgeError::Engine(error), + }); + } + if reply["rows"].as_u64().unwrap_or_default() == 0 { + return Ok(rows); + } + let batches = tokio::task::spawn_blocking(move || { + let reader = FileReader::try_new(file.reopen()?, None)?; + reader.collect::, _>>() + }) + .await + .map_err(|error| BridgeError::Stopped(error.to_string()))? + .map_err(|error: arrow_schema::ArrowError| { + BridgeError::Engine(format!("cannot read a MotherDuck batch: {error}")) + })?; + for batch in batches { + rows += batch.num_rows() as u64; + on_batch(batch).await.map_err(BridgeError::Consumer)?; + } + } +} diff --git a/crates/sqlload/src/duckdb_source/_helpers/catalog.rs b/crates/sqlload/src/duckdb_source/_helpers/catalog.rs index 18a7d13a7..3fbf51a1c 100644 --- a/crates/sqlload/src/duckdb_source/_helpers/catalog.rs +++ b/crates/sqlload/src/duckdb_source/_helpers/catalog.rs @@ -4,31 +4,31 @@ use std::collections::BTreeMap; use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; +use crate::duckdb_source::_helpers::connection::TextRows; use crate::duckdb_source::_helpers::type_map::source_column; +use crate::duckdb_source::constants::{CATALOG_QUERIES, NOT_NULL, TRUE_TEXT}; use crate::duckdb_source::errors::DuckDbSourceError; use crate::postgres_source::models::SourceTable; -/// The database's tables and views whose `schema.table` matches `include` and not `exclude` (exactly, as DuckDB keeps names), sorted: from `duckdb_tables()` and `duckdb_views()`, their columns from `duckdb_columns()` and their primary keys from `duckdb_constraints()`, all of the file's own database. +/// The database's tables and views whose `schema.table` matches `include` and not `exclude` (exactly, as DuckDB keeps names), sorted, from the answers to `CATALOG_QUERIES` in order: the tables and views, their columns, and their primary keys. pub(crate) fn catalog_tables( - connection: &duckdb::Connection, + answers: &[TextRows], (include, exclude): (&[String], &[String]), ) -> Result, DuckDbSourceError> { let (included, excluded) = (matcher(include)?, matcher(exclude)?); + let [listed, columns, keys] = answers else { + return Err(DuckDbSourceError::Invalid(format!( + "the catalog answered {} of its {} queries", + answers.len(), + CATALOG_QUERIES.len() + ))); + }; + let text = |row: &[Option], index: usize| { + row.get(index).cloned().flatten().unwrap_or_default() + }; let mut tables: BTreeMap<(String, String), SourceTable> = BTreeMap::new(); - let mut listed = connection.prepare( - "SELECT schema_name, table_name, comment, estimated_size FROM duckdb_tables() WHERE database_name = current_database() \ - UNION ALL SELECT schema_name, view_name, comment, -1 FROM duckdb_views() WHERE database_name = current_database() AND NOT internal", - )?; - let rows = listed.query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, i64>(3)?, - )) - })?; - for row in rows { - let (schema, name, comment, estimated_rows) = row?; + for row in listed { + let (schema, name) = (text(row, 0), text(row, 1)); let qualified = format!("{schema}.{name}"); if !included.is_match(&qualified) || excluded.is_match(&qualified) { continue; @@ -42,59 +42,45 @@ pub(crate) fn catalog_tables( columns: Vec::new(), primary_key: Vec::new(), indexed_columns: Vec::new(), - estimated_rows, + estimated_rows: text(row, 3).parse().unwrap_or(-1), estimated_bytes: -1, - comment: comment.filter(|comment| !comment.is_empty()), + comment: Some(text(row, 2)).filter(|comment| !comment.is_empty()), listed_keys: Vec::new(), }, ); } - if tables.is_empty() { - return Ok(Vec::new()); - } - let mut columns = connection.prepare( - "SELECT schema_name, table_name, column_name, data_type, is_nullable FROM duckdb_columns() WHERE database_name = current_database() ORDER BY schema_name, table_name, column_index", - )?; - let rows = columns.query_map([], |row| { - Ok(( - (row.get::<_, String>(0)?, row.get::<_, String>(1)?), - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - row.get::<_, bool>(4)?, - )) - })?; - for row in rows { - let (key, name, data_type, nullable) = row?; - if let Some(table) = tables.get_mut(&key) { + for row in columns { + if let Some(table) = tables.get_mut(&(text(row, 0), text(row, 1))) { + let nullable = text(row, 4) == TRUE_TEXT; table .columns - .push(source_column(&name, &data_type, nullable)); + .push(source_column(&text(row, 2), &text(row, 3), nullable)); } } - let mut keys = connection.prepare( - "SELECT schema_name, table_name, constraint_column_names FROM duckdb_constraints() WHERE database_name = current_database() AND constraint_type = 'PRIMARY KEY'", - )?; - let rows = keys.query_map([], |row| { - Ok(( - (row.get::<_, String>(0)?, row.get::<_, String>(1)?), - row.get::<_, duckdb::types::Value>(2)?, - )) - })?; - for row in rows { - let (key, names) = row?; - if let (Some(table), duckdb::types::Value::List(names)) = (tables.get_mut(&key), names) { - table.primary_key = names - .into_iter() - .filter_map(|name| match name { - duckdb::types::Value::Text(name) => Some(name), - _ => None, - }) - .collect(); + for row in keys { + if let Some(table) = tables.get_mut(&(text(row, 0), text(row, 1))) { + table.primary_key = serde_json::from_str(&text(row, 2)).map_err(|error| { + DuckDbSourceError::Invalid(format!("{}'s primary key: {error}", table.name)) + })?; } } Ok(tables.into_values().collect()) } +/// Each result column a `DESCRIBE` answered: its name, DuckDB type and whether it may be NULL. +pub(crate) fn described_columns(rows: &TextRows) -> Vec<(String, String, bool)> { + rows.iter() + .map(|row| { + let text = |index: usize| row.get(index).cloned().flatten(); + ( + text(0).unwrap_or_default(), + text(1).unwrap_or_default(), + text(2).is_none_or(|null| null != NOT_NULL), + ) + }) + .collect() +} + /// One glob set for `schema.table` patterns, matched exactly. fn matcher(patterns: &[String]) -> Result { let mut builder = GlobSetBuilder::new(); diff --git a/crates/sqlload/src/duckdb_source/_helpers/connection.rs b/crates/sqlload/src/duckdb_source/_helpers/connection.rs index be2146337..f5c584662 100644 --- a/crates/sqlload/src/duckdb_source/_helpers/connection.rs +++ b/crates/sqlload/src/duckdb_source/_helpers/connection.rs @@ -1,8 +1,15 @@ -//! A read-only connection to a DuckDB source's database. +//! Connections to a DuckDB source's database: a file read-only, or a MotherDuck database on the MotherDuck bridge. +use crate::adapters::dispatch::main::session::open_source_bridge::open_source_bridge; +use crate::adapters::motherduck_bridge::main::fork_connection::fork_connection; +use crate::adapters::motherduck_bridge::main::run_statement::run_statement; +use crate::adapters::motherduck_bridge::models::BridgeConnection; use crate::duckdb_source::errors::DuckDbSourceError; use crate::duckdb_source::models::DuckDbReader; +/// Rows of text values, `None` for `NULL`. +pub(crate) type TextRows = Vec>>; + /// The reader's database opened read-only, so a file another process writes is never changed and SQLLoad holds no write lock on it, with the reader's `memory_limit`. pub(crate) fn open_read_only( reader: &DuckDbReader, @@ -14,3 +21,87 @@ pub(crate) fn open_read_only( } Ok(connection) } + +/// A connection of its own to a `motherduck` reader's database, on the reader's bridge: opened on its first use, signed in as a MotherDuck target is, its DuckDB given the reader's `memory_limit`. +pub(crate) async fn open_bridge( + reader: &DuckDbReader, +) -> Result { + let Some(connection) = reader.motherduck.clone() else { + return Err(DuckDbSourceError::Invalid(format!( + "{} is a file, not a MotherDuck database", + reader.database + ))); + }; + let memory_limit = reader.memory_limit; + let shared = reader + .bridge + .get_or_try_init(|| async move { + let bridge = tokio::task::spawn_blocking(move || open_source_bridge(&connection)) + .await + .map_err(|error| { + DuckDbSourceError::Invalid(format!("opening MotherDuck stopped: {error}")) + })? + .map_err(|error| DuckDbSourceError::Invalid(error.to_string()))?; + if let Some(limit) = memory_limit { + let set = format!("SET memory_limit = '{}KiB'", limit / 1024); + run_statement(&bridge, set, Vec::new()) + .await + .map_err(DuckDbSourceError::Bridge)?; + } + Ok::<_, DuckDbSourceError>(bridge) + }) + .await?; + Ok(fork_connection(shared)) +} + +/// Each of `queries`' rows, run in order on one connection to the reader's database, every value as text; a failure names the index of the query that failed. Each query casts its columns to `VARCHAR`. +pub(crate) async fn text_rows( + reader: &DuckDbReader, + queries: Vec, +) -> Result, (usize, DuckDbSourceError)> { + if reader.motherduck.is_none() { + let reader = reader.clone(); + return tokio::task::spawn_blocking(move || { + let connection = open_read_only(&reader).map_err(|error| (0, error))?; + queries + .iter() + .enumerate() + .map(|(index, sql)| { + local_rows(&connection, sql).map_err(|error| (index, error.into())) + }) + .collect() + }) + .await + .map_err(|error| { + ( + 0, + DuckDbSourceError::Invalid(format!("the query stopped: {error}")), + ) + })?; + }; + let bridge = open_bridge(reader).await.map_err(|error| (0, error))?; + let mut answers = Vec::with_capacity(queries.len()); + for (index, sql) in queries.into_iter().enumerate() { + let rows = run_statement(&bridge, sql, Vec::new()) + .await + .map_err(|error| (index, DuckDbSourceError::Bridge(error)))?; + answers.push(rows); + } + Ok(answers) +} + +/// `sql`'s rows on a file's connection, each value as text. +fn local_rows(connection: &duckdb::Connection, sql: &str) -> Result { + let mut statement = connection.prepare(sql)?; + let mut rows = statement.query([])?; + let mut answer: TextRows = Vec::new(); + while let Some(row) = rows.next()? { + let width = row.as_ref().column_count(); + answer.push( + (0..width) + .map(|index| row.get::<_, Option>(index)) + .collect::, _>>()?, + ); + } + Ok(answer) +} diff --git a/crates/sqlload/src/duckdb_source/_helpers/query.rs b/crates/sqlload/src/duckdb_source/_helpers/query.rs index 575111f14..39a722b1c 100644 --- a/crates/sqlload/src/duckdb_source/_helpers/query.rs +++ b/crates/sqlload/src/duckdb_source/_helpers/query.rs @@ -39,7 +39,7 @@ pub(crate) fn select_query(request: &DuckDbReadRequest<'_>) -> String { /// `SELECT COUNT(*) FROM WHERE `, the table and query written as `select_query` writes them: how many rows `filter` selects. pub(crate) fn count_query((table, sql): (&SourceTable, Option<&str>), filter: &str) -> String { format!( - "SELECT COUNT(*) FROM {} WHERE {filter}", + "SELECT CAST(COUNT(*) AS VARCHAR) FROM {} WHERE {filter}", from_item((table, sql)) ) } diff --git a/crates/sqlload/src/duckdb_source/constants.rs b/crates/sqlload/src/duckdb_source/constants.rs index 58d1a1412..355091fa8 100644 --- a/crates/sqlload/src/duckdb_source/constants.rs +++ b/crates/sqlload/src/duckdb_source/constants.rs @@ -1,6 +1,9 @@ /// The system name a DuckDB source's tables carry in plans. pub const DUCKDB_SOURCE: &str = "duckdb"; +/// The system name a MotherDuck source's tables carry in plans. +pub const MOTHERDUCK_SOURCE: &str = "motherduck"; + /// The database path DuckDB opens in memory, which holds no tables to read. pub const MEMORY_DATABASE: &str = ":memory:"; @@ -9,3 +12,14 @@ pub const NOT_NULL: &str = "NO"; /// How many result batches the read holds ahead of the consumer. pub const BATCHES_AHEAD: usize = 4; + +/// How DuckDB writes `true` cast to `VARCHAR`. +pub const TRUE_TEXT: &str = "true"; + +/// The catalog queries, every value cast to `VARCHAR` so a file and the MotherDuck bridge answer alike, all of the opened database's own: its tables and views (schema, name, comment, estimated rows), their columns in order (schema, table, name, type, nullable), and their primary keys (schema, table, the key's columns as a JSON list). +pub const CATALOG_QUERIES: [&str; 3] = [ + "SELECT schema_name, table_name, comment, CAST(estimated_size AS VARCHAR) FROM duckdb_tables() WHERE database_name = current_database() \ + UNION ALL SELECT schema_name, view_name, comment, '-1' FROM duckdb_views() WHERE database_name = current_database() AND NOT internal", + "SELECT schema_name, table_name, column_name, data_type, CAST(is_nullable AS VARCHAR) FROM duckdb_columns() WHERE database_name = current_database() ORDER BY schema_name, table_name, column_index", + "SELECT schema_name, table_name, CAST(to_json(constraint_column_names) AS VARCHAR) FROM duckdb_constraints() WHERE database_name = current_database() AND constraint_type = 'PRIMARY KEY'", +]; diff --git a/crates/sqlload/src/duckdb_source/errors.rs b/crates/sqlload/src/duckdb_source/errors.rs index f270af918..3efb14891 100644 --- a/crates/sqlload/src/duckdb_source/errors.rs +++ b/crates/sqlload/src/duckdb_source/errors.rs @@ -5,6 +5,8 @@ pub enum DuckDbSourceError { DuckDb(#[from] duckdb::Error), #[error("{0}")] Invalid(String), + #[error("MotherDuck: {0}")] + Bridge(crate::adapters::motherduck_bridge::errors::BridgeError), /// What the caller's batch consumer failed with, as it said it. #[error("{0}")] Consumer(String), diff --git a/crates/sqlload/src/duckdb_source/main/count_rows.rs b/crates/sqlload/src/duckdb_source/main/count_rows.rs index c4c7ca4f3..1bff89b72 100644 --- a/crates/sqlload/src/duckdb_source/main/count_rows.rs +++ b/crates/sqlload/src/duckdb_source/main/count_rows.rs @@ -1,6 +1,6 @@ //! Count a table's rows that a condition selects. -use crate::duckdb_source::_helpers::connection::open_read_only; +use crate::duckdb_source::_helpers::connection::text_rows; use crate::duckdb_source::_helpers::query::count_query; use crate::duckdb_source::errors::DuckDbSourceError; use crate::duckdb_source::models::DuckDbReader; @@ -12,14 +12,16 @@ pub async fn count_rows( (table, sql): (&SourceTable, Option<&str>), filter: &str, ) -> Result { - let (reader, query) = (reader.clone(), count_query((table, sql), filter)); - tokio::task::spawn_blocking(move || { - let connection = open_read_only(&reader)?; - let count: i64 = connection.query_row(&query, [], |row| row.get(0))?; - u64::try_from(count).map_err(|error| { - DuckDbSourceError::Invalid(format!("{query} returned {count}: {error}")) - }) + let query = count_query((table, sql), filter); + let answers = text_rows(reader, vec![query.clone()]) + .await + .map_err(|(_, error)| error)?; + let count = answers + .first() + .and_then(|rows| rows.first()) + .and_then(|row| row.first().cloned().flatten()) + .unwrap_or_default(); + count.parse::().map_err(|error| { + DuckDbSourceError::Invalid(format!("{query} returned the count {count:?}: {error}")) }) - .await - .map_err(|error| DuckDbSourceError::Invalid(format!("counting rows stopped: {error}")))? } diff --git a/crates/sqlload/src/duckdb_source/main/describe_source.rs b/crates/sqlload/src/duckdb_source/main/describe_source.rs index f39435a6f..888722fdf 100644 --- a/crates/sqlload/src/duckdb_source/main/describe_source.rs +++ b/crates/sqlload/src/duckdb_source/main/describe_source.rs @@ -1,17 +1,24 @@ -//! What a DuckDB source reads, for `debug`. +//! What a DuckDB or MotherDuck source reads, for `debug`. -use crate::duckdb_source::_helpers::connection::open_read_only; +use crate::duckdb_source::_helpers::connection::text_rows; use crate::duckdb_source::errors::DuckDbSourceError; use crate::duckdb_source::models::DuckDbReader; -/// The reader's database file and the DuckDB version reading it, once the file has opened read-only. +/// The reader's database and the DuckDB version reading it, once the database has opened: a file read-only, a MotherDuck database on the bridge. pub async fn describe_source(reader: &DuckDbReader) -> Result { - let reader = reader.clone(); - tokio::task::spawn_blocking(move || { - let connection = open_read_only(&reader)?; - let version: String = connection.query_row("SELECT version()", [], |row| row.get(0))?; - Ok(format!("{}, read-only, DuckDB {version}", reader.database)) + let answers = text_rows(reader, vec!["SELECT version()".to_owned()]) + .await + .map_err(|(_, error)| error)?; + let version = answers + .first() + .and_then(|rows| rows.first()) + .and_then(|row| row.first().cloned().flatten()) + .unwrap_or_default(); + Ok(match reader.motherduck { + Some(_) => format!( + "{} on MotherDuck, through DuckDB {version}", + reader.database + ), + None => format!("{}, read-only, DuckDB {version}", reader.database), }) - .await - .map_err(|error| DuckDbSourceError::Invalid(format!("opening the database stopped: {error}")))? } diff --git a/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs b/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs index c95f96f9b..406449e42 100644 --- a/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs +++ b/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs @@ -1,4 +1,4 @@ -//! A reader for a `duckdb` source. +//! A reader for a `duckdb` or `motherduck` source. use crate::duckdb_source::constants::MEMORY_DATABASE; use crate::duckdb_source::errors::DuckDbSourceError; @@ -6,17 +6,37 @@ use crate::duckdb_source::models::DuckDbReader; use crate::project::main::motherduck_path::motherduck_path; use crate::project::models::{ConnectionConfig, ProjectConfig, SourceConfig}; -/// A reader for `source`'s `duckdb` connection's database file; each read opens it read-only, with a `memory_limit` of half a table's share of the project's memory budget (DuckDB's own default without one). `:memory:` and MotherDuck paths are refused: an empty in-memory database has nothing to read, and a MotherDuck database is a `motherduck` source. Nothing is opened yet. +/// A reader for `source`'s connection: a `duckdb` connection's database file, which each read opens read-only with a `memory_limit` of half a table's share of the project's memory budget (DuckDB's own default without one), or a `motherduck` connection's database, which each read opens on the MotherDuck bridge. Refused: `:memory:`, which holds no tables to read; a MotherDuck path on a `duckdb` connection, which is a `motherduck` source; and a `motherduck` connection naming no database, as a source reads one. Nothing is opened yet. pub fn duckdb_reader( project: &ProjectConfig, source: &SourceConfig, ) -> Result { - let Some(ConnectionConfig::DuckDb { database }) = project.connections.get(&source.connection) - else { - return Err(DuckDbSourceError::Invalid(format!( - "connection {:?} is not a duckdb connection", - source.connection - ))); + let memory_limit = project.memory.budget.map(|budget| budget.table_share / 2); + let database = match project.connections.get(&source.connection) { + Some(ConnectionConfig::DuckDb { database }) => database, + Some(connection @ ConnectionConfig::MotherDuck { database, .. }) => { + let database = database + .as_deref() + .filter(|database| motherduck_path(database).name.is_some()) + .ok_or_else(|| { + DuckDbSourceError::Invalid(format!( + "a motherduck source reads one database: name it in connection {:?}'s database", + source.connection + )) + })?; + return Ok(DuckDbReader { + database: database.to_owned(), + memory_limit, + motherduck: Some(connection.clone()), + bridge: std::sync::Arc::default(), + }); + } + _ => { + return Err(DuckDbSourceError::Invalid(format!( + "connection {:?} is not a duckdb or motherduck connection", + source.connection + ))); + } }; if database == MEMORY_DATABASE { return Err(DuckDbSourceError::Invalid( @@ -30,6 +50,8 @@ pub fn duckdb_reader( } Ok(DuckDbReader { database: database.clone(), - memory_limit: project.memory.budget.map(|budget| budget.table_share / 2), + memory_limit, + motherduck: None, + bridge: std::sync::Arc::default(), }) } diff --git a/crates/sqlload/src/duckdb_source/main/read_catalog.rs b/crates/sqlload/src/duckdb_source/main/read_catalog.rs index a0caa5cd6..7b87ab0f8 100644 --- a/crates/sqlload/src/duckdb_source/main/read_catalog.rs +++ b/crates/sqlload/src/duckdb_source/main/read_catalog.rs @@ -1,77 +1,68 @@ -//! Discover a DuckDB source's tables: tables and views matching `include`, and query tables described by their result's columns. +//! Discover a DuckDB or MotherDuck source's tables: tables and views matching `include`, and query tables described by their result's columns. -use crate::duckdb_source::_helpers::catalog::catalog_tables; -use crate::duckdb_source::_helpers::connection::open_read_only; +use crate::duckdb_source::_helpers::catalog::{catalog_tables, described_columns}; +use crate::duckdb_source::_helpers::connection::text_rows; use crate::duckdb_source::_helpers::query::subquery; use crate::duckdb_source::_helpers::type_map::source_column; -use crate::duckdb_source::constants::NOT_NULL; +use crate::duckdb_source::constants::CATALOG_QUERIES; use crate::duckdb_source::errors::DuckDbSourceError; use crate::duckdb_source::models::DuckDbReader; use crate::postgres_source::models::SourceTable; use crate::project::models::SourceConfig; -/// Every table and view of the reader's database whose `schema.table` matches `source`'s `include` and not its `exclude`; then one table per query table of `source`, its columns those of the query as a load wraps it, described with `DESCRIBE` so it reads no rows. +/// Every table and view of the reader's database whose `schema.table` matches `source`'s `include` and not its `exclude`; then one table per query table of `source`, its columns those of the query as a load wraps it, described with `DESCRIBE` so it reads no rows. Every query runs on one connection. pub async fn read_catalog( reader: &DuckDbReader, source: &SourceConfig, ) -> Result, DuckDbSourceError> { - let queries: Vec<(String, String)> = source + let named: Vec<(&String, &str)> = source .tables .iter() - .filter_map(|(name, table)| Some((name.clone(), table.sql.clone()?))) + .filter_map(|(name, table)| Some((name, table.sql.as_deref()?))) .collect(); - let (reader, include, exclude) = ( - reader.clone(), - source.include.clone(), - source.exclude.clone(), - ); - tokio::task::spawn_blocking(move || { - let connection = open_read_only(&reader)?; - let mut tables: Vec = match include.is_empty() { - true => Vec::new(), - false => catalog_tables(&connection, (&include, &exclude))?, - }; - for (name, sql) in &queries { - let described = describe(&connection, sql).map_err(|error| { - DuckDbSourceError::Invalid(format!("query table {name}: {error}")) - })?; - tables.push(SourceTable { - split_pages: None, - schema: String::new(), - name: name.clone(), - columns: described - .iter() - .map(|(column, data_type, nullable)| { - source_column(column, data_type, *nullable) - }) - .collect(), - primary_key: Vec::new(), - indexed_columns: Vec::new(), - estimated_rows: -1, - estimated_bytes: -1, - comment: None, - listed_keys: Vec::new(), - }); + let catalog = match source.include.is_empty() { + true => 0, + false => CATALOG_QUERIES.len(), + }; + let queries: Vec = CATALOG_QUERIES[..catalog] + .iter() + .map(|query| (*query).to_owned()) + .chain( + named + .iter() + .map(|(_, sql)| format!("DESCRIBE SELECT * FROM {}", subquery(sql))), + ) + .collect(); + let answers = text_rows(reader, queries).await.map_err(|(index, error)| { + match index + .checked_sub(catalog) + .and_then(|query| named.get(query)) + { + Some((name, _)) => DuckDbSourceError::Invalid(format!("query table {name}: {error}")), + None => error, } - Ok(tables) - }) - .await - .map_err(|error| DuckDbSourceError::Invalid(format!("reading the catalog stopped: {error}")))? -} - -/// Each result column of `sql`: its name, DuckDB type and whether it may be NULL. -fn describe( - connection: &duckdb::Connection, - sql: &str, -) -> Result, duckdb::Error> { - let mut statement = connection.prepare(&format!("DESCRIBE SELECT * FROM {}", subquery(sql)))?; - let rows = statement.query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)? - .is_none_or(|null| null != NOT_NULL), - )) })?; - rows.collect() + let (listed, described) = answers.split_at(catalog); + let mut tables: Vec = match catalog { + 0 => Vec::new(), + _ => catalog_tables(listed, (&source.include, &source.exclude))?, + }; + for ((name, _), rows) in named.iter().zip(described) { + tables.push(SourceTable { + split_pages: None, + schema: String::new(), + name: (*name).clone(), + columns: described_columns(rows) + .iter() + .map(|(column, data_type, nullable)| source_column(column, data_type, *nullable)) + .collect(), + primary_key: Vec::new(), + indexed_columns: Vec::new(), + estimated_rows: -1, + estimated_bytes: -1, + comment: None, + listed_keys: Vec::new(), + }); + } + Ok(tables) } diff --git a/crates/sqlload/src/duckdb_source/main/read_rows.rs b/crates/sqlload/src/duckdb_source/main/read_rows.rs index 798be486b..b86fd8c38 100644 --- a/crates/sqlload/src/duckdb_source/main/read_rows.rs +++ b/crates/sqlload/src/duckdb_source/main/read_rows.rs @@ -2,7 +2,8 @@ use arrow_array::RecordBatch; -use crate::duckdb_source::_helpers::connection::open_read_only; +use crate::adapters::motherduck_bridge::main::stream_rows::stream_rows; +use crate::duckdb_source::_helpers::connection::{open_bridge, open_read_only}; use crate::duckdb_source::_helpers::convert::{convert_batch, output_schema}; use crate::duckdb_source::_helpers::query::select_query; use crate::duckdb_source::constants::BATCHES_AHEAD; @@ -10,7 +11,7 @@ use crate::duckdb_source::errors::DuckDbSourceError; use crate::duckdb_source::models::{DuckDbReadRequest, DuckDbReader}; use crate::row_batches::main::loaded_batch::loaded_batch; -/// Read `request`'s rows and pass them to `on_batch` in batches of about `batch_rows`, each converted to the read columns' types; returns the row count. One statement streams the rows, so the read is consistent on its own; a thread reads at most a few batches ahead of `on_batch`, and stops when `on_batch` fails. +/// Read `request`'s rows and pass them to `on_batch` in batches of about `batch_rows`, each converted to the read columns' types; returns the row count. One statement streams the rows, so the read is consistent on its own. From a file, a thread reads at most a few batches ahead of `on_batch`; from MotherDuck, the bridge reads one batch at a time as it is asked. Either stops when `on_batch` fails. pub async fn read_rows( reader: &DuckDbReader, request: &DuckDbReadRequest<'_>, @@ -18,6 +19,22 @@ pub async fn read_rows( mut on_batch: impl AsyncFnMut(RecordBatch) -> Result<(), String>, ) -> Result { let shape = output_schema(request.columns).map_err(DuckDbSourceError::Invalid)?; + if reader.motherduck.is_some() { + let bridge = open_bridge(reader).await?; + let converted = async |batch: RecordBatch| { + let batch = loaded_batch(&batch, request.table) + .and_then(|batch| convert_batch(&batch, (&shape.0, &shape.1)))?; + on_batch(batch).await + }; + return stream_rows(&bridge, select_query(request), batch_rows, converted) + .await + .map_err(|error| match error { + crate::adapters::motherduck_bridge::errors::BridgeError::Consumer(said) => { + DuckDbSourceError::Consumer(said) + } + other => DuckDbSourceError::Bridge(other), + }); + } let (reader, query) = (reader.clone(), select_query(request)); let (sender, mut receiver) = tokio::sync::mpsc::channel::>(BATCHES_AHEAD); diff --git a/crates/sqlload/src/duckdb_source/models.rs b/crates/sqlload/src/duckdb_source/models.rs index 858bb8c2a..2422dd091 100644 --- a/crates/sqlload/src/duckdb_source/models.rs +++ b/crates/sqlload/src/duckdb_source/models.rs @@ -1,10 +1,17 @@ +use crate::adapters::motherduck_bridge::models::BridgeConnection; use crate::postgres_source::models::{ReadColumn, SourceTable}; +use crate::project::models::ConnectionConfig; -/// A DuckDB source: the database file it reads, opened read-only for each read, and the `memory_limit` each of its connections gets (half of a table's share of the memory budget; DuckDB's own default without a budget). +/// A DuckDB or MotherDuck source: the database it reads, opened for each read (a file read-only, a MotherDuck database on the MotherDuck bridge), and the `memory_limit` each connection to a file gets (half of a table's share of the memory budget; DuckDB's own default without a budget). #[derive(Debug, Clone)] pub struct DuckDbReader { + /// The database file, or a MotherDuck database's `md:` path as written. pub(crate) database: String, pub(crate) memory_limit: Option, + /// A `motherduck` source's connection; `None` for a database file. + pub(crate) motherduck: Option, + /// A `motherduck` source's bridge, shared by the reader's clones and opened on its first read; each read takes a connection of its own on it. + pub(crate) bridge: std::sync::Arc>, } /// What one read selects: a table or a query, its columns, and the rows' filter. diff --git a/crates/sqlload/src/loader/_helpers/sources.rs b/crates/sqlload/src/loader/_helpers/sources.rs index 8aecd2a45..3d7d11de2 100644 --- a/crates/sqlload/src/loader/_helpers/sources.rs +++ b/crates/sqlload/src/loader/_helpers/sources.rs @@ -238,11 +238,13 @@ pub(crate) async fn open_source( project, source, )?))), #[cfg(feature = "duckdb")] - SourceKind::DuckDb => Ok(SourceReader::DuckDb(Box::new(duckdb_reader( - project, source, - )?))), + SourceKind::DuckDb | SourceKind::MotherDuck => Ok(SourceReader::DuckDb(Box::new( + duckdb_reader(project, source)?, + ))), #[cfg(not(feature = "duckdb"))] - SourceKind::DuckDb => Err(LoaderError::Invalid(NO_DUCKDB_SOURCE.to_owned())), + SourceKind::DuckDb | SourceKind::MotherDuck => { + Err(LoaderError::Invalid(NO_DUCKDB_SOURCE.to_owned())) + } SourceKind::Python => Err(LoaderError::Invalid( "a Python source is read by calling its table functions".to_owned(), )), @@ -388,7 +390,7 @@ pub(crate) async fn source_tables( .map_err(|error| LoaderError::from(error).into())) } #[cfg(feature = "duckdb")] - SourceKind::DuckDb => { + SourceKind::DuckDb | SourceKind::MotherDuck => { let read = async { read_duckdb_catalog(&duckdb_reader(project, source)?, source).await }; Ok(read @@ -397,7 +399,9 @@ pub(crate) async fn source_tables( .map_err(|error| LoaderError::from(error).into())) } #[cfg(not(feature = "duckdb"))] - SourceKind::DuckDb => Ok(Err(LoaderError::Invalid(NO_DUCKDB_SOURCE.to_owned()).into())), + SourceKind::DuckDb | SourceKind::MotherDuck => { + Ok(Err(LoaderError::Invalid(NO_DUCKDB_SOURCE.to_owned()).into())) + } SourceKind::Python => Ok(Ok((python_tables(source), None, Vec::new()))), SourceKind::GoogleSheets => Ok(match sheets_tables(project, source).await { Ok((tables, warnings)) => Ok((tables, None, warnings)), diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs index 729ba03d2..e9e0d0ca8 100644 --- a/crates/sqlload/src/loader/_helpers/specs.rs +++ b/crates/sqlload/src/loader/_helpers/specs.rs @@ -18,7 +18,7 @@ use crate::column_types::main::is_variant::is_variant; use crate::column_types::main::parse_type::parse_type; use crate::column_types::models::{ColumnType, Warehouse}; use crate::databricks_source::constants::DATABRICKS_SOURCE; -use crate::duckdb_source::constants::DUCKDB_SOURCE; +use crate::duckdb_source::constants::{DUCKDB_SOURCE, MOTHERDUCK_SOURCE}; use crate::files_source::main::constant_columns::constant_columns; use crate::files_source::main::path_template::path_template; use crate::files_source::models::FileColumn; @@ -532,12 +532,14 @@ pub(crate) fn build_spec( kind @ (SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks - | SourceKind::DuckDb) => Some(WarehouseSource { + | SourceKind::DuckDb + | SourceKind::MotherDuck) => Some(WarehouseSource { system: match kind { SourceKind::Odbc => ODBC_SOURCE, SourceKind::BigQuery => BIGQUERY_SOURCE, SourceKind::Databricks => DATABRICKS_SOURCE, - _ => DUCKDB_SOURCE, + SourceKind::DuckDb => DUCKDB_SOURCE, + _ => MOTHERDUCK_SOURCE, }, sql: overrides.sql.clone(), route: WarehouseRoute::Query, @@ -783,7 +785,8 @@ pub(crate) fn missing_description( | SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks - | SourceKind::DuckDb => format!( + | SourceKind::DuckDb + | SourceKind::MotherDuck => format!( "no description; add description = \"...\" under [sources.{source_name}.tables.\"{qualified}\"]" ), }; diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index 35b08230f..c8a357d47 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -433,7 +433,10 @@ pub(crate) fn read_columns(spec: &TableSpec) -> Result, Vec { + (None, None) + if system(crate::duckdb_source::constants::DUCKDB_SOURCE) + || system(crate::duckdb_source::constants::MOTHERDUCK_SOURCE) => + { crate::duckdb_source::main::read_columns::read_columns( &selected_columns(spec), &spec.pinned, diff --git a/crates/sqlload/src/loader/main/_check_source.rs b/crates/sqlload/src/loader/main/_check_source.rs index abb71ec9b..b398a4754 100644 --- a/crates/sqlload/src/loader/main/_check_source.rs +++ b/crates/sqlload/src/loader/main/_check_source.rs @@ -48,12 +48,12 @@ pub(in crate::loader) async fn check_source( } SourceKind::Databricks => check_databricks(project, source).await, #[cfg(feature = "duckdb")] - SourceKind::DuckDb => { + SourceKind::DuckDb | SourceKind::MotherDuck => { let reader = crate::duckdb_source::main::duckdb_reader::duckdb_reader(project, source)?; Ok(crate::duckdb_source::main::describe_source::describe_source(&reader).await?) } #[cfg(not(feature = "duckdb"))] - SourceKind::DuckDb => Err(LoaderError::Invalid( + SourceKind::DuckDb | SourceKind::MotherDuck => Err(LoaderError::Invalid( crate::loader::constants::NO_DUCKDB_SOURCE.to_owned(), )), #[cfg(feature = "odbc")] diff --git a/crates/sqlload/src/project/_helpers/references.rs b/crates/sqlload/src/project/_helpers/references.rs index b18cc4592..53acfba9a 100644 --- a/crates/sqlload/src/project/_helpers/references.rs +++ b/crates/sqlload/src/project/_helpers/references.rs @@ -107,6 +107,7 @@ fn source_problems(project: &ProjectConfig, name: &str, source: &SourceConfig) - SourceKind::BigQuery => (&["bigquery"], "bigquery"), SourceKind::Databricks => (&["databricks"], "databricks"), SourceKind::DuckDb => (&["duckdb"], "duckdb"), + SourceKind::MotherDuck => (&["motherduck"], "motherduck"), SourceKind::Files => (&["s3", "filesystem", "sftp"], "s3, filesystem or sftp"), SourceKind::Python => (&[], "any"), SourceKind::GoogleSheets => (&["google_sheets"], "google_sheets"), @@ -230,10 +231,11 @@ fn kind_problems(place: &str, source: &SourceConfig) -> Vec { | SourceKind::BigQuery | SourceKind::Databricks | SourceKind::DuckDb + | SourceKind::MotherDuck ) && queries { problems.push(format!( - "{place} sql is for kind = \"snowflake\", \"odbc\", \"bigquery\", \"databricks\" and \"duckdb\"" + "{place} sql is for kind = \"snowflake\", \"odbc\", \"bigquery\", \"databricks\", \"duckdb\" and \"motherduck\"" )); } if source.kind != SourceKind::BigQuery && source.project.is_some() { @@ -281,10 +283,11 @@ fn kind_problems(place: &str, source: &SourceConfig) -> Vec { | SourceKind::BigQuery | SourceKind::Databricks | SourceKind::DuckDb + | SourceKind::MotherDuck ); if !selects && sheet_settings(source).any(|settings| settings.row_filter.is_some()) { problems.push(format!( - "{place} where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\", \"databricks\" and \"duckdb\", whose tables SQLLoad reads with a SELECT" + "{place} where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\", \"databricks\", \"duckdb\" and \"motherduck\", whose tables SQLLoad reads with a SELECT" )); } problems.extend( @@ -311,14 +314,18 @@ fn kind_problems(place: &str, source: &SourceConfig) -> Vec { | SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks - | SourceKind::DuckDb => { + | SourceKind::DuckDb + | SourceKind::MotherDuck => { if source.include.is_empty() && !queries { let tables = match source.kind { SourceKind::Snowflake | SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks - | SourceKind::DuckDb => "; list tables in include, or give a table a sql query", + | SourceKind::DuckDb + | SourceKind::MotherDuck => { + "; list tables in include, or give a table a sql query" + } _ => "", }; problems.push(format!( diff --git a/crates/sqlload/src/project/models.rs b/crates/sqlload/src/project/models.rs index 0025cfcb2..2869c819f 100644 --- a/crates/sqlload/src/project/models.rs +++ b/crates/sqlload/src/project/models.rs @@ -620,6 +620,7 @@ impl SourceConfig { | SourceKind::BigQuery | SourceKind::Databricks | SourceKind::DuckDb + | SourceKind::MotherDuck ) } @@ -679,6 +680,7 @@ impl SourceConfig { | SourceKind::BigQuery | SourceKind::Databricks | SourceKind::DuckDb + | SourceKind::MotherDuck | SourceKind::Python, ) => DEFAULT_TABLE_NAME, (None, SourceKind::Files) => DEFAULT_FILES_TABLE_NAME, @@ -708,6 +710,9 @@ pub enum SourceKind { /// Tables, views and queries of a DuckDB database file, read as Arrow from a read-only connection. #[serde(rename = "duckdb")] DuckDb, + /// Tables, views and queries of a MotherDuck database, read as Arrow through the MotherDuck bridge. + #[serde(rename = "motherduck")] + MotherDuck, /// Declared in Python with `sl.source` or `@sl.table`; never written in `sqlload.toml`. Python, /// Tabs or ranges of a Google Sheets spreadsheet, one table each. @@ -726,6 +731,7 @@ impl SourceKind { Self::BigQuery => "bigquery", Self::Databricks => "databricks", Self::DuckDb => "duckdb", + Self::MotherDuck => "motherduck", Self::Python => "python", Self::GoogleSheets => "google_sheets", } diff --git a/crates/sqlload/src/row_deletes/main/deletes_problems.rs b/crates/sqlload/src/row_deletes/main/deletes_problems.rs index 3083df54f..9b2e291db 100644 --- a/crates/sqlload/src/row_deletes/main/deletes_problems.rs +++ b/crates/sqlload/src/row_deletes/main/deletes_problems.rs @@ -28,6 +28,7 @@ pub fn deletes_problems( | SourceKind::BigQuery | SourceKind::Databricks | SourceKind::DuckDb + | SourceKind::MotherDuck ) { problems.push(format!( "deletes reads every key of the source table to find the ones it no longer holds, which only database and warehouse sources list; a {} source can't. Fix: remove deletes", diff --git a/packages/sqlload/pyproject.toml b/packages/sqlload/pyproject.toml index a72863c18..f5cc4ed52 100644 --- a/packages/sqlload/pyproject.toml +++ b/packages/sqlload/pyproject.toml @@ -21,10 +21,10 @@ classifiers = [ dependencies = ["orjson>=3.10", "tzdata; sys_platform == \"win32\""] [project.optional-dependencies] -# MotherDuck targets: MotherDuck accepts DuckDB 1.4.1 to 1.5.6 (raise the bound as MotherDuck does), not the DuckDB -# 2.0 SQLLoad bundles, so SQLLoad runs their SQL on this package in a separate Python process. Remove once MotherDuck -# accepts SQLLoad's DuckDB. -motherduck = ["duckdb>=1.4.1,<=1.5.6"] +# MotherDuck targets and sources: MotherDuck accepts DuckDB 1.4.1 to 1.5.6 (raise the bound as MotherDuck does), not the +# DuckDB 2.0 SQLLoad bundles, so SQLLoad runs their SQL on this package in a separate Python process; a source's rows +# stream from it as Arrow batches through pyarrow. Remove once MotherDuck accepts SQLLoad's DuckDB. +motherduck = ["duckdb>=1.4.1,<=1.5.6", "pyarrow>=14"] [project.scripts] sqlload = "sqlload._cli:main" diff --git a/packages/sqlload/tests/test_motherduck.py b/packages/sqlload/tests/test_motherduck.py index 8d2e27426..6d7e00388 100644 --- a/packages/sqlload/tests/test_motherduck.py +++ b/packages/sqlload/tests/test_motherduck.py @@ -45,7 +45,8 @@ #: after checking the token, with another database attached as MotherDuck attaches the account's others. MD_REFUSE makes #: it refuse as MotherDuck's extension does on a DuckDB version it doesn't support; MD_DIE_ON ends the bridge when a #: statement holds that text; MD_LOG records every statement. MD_CLIENT_VERSION is the duckdb version the bridge reports, -#: and MD_SERVER_VERSION what `md_version()` answers (without it, the function doesn't exist). +#: and MD_SERVER_VERSION what `md_version()` answers (without it, the function doesn't exist). MD_READ_ONLY opens each +#: database read-only, so a source's bridges (one per table) can hold it at once as MotherDuck allows. STAND_IN = r''' import os import sys @@ -102,7 +103,10 @@ def connect(database=":memory:", *args, config=None, **kwargs): 'threw an exception: "\nYour DuckDB version (v2.0.0) is not yet supported by MotherDuck. The latest supported ' f'version is v1.5.6. Please downgrade to use MotherDuck. {database}\n"' ) - inner = _connect(os.path.join(FOLDER, f"{name}.duckdb"), config=config or {}) + read_only = {"access_mode": "READ_ONLY"} if os.environ.get("MD_READ_ONLY") else {} + inner = _connect(os.path.join(FOLDER, f"{name}.duckdb"), config={**(config or {}), **read_only}) + if read_only: + return Database(inner) inner.execute(f"ATTACH '{os.path.join(FOLDER, 'other.duckdb')}' AS other") inner.execute("CREATE SCHEMA IF NOT EXISTS other.raw; CREATE TABLE IF NOT EXISTS other.raw.cloud__cloud_rows (stray INTEGER)") if os.environ.get("MD_SERVER_VERSION"): @@ -487,3 +491,83 @@ def test_given_json_type_variant_on_motherduck_when_planning_and_loading_then_th loaded = load(project, "cloud_rows", **env) assert ("unshredded" in loaded.stdout + loaded.stderr) == ("unshredded" in expected), loaded.stdout + loaded.stderr assert cloud(project, 'SELECT variant_typeof(payload), CAST(payload.user.id AS BIGINT) FROM raw.cloud__cloud_rows') == [("OBJECT(user)", 42)] + + +#: A project reading the stand-in's `shop` database as a motherduck source into the local DuckDB file. +SOURCE_PROJECT = """\ +[connections.wh] +kind = "duckdb" +database = "wh.duckdb" + +[connections.md] +kind = "motherduck" +database = "shop" +token = "${ENV:SHOP_MD_TOKEN}" + +[targets.local] +connection = "wh" +loader_schema = "raw" + +[sources.shop] +kind = "motherduck" +connection = "md" +destination = "local" +description = "The shop's MotherDuck database." +include = ["sales.*"] +exclude = ["sales.tmp_*"] +exclude_columns = ["card"] +write_strategy = "replace" + +[sources.shop.tables."sales.orders"] +description = "Orders, merged after their cursor." +write_strategy = "merge" +unique_key = ["id"] +cursor_column = "updated_at" +where = "region = 'EU'" + +[sources.shop.tables.big_query] +description = "Orders over 25, by query." +sql = "SELECT id, amount FROM sales.orders WHERE amount > 25 -- big ones" +""" + +SHOP = """ +CREATE SCHEMA sales; +CREATE TABLE sales.orders (id INTEGER PRIMARY KEY, amount DECIMAL(12,2), updated_at TIMESTAMP, tags VARCHAR[], card VARCHAR, region VARCHAR); +INSERT INTO sales.orders VALUES + (1, 10.50, '2026-01-01 00:00:00', ['a'], '4111', 'EU'), + (2, 20.00, '2026-01-02 00:00:00', [], '4222', 'EU'), + (3, 30.25, '2026-01-03 00:00:00', NULL, '4333', 'US'); +CREATE VIEW sales.big_orders AS SELECT id, amount FROM sales.orders WHERE amount > 15; +CREATE TABLE sales.tmp_scratch (x INTEGER); +""" + +ORDERS = "SELECT id, CAST(amount AS VARCHAR), CAST(tags AS VARCHAR) FROM raw.shop__orders ORDER BY id" + + +def test_given_a_motherduck_source_when_loading_again_then_its_tables_views_and_queries_load_through_the_bridge_and_the_cursor_merges_changes( + tmp_path: Path, +) -> None: + project = make_project(tmp_path, "") + (project / "sqlload.toml").write_text(SOURCE_PROJECT) + with duckdb.connect(str(project / "md" / "shop.duckdb")) as shop: + shop.execute(SHOP) + statements = project / "statements.log" + debug = sqlload(project, "debug", MD_LOG=str(statements), MD_READ_ONLY="1") + assert debug.returncode == 0 and "shop on MotherDuck, through DuckDB" in debug.stdout, debug.stdout + debug.stderr + assert TOKEN not in debug.stdout + debug.stderr + planned = sqlload(project, "plan", MD_READ_ONLY="1") + assert planned.returncode == 0, planned.stdout + planned.stderr + assert "tmp_scratch" not in planned.stdout, planned.stdout + load(project, MD_READ_ONLY="1") + assert local(project, ORDERS) == [(1, "10.50", "[\"a\"]"), (2, "20.00", "[]")] + assert local(project, "SELECT count(*) FROM raw.shop__big_orders") == [(2,)] + assert local(project, "SELECT id FROM raw.shop__big_query ORDER BY id") == [(3,)] + columns = "SELECT column_name FROM information_schema.columns WHERE table_name = 'shop__orders' AND column_name NOT LIKE '\\_sqlload%' ESCAPE '\\'" + assert sorted(row[0] for row in local(project, columns)) == ["amount", "id", "region", "tags", "updated_at"] + with duckdb.connect(str(project / "md" / "shop.duckdb")) as shop: + shop.execute("UPDATE sales.orders SET amount = 21.00, updated_at = '2026-01-05 00:00:00' WHERE id = 2") + shop.execute("INSERT INTO sales.orders VALUES (4, 40.00, '2026-01-06 00:00:00', ['b', 'c'], '4444', 'EU')") + load(project, MD_LOG=str(statements), MD_READ_ONLY="1") + assert local(project, ORDERS) == [(1, "10.50", "[\"a\"]"), (2, "21.00", "[]"), (4, "40.00", "[\"b\",\"c\"]")] + reads = [line for line in statements.read_text().splitlines() if line.startswith("SELECT \"id\"") and "sales\".\"orders\"" in line] + assert reads and all("\"updated_at\" >" in read for read in reads), reads diff --git a/packages/sqlload/tests/test_motherduck_live.py b/packages/sqlload/tests/test_motherduck_live.py index 326db1b73..709720ae3 100644 --- a/packages/sqlload/tests/test_motherduck_live.py +++ b/packages/sqlload/tests/test_motherduck_live.py @@ -12,6 +12,7 @@ from pathlib import Path from typing import Any +import duckdb import pytest TOKEN_VARIABLE = "SQLLOAD_TEST_MOTHERDUCK_TOKEN" @@ -80,13 +81,13 @@ def rows(ctx): @pytest.fixture(scope="module") def python(tmp_path_factory: pytest.TempPathFactory) -> str: - """A Python with duckdb 1.5.6, the newest version MotherDuck accepts.""" + """A Python with duckdb 1.5.6, the newest version MotherDuck accepts, and pyarrow, which a source's rows stream through.""" folder = tmp_path_factory.mktemp("duckdb15") uv = shutil.which("uv") assert uv, "the live MotherDuck tests need uv to build a duckdb 1.5.6 environment" subprocess.run([uv, "venv", "--quiet", "--python", "3.12", str(folder / "venv")], check=True) executable = folder / "venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python") - subprocess.run([uv, "pip", "install", "--quiet", "--python", str(executable), "duckdb==1.5.6"], check=True) + subprocess.run([uv, "pip", "install", "--quiet", "--python", str(executable), "duckdb==1.5.6", "pyarrow"], check=True) return str(executable) @@ -200,3 +201,67 @@ def test_given_json_type_variant_when_loading_into_live_motherduck_then_dicts_la assert admin( python, f"SELECT variant_typeof(payload), CAST(payload.user.id AS BIGINT) FROM {database}.raw.cloud__customers" ) == [["OBJECT(user)", 42]] + + +SOURCE_PROJECT = """\ +[connections.md] +kind = "motherduck" +database = "{database}" +token = "${{ENV:{variable}}}" + +[connections.wh] +kind = "duckdb" +database = "wh.duckdb" + +[targets.local] +connection = "wh" +loader_schema = "raw" + +[sources.shop] +kind = "motherduck" +connection = "md" +destination = "local" +description = "The shop's MotherDuck database." +include = ["sales.*"] +exclude_columns = ["card"] +write_strategy = "replace" + +[sources.shop.tables."sales.orders"] +description = "Orders, merged after their cursor." +write_strategy = "merge" +unique_key = ["id"] +cursor_column = "updated_at" +where = "region = 'EU'" + +[sources.shop.tables.big_query] +description = "Orders over 25, by query." +sql = "SELECT id, amount FROM sales.orders WHERE amount > 25" +""" + + +def test_given_a_motherduck_source_when_loading_again_then_its_tables_and_queries_land_in_duckdb_and_the_cursor_merges_changes( + tmp_path: Path, python: str, database: str +) -> None: + admin( + python, + f"CREATE SCHEMA {database}.sales", + f"CREATE TABLE {database}.sales.orders (id INTEGER PRIMARY KEY, amount DECIMAL(12,2), updated_at TIMESTAMP, tags VARCHAR[], card VARCHAR, region VARCHAR)", + f"INSERT INTO {database}.sales.orders VALUES (1, 10.50, '2026-01-01', ['a'], '4111', 'EU'), (2, 20.00, '2026-01-02', [], '4222', 'EU'), (3, 30.25, '2026-01-03', NULL, '4333', 'US')", + ) + (tmp_path / "sqlload.toml").write_text(SOURCE_PROJECT.format(database=database, variable=TOKEN_VARIABLE)) + debug = sqlload(tmp_path, python, "debug") + assert debug.returncode == 0 and f"{database} on MotherDuck, through DuckDB v1.5.6" in debug.stdout, debug.stdout + debug.stderr + load(tmp_path, python) + admin( + python, + f"UPDATE {database}.sales.orders SET amount = 21.00, updated_at = '2026-01-05' WHERE id = 2", + f"INSERT INTO {database}.sales.orders VALUES (4, 40.00, '2026-01-06', ['b', 'c'], '4444', 'EU')", + ) + load(tmp_path, python) + with duckdb.connect(str(tmp_path / "wh.duckdb"), read_only=True) as local: + orders = local.execute("SELECT id, CAST(amount AS VARCHAR), CAST(tags AS VARCHAR) FROM raw.shop__orders ORDER BY id").fetchall() + big = local.execute("SELECT id FROM raw.shop__big_query ORDER BY id").fetchall() + columns = local.execute("SELECT column_name FROM information_schema.columns WHERE table_name = 'shop__orders' ORDER BY ordinal_position").fetchall() + assert orders == [(1, "10.50", '["a"]'), (2, "21.00", "[]"), (4, "40.00", '["b","c"]')] + assert big == [(3,), (4,)] + assert "card" not in [column[0] for column in columns] diff --git a/website/src/content/docs/docs/configuration.mdx b/website/src/content/docs/docs/configuration.mdx index d06dffe4c..4bbd4ba6d 100644 --- a/website/src/content/docs/docs/configuration.mdx +++ b/website/src/content/docs/docs/configuration.mdx @@ -110,7 +110,7 @@ See [ClickHouse](/docs/targets/clickhouse/). ### `motherduck` -In beta. It connects through a separate +In beta, as a target and as a source. It connects through a separate DuckDB 1.5 process: install `sqlload[motherduck]`; see [MotherDuck](/docs/targets/motherduck/). | Key | Meaning | diff --git a/website/src/content/docs/docs/index.mdx b/website/src/content/docs/docs/index.mdx index 56bdb5dd6..da89d098a 100644 --- a/website/src/content/docs/docs/index.mdx +++ b/website/src/content/docs/docs/index.mdx @@ -16,7 +16,7 @@ SQLLoad loads data into your warehouse, and shows you how before it changes anyt | Databricks | beta | beta | | ClickHouse | | ✓ one self-hosted server¹ | | DuckDB | ✓ | ✓ | -| MotherDuck | | beta² | +| MotherDuck | beta² | beta² | | SQL Server, MySQL, MariaDB, OpenEdge and other databases through ODBC³ | ✓ | | | Files | ✓ Parquet, CSV, JSON, NDJSON, Excel (xlsx), XML; gzip, zstd and ZIP; any file name through `pattern` (S3, local folder, SFTP) | ✓ Parquet, CSV (S3, local folder) | | Google Sheets | ✓ | | diff --git a/website/src/content/docs/docs/sources/index.mdx b/website/src/content/docs/docs/sources/index.mdx index b20231d85..1b4f45275 100644 --- a/website/src/content/docs/docs/sources/index.mdx +++ b/website/src/content/docs/docs/sources/index.mdx @@ -1,9 +1,9 @@ --- title: "Sources" -description: "Postgres tables, Postgres changes, Snowflake tables and queries, ODBC databases, BigQuery and Databricks tables and queries (beta), DuckDB files, files in S3 or a folder, Google Sheets, and Python sources: the kinds of source and the settings they share." +description: "Postgres tables, Postgres changes, Snowflake tables and queries, ODBC databases, BigQuery, Databricks and MotherDuck tables and queries (beta), DuckDB files, files in S3 or a folder, Google Sheets, and Python sources: the kinds of source and the settings they share." --- -A source is where data comes from. SQLLoad has ten kinds: +A source is where data comes from. SQLLoad has eleven kinds: | `kind` | Reads | Tables | Declared in | |--------|-------|--------|-------------| @@ -14,6 +14,7 @@ A source is where data comes from. SQLLoad has ten kinds: | [`bigquery`](/docs/sources/bigquery/) (beta) | Tables, views and query results of BigQuery, read as Arrow through the Storage Read API | One per matching table, and one per query | `sqlload.toml` | | [`databricks`](/docs/sources/databricks/) (beta) | Tables, views and query results of a Databricks SQL warehouse, read as Arrow through the Statement Execution API | One per matching table, and one per query | `sqlload.toml` | | [`duckdb`](/docs/sources/duckdb/) | Tables, views and query results of a DuckDB database file, read as Arrow | One per matching table, and one per query | `sqlload.toml` | +| [`motherduck`](/docs/sources/motherduck/) (beta) | Tables, views and query results of a MotherDuck database, read as Arrow through a separate DuckDB 1.5 process | One per matching table, and one per query | `sqlload.toml` | | [`files`](/docs/sources/files/) | Parquet, CSV, JSON or NDJSON files in S3 or a local folder | One per source | `sqlload.toml` | | [`google_sheets`](/docs/sources/google-sheets/) | Tabs or ranges of a Google Sheets spreadsheet, typed | One per tab or range | `sqlload.toml` | | [Python](/docs/sources/python/) | Whatever a Python function yields: dicts, Arrow data, JSON bytes, HTTP jobs | One per decorated function | Python, under `python_paths` | @@ -39,6 +40,8 @@ Each kind has its own page: cost and permissions. - [DuckDB](/docs/sources/duckdb/): choosing tables, query tables, types, and reading a file another process writes. +- [MotherDuck](/docs/sources/motherduck/) (beta): installing the separate DuckDB 1.5 process, and + what reads as for a DuckDB source. - [Files](/docs/sources/files/): formats, partitions, which files load, and files the destination reads itself. - [Google Sheets](/docs/sources/google-sheets/): tabs and ranges, headers, and how cells are typed. diff --git a/website/src/content/docs/docs/sources/motherduck.mdx b/website/src/content/docs/docs/sources/motherduck.mdx new file mode 100644 index 000000000..508c0a325 --- /dev/null +++ b/website/src/content/docs/docs/sources/motherduck.mdx @@ -0,0 +1,77 @@ +--- +title: "MotherDuck" +description: "Load tables, views and query results from a MotherDuck database (beta), read as Arrow through a separate DuckDB 1.5 process: connection, choosing tables, query tables, cursors and types." +sidebar: + badge: + text: Beta + variant: caution +--- + +This source is in beta: it is tested, including live against MotherDuck, but gets less everyday use than the other connectors, so expect rough edges, and please report them in [GitHub issues](https://github.com/sqlstack-io/sqlload/issues). + +A `motherduck` source loads tables, views and query results from a MotherDuck database. It reads +the way a [DuckDB source](/docs/sources/duckdb/) does: one `SELECT` per read, streamed as Arrow, +with the same table choice, cursors, `where` and types. + +```toml +[connections.cloud] +kind = "motherduck" +database = "shop" +token = "${ENV:MOTHERDUCK_TOKEN}" + +[sources.shop] +kind = "motherduck" +connection = "cloud" +description = "Tables of the shop's MotherDuck database." +include = ["sales.*"] +write_strategy = "replace" + +[sources.shop.tables."sales.orders"] +description = "Orders, one row each." +cursor_column = "updated_at" +write_strategy = "merge" +unique_key = "{primary_key}" + +[sources.shop.tables.big_orders] +description = "Orders over 100." +sql = "SELECT * FROM sales.orders WHERE amount > 100" +``` + +## Install + +MotherDuck doesn't accept the DuckDB 2.0 that SQLLoad bundles yet, so SQLLoad reads a MotherDuck +source through the same separate Python process a [MotherDuck target](/docs/targets/motherduck/#install) +uses, running Python's `duckdb` 1.4.1 to 1.5.6. That process hands each batch to SQLLoad as Arrow, so +it also needs `pyarrow`. The `motherduck` extra installs both: + +```bash +pip install 'sqlload[motherduck]' +``` + +With `sqlload-cli`, set `SQLLOAD_MOTHERDUCK_PYTHON` to a Python that has `duckdb` 1.4.1 to 1.5.6 and +`pyarrow`. + +## Connection + +A source uses a [`motherduck` connection](/docs/configuration/#motherduck), the same kind a +MotherDuck target uses. Its `database` names the one database the source reads; a connection +without one is refused, as MotherDuck would otherwise attach every database in the account. The +token comes from `token`, else the `motherduck_token` or `MOTHERDUCK_TOKEN` environment variable, +and never appears in messages. + +## Choosing tables and reading them + +`include`, `exclude`, query tables, `cursor_column`, `where`, `exclude_columns` and pinned +`columns` work as for a [DuckDB source](/docs/sources/duckdb/#choosing-tables): SQLLoad lists the +database's own tables and views from DuckDB's catalog functions, and writes every filter in DuckDB +SQL. Each read is one statement on its own connection; the process reads one batch at a time, as +SQLLoad asks for it, so neither holds the whole table. + +Types map as on a [DuckDB source](/docs/sources/duckdb/#types). + +## Tested + +Live against a MotherDuck account on DuckDB 1.5.6: a table, a cursor merge that reads only changed +rows, `where`, `exclude_columns`, a list column and a query table, loaded twice into DuckDB. +Offline, the same load runs through the separate process against a stand-in for MotherDuck that +keeps the database in a local DuckDB file, with the token checked and kept out of every message. diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx index 09b3d759e..3fc7dee3c 100644 --- a/website/src/content/docs/docs/supported-systems.mdx +++ b/website/src/content/docs/docs/supported-systems.mdx @@ -14,6 +14,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes | BigQuery (beta) | Tables and views, by `include` patterns, and query results | The Storage Read API, as Arrow in parallel streams; views and queries through a query job. See [BigQuery](/docs/sources/bigquery/). | | Databricks (beta) | Tables and views, by `include` patterns, and query results | One `SELECT` on the SQL warehouse, its result downloaded as Arrow files in parallel through the Statement Execution API's links. See [Databricks](/docs/sources/databricks/). | | DuckDB | Tables and views of a database file, by `include` patterns, and query results | One `SELECT` on a read-only connection, streamed as Arrow. See [DuckDB](/docs/sources/duckdb/). | +| MotherDuck (beta) | Tables and views of one database, by `include` patterns, and query results | One `SELECT` run by a separate DuckDB 1.5 process (`sqlload[motherduck]`), its Arrow batches handed over one at a time. See [MotherDuck](/docs/sources/motherduck/). | | Files in S3 | Parquet, CSV, JSON, NDJSON, Excel (xlsx) and XML; gzip or zstd for text formats; ZIP archives of any of them; any file name through `pattern`. See [Files](/docs/sources/files/#excel-xml-and-zip-files). | The destination itself where it can load them exactly, otherwise SQLLoad's reader | | Files in a local folder | As for S3 | As for S3 | | Files on an SFTP server | As for S3 | SQLLoad's reader, over SFTP. See [Files](/docs/sources/files/#files-on-an-sftp-server). | @@ -47,6 +48,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes | BigQuery (beta) | Yes, untested | Yes, tested live | Yes, untested | Yes, untested | Yes, untested | Yes | Yes, untested | | Databricks (beta) | Yes, untested | Yes, untested | Yes, untested | Yes, tested live | Yes, tested live | Yes, tested live | Yes, tested live | | DuckDB | Yes, untested | Yes, untested | Yes, untested | Yes, untested | Yes, untested | Yes | Yes, untested | +| MotherDuck (beta) | Yes, untested | Yes, untested | Yes, untested | Yes, untested | Yes, untested | Yes, tested live | Yes, untested | | Files | Yes, read by Snowflake from an external stage | Yes, staged by SQLLoad in the volume | Yes, read by SQLLoad and streamed | Yes; Parquet in S3 read by ClickHouse | Yes, read by SQLLoad | Yes; Parquet read by DuckDB | Read and rewritten by SQLLoad | | Files on an SFTP server | Yes, read by SQLLoad, tested live | Yes, read by SQLLoad, tested live | Yes, untested | Yes, untested | Yes, untested | Yes | Yes, untested | | Google Sheets | Yes | Yes | Yes | Yes | Yes | Yes | Yes | @@ -67,7 +69,7 @@ them into DuckDB. | Postgres | Real Postgres in containers, as a source and as a target. `postgres_cdc` on Postgres 15, 16 and 17, plus live loads into DuckDB, Snowflake and Databricks. See [Postgres](/docs/targets/postgres/#tested). | | ClickHouse | Real ClickHouse 25.3, 25.8 and 26.10 in containers, with an S3-compatible store for the files it reads itself. See [ClickHouse](/docs/targets/clickhouse/#tested). | | DuckDB | Real, in process, as a source and a target. | -| MotherDuck | Live against a MotherDuck account on DuckDB 1.5.6: every write strategy, an incremental load and rerun, a new column and every portable type. Offline, every change loads through the separate process into a stand-in for MotherDuck and compares with DuckDB. See [MotherDuck](/docs/targets/motherduck/#tested). | +| MotherDuck | Live against a MotherDuck account on DuckDB 1.5.6: every write strategy, an incremental load and rerun, a new column and every portable type. Offline, every change loads through the separate process into a stand-in for MotherDuck and compares with DuckDB. As a source, live and offline: tables, a query table, `where`, `exclude_columns` and a cursor merge into DuckDB. See [MotherDuck](/docs/targets/motherduck/#tested). | | ODBC | psqlODBC against real Postgres in containers, into DuckDB, plus a live round from SQL Server and MySQL into Databricks. | | Databricks source | A fake Statement Execution API in tests (catalog discovery, result files in several chunks, an expired link fetched again, cursor reads, no token on downloads), plus a live round from a real SQL warehouse into DuckDB, Parquet files, Postgres and ClickHouse: tables, a view, a windowed query table, merges and every column shape. See [Databricks](/docs/sources/databricks/#tested). | | BigQuery source | A fake Storage Read API in tests (parallel streams, a broken stream resumed, cursor filters), and live loads from a sandbox project into DuckDB: tables, views, query tables, and integer, timestamp, text, `NUMERIC` and `BIGNUMERIC` cursors, plus a live round into Databricks. | diff --git a/website/src/content/docs/docs/targets/motherduck.mdx b/website/src/content/docs/docs/targets/motherduck.mdx index 1af549127..bfedab1fa 100644 --- a/website/src/content/docs/docs/targets/motherduck.mdx +++ b/website/src/content/docs/docs/targets/motherduck.mdx @@ -20,7 +20,8 @@ So SQLLoad sends a MotherDuck target's SQL to a separate Python process running package at a version MotherDuck accepts. The two DuckDB versions never share a process, and every other target still uses SQLLoad's own DuckDB. -Install the `motherduck` extra, which adds `duckdb>=1.4.1,<=1.5.6`: +Install the `motherduck` extra, which adds `duckdb>=1.4.1,<=1.5.6`, and `pyarrow`, which a +[MotherDuck source](/docs/sources/motherduck/) reads through: ```bash uv tool install 'sqlload[motherduck]' diff --git a/website/src/sidebar.json b/website/src/sidebar.json index da3011bfa..ce78762c9 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -70,6 +70,8 @@ "docs/sources/snowflake", "docs/sources/bigquery", "docs/sources/databricks", + "docs/sources/duckdb", + "docs/sources/motherduck", "docs/sources/files", "docs/sources/google-sheets", "docs/sources/odbc", From ad3c6fbe20fa285b0946202c77b2bf2620785166 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:36:33 +0000 Subject: [PATCH 6/9] fix: keep a DuckDB source file locked while any read is open, count MotherDuck source bridges in the memory budget Reads of one file share one read-only instance, so ending one read no longer drops the process's lock on the file, and spills go to a temporary folder instead of beside the file. A MotherDuck source's bridge process is counted by the memory watchdog while it is open. The bridge never writes a batch file SQLLoad has already removed. --- .../motherduck_bridge/_helpers/bridge.py | 9 ++- .../src/duckdb_source/_helpers/connection.rs | 80 +++++++++++++++---- .../src/duckdb_source/main/duckdb_reader.rs | 2 + .../src/duckdb_source/main/read_rows.rs | 2 +- crates/sqlload/src/duckdb_source/models.rs | 34 +++++++- .../src/duckdb_source/tests/helpers.rs | 32 ++++++++ .../duckdb_source/tests/test_read_source.rs | 50 +++++++++++- .../src/duckdb_source/tests/test_types.rs | 10 +++ crates/sqlload/src/memory_budget/main/mod.rs | 1 + .../src/memory_budget/main/watch_child.rs | 18 +++++ crates/sqlload/src/memory_budget/models.rs | 20 +++++ .../src/memory_budget/tests/test_types.rs | 8 ++ .../src/memory_budget/tests/test_watch.rs | 57 ++++++++++++- 13 files changed, 300 insertions(+), 23 deletions(-) create mode 100644 crates/sqlload/src/memory_budget/main/watch_child.rs diff --git a/crates/sqlload/src/adapters/motherduck_bridge/_helpers/bridge.py b/crates/sqlload/src/adapters/motherduck_bridge/_helpers/bridge.py index 1d937a934..15dea9c41 100644 --- a/crates/sqlload/src/adapters/motherduck_bridge/_helpers/bridge.py +++ b/crates/sqlload/src/adapters/motherduck_bridge/_helpers/bridge.py @@ -46,15 +46,18 @@ def plain(value): def write_next(batches, path): - """The next batch `batches` holds, written to `path` as an Arrow IPC file; its row count, 0 once every batch is read.""" + """The next batch `batches` holds, written to the existing file `path` as an Arrow IPC file; its row count, 0 once every batch is read. A missing `path` fails.""" import pyarrow.ipc try: batch = batches.read_next_batch() except StopIteration: return 0 - with pyarrow.ipc.new_file(path, batch.schema) as writer: - writer.write_batch(batch) + # SQLLoad created the file, readable by its user only; opening it rather than creating it means a file SQLLoad has already removed (its read stopped) is never written again. + with open(path, "r+b") as sink: + sink.truncate() + with pyarrow.ipc.new_file(sink, batch.schema) as writer: + writer.write_batch(batch) return batch.num_rows diff --git a/crates/sqlload/src/duckdb_source/_helpers/connection.rs b/crates/sqlload/src/duckdb_source/_helpers/connection.rs index f5c584662..8dbbccfc2 100644 --- a/crates/sqlload/src/duckdb_source/_helpers/connection.rs +++ b/crates/sqlload/src/duckdb_source/_helpers/connection.rs @@ -1,28 +1,74 @@ //! Connections to a DuckDB source's database: a file read-only, or a MotherDuck database on the MotherDuck bridge. +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::path::PathBuf; +use std::sync::{Mutex, OnceLock, PoisonError}; + use crate::adapters::dispatch::main::session::open_source_bridge::open_source_bridge; use crate::adapters::motherduck_bridge::main::fork_connection::fork_connection; use crate::adapters::motherduck_bridge::main::run_statement::run_statement; use crate::adapters::motherduck_bridge::models::BridgeConnection; use crate::duckdb_source::errors::DuckDbSourceError; -use crate::duckdb_source::models::DuckDbReader; +use crate::duckdb_source::models::{DuckDbReader, FileHold, OpenFile, ReadOnlyFile, SourceBridge}; +use crate::memory_budget::main::watch_child::watch_child; /// Rows of text values, `None` for `NULL`. pub(crate) type TextRows = Vec>>; -/// The reader's database opened read-only, so a file another process writes is never changed and SQLLoad holds no write lock on it, with the reader's `memory_limit`. -pub(crate) fn open_read_only( - reader: &DuckDbReader, -) -> Result { - let config = duckdb::Config::default().access_mode(duckdb::AccessMode::ReadOnly)?; - let connection = duckdb::Connection::open_with_flags(&reader.database, config)?; - if let Some(limit) = reader.memory_limit { - connection.execute_batch(&format!("SET memory_limit = '{}KiB'", limit / 1024))?; +/// Each database file a source reads now, by its canonical path, with the one instance this process holds on it. DuckDB locks a file with POSIX locks, which belong to the process: closing a second instance on the file would drop the first one's lock too, letting another process write the file mid-read. +static OPEN_FILES: OnceLock>> = OnceLock::new(); + +/// A connection to the reader's database file, opened read-only, so the file is never changed and SQLLoad holds no write lock on it. Reads open at the same time share one instance, given the reader's `memory_limit` and a temporary folder of its own to spill into (not the default folder beside the file). +pub(crate) fn open_read_only(reader: &DuckDbReader) -> Result { + let path = std::fs::canonicalize(&reader.database) + .map_err(|error| DuckDbSourceError::Invalid(format!("{}: {error}", reader.database)))?; + let files = OPEN_FILES.get_or_init(Mutex::default); + let mut open = files.lock().unwrap_or_else(PoisonError::into_inner); + let file = match open.entry(path.clone()) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + let spill = tempfile::Builder::new() + .prefix("sqlload-duckdb-source-") + .tempdir() + .map_err(|error| DuckDbSourceError::Invalid(error.to_string()))?; + let config = duckdb::Config::default() + .access_mode(duckdb::AccessMode::ReadOnly)? + .with("temp_directory", spill.path().display().to_string())?; + let connection = duckdb::Connection::open_with_flags(&path, config)?; + if let Some(limit) = reader.memory_limit { + connection.execute_batch(&format!("SET memory_limit = '{}KiB'", limit / 1024))?; + } + entry.insert(OpenFile { + reads: 0, + connection, + _spill: spill, + }) + } + }; + let connection = file.connection.try_clone()?; + file.reads += 1; + Ok(ReadOnlyFile { + connection, + _hold: FileHold { path }, + }) +} + +/// The last read of a file to end closes its instance, while it holds the open files' lock, so no new instance on the file opens before this one has closed. +impl Drop for FileHold { + fn drop(&mut self) { + let files = OPEN_FILES.get_or_init(Mutex::default); + let mut open = files.lock().unwrap_or_else(PoisonError::into_inner); + if let Entry::Occupied(mut entry) = open.entry(self.path.clone()) { + entry.get_mut().reads -= 1; + if entry.get().reads == 0 { + drop(entry.remove()); + } + } } - Ok(connection) } -/// A connection of its own to a `motherduck` reader's database, on the reader's bridge: opened on its first use, signed in as a MotherDuck target is, its DuckDB given the reader's `memory_limit`. +/// A connection of its own to a `motherduck` reader's database, on the reader's bridge: opened on its first use, signed in as a MotherDuck target is, its DuckDB given the reader's `memory_limit`, and its memory counted by the load's watchdog while it is open. pub(crate) async fn open_bridge( reader: &DuckDbReader, ) -> Result { @@ -32,7 +78,7 @@ pub(crate) async fn open_bridge( reader.database ))); }; - let memory_limit = reader.memory_limit; + let (memory_limit, guard) = (reader.memory_limit, reader.guard.clone()); let shared = reader .bridge .get_or_try_init(|| async move { @@ -48,10 +94,14 @@ pub(crate) async fn open_bridge( .await .map_err(DuckDbSourceError::Bridge)?; } - Ok::<_, DuckDbSourceError>(bridge) + let watched = guard.map(|guard| watch_child(&guard, bridge.cursor.process.pid)); + Ok::<_, DuckDbSourceError>(SourceBridge { + connection: bridge, + _watched: watched, + }) }) .await?; - Ok(fork_connection(shared)) + Ok(fork_connection(&shared.connection)) } /// Each of `queries`' rows, run in order on one connection to the reader's database, every value as text; a failure names the index of the query that failed. Each query casts its columns to `VARCHAR`. @@ -67,7 +117,7 @@ pub(crate) async fn text_rows( .iter() .enumerate() .map(|(index, sql)| { - local_rows(&connection, sql).map_err(|error| (index, error.into())) + local_rows(&connection.connection, sql).map_err(|error| (index, error.into())) }) .collect() }) diff --git a/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs b/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs index 406449e42..a57cc3f88 100644 --- a/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs +++ b/crates/sqlload/src/duckdb_source/main/duckdb_reader.rs @@ -29,6 +29,7 @@ pub fn duckdb_reader( memory_limit, motherduck: Some(connection.clone()), bridge: std::sync::Arc::default(), + guard: project.memory.guard.clone(), }); } _ => { @@ -53,5 +54,6 @@ pub fn duckdb_reader( memory_limit, motherduck: None, bridge: std::sync::Arc::default(), + guard: project.memory.guard.clone(), }) } diff --git a/crates/sqlload/src/duckdb_source/main/read_rows.rs b/crates/sqlload/src/duckdb_source/main/read_rows.rs index b86fd8c38..ade3880d1 100644 --- a/crates/sqlload/src/duckdb_source/main/read_rows.rs +++ b/crates/sqlload/src/duckdb_source/main/read_rows.rs @@ -41,7 +41,7 @@ pub async fn read_rows( let streaming = tokio::task::spawn_blocking(move || { let read = || -> Result<(), DuckDbSourceError> { let connection = open_read_only(&reader)?; - let mut statement = connection.prepare(&query)?; + let mut statement = connection.connection.prepare(&query)?; let mut held: Vec = Vec::new(); let mut held_rows = 0; for chunk in statement.stream_arrow([])? { diff --git a/crates/sqlload/src/duckdb_source/models.rs b/crates/sqlload/src/duckdb_source/models.rs index 2422dd091..e3d90c701 100644 --- a/crates/sqlload/src/duckdb_source/models.rs +++ b/crates/sqlload/src/duckdb_source/models.rs @@ -1,4 +1,5 @@ use crate::adapters::motherduck_bridge::models::BridgeConnection; +use crate::memory_budget::models::{MemoryGuard, WatchedChild}; use crate::postgres_source::models::{ReadColumn, SourceTable}; use crate::project::models::ConnectionConfig; @@ -11,7 +12,16 @@ pub struct DuckDbReader { /// A `motherduck` source's connection; `None` for a database file. pub(crate) motherduck: Option, /// A `motherduck` source's bridge, shared by the reader's clones and opened on its first read; each read takes a connection of its own on it. - pub(crate) bridge: std::sync::Arc>, + pub(crate) bridge: std::sync::Arc>, + /// The load's memory watchdog, which counts the bridge's memory once it opens; `None` without a budget. + pub(crate) guard: Option>, +} + +/// A `motherduck` source's open bridge, its memory counted by the load's watchdog while it is open. +#[derive(Debug)] +pub(crate) struct SourceBridge { + pub(crate) connection: BridgeConnection, + pub(crate) _watched: Option, } /// What one read selects: a table or a query, its columns, and the rows' filter. @@ -24,3 +34,25 @@ pub struct DuckDbReadRequest<'a> { /// A DuckDB condition on the read columns, already rendered. pub filter: Option<&'a str>, } + +/// A read-only connection to a source's database file, on the one DuckDB instance this process holds on the file while any read of it is open. +#[derive(Debug)] +pub(crate) struct ReadOnlyFile { + /// Declared first, so it closes before `_hold` lets the instance go. + pub(crate) connection: duckdb::Connection, + pub(crate) _hold: FileHold, +} + +/// One open read's share of the instance on a database file, by the file's canonical path; the last share to end closes the instance. +#[derive(Debug)] +pub(crate) struct FileHold { + pub(crate) path: std::path::PathBuf, +} + +/// The instance this process holds on a database file: the reads sharing it, its own connection (closed last), and the folder its queries spill into. +#[derive(Debug)] +pub(crate) struct OpenFile { + pub(crate) reads: usize, + pub(crate) connection: duckdb::Connection, + pub(crate) _spill: tempfile::TempDir, +} diff --git a/crates/sqlload/src/duckdb_source/tests/helpers.rs b/crates/sqlload/src/duckdb_source/tests/helpers.rs index 7b8706e20..f5433dcb1 100644 --- a/crates/sqlload/src/duckdb_source/tests/helpers.rs +++ b/crates/sqlload/src/duckdb_source/tests/helpers.rs @@ -40,3 +40,35 @@ pub(crate) fn shop_project(database: &str, source_lines: &str) -> Result Result { + let exe = std::env::current_exe().map_err(|error| error.to_string())?; + let test = "duckdb_source::tests::read_source::given_open_reads_of_a_file_when_another_process_writes_it_then_it_is_locked_until_the_last_read_ends"; + let output = std::process::Command::new(exe) + .args(["--exact", test, "--nocapture"]) + .env(WRITE_FROM_ANOTHER_PROCESS, path) + .output() + .map_err(|error| error.to_string())?; + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +/// In the process [`write_from_another_process`] starts: try to write the file it names, say whether that worked, and end the process. +pub(crate) fn write_when_asked() { + let Some(path) = std::env::var_os(WRITE_FROM_ANOTHER_PROCESS) else { + return; + }; + let written = duckdb::Connection::open(path) + .and_then(|connection| connection.execute_batch("CREATE TABLE written (x INTEGER)")); + println!( + "{}", + written.map_or_else( + |error| format!("the file was locked: {error}"), + |()| "the file was written".to_owned() + ) + ); + std::process::exit(0); +} diff --git a/crates/sqlload/src/duckdb_source/tests/test_read_source.rs b/crates/sqlload/src/duckdb_source/tests/test_read_source.rs index 24685b3b8..29841982e 100644 --- a/crates/sqlload/src/duckdb_source/tests/test_read_source.rs +++ b/crates/sqlload/src/duckdb_source/tests/test_read_source.rs @@ -4,14 +4,17 @@ use arrow_cast::display::array_value_to_string; use arrow_select::concat::concat_batches; use crate::column_types::models::ColumnType; +use crate::duckdb_source::_helpers::connection::open_read_only; use crate::duckdb_source::main::duckdb_reader::duckdb_reader; use crate::duckdb_source::main::read_catalog::read_catalog; use crate::duckdb_source::main::read_columns::read_columns; use crate::duckdb_source::main::read_rows::read_rows; use crate::duckdb_source::models::DuckDbReadRequest; -use crate::duckdb_source::tests::helpers::{SHOP, runtime, shop_project}; +use crate::duckdb_source::tests::helpers::{ + SHOP, runtime, shop_project, write_from_another_process, write_when_asked, +}; use crate::duckdb_source::tests::test_types::{ - ReadCatalogTestCase, ReadRowsTestCase, RefusedReaderTestCase, + LockedFileTestCase, ReadCatalogTestCase, ReadRowsTestCase, RefusedReaderTestCase, }; #[test] @@ -189,3 +192,46 @@ fn given_a_memory_or_motherduck_database_when_making_a_reader_then_it_is_refused } Ok(()) } + +#[test] +fn given_open_reads_of_a_file_when_another_process_writes_it_then_it_is_locked_until_the_last_read_ends() +-> Result<(), String> { + write_when_asked(); + let test_cases = [ + LockedFileTestCase { + description: "one of two reads ended: the other's lock still holds", + reads: 2, + ended: 1, + expected_said: "the file was locked", + }, + LockedFileTestCase { + description: "every read ended: the file is free to write", + reads: 1, + ended: 1, + expected_said: "the file was written", + }, + ]; + for test_case in test_cases { + let folder = tempfile::tempdir().map_err(|error| error.to_string())?; + let path = folder.path().join("shop.duckdb"); + duckdb::Connection::open(&path) + .and_then(|connection| connection.execute_batch(SHOP)) + .map_err(|error| error.to_string())?; + let project = shop_project(&path.display().to_string(), "")?; + let reader = + duckdb_reader(&project, &project.sources["shop"]).map_err(|error| error.to_string())?; + let mut reads = (0..test_case.reads) + .map(|_| open_read_only(&reader)) + .collect::, _>>() + .map_err(|error| error.to_string())?; + reads.drain(..test_case.ended); + let said = write_from_another_process(&path)?; + drop(reads); + assert!( + said.contains(test_case.expected_said), + "{}: {said}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlload/src/duckdb_source/tests/test_types.rs b/crates/sqlload/src/duckdb_source/tests/test_types.rs index a4cd40312..d02bb8101 100644 --- a/crates/sqlload/src/duckdb_source/tests/test_types.rs +++ b/crates/sqlload/src/duckdb_source/tests/test_types.rs @@ -36,3 +36,13 @@ pub(crate) struct RefusedReaderTestCase { /// Part of the refusal. pub(crate) expected_error: &'static str, } + +/// Reads of a database file open in this process, some of them ended, while another process tries to write the file. +pub(crate) struct LockedFileTestCase { + pub(crate) description: &'static str, + pub(crate) reads: usize, + /// How many of the reads have ended when the other process writes. + pub(crate) ended: usize, + /// What the other process says. + pub(crate) expected_said: &'static str, +} diff --git a/crates/sqlload/src/memory_budget/main/mod.rs b/crates/sqlload/src/memory_budget/main/mod.rs index 28d1f9637..d2ecdd468 100644 --- a/crates/sqlload/src/memory_budget/main/mod.rs +++ b/crates/sqlload/src/memory_budget/main/mod.rs @@ -5,4 +5,5 @@ pub mod run_memory; pub mod run_stopped; pub mod start_watchdog; pub mod table_memory; +pub mod watch_child; pub mod watch_table; diff --git a/crates/sqlload/src/memory_budget/main/watch_child.rs b/crates/sqlload/src/memory_budget/main/watch_child.rs new file mode 100644 index 000000000..7d0a2d17a --- /dev/null +++ b/crates/sqlload/src/memory_budget/main/watch_child.rs @@ -0,0 +1,18 @@ +//! Count a helper process's memory with the load's. + +use std::sync::Arc; + +use crate::memory_budget::models::{MemoryGuard, WatchedChild}; + +/// Have `guard` count process `pid`'s memory with SQLLoad's until the returned handle is dropped, so a process that has ended (and whose id may be reused) stops counting. +pub fn watch_child(guard: &Arc, pid: u32) -> WatchedChild { + guard + .children + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(pid); + WatchedChild { + guard: guard.clone(), + pid, + } +} diff --git a/crates/sqlload/src/memory_budget/models.rs b/crates/sqlload/src/memory_budget/models.rs index 30b7da4ee..577d92141 100644 --- a/crates/sqlload/src/memory_budget/models.rs +++ b/crates/sqlload/src/memory_budget/models.rs @@ -169,3 +169,23 @@ pub struct WatchedTable { pub guard: std::sync::Arc, pub table: std::sync::Arc, } + +/// A helper process whose memory a guard counts with SQLLoad's, until this is dropped. +#[derive(Debug)] +pub struct WatchedChild { + pub guard: std::sync::Arc, + pub pid: u32, +} + +impl Drop for WatchedChild { + fn drop(&mut self) { + let mut children = self + .guard + .children + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(index) = children.iter().position(|pid| *pid == self.pid) { + children.swap_remove(index); + } + } +} diff --git a/crates/sqlload/src/memory_budget/tests/test_types.rs b/crates/sqlload/src/memory_budget/tests/test_types.rs index 4bcc69443..e584a1d0b 100644 --- a/crates/sqlload/src/memory_budget/tests/test_types.rs +++ b/crates/sqlload/src/memory_budget/tests/test_types.rs @@ -47,3 +47,11 @@ pub(crate) struct MemorySettingTestCase { pub(crate) text: &'static str, pub(crate) expected_setting: Result, } + +/// Helper processes a guard watches, the first `dropped` of them ended, and the ids it still counts. +pub(crate) struct WatchedChildTestCase { + pub(crate) description: &'static str, + pub(crate) pids: &'static [u32], + pub(crate) dropped: usize, + pub(crate) expected_counted: &'static [u32], +} diff --git a/crates/sqlload/src/memory_budget/tests/test_watch.rs b/crates/sqlload/src/memory_budget/tests/test_watch.rs index bea3f75d5..629163907 100644 --- a/crates/sqlload/src/memory_budget/tests/test_watch.rs +++ b/crates/sqlload/src/memory_budget/tests/test_watch.rs @@ -1,5 +1,8 @@ use crate::memory_budget::_helpers::watch::run_stop_due; -use crate::memory_budget::tests::test_types::RunStopTestCase; +use crate::memory_budget::main::start_watchdog::start_watchdog; +use crate::memory_budget::main::watch_child::watch_child; +use crate::memory_budget::models::{BudgetOrigin, MemoryBudget}; +use crate::memory_budget::tests::test_types::{RunStopTestCase, WatchedChildTestCase}; #[test] fn given_readings_past_95_percent_when_watching_then_the_run_stops_only_when_no_table_stopping_frees_memory_in_time() @@ -63,3 +66,55 @@ fn given_readings_past_95_percent_when_watching_then_the_run_stops_only_when_no_ ); } } + +#[test] +fn given_watched_children_when_some_handles_drop_then_the_guard_counts_only_the_others() { + let test_cases = [ + WatchedChildTestCase { + description: "both children count while open", + pids: &[41, 42], + dropped: 0, + expected_counted: &[41, 42], + }, + WatchedChildTestCase { + description: "an ended child's id stops counting", + pids: &[41, 42], + dropped: 1, + expected_counted: &[42], + }, + WatchedChildTestCase { + description: "no child counts once all have ended", + pids: &[41, 42], + dropped: 2, + expected_counted: &[], + }, + ]; + for test_case in test_cases { + let guard = start_watchdog(MemoryBudget { + bytes: u64::MAX, + origin: BudgetOrigin::Setting, + limit: None, + overhead: 0, + duckdb: None, + duckdb_targets: 0, + table_share: 0, + tables_at_once: 1, + }); + let mut handles: Vec<_> = test_case + .pids + .iter() + .map(|pid| watch_child(&guard, *pid)) + .collect(); + handles.drain(..test_case.dropped); + let counted = guard + .children + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + assert_eq!( + counted, test_case.expected_counted, + "{}", + test_case.description + ); + } +} From 1e76b16eb7063afbaa1a55c9fa5cf97255a30973 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:54:02 +0000 Subject: [PATCH 7/9] fix: give a shared DuckDB source instance one memory_limit per open read Reads of one file sharing an instance now get the limit separate instances had between them, and an instance whose first connection fails is never left open. --- .../src/duckdb_source/_helpers/connection.rs | 47 +++++++++++----- crates/sqlload/src/duckdb_source/models.rs | 2 + .../src/duckdb_source/tests/helpers.rs | 5 +- .../duckdb_source/tests/test_read_source.rs | 54 ++++++++++++++++++- .../src/duckdb_source/tests/test_types.rs | 9 ++++ 5 files changed, 101 insertions(+), 16 deletions(-) diff --git a/crates/sqlload/src/duckdb_source/_helpers/connection.rs b/crates/sqlload/src/duckdb_source/_helpers/connection.rs index 8dbbccfc2..57fcf2aee 100644 --- a/crates/sqlload/src/duckdb_source/_helpers/connection.rs +++ b/crates/sqlload/src/duckdb_source/_helpers/connection.rs @@ -19,14 +19,18 @@ pub(crate) type TextRows = Vec>>; /// Each database file a source reads now, by its canonical path, with the one instance this process holds on it. DuckDB locks a file with POSIX locks, which belong to the process: closing a second instance on the file would drop the first one's lock too, letting another process write the file mid-read. static OPEN_FILES: OnceLock>> = OnceLock::new(); -/// A connection to the reader's database file, opened read-only, so the file is never changed and SQLLoad holds no write lock on it. Reads open at the same time share one instance, given the reader's `memory_limit` and a temporary folder of its own to spill into (not the default folder beside the file). +/// A connection to the reader's database file, opened read-only, so the file is never changed and SQLLoad holds no write lock on it. Reads open at the same time share one instance, with a temporary folder of its own to spill into (not the default folder beside the file) and a `memory_limit` of the reader's limit for each of them, as separate instances would have had. pub(crate) fn open_read_only(reader: &DuckDbReader) -> Result { let path = std::fs::canonicalize(&reader.database) .map_err(|error| DuckDbSourceError::Invalid(format!("{}: {error}", reader.database)))?; let files = OPEN_FILES.get_or_init(Mutex::default); let mut open = files.lock().unwrap_or_else(PoisonError::into_inner); - let file = match open.entry(path.clone()) { - Entry::Occupied(entry) => entry.into_mut(), + let (file, connection) = match open.entry(path.clone()) { + Entry::Occupied(entry) => { + let file = entry.into_mut(); + let connection = file.connection.try_clone()?; + (file, connection) + } Entry::Vacant(entry) => { let spill = tempfile::Builder::new() .prefix("sqlload-duckdb-source-") @@ -35,26 +39,39 @@ pub(crate) fn open_read_only(reader: &DuckDbReader) -> Result Result<(), duckdb::Error> { + let Some(limit) = file.read_limit else { + return Ok(()); + }; + let total = limit.saturating_mul(file.reads as u64) / 1024; + file.connection + .execute_batch(&format!("SET memory_limit = '{total}KiB'")) } -/// The last read of a file to end closes its instance, while it holds the open files' lock, so no new instance on the file opens before this one has closed. +/// A read ending lowers its file's shared `memory_limit` (a limit DuckDB can't lower yet stays as it was); the last read to end closes the instance, while it holds the open files' lock, so no new instance on the file opens before this one has closed. impl Drop for FileHold { fn drop(&mut self) { let files = OPEN_FILES.get_or_init(Mutex::default); @@ -63,6 +80,8 @@ impl Drop for FileHold { entry.get_mut().reads -= 1; if entry.get().reads == 0 { drop(entry.remove()); + } else { + let _ = share_limit(entry.get()); } } } diff --git a/crates/sqlload/src/duckdb_source/models.rs b/crates/sqlload/src/duckdb_source/models.rs index e3d90c701..a57a4dae9 100644 --- a/crates/sqlload/src/duckdb_source/models.rs +++ b/crates/sqlload/src/duckdb_source/models.rs @@ -53,6 +53,8 @@ pub(crate) struct FileHold { #[derive(Debug)] pub(crate) struct OpenFile { pub(crate) reads: usize, + /// Each read's `memory_limit`; the instance gets it once for each open read. + pub(crate) read_limit: Option, pub(crate) connection: duckdb::Connection, pub(crate) _spill: tempfile::TempDir, } diff --git a/crates/sqlload/src/duckdb_source/tests/helpers.rs b/crates/sqlload/src/duckdb_source/tests/helpers.rs index f5433dcb1..99b4eef77 100644 --- a/crates/sqlload/src/duckdb_source/tests/helpers.rs +++ b/crates/sqlload/src/duckdb_source/tests/helpers.rs @@ -66,7 +66,10 @@ pub(crate) fn write_when_asked() { println!( "{}", written.map_or_else( - |error| format!("the file was locked: {error}"), + |error| format!( + "the file was {}: {error}", + ["not opened", "locked"][usize::from(error.to_string().contains("lock"))] + ), |()| "the file was written".to_owned() ) ); diff --git a/crates/sqlload/src/duckdb_source/tests/test_read_source.rs b/crates/sqlload/src/duckdb_source/tests/test_read_source.rs index 29841982e..c1567ea0d 100644 --- a/crates/sqlload/src/duckdb_source/tests/test_read_source.rs +++ b/crates/sqlload/src/duckdb_source/tests/test_read_source.rs @@ -9,12 +9,13 @@ use crate::duckdb_source::main::duckdb_reader::duckdb_reader; use crate::duckdb_source::main::read_catalog::read_catalog; use crate::duckdb_source::main::read_columns::read_columns; use crate::duckdb_source::main::read_rows::read_rows; -use crate::duckdb_source::models::DuckDbReadRequest; +use crate::duckdb_source::models::{DuckDbReadRequest, DuckDbReader}; use crate::duckdb_source::tests::helpers::{ SHOP, runtime, shop_project, write_from_another_process, write_when_asked, }; use crate::duckdb_source::tests::test_types::{ LockedFileTestCase, ReadCatalogTestCase, ReadRowsTestCase, RefusedReaderTestCase, + SharedLimitTestCase, }; #[test] @@ -235,3 +236,54 @@ fn given_open_reads_of_a_file_when_another_process_writes_it_then_it_is_locked_u } Ok(()) } + +#[test] +fn given_reads_sharing_a_file_when_they_open_and_end_then_its_memory_limit_is_one_read_limit_each() +-> Result<(), String> { + let test_cases = [ + SharedLimitTestCase { + description: "one read gets its own limit", + reads: 1, + ended: 0, + expected_limit: "128.0 MiB", + }, + SharedLimitTestCase { + description: "three reads at once get three limits", + reads: 3, + ended: 0, + expected_limit: "384.0 MiB", + }, + SharedLimitTestCase { + description: "two of three ended: one limit is left", + reads: 3, + ended: 2, + expected_limit: "128.0 MiB", + }, + ]; + for test_case in test_cases { + let folder = tempfile::tempdir().map_err(|error| error.to_string())?; + let path = folder.path().join("shop.duckdb"); + duckdb::Connection::open(&path) + .and_then(|connection| connection.execute_batch(SHOP)) + .map_err(|error| error.to_string())?; + let project = shop_project(&path.display().to_string(), "")?; + let reader = DuckDbReader { + memory_limit: Some(128 * 1024 * 1024), + ..duckdb_reader(&project, &project.sources["shop"]) + .map_err(|error| error.to_string())? + }; + let mut reads = (0..test_case.reads) + .map(|_| open_read_only(&reader)) + .collect::, _>>() + .map_err(|error| error.to_string())?; + reads.drain(..test_case.ended); + let limit: String = reads[0] + .connection + .query_row("SELECT current_setting('memory_limit')", [], |row| { + row.get(0) + }) + .map_err(|error| error.to_string())?; + assert_eq!(limit, test_case.expected_limit, "{}", test_case.description); + } + Ok(()) +} diff --git a/crates/sqlload/src/duckdb_source/tests/test_types.rs b/crates/sqlload/src/duckdb_source/tests/test_types.rs index d02bb8101..6bdc7bc44 100644 --- a/crates/sqlload/src/duckdb_source/tests/test_types.rs +++ b/crates/sqlload/src/duckdb_source/tests/test_types.rs @@ -46,3 +46,12 @@ pub(crate) struct LockedFileTestCase { /// What the other process says. pub(crate) expected_said: &'static str, } + +/// Reads of one database file open in this process, some of them ended, each with a 128 MiB `memory_limit`, and the limit their shared instance has. +pub(crate) struct SharedLimitTestCase { + pub(crate) description: &'static str, + pub(crate) reads: usize, + pub(crate) ended: usize, + /// `current_setting('memory_limit')` as DuckDB prints it. + pub(crate) expected_limit: &'static str, +} From c09e2d8464445f5a5a609dd9e894424ab907a07f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:00:12 +0000 Subject: [PATCH 8/9] test: expect duckdb and motherduck in the sql and where refusals --- crates/sqlload/src/project/tests/test_parse_project.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/sqlload/src/project/tests/test_parse_project.rs b/crates/sqlload/src/project/tests/test_parse_project.rs index f3c36d002..3bc750fe0 100644 --- a/crates/sqlload/src/project/tests/test_parse_project.rs +++ b/crates/sqlload/src/project/tests/test_parse_project.rs @@ -435,9 +435,9 @@ fn given_an_invalid_project_when_parsing_then_every_problem_is_named() { expected_message: "unknown variant `avro`", }, ParseProjectErrorTestCase { - description: "a query table is for snowflake, odbc, bigquery and databricks sources", + description: "a query table is for snowflake, odbc, bigquery, databricks, duckdb and motherduck sources", text: "[sources.app]\nkind = \"postgres\"\nconnection = \"pg\"\ninclude = [\"*\"]\n[sources.app.tables.\"public.a\"]\nsql = \"SELECT 1\"\n", - expected_message: "[sources.app] sql is for kind = \"snowflake\", \"odbc\", \"bigquery\" and \"databricks\"", + expected_message: "[sources.app] sql is for kind = \"snowflake\", \"odbc\", \"bigquery\", \"databricks\", \"duckdb\" and \"motherduck\"", }, ParseProjectErrorTestCase { description: "a databricks connection a target loads into needs a volume to stage files in", @@ -447,12 +447,12 @@ fn given_an_invalid_project_when_parsing_then_every_problem_is_named() { ParseProjectErrorTestCase { description: "where narrows a database table's SELECT, so a files source refuses it", text: "[connections.lake]\nkind = \"filesystem\"\nroot = \"/data\"\n[sources.raw]\nkind = \"files\"\nconnection = \"lake\"\npath = \"raw/\"\nformat = \"csv\"\nwhere = \"a = 1\"\n", - expected_message: "[sources.raw] where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\" and \"databricks\"", + expected_message: "[sources.raw] where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\", \"databricks\", \"duckdb\" and \"motherduck\"", }, ParseProjectErrorTestCase { description: "a postgres_cdc source captures every change, so it refuses where on a table", text: "[sources.app]\nkind = \"postgres_cdc\"\nconnection = \"pg\"\ninclude = [\"public.a\"]\n[sources.app.tables.\"public.a\"]\nwhere = \"a = 1\"\n", - expected_message: "[sources.app] where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\" and \"databricks\"", + expected_message: "[sources.app] where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\", \"databricks\", \"duckdb\" and \"motherduck\"", }, ParseProjectErrorTestCase { description: "a query table takes its condition in its sql", From e2b2fc159b810783fe1e06617870f4852a57d05d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:13:00 +0000 Subject: [PATCH 9/9] fix(duckdb): read primary keys without the json extension The catalog's primary-key query used to_json, which DuckDB's bundled build doesn't include, so listing a file's tables downloaded the json extension first. That failed offline, and on Windows two reads installing it at once collided. The query now returns one key column a row, in key order, using only DuckDB's own functions. The lock test also counts Windows' sharing violation as the lock. --- .../src/duckdb_source/_helpers/catalog.rs | 4 +- crates/sqlload/src/duckdb_source/constants.rs | 7 +- .../src/duckdb_source/tests/helpers.rs | 7 +- .../duckdb_source/tests/test_read_source.rs | 79 ++++++++++++++++++- .../src/duckdb_source/tests/test_types.rs | 8 ++ 5 files changed, 96 insertions(+), 9 deletions(-) diff --git a/crates/sqlload/src/duckdb_source/_helpers/catalog.rs b/crates/sqlload/src/duckdb_source/_helpers/catalog.rs index 3fbf51a1c..cf6158dc8 100644 --- a/crates/sqlload/src/duckdb_source/_helpers/catalog.rs +++ b/crates/sqlload/src/duckdb_source/_helpers/catalog.rs @@ -59,9 +59,7 @@ pub(crate) fn catalog_tables( } for row in keys { if let Some(table) = tables.get_mut(&(text(row, 0), text(row, 1))) { - table.primary_key = serde_json::from_str(&text(row, 2)).map_err(|error| { - DuckDbSourceError::Invalid(format!("{}'s primary key: {error}", table.name)) - })?; + table.primary_key.push(text(row, 2)); } } Ok(tables.into_values().collect()) diff --git a/crates/sqlload/src/duckdb_source/constants.rs b/crates/sqlload/src/duckdb_source/constants.rs index 355091fa8..f26dbe036 100644 --- a/crates/sqlload/src/duckdb_source/constants.rs +++ b/crates/sqlload/src/duckdb_source/constants.rs @@ -16,10 +16,13 @@ pub const BATCHES_AHEAD: usize = 4; /// How DuckDB writes `true` cast to `VARCHAR`. pub const TRUE_TEXT: &str = "true"; -/// The catalog queries, every value cast to `VARCHAR` so a file and the MotherDuck bridge answer alike, all of the opened database's own: its tables and views (schema, name, comment, estimated rows), their columns in order (schema, table, name, type, nullable), and their primary keys (schema, table, the key's columns as a JSON list). +/// The catalog queries, every value cast to `VARCHAR` so a file and the MotherDuck bridge answer alike, all of the opened database's own: its tables and views (schema, name, comment, estimated rows), their columns in order (schema, table, name, type, nullable), and their primary keys (schema, table, one key column a row, in key order). They use only DuckDB's own functions, none of an extension it would have to download first. pub const CATALOG_QUERIES: [&str; 3] = [ "SELECT schema_name, table_name, comment, CAST(estimated_size AS VARCHAR) FROM duckdb_tables() WHERE database_name = current_database() \ UNION ALL SELECT schema_name, view_name, comment, '-1' FROM duckdb_views() WHERE database_name = current_database() AND NOT internal", "SELECT schema_name, table_name, column_name, data_type, CAST(is_nullable AS VARCHAR) FROM duckdb_columns() WHERE database_name = current_database() ORDER BY schema_name, table_name, column_index", - "SELECT schema_name, table_name, CAST(to_json(constraint_column_names) AS VARCHAR) FROM duckdb_constraints() WHERE database_name = current_database() AND constraint_type = 'PRIMARY KEY'", + "SELECT schema_name, table_name, column_name FROM (\ + SELECT schema_name, table_name, unnest(constraint_column_names) AS column_name, generate_subscripts(constraint_column_names, 1) AS position \ + FROM duckdb_constraints() WHERE database_name = current_database() AND constraint_type = 'PRIMARY KEY'\ + ) ORDER BY schema_name, table_name, position", ]; diff --git a/crates/sqlload/src/duckdb_source/tests/helpers.rs b/crates/sqlload/src/duckdb_source/tests/helpers.rs index 99b4eef77..07aa18bdf 100644 --- a/crates/sqlload/src/duckdb_source/tests/helpers.rs +++ b/crates/sqlload/src/duckdb_source/tests/helpers.rs @@ -68,7 +68,12 @@ pub(crate) fn write_when_asked() { written.map_or_else( |error| format!( "the file was {}: {error}", - ["not opened", "locked"][usize::from(error.to_string().contains("lock"))] + // DuckDB says "lock" on Unix; on Windows the lock is a sharing violation. + ["not opened", "locked"][usize::from( + ["lock", "being used by another process"] + .iter() + .any(|said| error.to_string().contains(said)) + )] ), |()| "the file was written".to_owned() ) diff --git a/crates/sqlload/src/duckdb_source/tests/test_read_source.rs b/crates/sqlload/src/duckdb_source/tests/test_read_source.rs index c1567ea0d..7e33b578d 100644 --- a/crates/sqlload/src/duckdb_source/tests/test_read_source.rs +++ b/crates/sqlload/src/duckdb_source/tests/test_read_source.rs @@ -4,7 +4,9 @@ use arrow_cast::display::array_value_to_string; use arrow_select::concat::concat_batches; use crate::column_types::models::ColumnType; -use crate::duckdb_source::_helpers::connection::open_read_only; +use crate::duckdb_source::_helpers::catalog::catalog_tables; +use crate::duckdb_source::_helpers::connection::{TextRows, open_read_only}; +use crate::duckdb_source::constants::CATALOG_QUERIES; use crate::duckdb_source::main::duckdb_reader::duckdb_reader; use crate::duckdb_source::main::read_catalog::read_catalog; use crate::duckdb_source::main::read_columns::read_columns; @@ -14,8 +16,8 @@ use crate::duckdb_source::tests::helpers::{ SHOP, runtime, shop_project, write_from_another_process, write_when_asked, }; use crate::duckdb_source::tests::test_types::{ - LockedFileTestCase, ReadCatalogTestCase, ReadRowsTestCase, RefusedReaderTestCase, - SharedLimitTestCase, + LockedFileTestCase, PrimaryKeyTestCase, ReadCatalogTestCase, ReadRowsTestCase, + RefusedReaderTestCase, SharedLimitTestCase, }; #[test] @@ -55,6 +57,77 @@ fn given_a_duckdb_file_when_listing_its_catalog_then_include_and_exclude_pick_ta Ok(()) } +#[test] +fn given_duckdb_without_extensions_when_listing_its_catalog_then_every_query_answers_and_keys_keep_their_order() +-> Result<(), String> { + let config = duckdb::Config::default() + .with("autoinstall_known_extensions", "false") + .and_then(|config| config.with("autoload_known_extensions", "false")) + .map_err(|error| error.to_string())?; + let connection = + duckdb::Connection::open_in_memory_with_flags(config).map_err(|error| error.to_string())?; + connection + .execute_batch(&format!( + "{SHOP} CREATE TABLE sales.pairs (a INTEGER, b INTEGER, c INTEGER, PRIMARY KEY (c, a));" + )) + .map_err(|error| error.to_string())?; + let mut answers: Vec = Vec::new(); + for query in CATALOG_QUERIES { + let mut statement = connection + .prepare(query) + .map_err(|error| format!("{query}: {error}"))?; + let width = statement.column_count(); + let rows = statement + .query_map([], |row| { + (0..width) + .map(|index| row.get::<_, Option>(index)) + .collect() + }) + .and_then(Iterator::collect) + .map_err(|error| format!("{query}: {error}"))?; + answers.push(rows); + } + let tables = catalog_tables(&answers, (&["sales.*".to_owned()], &[])) + .map_err(|error| error.to_string())?; + let test_cases = [ + PrimaryKeyTestCase { + description: "a one-column key", + table: "sales.orders", + expected_key: &["id"], + }, + PrimaryKeyTestCase { + description: "a key declared out of column order keeps its declared order", + table: "sales.pairs", + expected_key: &["c", "a"], + }, + PrimaryKeyTestCase { + description: "a table without a key", + table: "sales.tmp_scratch", + expected_key: &[], + }, + ]; + for test_case in test_cases { + let key = tables + .iter() + .find(|table| table.qualified_name() == test_case.table) + .map(|table| table.primary_key.clone()); + assert_eq!( + key.as_deref(), + Some( + test_case + .expected_key + .iter() + .map(|column| (*column).to_owned()) + .collect::>() + .as_slice() + ), + "{}", + test_case.description + ); + } + Ok(()) +} + #[test] fn given_a_duckdb_file_when_reading_a_table_or_query_then_rows_load_typed_and_the_file_is_left_as_it_was() -> Result<(), String> { diff --git a/crates/sqlload/src/duckdb_source/tests/test_types.rs b/crates/sqlload/src/duckdb_source/tests/test_types.rs index 6bdc7bc44..39bc949cf 100644 --- a/crates/sqlload/src/duckdb_source/tests/test_types.rs +++ b/crates/sqlload/src/duckdb_source/tests/test_types.rs @@ -17,6 +17,14 @@ pub(crate) struct ReadCatalogTestCase { pub(crate) expected_tables: &'static [&'static str], } +/// The primary key the catalog reports for one table, in key order. +pub(crate) struct PrimaryKeyTestCase { + pub(crate) description: &'static str, + /// The table's qualified name as the catalog lists it. + pub(crate) table: &'static str, + pub(crate) expected_key: &'static [&'static str], +} + /// One read of a table or query table the shop source lists. pub(crate) struct ReadRowsTestCase { pub(crate) description: &'static str,