Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 75 additions & 0 deletions crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
//! Whether a destination reaches the files it would read itself.

use crate::adapters::databricks::main::stage::_check_folder::check_folder;
use crate::adapters::databricks::main::stage::_external_locations::external_locations;
use crate::adapters::databricks::models::DatabricksDestination;
use crate::adapters::dispatch::models::Destination;
#[cfg(feature = "duckdb")]
use crate::adapters::duckdb::main::_check_files::check_files;
use crate::files_source::models::FilesRead;
use crate::loader::models::FilesSource;

/// Check, at plan, that `destination` reads the files of `source` in S3 where they lie, for a table nothing sends through SQLLoad, with `read` its planned read: Databricks looks up its Unity Catalog external locations once per run, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers, and lists the folder when one covers it and it holds files to load; MotherDuck reads the first file to load on its servers. The error says why it can't, with the fix.
pub async fn check_reach(
destination: &Destination,
source: &FilesSource,
read: Option<&FilesRead>,
) -> Result<(), String> {
let first = read.and_then(|read| read.files.first());
let folder = source.location.to_string();
match destination {
Destination::Databricks(databricks) => {
databricks_reach(databricks, first.map(|_| folder.as_str())).await
}
#[cfg(feature = "duckdb")]
Destination::DuckDb(duckdb) => {
let Some(data) = first else {
return Ok(());
};
let url = format!("{folder}{}", data.file.relative_key);
check_files(duckdb, &source.location.store, &url)
.await
.map_err(|error| {
format!(
"{url} cannot be read by MotherDuck on its servers: {}. Fix: MotherDuck's servers read the bucket from their own network, with the keys SQLLoad's S3 reads use (the AWS_* variables, else the connection's profile or AWS_PROFILE, else the AWS credential chain on this machine): give those keys s3:GetObject and s3:ListBucket on it, and kms:Decrypt on its KMS key if it has one, and let the bucket policy allow reads from outside your network or VPC endpoint; a bucket only your network may reach can't load into MotherDuck",
first_line(&error.to_string())
)
})
}
_ => Ok(()),
}
}

/// Databricks' external locations, and whether it can list `folder`, which holds files to load, when one of them covers it (a folder holding no files yet does not list).
async fn databricks_reach(
databricks: &DatabricksDestination,
folder: Option<&str>,
) -> Result<(), String> {
let locations = external_locations(databricks)
.await
.map_err(|error| format!("cannot list Databricks' external locations: {error}"))?;
let Some(folder) = folder else {
return Ok(());
};
let Some(url) = locations
.iter()
.find(|url| folder.starts_with(url.as_str()))
else {
return Ok(());
};
check_folder(databricks, folder).await.map_err(|error| {
format!(
"{folder} cannot be read by Databricks through the external location at {url}: {}. Fix: when Databricks refuses access, grant the service principal READ FILES on that external location",
first_line(&error.to_string())
)
})
}

/// The first line of `error`, without a trailing colon: the services follow it with pages of detail.
fn first_line(error: &str) -> &str {
error
.lines()
.next()
.unwrap_or_default()
.trim_end_matches([':', ' '])
}
32 changes: 0 additions & 32 deletions crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs

This file was deleted.

3 changes: 2 additions & 1 deletion crates/sqlload/src/adapters/dispatch/main/route/mod.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
pub mod check_reach;
pub mod choose_stage;
pub mod direct_blocked;
pub mod list_reach;
pub mod route_warning;
pub mod sends_rows;
pub mod stage_misses;
pub mod unload_blocked;
pub mod unload_refused;
pub mod writer_accepts;
19 changes: 19 additions & 0 deletions crates/sqlload/src/adapters/dispatch/main/route/unload_refused.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
//! Whether a destination reads a warehouse's unloaded files itself.

use crate::adapters::dispatch::models::Destination;
use crate::routing::models::Store;

/// Why `destination` doesn't read the files a warehouse source unloads to `unload_to` itself, so SQLLoad reads them: MotherDuck reads only a files source's Parquet on its servers so far; `None` otherwise.
pub fn unload_refused(destination: &Destination, unload_to: &[Store]) -> Option<String> {
if unload_to.is_empty() {
return None;
}
match destination {
#[cfg(feature = "duckdb")]
Destination::DuckDb(duckdb) if duckdb.motherduck => Some(
"MotherDuck reads only a files source's Parquet in S3 itself, not a warehouse's unload"
.to_owned(),
),
_ => None,
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,14 @@ use crate::adapters::snowflake::models::StageRoute;
use crate::routing::models::WriterAccepts;
use crate::routing::models::{FileFormat, Store};

/// What `destination` takes over the chosen `stage`: Arrow batches (DuckDB also reads Parquet in S3 or on this machine itself, but MotherDuck does not; ClickHouse reads Parquet in S3), or what its stage route takes.
/// What `destination` takes over the chosen `stage`: Arrow batches (DuckDB also reads Parquet in S3 or on this machine itself, MotherDuck Parquet in S3 on its servers, and ClickHouse Parquet in S3), or what its stage route takes.
pub fn writer_accepts(destination: &Destination, stage: Option<&StageRoute>) -> WriterAccepts {
match destination {
#[cfg(feature = "duckdb")]
Destination::DuckDb(duckdb) if duckdb.motherduck => WriterAccepts {
arrow: true,
pulls_from: vec![Store::ObjectStore("s3://".to_owned())],
pull_formats: vec![FileFormat::Parquet],
..WriterAccepts::default()
},
#[cfg(feature = "duckdb")]
Expand Down
6 changes: 4 additions & 2 deletions crates/sqlload/src/adapters/duckdb/_helpers/direct_load.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use crate::adapters::contract::models::{DirectBatch, DirectLoaded};
use crate::adapters::duckdb::_helpers::pull_sql::{PulledFile, pull_insert};
use crate::adapters::duckdb::main::_direct_problem::direct_problem;
use crate::adapters::duckdb::main::_execute_statements::execute_statements;
use crate::adapters::duckdb::main::_open_bucket::open_bucket;
use crate::adapters::duckdb::main::_pull_files::pull_files;
use crate::adapters::duckdb::models::DuckDbWriter;
use crate::files_source::constants::{SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN};
Expand Down Expand Up @@ -54,15 +55,16 @@ impl DuckDbWriter {
let insert = pull_insert(
target.staging.unwrap_or(target.table),
(carried, &in_files),
&files,
(&files, self.destination.motherduck),
(target.compat, target.load_id, batch.loaded_at),
);
open_bucket(&self.destination, &source.location.store).await?;
let transaction = target.direct || target.builds;
if transaction && !self.open_transaction {
execute_statements(&self.destination, vec![BEGIN_STATEMENT.to_owned()]).await?;
self.open_transaction = true;
}
let rows = pull_files(&self.destination, &source.location.store, insert).await?;
let rows = pull_files(&self.destination, insert).await?;
let expected = footer_rows(batch.footers);
if rows != expected {
return Err(LoaderError::Invalid(format!(
Expand Down
96 changes: 90 additions & 6 deletions crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs
Original file line number Diff line number Diff line change
@@ -1,18 +1,26 @@
//! The statement that has DuckDB load Parquet files itself.

use std::hash::{DefaultHasher, Hash, Hasher};
use std::time::Instant;

use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS};
use crate::adapters::contract::models::TableName;
use crate::adapters::contract::types::SqlAdapter;
use crate::adapters::duckdb::constants::UTC_OFFSET;
use crate::adapters::duckdb::models::DuckDbSql;
use crate::adapters::duckdb::_helpers::motherduck_connection::environment_variable;
use crate::adapters::duckdb::constants::{CHAIN_SECRET_REFRESH, UTC_OFFSET};
use crate::adapters::duckdb::errors::DuckDbError;
use crate::adapters::duckdb::models::{DuckDbSql, SetSecret};
use crate::column_types::main::is_variant::is_variant;
use crate::column_types::models::ColumnType;
use crate::file_store::main::s3_keys::s3_keys;
use crate::files_source::constants::SOURCE_ROW_COLUMN;
use crate::load_naming::main::metadata_role::metadata_role;
use crate::load_naming::models::{Compat, MetadataRole};
use crate::postgres_source::models::ReadColumn;
use crate::schema_plan::constants::MERGE_ROW_COLUMN;
use crate::schema_plan::models::DesiredColumn;
use crate::stage_api::models::AwsCredentials;
use crate::store_paths::models::StoreKind;

/// One file DuckDB reads: where it reads it, and the values SQLLoad gives every row of it (its name and version as the files source records them, and its partition values), by column (`None` for a partition folder the file lacks).
#[derive(Debug, Clone, PartialEq, Eq)]
Expand All @@ -21,11 +29,11 @@ pub(crate) struct PulledFile {
pub constants: Vec<(String, Option<String>)>,
}

/// `INSERT INTO into` every one of `columns` from the Parquet `files`, matched by name so a file that lacks a column gives NULL. A column comes from the file's constants when it has one of that name (a list of every file's value, indexed by the row's `file_index`: no join, so DuckDB streams the rows into the table instead of holding them), otherwise from the files when at least one of them holds it (`in_files`, each column with its name in the files, which `naming`, `rename` or numbering may have changed: `f."Order ID"` loads into `order_id`), and is cast to its type; `_sqlload_source_row` counts each file's rows from 1; the load id and load time are the batch's; any other column is NULL. Every file must carry the same constant columns.
/// `INSERT INTO into` every one of `columns` from the Parquet `files`, matched by name so a file that lacks a column gives NULL. A column comes from the file's constants when it has one of that name (a list of every file's value, indexed by the row's `file_index`: no join, so DuckDB streams the rows into the table instead of holding them), otherwise from the files when at least one of them holds it (`in_files`, each column with its name in the files, which `naming`, `rename` or numbering may have changed: `f."Order ID"` loads into `order_id`), and is cast to its type; `_sqlload_source_row` counts each file's rows from 1; the load id and load time are the batch's; any other column is NULL. Every file must carry the same constant columns. Into MotherDuck (`remote`) the files are read on its servers.
pub(crate) fn pull_insert(
into: &TableName,
(columns, in_files): (&[DesiredColumn], &[(String, String)]),
files: &[PulledFile],
(files, remote): (&[PulledFile], bool),
(compat, load_id, loaded_at): (Compat, &str, &str),
) -> String {
let literal = |text: &str| DuckDbSql.render_loader_value_literal(text);
Expand Down Expand Up @@ -75,11 +83,12 @@ pub(crate) fn pull_insert(
.map(|column| identifier(&column.name))
.collect();
format!(
"INSERT INTO {} ({}) SELECT {} FROM read_parquet([{}], union_by_name = true, file_row_number = true, hive_partitioning = false) AS f",
"INSERT INTO {} ({}) SELECT {} FROM read_parquet([{}], union_by_name = true, file_row_number = true, hive_partitioning = false{}) AS f",
DuckDbSql.render_qualified_name(into),
targets.join(", "),
values.join(", "),
urls.join(", ")
urls.join(", "),
on_servers(remote)
)
}

Expand Down Expand Up @@ -215,6 +224,14 @@ fn unloaded_value(output: &ColumnType, quoted: &str) -> String {
}
}

/// The `read_parquet` option that has MotherDuck read the files on its servers (`remote`), so no row passes through this machine; none for SQLLoad's own DuckDB.
fn on_servers(remote: bool) -> &'static str {
match remote {
true => ", MD_RUN = REMOTE",
false => "",
}
}

/// The statement, with its parameters, that gives DuckDB a temporary secret for `bucket`: scoped to it and named after it (hex, so two buckets never share a name), in `region`, with `keys` (key id, secret and session token, empty when there is none) passed as parameters and never in SQL, or else the AWS credential chain.
pub(crate) fn s3_secret(
bucket: &str,
Expand Down Expand Up @@ -246,3 +263,70 @@ pub(crate) fn default_region(variable: &dyn Fn(&str) -> Option<String>) -> Strin
.or_else(|| set("AWS_DEFAULT_REGION"))
.unwrap_or_else(|| "us-east-1".to_owned())
}

/// What DuckDB needs to read files in an S3 bucket itself: the extensions to load (installing them on first use) and the [temporary secret](s3_secret), with its parameters, whose keys come from the AWS credential chain when `chain`.
pub(crate) struct BucketAccess {
pub bucket: String,
pub extensions: &'static str,
pub statement: String,
pub parameters: Vec<String>,
pub chain: bool,
}

impl BucketAccess {
/// The secret as it is set `at`, with a hash of its statement and parameters in place of the keys.
pub(crate) fn set_at(&self, at: Instant) -> SetSecret {
let mut hasher = DefaultHasher::new();
(&self.statement, &self.parameters).hash(&mut hasher);
SetSecret {
hash: hasher.finish(),
chain: self.chain,
at,
}
}
}

/// Whether the secret `set` on the database for a bucket still serves `wanted`, the one a table would set `now`: the same statement and parameters, and, for keys DuckDB resolved through the AWS credential chain when it was set, set less than [`CHAIN_SECRET_REFRESH`] ago, as they may expire.
pub(crate) fn secret_current(set: Option<&SetSecret>, wanted: &SetSecret, now: Instant) -> bool {
set.is_some_and(|set| {
set.hash == wanted.hash
&& (!set.chain || now.saturating_duration_since(set.at) < CHAIN_SECRET_REFRESH)
})
}

/// For files in S3, [`BucketAccess`] for their bucket: the secret in the connection's region, else `AWS_REGION` / `AWS_DEFAULT_REGION` (unset when empty), else us-east-1 as SQLLoad's reader does; with the keys SQLLoad's own S3 reads use (the `AWS_*` variables, else the shared credentials file under the connection's `profile` or `AWS_PROFILE`), passed as parameters and never in SQL, otherwise the AWS credential chain, resolved where DuckDB runs. `None` for files elsewhere.
pub(crate) fn bucket_access(store: &StoreKind) -> Result<Option<BucketAccess>, DuckDbError> {
let StoreKind::S3 {
bucket,
region,
profile,
..
} = store
else {
return Ok(None);
};
let keys = s3_keys(profile.as_deref())
.map_err(DuckDbError::Task)?
.map(secret_parameters);
let extensions = match keys {
Some(_) => "INSTALL httpfs; LOAD httpfs;",
None => "INSTALL httpfs; LOAD httpfs; INSTALL aws; LOAD aws;",
};
let region = region
.clone()
.unwrap_or_else(|| default_region(&environment_variable));
let (statement, parameters) = s3_secret(bucket, region, keys.as_ref());
Ok(Some(BucketAccess {
bucket: bucket.clone(),
extensions,
statement,
parameters,
chain: keys.is_none(),
}))
}

/// `keys` as the `KEY_ID`, `SECRET` and `SESSION_TOKEN` parameters of a DuckDB secret.
fn secret_parameters(keys: AwsCredentials) -> [String; 3] {
let token = keys.session_token.map(|token| token.0).unwrap_or_default();
[keys.key_id, keys.secret.0, token]
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,6 @@ pub(in crate::adapters::duckdb) fn shared_destination(
memory: None,
own_limit: None,
unflushed: Arc::default(),
secrets: Arc::default(),
})
}
4 changes: 3 additions & 1 deletion crates/sqlload/src/adapters/duckdb/_helpers/unloaded_load.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use crate::adapters::contract::constants::{BEGIN_STATEMENT, COMMIT_STATEMENT};
use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch, UnloadedSource};
use crate::adapters::duckdb::_helpers::pull_sql::{UnloadedColumns, unloaded_insert};
use crate::adapters::duckdb::main::_execute_statements::execute_statements;
use crate::adapters::duckdb::main::_open_bucket::open_bucket;
use crate::adapters::duckdb::main::_pull_files::pull_files;
use crate::adapters::duckdb::models::DuckDbWriter;
use crate::loader::errors::LoaderError;
Expand Down Expand Up @@ -48,12 +49,13 @@ impl DuckDbWriter {
&urls,
(target.load_id, batch.loaded_at),
);
open_bucket(&self.destination, &files.folder.store).await?;
let transaction = target.direct || target.builds;
if transaction && !self.open_transaction {
execute_statements(&self.destination, vec![BEGIN_STATEMENT.to_owned()]).await?;
self.open_transaction = true;
}
let rows = pull_files(&self.destination, &files.folder.store, insert).await?;
let rows = pull_files(&self.destination, insert).await?;
if rows != files.rows {
return Err(LoaderError::Invalid(format!(
"DuckDB loaded {rows} rows from the unloaded files in {}, but the unload reports {}; the batch is not committed",
Expand Down
2 changes: 2 additions & 0 deletions crates/sqlload/src/adapters/duckdb/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,5 @@ pub const CHECKPOINT_WAIT: std::time::Duration = std::time::Duration::from_secs(
pub const NO_AUTOCHECKPOINT: &str = "SET checkpoint_threshold = '1TB'";
/// The suffix of a DuckDB file's write-ahead log.
pub const WAL_SUFFIX: &str = ".wal";
/// How long a bucket secret on the AWS credential chain is used before it is set again: DuckDB resolves the chain's keys once, when the secret is set, and temporary keys from a role may have only five minutes left when fetched.
pub const CHAIN_SECRET_REFRESH: std::time::Duration = std::time::Duration::from_secs(240);
Loading
Loading